From 62b266e51996c7ef70397fb5ca55dfd7575061e5 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Thu, 17 Sep 2026 19:23:03 -0400 Subject: [PATCH 001/125] fix(api): check delegated draft permissions before running (#866) The daemon checks `draft.create` before an agent's `draft-reply` request can run or make other work wait. The command handler and operation gate share this check. The draft command also checks that the token covers the requested account. Only `draft.create` is accepted. Tests cover allowed and rejected requests and drafting for the correct account. Refs #666. Co-authored-by: Rod Boev --- cmd/msgvault/cmd/agent_token_test.go | 92 +++++++++++++++++++++++ internal/agentgrant/agentgrant.go | 6 +- internal/api/cli_allowlist_drafts_test.go | 37 +++++++++ internal/api/cli_handlers.go | 2 +- internal/api/cli_run_draft.go | 6 ++ internal/api/operation_gate.go | 19 +++-- internal/api/operation_gate_test.go | 36 +++++++++ 7 files changed, 186 insertions(+), 12 deletions(-) diff --git a/cmd/msgvault/cmd/agent_token_test.go b/cmd/msgvault/cmd/agent_token_test.go index 3d3c6152b..e406104eb 100644 --- a/cmd/msgvault/cmd/agent_token_test.go +++ b/cmd/msgvault/cmd/agent_token_test.go @@ -1,13 +1,16 @@ package cmd import ( + "bufio" "bytes" + "context" "encoding/json" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -19,6 +22,8 @@ import ( "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -105,6 +110,93 @@ type agentTokenListFixture struct { Tokens []agentTokenFixtureView `json:"tokens"` } +func TestDelegatedDraftSourceScopeThroughHTTP(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + providerCalls := 0 + clientFactory := adapter.draftClientFactory + adapter.draftClientFactory = func(ctx context.Context, source *store.Source) (*imaplib.Client, error) { + providerCalls++ + return clientFactory(ctx, source) + } + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key", AgentAccess: true}, + }, + Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(sourceID int64) string { + body, err := json.Marshal(map[string]any{ + "label": "test-agent", + "permissions": []string{"draft.create"}, + "source_ids": []int64{sourceID}, + }) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-test-key") + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + require.Equal(http.StatusCreated, resp.StatusCode) + var issued agentTokenIssueFixture + require.NoError(json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret + } + + run := func(secret string) []api.CLIRunEvent { + args := []string{ + "draft-reply", strconv.FormatInt(fixture.parentID, 10), + "--from", testutil.IMAPTestUsername, "--body", "reply body", "--json", + } + body, err := json.Marshal(map[string]any{"args": args}) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + require.Equal(http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + require.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + require.NoError(scanner.Err()) + return events + } + + events := run(issue(fixture.source.ID)) + require.Len(events, 2) + assert.Equal(cliStreamStdout, events[0].Type) + var result draftReplyOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assert.Equal(draftReplyStatusCreated, result.Status) + assert.Equal(fixture.source.ID, result.SourceID) + assert.Equal("Drafts", result.Mailbox) + assert.Equal("complete", events[1].Type) + assert.Equal(1, providerCalls) + + secondSource, err := fixture.store.GetOrCreateSource("imap", "other@example.com") + require.NoError(err) + events = run(issue(secondSource.ID)) + require.Len(events, 1) + assert.Equal("error", events[0].Type) + assert.Equal("not_permitted", events[0].Error) + assert.Equal(1, providerCalls, "an out-of-grant source must be rejected before provider work") +} + // TestAgentTokenIssueOutputsSecret verifies that the issue subcommand (row 6): // - sends POST /api/v1/agent-tokens with the correct JSON body // - displays the token ID, label, permissions, and one-time secret in plain diff --git a/internal/agentgrant/agentgrant.go b/internal/agentgrant/agentgrant.go index 4474e0f31..2d7904e0f 100644 --- a/internal/agentgrant/agentgrant.go +++ b/internal/agentgrant/agentgrant.go @@ -44,9 +44,13 @@ type Grant struct { CreatedAt time.Time } +func (g Grant) HasPermission(p Permission) bool { + return slices.Contains(g.Permissions, p) +} + // Allows returns true only when p is in the grant AND some SourceRef matches Type and Identifier. func (g Grant) Allows(p Permission, src SourceRef) bool { - if !slices.Contains(g.Permissions, p) { + if !g.HasPermission(p) { return false } for _, s := range g.Sources { diff --git a/internal/api/cli_allowlist_drafts_test.go b/internal/api/cli_allowlist_drafts_test.go index a2135f093..6b8a7f3e4 100644 --- a/internal/api/cli_allowlist_drafts_test.go +++ b/internal/api/cli_allowlist_drafts_test.go @@ -144,6 +144,43 @@ func TestDelegatedCLIRunAdmission(t *testing.T) { }) } +func TestDelegatedCLIRunRequiresGrantedPermission(t *testing.T) { + for _, tc := range []struct { + name string + grant *agentgrant.Grant + code int + calls int + }{ + {name: "draft.create", grant: &agentgrant.Grant{Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}}, code: http.StatusOK, calls: 1}, + {name: "missing permission", grant: &agentgrant.Grant{}, code: http.StatusBadRequest}, + {name: "nil grant", code: http.StatusBadRequest}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + runnerCalls := 0 + stub := &stubSourceStore{} + stub.runFunc = func(context.Context, CLIRunRequest, func(CLIRunEvent) error) error { + runnerCalls++ + return nil + } + srv := &Server{store: stub, logger: testLogger()} + req := httptest.NewRequest(http.MethodPost, "/api/v1/cli/run", bytes.NewBufferString(`{"args":["draft-reply","42"]}`)) + req = req.WithContext(context.WithValue(req.Context(), requestSecurityContextKey{}, requestSecurity{ + auth: requestAuthentication{Mode: AuthModeDelegated, Grant: tc.grant}, + })) + resp := httptest.NewRecorder() + srv.handleCLIRun(resp, req) + assert.Equal(tc.code, resp.Code) + assert.Equal(tc.calls, runnerCalls) + if tc.code != http.StatusOK { + var response ErrorResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&response)) + assert.Equal("command_not_allowed", response.Error) + } + }) + } +} + // TestDelegatedGrantScopesSource is the mutation probe for cli_handlers.go:1315. // It drives a delegated draft-reply through the real handler against a source // that is not in the grant, and asserts the request is refused. diff --git a/internal/api/cli_handlers.go b/internal/api/cli_handlers.go index 379374941..3d64f7ee6 100644 --- a/internal/api/cli_handlers.go +++ b/internal/api/cli_handlers.go @@ -1310,7 +1310,7 @@ func (s *Server) handleCLIRun(w http.ResponseWriter, r *http.Request) { } auth := s.requestAuthentication(r) if auth.Mode == AuthModeDelegated { - if !IsCLIRunDraftReply(req.Args) { + if !delegatedCLIRunAdmitted(req.Args, auth.Grant) { writeError(w, http.StatusBadRequest, "command_not_allowed", "command is not allowed through the daemon CLI runner") return } diff --git a/internal/api/cli_run_draft.go b/internal/api/cli_run_draft.go index a5a94e772..a47ec6cb2 100644 --- a/internal/api/cli_run_draft.go +++ b/internal/api/cli_run_draft.go @@ -1,5 +1,7 @@ package api +import "go.kenn.io/msgvault/internal/agentgrant" + // CLIRunDraftReplyCommand names the daemon CLI command that the daemon runs // in-process instead of spawning a subprocess. const CLIRunDraftReplyCommand = "draft-reply" @@ -10,6 +12,10 @@ func IsCLIRunDraftReply(args []string) bool { return len(args) > 0 && args[0] == CLIRunDraftReplyCommand } +func delegatedCLIRunAdmitted(args []string, grant *agentgrant.Grant) bool { + return grant != nil && IsCLIRunDraftReply(args) && grant.HasPermission(agentgrant.PermissionDraftCreate) +} + // CLIRunCodedError carries a fixed code for the client and the underlying // cause for the daemon log. Clients only ever see Code. type CLIRunCodedError struct { diff --git a/internal/api/operation_gate.go b/internal/api/operation_gate.go index 525de3762..6dc249ecf 100644 --- a/internal/api/operation_gate.go +++ b/internal/api/operation_gate.go @@ -286,8 +286,8 @@ func operationGateMiddleware(gate OperationGate, authorized func(*http.Request) return } security, _ := securityFromRequest(r) - delegated := security.auth.Mode == AuthModeDelegated - shouldGate, label, err := operationGateRequest(r, delegated) + auth := security.auth + shouldGate, label, err := operationGateRequest(r, auth) if err != nil { if errors.Is(err, errCLIRunGateInspectionBodyTooLarge) { writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", @@ -451,7 +451,7 @@ func readOnlyPostRouteRequest(r *http.Request) bool { return pattern != "" } -func operationGateRequest(r *http.Request, delegated bool) (bool, string, error) { +func operationGateRequest(r *http.Request, auth requestAuthentication) (bool, string, error) { if r.URL.Path == DaemonShutdownPath { return false, "", nil } @@ -476,7 +476,7 @@ func operationGateRequest(r *http.Request, delegated bool) (bool, string, error) return true, label, nil } if r.URL.Path == "/api/v1/cli/run" { - label, skip, err := cliRunGateDecision(r, delegated) + label, skip, err := cliRunGateDecision(r, auth) if err != nil { return false, "", err } @@ -533,7 +533,7 @@ var cliRunSelfGatedCommands = map[string]bool{ "backup create": true, } -func cliRunGateDecision(r *http.Request, delegated bool) (label string, skip bool, err error) { +func cliRunGateDecision(r *http.Request, auth requestAuthentication) (label string, skip bool, err error) { if r == nil || r.Body == nil { return "", false, nil } @@ -552,10 +552,9 @@ func cliRunGateDecision(r *http.Request, delegated bool) (label string, skip boo Args []string `json:"args"` } if json.Unmarshal(body, &req) == nil && len(req.Args) > 0 { - // Delegated callers may only reach draft-reply; any other command is - // rejected by the handler before it does any work, so do not take a - // gate slot or surface a label to the owner. - if delegated && !IsCLIRunDraftReply(req.Args) { + // Delegated callers are admitted by the same command and permission + // predicate as the handler, so rejected work never takes a gate slot. + if auth.Mode == AuthModeDelegated && !delegatedCLIRunAdmitted(req.Args, auth.Grant) { return "", true, nil } command := cliRunCommandWords(req.Args) @@ -566,7 +565,7 @@ func cliRunGateDecision(r *http.Request, delegated bool) (label string, skip boo return "msgvault " + command, false, nil } } - if delegated { + if auth.Mode == AuthModeDelegated { // Unparseable or empty-args body: the handler rejects it; do not gate. return "", true, nil } diff --git a/internal/api/operation_gate_test.go b/internal/api/operation_gate_test.go index 56071e9b4..394f5c9fc 100644 --- a/internal/api/operation_gate_test.go +++ b/internal/api/operation_gate_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/personenrichment" "go.kenn.io/msgvault/internal/query" @@ -135,6 +136,41 @@ func TestOperationGateMiddlewareGatesMutatingMethods(t *testing.T) { } } +func TestOperationGateMiddlewareSkipsUnauthorizedDelegatedCLIRun(t *testing.T) { + grant := &agentgrant.Grant{Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}} + for _, tc := range []struct { + name string + grant *agentgrant.Grant + command string + calls int + }{ + {name: "draft.create", grant: grant, command: CLIRunDraftReplyCommand, calls: 1}, + {name: "missing permission", grant: &agentgrant.Grant{}, command: CLIRunDraftReplyCommand}, + {name: "nil grant", command: CLIRunDraftReplyCommand}, + {name: "owner command", grant: grant, command: "remove-account"}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + gate := &recordingOperationGate{allow: true} + handler := operationGateMiddleware(gate, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + body, err := json.Marshal(CLIRunRequest{Args: []string{tc.command, "42"}}) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/api/v1/cli/run", strings.NewReader(string(body))) + req = req.WithContext(context.WithValue(req.Context(), requestSecurityContextKey{}, requestSecurity{ + auth: requestAuthentication{Mode: AuthModeDelegated, Grant: tc.grant}, + })) + resp := httptest.NewRecorder() + handler.ServeHTTP(resp, req) + assert.Equal(http.StatusNoContent, resp.Code) + begin, done := gate.counts() + assert.Equal(tc.calls, begin) + assert.Equal(tc.calls, done) + }) + } +} + func TestOperationGateMiddlewareSkipsDaemonShutdown(t *testing.T) { assert := assert.New(t) From 0316539967df751524d5f9ff7092f7b0f4a05018 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 18 Sep 2026 11:39:34 -0500 Subject: [PATCH 002/125] fix(web): shorten exploration URLs (#853) Shorten Web UI links by omitting defaults, keyboard focus, scroll position, and inactive workspace choices. Ordinary tabs now use readable links such as `?workspace=files&mode=full_text`; filters, layout changes, and selected items use a smaller exploration payload when needed. Keep the complete session state in browser history so Back and Forward restore focus and choices from other tabs. Search mode stays explicit so a shared link does not pick up a different browser's saved preference. Co-authored-by: Wes McKinney --- docs/changelog.md | 3 + docs/web-ui.md | 10 +- web/src/lib/components/shell/AppShell.svelte | 5 +- web/src/lib/explore/state.svelte.ts | 96 ++++++++++++++-- web/src/lib/explore/state.test.ts | 112 ++++++++++++------- web/src/lib/search/modes.ts | 4 +- web/tests/e2e/keyboard.spec.ts | 10 +- web/tests/explore-state.ts | 10 ++ web/tests/files-viewer.spec.ts | 12 +- web/tests/presentations.spec.ts | 7 +- web/tests/relationships.spec.ts | 5 +- web/tests/theme-keyboard.spec.ts | 15 +++ 12 files changed, 211 insertions(+), 78 deletions(-) create mode 100644 web/tests/explore-state.ts diff --git a/docs/changelog.md b/docs/changelog.md index 9b9ba9caa..750e4c4a9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -135,6 +135,9 @@ See [searching](usage/searching.md), [vector search](usage/vector-search.md), ### Browser, terminal, and integrations +- Share shorter Web UI links. Ordinary tabs use readable workspace and search + mode parameters. Links omit defaults, keyboard focus, scroll position, and + choices from other tabs; browser Back and Forward retain that session context. - Use Web Directory to edit profiles, relationships, and employment, review identity matches and facts, merge or split profiles, publish CardDAV contacts, and inspect curated networks and person attachment galleries. diff --git a/docs/web-ui.md b/docs/web-ui.md index 3fe91e3dc..af930ce00 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-09" +last_edited: "2026-09-15" title: Web UI description: Browse messages and files, maintain people, and monitor archive work from your browser. --- @@ -107,8 +107,12 @@ warns that its session cookie travels without TLS. `HttpOnly` and Everything opens as a compact, sortable table of logical entries: one row per email, calendar event, meeting note, other durable item, or chat conversation. Raw chat fragments appear only after drilling into a conversation. Filter, -Group by, Show as, and Search compose into one URL-backed context, so browser -Back and Forward restore the analytical slice and focused item. +Group by, Show as, and Search form a shareable view. Ordinary tabs use short +URLs such as `?workspace=everything&mode=full_text`. Filters, layout changes, +and the selected item appear in the link only when they differ from the +defaults. Keyboard focus, scroll position, and choices from other workspaces +stay out of the link; browser history keeps them so Back and Forward restore +them. Search mode is always explicit: diff --git a/web/src/lib/components/shell/AppShell.svelte b/web/src/lib/components/shell/AppShell.svelte index 111765fca..d284ef440 100644 --- a/web/src/lib/components/shell/AppShell.svelte +++ b/web/src/lib/components/shell/AppShell.svelte @@ -350,7 +350,10 @@ // popstate (see `handleHistoryFocus` below) — so a user who lands by // default, navigates elsewhere, then explicitly clicks back into // Relationships later is never silently bounced away again. - let arrivedWithoutExploreParam = untrack(() => new URLSearchParams(window.location.search).get('explore') === null); + let arrivedWithoutExploreParam = untrack(() => { + const parameters = new URLSearchParams(window.location.search); + return !parameters.has('workspace') && !parameters.has('explore'); + }); let landingFallbackApplied = false; let contextualViewerFile = $state(); let contextualViewerReturnFocus = $state(); diff --git a/web/src/lib/explore/state.svelte.ts b/web/src/lib/explore/state.svelte.ts index 7a186c8c3..d85291743 100644 --- a/web/src/lib/explore/state.svelte.ts +++ b/web/src/lib/explore/state.svelte.ts @@ -187,7 +187,7 @@ export const defaultExploreURLState: ExploreURLState = { interface ExploreWindow { location: Pick; - history: Pick; + history: Pick; addEventListener(type: 'popstate', listener: () => void): void; removeEventListener(type: 'popstate', listener: () => void): void; } @@ -513,21 +513,88 @@ function normalize(value: unknown): ExploreURLState { } as ExploreURLState; } +// Fields that only describe one workspace stay out of the link when another +// workspace is shared; browser history still carries them for Back/Forward. +const WORKSPACE_FIELDS: Partial>> = { + directoryQuery: ['directory'], + directoryContactState: ['directory'], + directoryCategory: ['directory'], + directoryOrganization: ['directory'], + directoryPrimaryChannel: ['directory'], + directoryLastContactAfter: ['directory'], + directoryLastContactBefore: ['directory'], + directorySort: ['directory'], + directoryPersonID: ['directory', 'directory_review'], + reviewKind: ['directory_review'], + identityState: ['directory_review'], + relationshipReviewState: ['directory_review'], + fileSort: ['files'], + fileFilenameQuery: ['files'], + fileMIMEFamilies: ['files'], + personFilePresentation: ['relationships'], + personFileDirections: ['relationships'], + identityQuery: ['relationships'], + identitySort: ['relationships'], + analysisTarget: ['relationships'], + selectedIdentifier: ['relationships'], + relationshipFacet: ['relationships'], + relationshipTarget: ['relationships'], + relationshipShowAll: ['relationships'], + relationshipFiles: ['relationships'], + operationLane: ['operations'], + operationKind: ['operations'], + operationState: ['operations'], + operationStartedFrom: ['operations'], + operationStartedBefore: ['operations'], + operationRunID: ['operations'], + operationStatus: ['operations'], + settingsAuthority: ['settings'] +}; +// Keyboard focus and scroll position live only in browser history. +const SESSION_ONLY_FIELDS = new Set(['activeRow', 'scrollAnchor']); + +function sharedDetails(state: ExploreURLState): Record { + return Object.fromEntries(Object.entries(state).filter(([key, value]) => { + const field = key as keyof ExploreURLState; + if (field === 'schemaVersion' || field === 'workspace' || field === 'searchMode') return false; + if (SESSION_ONLY_FIELDS.has(field)) return false; + const owners = WORKSPACE_FIELDS[field]; + if (owners && !owners.includes(state.workspace)) return false; + return JSON.stringify(value) !== JSON.stringify(defaultExploreURLState[field]); + })); +} + export function serializeExploreURLState(state: ExploreURLState, baseSearch = ''): string { const parameters = new URLSearchParams(baseSearch.startsWith('?') ? baseSearch.slice(1) : baseSearch); - parameters.set(STATE_PARAMETER, JSON.stringify(normalize(state))); + const normalized = normalize(state); + parameters.set('workspace', normalized.workspace); + // An explicit mode keeps a shared link independent of browser preferences. + parameters.set('mode', normalized.searchMode); + const details = sharedDetails(normalized); + if (Object.keys(details).length === 0) parameters.delete(STATE_PARAMETER); + else parameters.set(STATE_PARAMETER, JSON.stringify({ schemaVersion: normalized.schemaVersion, ...details })); return `?${parameters.toString()}`; } +function historyEntry(search: string, state: ExploreURLState): { exploreSearch: string; exploreState: unknown } { + // History entries must be structured-cloneable, so strip reactive proxies. + return { exploreSearch: search, exploreState: JSON.parse(JSON.stringify(state)) }; +} + export function parseExploreURLState(search: string): ExploreURLState { const parameters = new URLSearchParams(search.startsWith('?') ? search.slice(1) : search); const encoded = parameters.get(STATE_PARAMETER); - if (encoded === null) return freshDefaults(); + let details: unknown = {}; try { - return normalize(JSON.parse(encoded)); + if (encoded !== null) details = JSON.parse(encoded); } catch { - return freshDefaults(); + // A malformed detail payload must not discard the selected workspace. } + return normalize({ + ...(isRecord(details) ? details : {}), + ...(parameters.has('workspace') ? { workspace: parameters.get('workspace') } : {}), + ...(parameters.has('mode') ? { searchMode: parameters.get('mode') } : {}), + }); } export class ExploreState { @@ -696,7 +763,11 @@ export class ExploreState { } private readURLState(): ExploreURLState { - const parsed = parseExploreURLState(this.browser.location.search); + const history = this.browser.history.state; + const parsed = isRecord(history) && history.exploreSearch === this.browser.location.search && + isRecord(history.exploreState) + ? normalize(history.exploreState) + : parseExploreURLState(this.browser.location.search); parsed.searchMode = resolveInitialSearchMode( explicitSearchModeFromURL(this.browser.location.search), this.preferenceStorage, @@ -729,8 +800,9 @@ export class ExploreState { .map((key) => [key, this.current[key]]) ) as Partial; const priorEntry = normalize({ ...this.committed, ...transient, ...priorFocus }); - const committedURL = `${this.browser.location.pathname}${serializeExploreURLState(priorEntry, baseSearch)}${this.browser.location.hash}`; - this.browser.history.replaceState(null, '', committedURL); + const priorSearch = serializeExploreURLState(priorEntry, baseSearch); + const committedURL = `${this.browser.location.pathname}${priorSearch}${this.browser.location.hash}`; + this.browser.history.replaceState(historyEntry(priorSearch, priorEntry), '', committedURL); } const next = normalize({ ...this.current, ...effectivePatch }); // Preserve per-field reactivity: transient scroll/column changes must not @@ -742,12 +814,14 @@ export class ExploreState { for (const key of keysToApply) { if (key in next) this.current[key] = next[key]; } - const url = `${this.browser.location.pathname}${serializeExploreURLState(this.current, baseSearch)}${this.browser.location.hash}`; + const search = serializeExploreURLState(this.current, baseSearch); + const url = `${this.browser.location.pathname}${search}${this.browser.location.hash}`; + const history = historyEntry(search, this.current); if (mode === 'push') { - this.browser.history.pushState(null, '', url); + this.browser.history.pushState(history, '', url); this.committed = normalize(this.current); this.pendingSearchPriorFocus = undefined; - } else this.browser.history.replaceState(null, '', url); + } else this.browser.history.replaceState(history, '', url); } } diff --git a/web/src/lib/explore/state.test.ts b/web/src/lib/explore/state.test.ts index c2a822b5b..8a90c3429 100644 --- a/web/src/lib/explore/state.test.ts +++ b/web/src/lib/explore/state.test.ts @@ -17,6 +17,53 @@ import { createAllMatchingSelection, predicateFingerprint } from './selection'; import { SEARCH_MODE_PREFERENCE_KEY } from '../search/modes'; describe('Explore URL state', () => { + it('shares an ordinary tab without defaults or another workspace selection', () => { + const search = serializeExploreURLState({ + ...defaultExploreURLState, + workspace: 'everything', + relationshipTarget: 'cluster:42', + directoryQuery: 'Alex', + fileFilenameQuery: 'invoice', + activeRow: 'message:7', + scrollAnchor: { key: 'message:7', offset: 10 }, + }); + + expect(search).toBe('?workspace=everything&mode=full_text'); + expect(parseExploreURLState(search)).toMatchObject({ + workspace: 'everything', relationshipTarget: null, directoryQuery: '', fileFilenameQuery: '', + }); + }); + + it('restores readable workspace and search-mode parameters with selected filters', () => { + const search = '?workspace=files&mode=hybrid&explore=' + encodeURIComponent(JSON.stringify({ + schemaVersion: 2, + query: 'project notes', + filters: [{ dimension: 'source', values: ['7'] }], + fileMIMEFamilies: ['pdf'], + })); + expect(parseExploreURLState(search)).toMatchObject({ + workspace: 'files', searchMode: 'hybrid', query: 'project notes', + filters: [{ dimension: 'source', values: ['7'] }], fileMIMEFamilies: ['pdf'], + }); + }); + + it('keeps inactive workspace choices in browser history while sharing only the current view', async () => { + window.history.replaceState(null, '', '/'); + const state = new ExploreState(window); + state.commitNavigation({ relationshipTarget: 'cluster:42' }); + state.commitWorkspace('everything'); + expect(window.location.search).toBe('?workspace=everything&mode=full_text'); + + window.history.back(); + await new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); + expect(state.current).toMatchObject({ workspace: 'relationships', relationshipTarget: 'cluster:42' }); + window.history.forward(); + await new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); + state.commitWorkspace('relationships'); + expect(state.current.relationshipTarget).toBe('cluster:42'); + state.destroy(); + }); + it('restores the conflict identity review queue from URL state', () => { const restored = parseExploreURLState(serializeExploreURLState({ ...defaultExploreURLState, @@ -32,6 +79,20 @@ describe('Explore URL state', () => { }); }); + it('keeps the selected person when sharing a Fact review', () => { + const restored = parseExploreURLState(serializeExploreURLState({ + ...defaultExploreURLState, + workspace: 'directory_review', + reviewKind: 'fact', + directoryPersonID: 7, + directoryQuery: 'Alex', + })); + + expect(restored).toMatchObject({ + workspace: 'directory_review', reviewKind: 'fact', directoryPersonID: 7, directoryQuery: '', + }); + }); + it('round-trips the imported relationship queue and normalizes its state independently', () => { const restored = parseExploreURLState(serializeExploreURLState({ ...defaultExploreURLState, @@ -246,22 +307,10 @@ describe('Explore URL state', () => { } }); - it('round-trips every durable field in the versioned envelope', () => { + it('preserves Files filters, layout, and the selected item in shared URLs', () => { const state: ExploreURLState = { - schemaVersion: 2, + ...defaultExploreURLState, workspace: 'files', - directoryQuery: '', - directoryContactState: '', - directoryCategory: '', - directoryOrganization: '', - directoryPrimaryChannel: '', - directoryLastContactAfter: '', - directoryLastContactBefore: '', - directorySort: 'name', - directoryPersonID: null, - reviewKind: 'identity', - identityState: 'candidate', - relationshipReviewState: 'pending', query: 'from:alice quarterly plan', searchMode: 'hybrid', filters: [ @@ -274,32 +323,12 @@ describe('Explore URL state', () => { fileSort: { field: 'filename', direction: 'asc' }, fileFilenameQuery: 'invoice', fileMIMEFamilies: ['pdf', 'image'], - personFilePresentation: 'media', - personFileDirections: ['from_person', 'group'], - identityQuery: 'Shared Name', - identitySort: { field: 'display_label', direction: 'asc' }, - analysisTarget: 'person:42', - selectedIdentifier: 'email:alice@example.com', - relationshipFacet: 'domains', - relationshipTarget: 'domain:example.com', - relationshipShowAll: true, - relationshipFiles: true, - operationLane: '', - operationKind: '', - operationState: '', - operationStartedFrom: '', - operationStartedBefore: '', - operationRunID: null, - operationStatus: '', - settingsAuthority: '', columns: ['kind', 'people', 'title', 'excerpt', 'time', 'attachments', 'size'], columnWidths: { people: 240, title: 360 }, selectedRow: 'message:42', inspectorPinned: true, inspectorWidth: 456, - conversationAnchor: 'message:37', - scrollAnchor: { key: 'message:31', offset: 12 }, - activeRow: 'message:33' + conversationAnchor: 'message:37' }; expect(parseExploreURLState(serializeExploreURLState(state))).toEqual(state); @@ -1039,7 +1068,7 @@ describe('ExploreState history ownership', () => { state.destroy(); }); - it('restores row, scroll, inspector, grouping, and mode on popstate', () => { + it('restores row, scroll, inspector, grouping, and mode on popstate', async () => { const state = new ExploreState(window); const restored: ExploreURLState = { ...defaultExploreURLState, @@ -1051,9 +1080,10 @@ describe('ExploreState history ownership', () => { searchMode: 'hybrid', activeRow: 'conversation:8' }; - window.history.replaceState(null, '', serializeExploreURLState(restored)); - - window.dispatchEvent(new PopStateEvent('popstate')); + state.commitNavigation(restored); + state.commitWorkspace('settings'); + window.history.back(); + await new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); expect(state.current).toMatchObject(restored); state.destroy(); @@ -1201,9 +1231,7 @@ describe('ExploreState history ownership', () => { const replacement = { ...defaultExploreURLState, - query: 'replacement', - activeRow: 'message:replacement', - scrollAnchor: { key: 'message:replacement', offset: 12 } + query: 'replacement' }; window.history.replaceState(null, '', serializeExploreURLState(replacement)); window.dispatchEvent(new PopStateEvent('popstate')); diff --git a/web/src/lib/search/modes.ts b/web/src/lib/search/modes.ts index aabf3a355..091c74f15 100644 --- a/web/src/lib/search/modes.ts +++ b/web/src/lib/search/modes.ts @@ -83,7 +83,9 @@ export function resolveInitialSearchMode( } export function explicitSearchModeFromURL(search: string, parameter = 'explore'): ExploreSearchMode | undefined { - const encoded = new URLSearchParams(search.startsWith('?') ? search.slice(1) : search).get(parameter); + const parameters = new URLSearchParams(search.startsWith('?') ? search.slice(1) : search); + if (parameters.has('mode')) return parseSearchMode(parameters.get('mode')); + const encoded = parameters.get(parameter); if (encoded === null) return undefined; try { const value = JSON.parse(encoded) as { searchMode?: unknown }; diff --git a/web/tests/e2e/keyboard.spec.ts b/web/tests/e2e/keyboard.spec.ts index 11188b3d3..1b13b566a 100644 --- a/web/tests/e2e/keyboard.spec.ts +++ b/web/tests/e2e/keyboard.spec.ts @@ -1,6 +1,7 @@ import { expect, test, type Page } from '@playwright/test'; import { loadMixedArchive } from './fixtures/mixed-archive'; import { installOperations } from './fixtures/operations'; +import { exploreHistoryState } from '../explore-state'; async function tabTo(page: Page, accessibleName: string, limit = 120, key: 'Tab' | 'Shift+Tab' = 'Tab') { const seen = new Set(); @@ -142,9 +143,7 @@ test('pointer-free archive journey preserves focus, announcements, and history', await expect(relationshipTimeline).toBeFocused(); await page.keyboard.press('Escape'); await expect(relationshipList).toBeFocused(); - await expect.poll(() => - JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}').relationshipTarget - ).toBeNull(); + await expect.poll(async () => (await exploreHistoryState(page)).relationshipTarget).toBeNull(); await expect(page.getByRole('heading', { name: 'Beta Person' })).toBeHidden(); await tabTo(page, 'Everything'); @@ -180,10 +179,7 @@ test('pointer-free archive journey preserves focus, announcements, and history', await expect(renderedActiveRow).toBeVisible(); await expect(grid).toHaveAttribute('aria-activedescendant', await renderedActiveRow.getAttribute('id') ?? ''); await page.keyboard.press('End'); - await expect.poll(() => { - const encoded = new URL(page.url()).searchParams.get('explore'); - return JSON.parse(encoded ?? '{}').activeRow; - }).toBe(terminalServedRow.key); + await expect.poll(async () => (await exploreHistoryState(page)).activeRow).toBe(terminalServedRow.key); await expect(grid.locator(`[data-row-key="${terminalServedRow.key}"]`)).toBeVisible(); await page.keyboard.press('Enter'); const everythingReading = page.getByRole('complementary', { name: /Reading pane/ }); diff --git a/web/tests/explore-state.ts b/web/tests/explore-state.ts new file mode 100644 index 000000000..0092ddaae --- /dev/null +++ b/web/tests/explore-state.ts @@ -0,0 +1,10 @@ +import type { Page } from '@playwright/test'; + +// The full exploration state, including keyboard focus and scroll position, +// lives in the browser history entry; the URL carries only the shareable part. +export function exploreHistoryState(page: Page): Promise> { + return page.evaluate(() => { + const entry = window.history.state as { exploreState?: Record } | null; + return entry?.exploreState ?? {}; + }); +} diff --git a/web/tests/files-viewer.spec.ts b/web/tests/files-viewer.spec.ts index f0ede883d..8c39a40fd 100644 --- a/web/tests/files-viewer.spec.ts +++ b/web/tests/files-viewer.spec.ts @@ -1,4 +1,5 @@ import { expect, test, type Page } from '@playwright/test'; +import { exploreHistoryState } from './explore-state'; function exploreURLState() { return { @@ -101,10 +102,8 @@ test('Escape closes an authenticated image once, suspends background shortcuts, await grid.focus(); await expect(grid).toHaveAttribute('aria-activedescendant', 'file-row-7'); await page.keyboard.press('Home'); - await expect.poll(() => { - const encoded = new URL(page.url()).searchParams.get('explore'); - if (!encoded) return undefined; - const state = JSON.parse(encoded) as { activeRow?: string; fileFilenameQuery?: string }; + await expect.poll(async () => { + const state = await exploreHistoryState(page); return { activeRow: state.activeRow, fileFilenameQuery: state.fileFilenameQuery }; }).toEqual({ activeRow: 'file:7', fileFilenameQuery: 'pixel' }); await expect(grid).toHaveAttribute('aria-busy', 'false'); @@ -208,10 +207,7 @@ test('Back from an open file viewer closes it and restores the Files list URL', const grid = page.getByRole('grid', { name: 'Files results' }); await grid.focus(); await expect(grid).toHaveAttribute('aria-activedescendant', 'file-row-7'); - await expect.poll(() => { - const encoded = new URL(page.url()).searchParams.get('explore'); - return encoded ? (JSON.parse(encoded) as { activeRow?: string | null }).activeRow : undefined; - }).toBe('file:7'); + await expect.poll(async () => (await exploreHistoryState(page)).activeRow).toBe('file:7'); const listURL = page.url(); await page.keyboard.press('Enter'); diff --git a/web/tests/presentations.spec.ts b/web/tests/presentations.spec.ts index d6554ab37..8782cc9ee 100644 --- a/web/tests/presentations.spec.ts +++ b/web/tests/presentations.spec.ts @@ -1,5 +1,6 @@ import { expect, test } from '@playwright/test'; import { selectKitOption } from './kit-ui'; +import { exploreHistoryState } from './explore-state'; const rows = [1, 2].map((id) => ({ key: `message:${id}`, kind: 'message', message_type: 'email', conversation_type: 'email_thread', @@ -78,7 +79,7 @@ test('Show as preserves analytical meaning, keyboard focus, history, and Saved V await page.keyboard.press('Enter'); const viewer = page.getByRole('dialog', { name: 'View pasta-analysis.pdf' }); await expect(viewer).toBeVisible(); - expect(JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}')) + expect(await exploreHistoryState(page)) .toMatchObject({ selectedRow: 'attachment:7', activeRow: 'message:1:file:7' }); await page.goBack(); @@ -92,14 +93,14 @@ test('Show as preserves analytical meaning, keyboard focus, history, and Saved V await viewer.getByRole('button', { name: 'Close file viewer' }).click(); await expect(viewer).not.toBeVisible(); await expect(files).toBeFocused(); - expect(JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}').selectedRow).toBeNull(); + expect((await exploreHistoryState(page)).selectedRow).toBeNull(); await files.focus(); await page.keyboard.press('Enter'); await expect(viewer).toBeVisible(); await viewer.getByRole('button', { name: 'Open containing item' }).click(); await expect(page.getByRole('complementary', { name: 'Reading pane: Presentation message 1' })).toBeVisible(); - const tableState = JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}'); + const tableState = await exploreHistoryState(page); expect(tableState).toMatchObject({ presentation: 'table', activeRow: 'message:1', scrollAnchor: null }); expect(tableState.activeRow).not.toContain(':file:'); diff --git a/web/tests/relationships.spec.ts b/web/tests/relationships.spec.ts index 5a0e7cbb9..a554ccd21 100644 --- a/web/tests/relationships.spec.ts +++ b/web/tests/relationships.spec.ts @@ -1,4 +1,5 @@ import { expect, test, type Page } from '@playwright/test'; +import { exploreHistoryState } from './explore-state'; const when = '2026-07-19T10:00:00Z'; @@ -176,7 +177,7 @@ test('legacy People URL lands on the Relationships hub and walks list, timeline, // pane's own open/close state lives outside the URL entirely — closing // the conversation reading pane that was still open underneath it too. await page.goBack(); - await expect.poll(() => JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}').relationshipTarget).toBeNull(); + await expect.poll(async () => (await exploreHistoryState(page)).relationshipTarget).toBeNull(); await expect(list.getByRole('row', { name: /Alice Example/ })).toHaveAttribute('aria-selected', 'false'); await expect(page.getByRole('heading', { name: 'Alice Example' })).toBeHidden(); await expect(reading).toBeHidden(); @@ -219,7 +220,7 @@ test('person attachment gallery preserves directions and Media state across sour await expect(page.getByRole('radio', { name: 'Media' })).toHaveAttribute('aria-checked', 'true'); await expect(page.getByRole('checkbox', { name: 'Group conversations' })).toBeChecked(); await expect(page.getByRole('button', { name: 'Open photo.png' })).toBeVisible(); - const restored = JSON.parse(new URL(page.url()).searchParams.get('explore') ?? '{}') as Record; + const restored = await exploreHistoryState(page); expect(restored).toMatchObject({ workspace: 'relationships', relationshipTarget: 'cluster:1', relationshipFiles: true, personFilePresentation: 'media', personFileDirections: ['from_person', 'group'] diff --git a/web/tests/theme-keyboard.spec.ts b/web/tests/theme-keyboard.spec.ts index 592da1901..37c02eaf2 100644 --- a/web/tests/theme-keyboard.spec.ts +++ b/web/tests/theme-keyboard.spec.ts @@ -65,6 +65,21 @@ test.beforeEach(async ({ page }) => { await expect(page.getByText('Synthetic archive subject')).toBeVisible(); }); +test('compact workspace links preserve browser navigation and reopen the selected tab', async ({ page }) => { + await selectKitTopBarTab(page, 'Files'); + await expect(page.getByText('synthetic.pdf', { exact: true })).toBeVisible(); + await expect(page).toHaveURL(/\?workspace=files&mode=full_text$/); + const filesURL = page.url(); + + await selectKitTopBarTab(page, 'Everything'); + await expect(page.getByText('Synthetic archive subject', { exact: true })).toBeVisible(); + await expect(page).toHaveURL(/\?workspace=everything&mode=full_text$/); + await page.goBack(); + await expect(page.getByText('synthetic.pdf', { exact: true })).toBeVisible(); + await page.goto(filesURL); + await expect(page.getByText('synthetic.pdf', { exact: true })).toBeVisible(); +}); + test('one registry drives selection, searchable help, palette, and editable suspension', async ({ page }) => { const grid = page.getByRole('grid', { name: 'Everything results' }); const renderedRow = page.locator('[data-row-key="message:1"]'); From d42272c2b2865f4eb2d4f15b8e7bae19428a170f Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 19 Sep 2026 15:25:05 -0400 Subject: [PATCH 003/125] feat(imap): retrieve, edit, and delete managed drafts (#861) Retrieve, edit, and delete managed IMAP reply drafts through the CLI, with the selected daemon owning provider access and local persistence. The daemon tracks draft ownership and revisions in SQLite or PostgreSQL. An edit saves replacement content before APPEND, publishes the new revision, then removes the exact old UID. Conditional deletion conflicts stop removal. Reused mailbox IDs preserve previously archived content. Deletion failures before a remote write leave the draft retryable. When remote removal is confirmed, a retry can finish the local operation. Uncertain writes stay pending and retain content and known IMAP copies; explicit recovery remains a separate follow-up. Discarded drafts retain local content for `draft-get`. Lifecycle commands remain owner-only. Committed changes refresh analytics even when cleanup or response delivery fails. Refs #666, slice 3b. Co-authored-by: Rod Boev --- cmd/msgvault/cmd/daemon_cli_http.go | 4 +- cmd/msgvault/cmd/draft_lifecycle.go | 48 + cmd/msgvault/cmd/serve.go | 3 + cmd/msgvault/cmd/serve_draft_lifecycle.go | 815 +++++++++ .../cmd/serve_draft_lifecycle_cache_test.go | 199 +++ .../cmd/serve_draft_lifecycle_retry_test.go | 58 + .../cmd/serve_draft_lifecycle_review_test.go | 1567 +++++++++++++++++ .../cmd/serve_draft_lifecycle_test.go | 64 + cmd/msgvault/cmd/serve_reply_draft.go | 47 +- cmd/msgvault/cmd/serve_reply_draft_test.go | 5 +- docs/changelog.md | 4 + docs/cli-reference.md | 25 + docs/usage/imap.md | 74 + go.mod | 2 +- go.sum | 4 +- internal/api/cli_allowlist_drafts_test.go | 140 +- internal/api/cli_handlers.go | 5 +- internal/api/cli_run_draft.go | 20 + internal/api/operation_gate.go | 1 + internal/imap/client.go | 45 +- internal/imap/draft.go | 11 +- internal/imap/draft_inspect.go | 198 +++ internal/imap/draft_remove.go | 209 +++ internal/imap/draft_remove_test.go | 36 + internal/imap/draft_replacement_test.go | 75 + internal/imap/draft_review_test.go | 1058 +++++++++++ internal/imap/reply.go | 157 ++ internal/store/dialect_pg.go | 2 +- internal/store/gc.go | 20 +- internal/store/imap_draft_lifecycle.go | 626 +++++++ internal/store/imap_draft_lifecycle_test.go | 194 ++ internal/store/imap_draft_review_test.go | 315 ++++ internal/store/imap_drafts.go | 133 +- internal/store/imap_drafts_test.go | 6 +- internal/store/imap_identity_test.go | 28 +- internal/store/messages.go | 162 +- internal/store/schema.sql | 68 + internal/store/schema_pg.sql | 65 + 38 files changed, 6325 insertions(+), 168 deletions(-) create mode 100644 cmd/msgvault/cmd/draft_lifecycle.go create mode 100644 cmd/msgvault/cmd/serve_draft_lifecycle.go create mode 100644 cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go create mode 100644 cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go create mode 100644 cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go create mode 100644 cmd/msgvault/cmd/serve_draft_lifecycle_test.go create mode 100644 internal/imap/draft_inspect.go create mode 100644 internal/imap/draft_remove.go create mode 100644 internal/imap/draft_remove_test.go create mode 100644 internal/imap/draft_replacement_test.go create mode 100644 internal/imap/draft_review_test.go create mode 100644 internal/store/imap_draft_lifecycle.go create mode 100644 internal/store/imap_draft_lifecycle_test.go create mode 100644 internal/store/imap_draft_review_test.go diff --git a/cmd/msgvault/cmd/daemon_cli_http.go b/cmd/msgvault/cmd/daemon_cli_http.go index 9a834c5b0..6bfb36fb9 100644 --- a/cmd/msgvault/cmd/daemon_cli_http.go +++ b/cmd/msgvault/cmd/daemon_cli_http.go @@ -80,7 +80,7 @@ func runDaemonCLICommandHTTPWithEnv( if err != nil { return err } - if api.IsCLIRunDraftReply(args) { + if api.IsCLIRunDraftReply(args) || api.IsCLIRunDraftLifecycle(args) { cwd = "" } @@ -97,7 +97,7 @@ func runDaemonCLICommandHTTPWithEnv( if _, err := fmt.Fprint(cmd.ErrOrStderr(), data); err != nil { return fmt.Errorf("write CLI stderr: %w", err) } - draftFailureReported = api.IsCLIRunDraftReply(args) && data != "" + draftFailureReported = (api.IsCLIRunDraftReply(args) || api.IsCLIRunDraftLifecycle(args)) && data != "" } return nil }) diff --git a/cmd/msgvault/cmd/draft_lifecycle.go b/cmd/msgvault/cmd/draft_lifecycle.go new file mode 100644 index 000000000..9fa293bde --- /dev/null +++ b/cmd/msgvault/cmd/draft_lifecycle.go @@ -0,0 +1,48 @@ +package cmd + +import "github.com/spf13/cobra" + +func init() { + rootCmd.AddCommand(newDraftGetCommand()) + rootCmd.AddCommand(newDraftEditCommand()) + rootCmd.AddCommand(newDraftDeleteCommand()) +} + +func newDraftGetCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-get ", + Short: "Read a managed IMAP draft from the archive", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftEditCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-edit ", + Short: "Replace the body of a managed IMAP draft", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + command.Flags().String("body", "", "replacement plain-text body") + _ = command.MarkFlagRequired("revision") + _ = command.MarkFlagRequired("body") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftDeleteCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-delete ", + Short: "Delete a managed IMAP draft", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + _ = command.MarkFlagRequired("revision") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index b9ec7a8be..57fc956c5 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -1839,6 +1839,9 @@ func (a *storeAPIAdapter) runCLICommandWithRunner( if api.IsCLIRunDraftReply(req.Args) { return a.runCLIReplyDraft(ctx, req, emit) } + if api.IsCLIRunDraftLifecycle(req.Args) { + return a.runCLIDraftLifecycle(ctx, req, emit) + } runSubprocess := func(ctx context.Context) error { args := req.Args if req.GrantDecided { diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle.go b/cmd/msgvault/cmd/serve_draft_lifecycle.go new file mode 100644 index 000000000..ca4696ec7 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle.go @@ -0,0 +1,815 @@ +package cmd + +import ( + "context" + "database/sql" + jsonv2 "encoding/json/v2" + "errors" + "fmt" + "strconv" + "strings" + "time" + "unicode/utf8" + + "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" +) + +type draftLifecycleIntent struct { + Operation string + DraftID string + Revision int64 + Body string + JSON bool +} + +const draftLifecycleActive = "active" + +type draftLifecycleReceipt struct { + Mailbox string `json:"mailbox"` + UIDValidity uint32 `json:"uidvalidity"` + UID uint32 `json:"uid"` +} + +type draftLifecycleObservation struct { + State string `json:"state"` + Code string `json:"code,omitempty"` + Mailbox string `json:"mailbox,omitempty"` + UIDValidity uint32 `json:"uidvalidity,omitempty"` + UID uint32 `json:"uid,omitempty"` + Flags []string `json:"flags,omitempty"` + Present bool `json:"present"` + Draft bool `json:"draft"` + Deleted bool `json:"deleted"` + Complete bool `json:"complete"` + UIDPlus bool `json:"uidplus"` +} + +type draftLifecycleOutput struct { + Status string `json:"status"` + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + Lifecycle string `json:"lifecycle"` + MessageID int64 `json:"message_id"` + SourceID int64 `json:"source_id"` + Receipt draftLifecycleReceipt `json:"receipt"` + Content string `json:"content,omitempty"` + RawMIME string `json:"raw_mime,omitempty"` + CandidateContent string `json:"candidate_content,omitempty"` + PendingOperation string `json:"pending_operation,omitempty"` + PendingCode string `json:"pending_code,omitempty"` + PendingReceipt *draftLifecycleReceipt `json:"pending_receipt,omitempty"` + ProviderObservation *draftLifecycleObservation `json:"provider_observation,omitempty"` + Observation *draftLifecycleObservation `json:"observation,omitempty"` + ManualReconciliation bool `json:"manual_reconciliation,omitempty"` +} + +func parseDraftLifecycleArgs(args []string) (draftLifecycleIntent, error) { + if !api.IsCLIRunDraftLifecycle(args) { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("expected a draft lifecycle command")) + } + intent := draftLifecycleIntent{Operation: args[0]} + var positional string + var revisionSet, bodySet, jsonSet bool + rest := args[1:] + for len(rest) > 0 { + arg := rest[0] + rest = rest[1:] + nameValue, ok := strings.CutPrefix(arg, "--") + if !ok { + if positional != "" { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("expected exactly one draft ID")) + } + positional = arg + continue + } + name, value, hasValue := strings.Cut(nameValue, "=") + switch name { + case "revision": + if !hasValue { + if len(rest) == 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision requires a value")) + } + value, rest = rest[0], rest[1:] + } + if revisionSet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision given more than once")) + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision must be a positive integer")) + } + intent.Revision, revisionSet = parsed, true + case "body": + if !hasValue { + if len(rest) == 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--body requires a value")) + } + value, rest = rest[0], rest[1:] + } + if bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--body given more than once")) + } + intent.Body, bodySet = value, true + case "json": + if jsonSet || (hasValue && value != "true") { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--json accepts one flag without a value")) + } + intent.JSON, jsonSet = true, true + case "log-level", "verbose", "log-sql", "log-sql-slow-ms": + if !hasValue && name != "verbose" && name != "log-sql" && len(rest) > 0 { + rest = rest[1:] + } + default: + return draftLifecycleIntent{}, draftReplyError("invalid_args", fmt.Errorf("unknown flag --%s", name)) + } + } + if positional == "" || !utf8.ValidString(positional) || strings.TrimSpace(positional) == "" || strings.ContainsAny(positional, "\x00\r\n") { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft ID is required")) + } + intent.DraftID = positional + switch intent.Operation { + case api.CLIRunDraftGetCommand: + if revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-get accepts only --json")) + } + case api.CLIRunDraftEditCommand: + if !revisionSet || !bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-edit requires --revision and --body")) + } + case api.CLIRunDraftDeleteCommand: + if !revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-delete requires --revision and no body")) + } + } + return intent, nil +} + +func draftLifecycleReceiptOutput(receipt store.IMAPDraftReceipt) draftLifecycleReceipt { + return draftLifecycleReceipt{Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID} +} + +func draftLifecycleObservationOutput(observation imaplib.DraftObservation) *draftLifecycleObservation { + flags := make([]string, len(observation.Flags)) + for i, flag := range observation.Flags { + flags[i] = string(flag) + } + return &draftLifecycleObservation{ + State: observation.State, Code: observation.Code, + Mailbox: observation.Mailbox, UIDValidity: observation.UIDValidity, + UID: observation.UID, Flags: flags, Present: observation.Present, + Draft: observation.Draft, Deleted: observation.Deleted, + Complete: observation.Complete, UIDPlus: observation.UIDPlus, + } +} + +func draftLifecycleObservationCode(observation imaplib.DraftObservation, fallback string) string { + if observation.Code != "" { + return observation.Code + } + return fallback +} + +func (a *storeAPIAdapter) draftLifecycleOutput( + ctx context.Context, + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, +) (draftLifecycleOutput, error) { + message, err := a.store.GetMessageContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftLifecycleOutput{}, fmt.Errorf("load managed draft message: %w", err) + } + raw, err := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftLifecycleOutput{}, fmt.Errorf("load managed draft MIME: %w", err) + } + lifecycle := draftLifecycleActive + if draft.DiscardedAt != nil { + lifecycle = "discarded" + } + output := draftLifecycleOutput{ + Status: status, DraftID: draft.DraftID, Revision: draft.Revision, + Lifecycle: lifecycle, MessageID: draft.CurrentMessageID, + SourceID: draft.SourceID, Receipt: draftLifecycleReceiptOutput(draft.CurrentReceipt), + Content: message.BodyText, RawMIME: string(raw), + ProviderObservation: providerObservation, Observation: observation, + } + if draft.Pending != nil { + output.PendingOperation = draft.Pending.Operation + output.PendingCode = draft.Pending.Code + if len(draft.Pending.Raw) > 0 { + output.CandidateContent = string(draft.Pending.Raw) + } + if draft.Pending.ReplacementReceipt != nil { + receipt := draftLifecycleReceiptOutput(*draft.Pending.ReplacementReceipt) + output.PendingReceipt = &receipt + } + } + return output, nil +} + +func emitDraftLifecycleOutput( + emit func(api.CLIRunEvent) error, + stream string, + asJSON bool, + output draftLifecycleOutput, +) error { + if emit == nil { + return nil + } + if asJSON { + data, err := jsonv2.Marshal(output) + if err != nil { + return err + } + return emit(api.CLIRunEvent{Type: stream, Data: string(data) + "\n"}) + } + var data strings.Builder + fmt.Fprintf(&data, "draft %s revision %d %s\n", + textutil.SanitizeTerminal(output.DraftID), output.Revision, textutil.SanitizeTerminal(output.Lifecycle)) + fmt.Fprintf(&data, "status: %s\n", textutil.SanitizeTerminal(output.Status)) + fmt.Fprintf(&data, "receipt (revision %d): %s\n", + output.Revision, textutil.SanitizeTerminal(formatDraftLifecycleReceipt(output.Receipt))) + fmt.Fprintf(&data, "content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.Content), "\n")) + if output.PendingOperation != "" { + fmt.Fprintf(&data, "pending operation: %s\n", textutil.SanitizeTerminal(output.PendingOperation)) + } + if output.CandidateContent != "" { + fmt.Fprintf(&data, "candidate content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.CandidateContent), "\n")) + } + if output.PendingReceipt != nil { + fmt.Fprintf(&data, "pending receipt (revision %d): %s\n", + output.Revision, textutil.SanitizeTerminal(formatDraftLifecycleReceipt(*output.PendingReceipt))) + } + if output.Status == "accepted_local_failed" && output.ProviderObservation != nil && + output.ProviderObservation.State == "present" && output.ProviderObservation.Present && + output.ProviderObservation.Mailbox != "" && output.ProviderObservation.UIDValidity != 0 && + output.ProviderObservation.UID != 0 { + fmt.Fprintf(&data, "acknowledged replacement receipt: %s\n", + textutil.SanitizeTerminal(formatDraftLifecycleObservationReceipt(*output.ProviderObservation))) + } + if output.Observation != nil && output.Status == "pending" { + fmt.Fprintf(&data, "old provider receipt: %s\n", + textutil.SanitizeTerminal(formatDraftLifecycleObservationReceipt(*output.Observation))) + } + providerOutcome := output.PendingCode + if providerOutcome == "" { + observations := []*draftLifecycleObservation{output.ProviderObservation, output.Observation} + if output.Status == "pending" { + observations = []*draftLifecycleObservation{output.Observation, output.ProviderObservation} + } + for _, observation := range observations { + if observation == nil { + continue + } + providerOutcome = observation.Code + if providerOutcome == "" { + providerOutcome = observation.State + } + if providerOutcome != "" { + break + } + } + } + if providerOutcome != "" { + fmt.Fprintf(&data, "provider outcome: %s\n", textutil.SanitizeTerminal(providerOutcome)) + } + if output.Status == "pending" || output.Status == "accepted_local_failed" || output.ManualReconciliation { + fmt.Fprintf(&data, "old draft ID remains blocked at revision %d\n", output.Revision) + fmt.Fprintln(&data, "manual action: reconcile the provider receipt and local state before retrying") + } + return emit(api.CLIRunEvent{Type: stream, Data: data.String()}) +} + +func formatDraftLifecycleReceipt(receipt draftLifecycleReceipt) string { + return fmt.Sprintf("%s uidvalidity=%d uid=%d", receipt.Mailbox, receipt.UIDValidity, receipt.UID) +} + +func formatDraftLifecycleObservationReceipt(observation draftLifecycleObservation) string { + return fmt.Sprintf("%s uidvalidity=%d uid=%d (%s)", + observation.Mailbox, observation.UIDValidity, observation.UID, observation.Code) +} + +func (a *storeAPIAdapter) loadManagedDraftSource(ctx context.Context, draft store.IMAPDraft) (*store.Source, error) { + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + return nil, draftReplyError("invalid_source", err) + } + mailbox, err := authorizeIMAPDraft(a.draftPolicy, source.ID, source.SourceType) + if err != nil { + return nil, err + } + if mailbox != draft.CurrentReceipt.Mailbox { + return nil, draftReplyError("invalid_mailbox", errors.New("draft receipt mailbox is outside the current owner grant")) + } + if !source.SyncConfig.Valid { + return nil, draftReplyError("invalid_source", errors.New("source has no sync config")) + } + config, err := imaplib.ConfigFromJSON(source.SyncConfig.String) + if err != nil || config.Identifier() != source.Identifier { + return nil, draftReplyError("invalid_source", errors.New("source sync config identity does not match the source")) + } + return source, nil +} + +func localDraftEvidenceContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) +} + +func (a *storeAPIAdapter) releaseDraftSourceAndRefreshCache(ctx context.Context, source *store.Source, execution *store.SyncExecution) { + if err := execution.Release(); err != nil { + logger.Error("release source after draft write", "source_id", source.ID, "error", err) + } + // Committed changes must reach the cache even if cleanup or output fails. + refreshCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + a.refreshDraftCache(refreshCtx, source) +} + +func (a *storeAPIAdapter) runCLIDraftLifecycle( + ctx context.Context, + req api.CLIRunRequest, + emit func(api.CLIRunEvent) error, +) error { + if len(req.Env) != 0 || req.Cwd != "" { + return draftReplyError("invalid_args", errors.New("draft lifecycle commands accept no environment or working directory")) + } + intent, err := parseDraftLifecycleArgs(req.Args) + if err != nil { + return err + } + draft, err := a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + return draftReplyError("draft_not_found", err) + } + if intent.Operation == api.CLIRunDraftGetCommand { + provider := &draftLifecycleObservation{State: "not_checked", Code: "not_checked"} + output, err := a.draftLifecycleOutput(ctx, draft, "ok", provider, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + return emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if draft.DiscardedAt != nil { + if intent.Operation == api.CLIRunDraftDeleteCommand { + output, err := a.draftLifecycleOutput(ctx, draft, "already_discarded", nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + return emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output) + } + return draftReplyError("draft_discarded", errors.New("discarded drafts cannot be edited")) + } + if draft.Pending != nil && draft.Pending.Code != store.IMAPDraftCodeRemoved { + return draftReplyError("pending_operation", store.ErrIMAPDraftPending) + } + source, err := a.loadManagedDraftSource(ctx, draft) + if err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + defer func() { _ = execution.Release() }() + draft, err = a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + return draftReplyError("draft_not_found", err) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", errors.New("draft changed while acquiring source ownership")) + } + if draft.Pending != nil && (draft.Pending.Code != store.IMAPDraftCodeRemoved || "draft-"+draft.Pending.Operation != intent.Operation) { + return draftReplyError("pending_operation", store.ErrIMAPDraftPending) + } + source, err = a.loadManagedDraftSource(ctx, draft) + if err != nil { + return err + } + currentRaw, err := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + var replacement imaplib.ReplyDraft + if intent.Operation == api.CLIRunDraftEditCommand { + replacement, err = imaplib.BuildDraftReplacement(currentRaw, intent.Body, time.Now(), "") + if err != nil { + return draftReplyError("invalid_draft", err) + } + } + if draft.Pending != nil { + if intent.Operation == api.CLIRunDraftEditCommand { + current, err := msgmime.Parse(currentRaw) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if current.BodyText != replacement.Parsed.BodyText { + return draftReplyError("pending_operation", errors.New("retry must use the already published edit body")) + } + } + finished, err := a.store.FinishIMAPDraftRemovalContext(ctx, intent.DraftID, intent.Revision) + if err != nil { + return draftReplyError("cleanup_local_failed", err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + status := "edited" + if intent.Operation == api.CLIRunDraftDeleteCommand { + status = "deleted" + } + output, err := a.draftLifecycleOutput(ctx, finished, status, nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil + } + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + + providerReceipt := imaplib.DraftReceipt{ + Mailbox: draft.CurrentReceipt.Mailbox, + UIDValidity: draft.CurrentReceipt.UIDValidity, + UID: draft.CurrentReceipt.UID, + } + inspection, err := client.InspectDraft(ctx, providerReceipt) + if err != nil || !inspection.Present || inspection.Deleted || !inspection.Draft || !inspection.UIDPlus { + provider := draftLifecycleObservationOutput(inspection) + output, outputErr := a.draftLifecycleOutput(ctx, draft, "refused", provider, nil) + if outputErr == nil { + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + } + if err != nil { + code := inspection.Code + if code == "" { + code = "provider_refused" + } + return draftReplyError(code, err) + } + code := inspection.Code + if code == "" { + code = "provider_refused" + } + return draftReplyError(code, errors.New("provider draft inspection refused mutation")) + } + if intent.Operation == api.CLIRunDraftEditCommand { + return a.runDraftEdit(ctx, intent, draft, source, client, currentRaw, replacement, inspection, execution, emit) + } + return a.runDraftDelete(ctx, intent, draft, source, client, inspection, execution, emit) +} + +func (a *storeAPIAdapter) runDraftEdit( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + source *store.Source, + client *imaplib.Client, + currentRaw []byte, + replacement imaplib.ReplyDraft, + inspection imaplib.DraftObservation, + execution *store.SyncExecution, + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimIMAPDraftContext(ctx, intent.DraftID, intent.Revision, store.IMAPDraftOperationEdit, replacement.Raw) + if err != nil { + return draftReplyError("claim_failed", err) + } + appendResult, err := client.AppendDraft(ctx, draft.CurrentReceipt.Mailbox, replacement.Raw) + if err != nil { + code := appendResult.Code + if code == "" { + code = "append_failed" + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + latest := claimed + var persistenceErr error + if appendResult.State == imaplib.DraftStateRejected || appendResult.State == imaplib.DraftStateCancelled { + latest, persistenceErr = a.store.AbortIMAPDraftContext(evidenceCtx, intent.DraftID, intent.Revision, appendResult.State) + if persistenceErr != nil { + latest = claimed + } + } else { + persistenceErr = a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, nil) + if loaded, loadErr := a.store.GetIMAPDraftContext(evidenceCtx, intent.DraftID); loadErr == nil { + latest = loaded + } + } + status := "pending" + if latest.Pending == nil && persistenceErr == nil { + status = draftLifecycleActive + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, latest, status, draftLifecycleObservationOutput(inspection), nil) + if outputErr != nil { + output = draftLifecycleOutput{ + Status: status, DraftID: latest.DraftID, Revision: latest.Revision, Lifecycle: draftLifecycleActive, + MessageID: latest.CurrentMessageID, SourceID: latest.SourceID, + Receipt: draftLifecycleReceiptOutput(latest.CurrentReceipt), RawMIME: string(currentRaw), + } + if latest.Pending != nil { + output.PendingOperation = latest.Pending.Operation + output.CandidateContent = string(latest.Pending.Raw) + } + } + output.PendingCode = code + output.ManualReconciliation = latest.Pending != nil || persistenceErr != nil + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + if persistenceErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, persistenceErr)) + } + if coded, ok := errors.AsType[*imaplib.DraftAppendError](err); ok { + err = coded.Err + } + return draftReplyError(code, err) + } + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: appendResult.UIDValidity, UID: appendResult.UID} + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, appendResult.Code, &receipt); err != nil { + appendObservation := &draftLifecycleObservation{ + State: "present", Code: appendResult.Code, Mailbox: receipt.Mailbox, + UIDValidity: receipt.UIDValidity, UID: receipt.UID, Present: true, + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, claimed, "accepted_local_failed", appendObservation, nil) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + } + return draftReplyError("accepted_local_failed", err) + } + reportAcceptedLocalFailure := func(cause error) error { + output := draftLifecycleOutput{ + Status: "accepted_local_failed", DraftID: claimed.DraftID, Revision: claimed.Revision, + Lifecycle: draftLifecycleActive, MessageID: claimed.CurrentMessageID, SourceID: claimed.SourceID, + Receipt: draftLifecycleReceiptOutput(claimed.CurrentReceipt), RawMIME: string(currentRaw), + PendingOperation: store.IMAPDraftOperationEdit, PendingCode: appendResult.Code, + CandidateContent: string(replacement.Raw), ManualReconciliation: true, + } + if loaded, loadErr := a.draftLifecycleOutput(evidenceCtx, claimed, "accepted_local_failed", nil, nil); loadErr == nil { + output = loaded + } + output.PendingCode = appendResult.Code + output.ManualReconciliation = true + pendingReceipt := draftLifecycleReceiptOutput(receipt) + output.PendingReceipt = &pendingReceipt + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + return draftReplyError("accepted_local_failed", cause) + } + currentMessage, currentMessageErr := a.store.GetMessageContext(evidenceCtx, draft.CurrentMessageID) + if currentMessageErr != nil { + return reportAcceptedLocalFailure(currentMessageErr) + } + replyTo, replyToErr := a.store.GetMessageReplyToMessageIDContext(evidenceCtx, draft.CurrentMessageID) + if replyToErr != nil { + return reportAcceptedLocalFailure(replyToErr) + } + participants, build := draftLifecyclePersistData(currentMessage.ConversationID, replyTo, replacement, receipt) + published, err := a.store.PublishIMAPDraftReplacementContext(evidenceCtx, intent.DraftID, intent.Revision, participants, build) + if err != nil { + return reportAcceptedLocalFailure(err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + if ctx.Err() != nil { + output, outputErr := a.draftLifecycleOutput(evidenceCtx, published, "pending", nil, nil) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + } + return draftReplyError("cancelled", ctx.Err()) + } + removed, err := client.RemoveDraft(ctx, imaplib.DraftReceipt{Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: draft.CurrentReceipt.UIDValidity, UID: draft.CurrentReceipt.UID}) + evidenceCtx, cancelCleanupEvidence := localDraftEvidenceContext(ctx) + defer cancelCleanupEvidence() + if err != nil || !removed.Complete { + code := draftLifecycleObservationCode(removed, "cleanup_incomplete") + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, published.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + if recordErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + return draftReplyError(code, err) + } + if err := a.store.RecordIMAPDraftOutcomeContext( + evidenceCtx, intent.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil, + ); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, published.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + return draftReplyError("cleanup_local_failed", err) + } + output, err := a.draftLifecycleOutput(evidenceCtx, finished, "edited", nil, draftLifecycleObservationOutput(removed)) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + +func (a *storeAPIAdapter) emitDraftLifecyclePending( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + providerObservation *draftLifecycleObservation, + observation imaplib.DraftObservation, + emit func(api.CLIRunEvent) error, +) { + if latest, err := a.store.GetIMAPDraftContext(ctx, draft.DraftID); err == nil { + draft = latest + } + output, err := a.draftLifecycleOutput(ctx, draft, "pending", providerObservation, draftLifecycleObservationOutput(observation)) + if err != nil { + output = draftLifecycleOutput{ + Status: "pending", DraftID: draft.DraftID, Revision: draft.Revision, Lifecycle: draftLifecycleActive, + MessageID: draft.CurrentMessageID, SourceID: draft.SourceID, Receipt: draftLifecycleReceiptOutput(draft.CurrentReceipt), + ProviderObservation: providerObservation, Observation: draftLifecycleObservationOutput(observation), + } + if draft.Pending != nil { + output.PendingOperation = draft.Pending.Operation + output.CandidateContent = string(draft.Pending.Raw) + if draft.Pending.ReplacementReceipt != nil { + receipt := draftLifecycleReceiptOutput(*draft.Pending.ReplacementReceipt) + output.PendingReceipt = &receipt + } + } + } + if observation.Code != "" { + output.PendingCode = observation.Code + } else if output.PendingCode == "" { + output.PendingCode = "cleanup_incomplete" + } + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) +} + +func (a *storeAPIAdapter) runDraftDelete( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + source *store.Source, + client *imaplib.Client, + inspection imaplib.DraftObservation, + execution *store.SyncExecution, + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimIMAPDraftContext(ctx, intent.DraftID, intent.Revision, store.IMAPDraftOperationDelete, nil) + if err != nil { + return draftReplyError("claim_failed", err) + } + removed, err := client.RemoveDraft(ctx, imaplib.DraftReceipt{Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: draft.CurrentReceipt.UIDValidity, UID: draft.CurrentReceipt.UID}) + if err != nil || !removed.Complete { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + code := draftLifecycleObservationCode(removed, "cleanup_incomplete") + if !removed.WriteAttempted { + active, abortErr := a.store.AbortIMAPDraftContext(evidenceCtx, intent.DraftID, intent.Revision, "not_attempted") + if abortErr != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("local_persistence_failed", errors.Join(err, abortErr)) + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, active, draftLifecycleActive, draftLifecycleObservationOutput(inspection), draftLifecycleObservationOutput(removed)) + if outputErr != nil { + return draftReplyError("draft_read_failed", outputErr) + } + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + return draftReplyError(code, err) + } + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + if recordErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + return draftReplyError(code, err) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext( + evidenceCtx, intent.DraftID, intent.Revision, store.IMAPDraftCodeRemoved, nil, + ); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, intent.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("cleanup_local_failed", err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + output, err := a.draftLifecycleOutput(evidenceCtx, finished, "deleted", nil, draftLifecycleObservationOutput(removed)) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + +func draftLifecyclePersistData( + conversationID int64, + replyTo sql.NullInt64, + replacement imaplib.ReplyDraft, + receipt store.IMAPDraftReceipt, +) ([]store.ParticipantPersistData, func([]int64) *store.MessagePersistData) { + parsed := replacement.Parsed + addresses := append([]msgmime.Address(nil), parsed.From...) + addresses = append(addresses, parsed.To...) + addresses = append(addresses, parsed.Cc...) + addresses = append(addresses, parsed.Bcc...) + participants := make([]store.ParticipantPersistData, len(addresses)) + for i, address := range addresses { + participants[i] = store.ParticipantPersistData{EmailAddress: address.Email, DisplayName: address.Name, Domain: address.Domain} + } + fromCount, toCount := len(parsed.From), len(parsed.To) + ccCount, bccCount := len(parsed.Cc), len(parsed.Bcc) + build := func(ids []int64) *store.MessagePersistData { + at := 0 + fromIDs := ids[at : at+fromCount] + at += fromCount + toIDs := ids[at : at+toCount] + at += toCount + ccIDs := ids[at : at+ccCount] + at += ccCount + bccIDs := ids[at : at+bccCount] + toAddresses := addressStrings(parsed.To) + ccAddresses := addressStrings(parsed.Cc) + bccAddresses := addressStrings(parsed.Bcc) + fromAddresses := addressStrings(parsed.From) + rfc822 := msgmime.NormalizeMessageID(parsed.MessageID) + if rfc822 != "" { + rfc822 = "<" + rfc822 + ">" + } + message := &store.Message{ + SourceID: receipt.SourceID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + RFC822MessageID: sql.NullString{String: rfc822, Valid: rfc822 != ""}, + ConversationID: conversationID, + MessageType: store.MessageTypeEmail, IsFromMe: true, IdentityDerivedIsFromMe: true, + SenderID: sql.NullInt64{Int64: fromIDs[0], Valid: len(fromIDs) > 0}, + ReplyToMessageID: replyTo, + ListID: sql.NullString{String: parsed.ListID, Valid: parsed.ListID != ""}, + Subject: sql.NullString{String: parsed.Subject, Valid: parsed.Subject != ""}, + Snippet: sql.NullString{String: strings.TrimSpace(parsed.BodyText), Valid: parsed.BodyText != ""}, + SentAt: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + InternalDate: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + SizeEstimate: int64(len(replacement.Raw)), ArchivedAt: time.Now(), + } + return &store.MessagePersistData{ + Message: message, BodyText: sql.NullString{String: parsed.BodyText, Valid: true}, + RawMIME: replacement.Raw, RawFormat: "mime", + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: fromIDs, EmailAddresses: fromAddresses}, + {Type: "to", ParticipantIDs: toIDs, EmailAddresses: toAddresses}, + {Type: "cc", ParticipantIDs: ccIDs, EmailAddresses: ccAddresses}, + {Type: "bcc", ParticipantIDs: bccIDs, EmailAddresses: bccAddresses}, + }, + FTS: &store.FTSDoc{Subject: parsed.Subject, Body: parsed.BodyText, FromAddr: firstAddress(parsed.From), ToAddrs: strings.Join(toAddresses, " ")}, + } + } + return participants, build +} + +func addressStrings(addresses []msgmime.Address) []string { + result := make([]string, len(addresses)) + for i, address := range addresses { + result[i] = address.Email + } + return result +} + +func firstAddress(addresses []msgmime.Address) string { + if len(addresses) == 0 { + return "" + } + return addresses[0].Email +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go new file mode 100644 index 000000000..92a9598e8 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go @@ -0,0 +1,199 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "log/slog" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftReplyOutputFailureRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var created draftReplyOutput + err := adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{ + Args: []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "createdcache", "--json"}, + }, func(event api.CLIRunEvent) error { + requirements.NoError(json.Unmarshal([]byte(event.Data), &created)) + cancel() + return errors.New("output disconnected") + }) + requirements.ErrorContains(err, "output_failed") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("createdcache"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1, "created draft must reach search despite output failure and cancellation") + assertions.Equal(draft.CurrentMessageID, results[0].ID) +} + +func TestDraftEditCancelledAfterPublicationRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original body") + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + if err == nil && draft.Revision == 2 { + cancel() + } + }, + })) + defer slog.SetDefault(previous) + err := adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "publishedcache", "--json"}, + }, func(api.CLIRunEvent) error { return nil }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + slog.SetDefault(previous) + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.Pending) + assertions.Equal(store.IMAPDraftCodeCleanup, draft.Pending.Code) + assertions.Equal(int64(2), draft.Revision) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("publishedcache"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + assertions.Equal(draft.CurrentMessageID, results[0].ID) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(2), draftCount, "both published and pending predecessor drafts remain visible") +} + +func TestDraftDeleteOutputFailureRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + for _, resume := range []bool{false, true} { + name := "normal deletion" + if resume { + name = "pending completion" + } + t.Run(name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "deletedcache") + if resume { + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), created.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + } + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + results, err := engine.SearchFast(t.Context(), search.Parse("deletedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + requirements.NoError(engine.Close()) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + err = adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json"}, + }, func(api.CLIRunEvent) error { + cancel() + return errors.New("output disconnected") + }) + requirements.ErrorContains(err, "output_failed") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.DiscardedAt) + engine, err = query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err = engine.SearchFast(t.Context(), search.Parse("deletedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Empty(results, "completed deletion must reach search despite output failure and cancellation") + }) + } +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go new file mode 100644 index 000000000..90807f714 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" +) + +func TestDraftDeleteCancelledBeforeStoreCanRetry(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "keep until retry") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + if err == nil && draft.Pending != nil { + cancel() + } + }, + })) + t.Cleanup(func() { slog.SetDefault(previous) }) + args := []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json"} + var events []api.CLIRunEvent + err := adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.Nil(draft.Pending) + assertions.Equal(int64(1), draft.Revision) + assertions.Nil(draft.DiscardedAt) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("active", output.Status) + assertions.False(output.ManualReconciliation) + slog.SetDefault(previous) + _, err = runReviewLifecycle(t, adapter, args...) + requirements.NoError(err) + draft, err = fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + assertions.NotNil(draft.DiscardedAt) +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go new file mode 100644 index 000000000..0515025ec --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go @@ -0,0 +1,1567 @@ +package cmd + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/emersion/go-imap/v2/imapserver" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/testutil" +) + +type reviewDraftCommitHandler struct { + slog.Handler + + onCommit func() +} + +func (h reviewDraftCommitHandler) Handle(_ context.Context, record slog.Record) error { + if record.Message == "sql tx commit" || record.Message == "sql tx slow" { + h.onCommit() + } + return nil +} + +func TestDraftLifecycleCancelledAfterClaimBeforeAppend(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if draft, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID); err == nil && draft.Pending != nil { + cancel() + } + }, + })) + defer slog.SetDefault(previous) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: []string{"draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "unsent", "--json"}}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + assertions.Zero(barrier.appendCalls.Load()) + pending, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(pending.Pending) + assertions.Equal(int64(1), pending.Revision) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, addr)) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(draftLifecycleActive, output.Status) + assertions.False(output.ManualReconciliation) +} + +func TestDraftLifecycleRejectedClaimPersistenceEvidence(t *testing.T) { + testutil.SkipIfPostgres(t, "claim failure injection uses SQLite triggers") + for _, fault := range []string{"record", "clear"} { + t.Run(fault, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{}), appendErr: &emersionimap.Error{Type: emersionimap.StatusResponseTypeNo, Text: "injected rejection"}} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + condition := "NEW.pending_code = 'append_rejected'" + if fault == "clear" { + condition = "OLD.pending_operation IS NOT NULL AND NEW.pending_operation IS NULL" + } + _, err := fixture.store.DB().Exec("CREATE TRIGGER fail_rejected_claim BEFORE UPDATE ON imap_drafts WHEN " + condition + " BEGIN SELECT RAISE(FAIL, 'injected rejection persistence failure'); END") + requirements.NoError(err) + events, err := runReviewLifecycle(t, fixture.adapter, "draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "candidate", "--json") + requirements.Error(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + if fault == "clear" { + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal("local_persistence_failed", err.Error()) + coded, ok := errors.AsType[*api.CLIRunCodedError](err) + requirements.True(ok) + requirements.ErrorContains(coded.Err, "injected rejection persistence failure") + } else { + assertions.Equal(draftLifecycleActive, output.Status) + assertions.False(output.ManualReconciliation) + } + pending, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + if fault == "clear" { + requirements.NotNil(pending.Pending) + assertions.Contains(string(pending.Pending.Raw), "candidate") + } else { + assertions.Nil(pending.Pending) + } + assertions.Equal(int64(1), pending.Revision) + assertions.Equal(int32(1), barrier.appendCalls.Load()) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, addr)) + }) + } +} + +func TestDraftReplyHumanLifecycleHandle(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftReplyCommand()) + silenceUsageInRunE(root) + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetArgs([]string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "handle"}) + requirements.NoError(root.ExecuteContext(t.Context())) + var draftID string + requirements.NoError(fixture.store.DB().QueryRow("SELECT draft_id FROM imap_drafts").Scan(&draftID)) + assertions.Contains(stdout.String(), "draft "+draftID+" revision 1") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), draftID) + requirements.NoError(err) + assertions.Equal(int64(1), draft.Revision) +} + +func TestEmitDraftLifecycleOutputHuman(t *testing.T) { + tests := []struct { + name string + output draftLifecycleOutput + want []string + }{ + { + name: "sanitizes metadata and multiline content", + output: draftLifecycleOutput{ + Status: "status\x1b[31m\r\ninjected", + DraftID: "draft-ID\x1b[31m\r\nspoofed", + Revision: 7, + Lifecycle: "life\x1b]0;title\x07cycle", + Receipt: draftLifecycleReceipt{Mailbox: "Mailbox\x1b]0;x\x07\r\nreceipt", UIDValidity: 11, UID: 12}, + Content: "line one\r\nline \x1b[31mred\x1b[0m\nline\u009bthree", + RawMIME: "raw-only-value", + CandidateContent: "candidate one\ncandidate \x1b]0;evil\x07two\r\nend", + PendingOperation: "edit\x1b[31m\r\noperation", + PendingCode: "pending\x1b[31m\r\ncode", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "Pending\x1b[31m\r\nmail", UIDValidity: 13, UID: 14}, + ManualReconciliation: true, + }, + want: []string{ + "draft draft-ID spoofed revision 7 lifecycle", + "status: status injected", + "receipt (revision 7): Mailbox receipt uidvalidity=11 uid=12", + "content:\nline one\nline red\nlinethree\n", + "pending operation: edit operation", + "candidate content:\ncandidate one\ncandidate two\nend\n", + "pending receipt (revision 7): Pending mail uidvalidity=13 uid=14", + "provider outcome: pending code", + "old draft ID remains blocked at revision 7", + "manual action: reconcile the provider receipt and local state before retrying", + }, + }, + { + name: "sanitizes pending old provider receipt", + output: draftLifecycleOutput{ + Status: "pending", + DraftID: "pending-id", + Revision: 8, + Lifecycle: "active", + Receipt: draftLifecycleReceipt{Mailbox: "Drafts", UIDValidity: 21, UID: 22}, + Content: "stored\nbody", + CandidateContent: "candidate\nbody", + PendingOperation: "edit", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "replacement", UIDValidity: 21, UID: 23}, + Observation: &draftLifecycleObservation{ + State: "present", Code: "old\x1b[31mcode", Mailbox: "old\x1b]0;evil\x07mail", + UIDValidity: 21, UID: 22, Present: true, + }, + }, + want: []string{ + "old provider receipt: oldmail uidvalidity=21 uid=22 (oldcode)", + "provider outcome: oldcode", + }, + }, + { + name: "sanitizes accepted replacement receipt", + output: draftLifecycleOutput{ + Status: "accepted_local_failed", + DraftID: "accepted-id", + Revision: 9, + Lifecycle: "active", + Receipt: draftLifecycleReceipt{Mailbox: "Drafts", UIDValidity: 31, UID: 32}, + Content: "stored", + PendingOperation: "edit", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "replacement", UIDValidity: 31, UID: 33}, + ProviderObservation: &draftLifecycleObservation{ + State: "present", Code: "replacement\x1b[31mcode", Mailbox: "new\x1b]0;evil\x07mail", + UIDValidity: 31, UID: 33, Present: true, + }, + Observation: &draftLifecycleObservation{ + State: "missing", Code: "old\x1b[31mcode", Mailbox: "old\x1b]0;evil\x07mail", + }, + }, + want: []string{ + "acknowledged replacement receipt: newmail uidvalidity=31 uid=33 (replacementcode)", + "provider outcome: replacementcode", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + var events []api.CLIRunEvent + err := emitDraftLifecycleOutput(func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }, cliStreamStderr, false, tt.output) + requirements.NoError(err) + requirements.Len(events, 1) + for _, want := range tt.want { + assertions.Contains(events[0].Data, want) + } + if tt.output.RawMIME != "" { + assertions.NotContains(events[0].Data, tt.output.RawMIME) + } + for _, control := range []string{"\x00", "\x07", "\x1b", "\r", "\u009b"} { + assertions.NotContains(events[0].Data, control) + } + }) + } +} + +func TestEmitDraftLifecycleOutputJSONRoundTrip(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + output := draftLifecycleOutput{ + Status: "accepted_local_failed\x1b[31m\r\nstatus", + DraftID: "draft\x1b]0;id\x07value", + Revision: 42, + Lifecycle: "active\u009bstate", + MessageID: 43, + SourceID: 44, + Receipt: draftLifecycleReceipt{Mailbox: "Drafts\r\nmailbox", UIDValidity: 45, UID: 46}, + Content: "stored\r\n\x1b[31mcontent\x1b[0m\u009b", + RawMIME: "From: sender@example.com\r\n\r\n\x1b[31mraw\x1b[0m\u009b", + CandidateContent: "candidate\r\n\x1b]0;candidate\x07content", + PendingOperation: "edit\r\noperation", + PendingCode: "pending\u009bcode", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "Replacement\x1b[31m\r\nmailbox", UIDValidity: 47, UID: 48}, + ProviderObservation: &draftLifecycleObservation{ + State: "present\x1b[31m", Code: "replacement\r\ncode", Mailbox: "Replacement\x1b]0;mail\x07box", + UIDValidity: 49, UID: 50, Flags: []string{"\\Seen", "flag\u009bvalue"}, Present: true, + Draft: true, Deleted: false, Complete: true, UIDPlus: true, + }, + Observation: &draftLifecycleObservation{ + State: "missing\x1b[31m", Code: "old\r\ncode", Mailbox: "Old\x1b]0;mail\x07box", + UIDValidity: 51, UID: 52, Flags: []string{"\\Draft", "old\u009bflag"}, Present: false, + Draft: false, Deleted: true, Complete: false, UIDPlus: false, + }, + ManualReconciliation: true, + } + + var events []api.CLIRunEvent + err := emitDraftLifecycleOutput(func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }, cliStreamStdout, true, output) + requirements.NoError(err) + requirements.Len(events, 1) + var decoded draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &decoded)) + assertions.Equal(output, decoded) +} + +func TestDraftLifecycleAcceptedReadFailureEvidence(t *testing.T) { + testutil.SkipIfPostgres(t, "local read fault injection alters SQLite tables") + for _, fault := range []string{"message", "reply_link"} { + for _, asJSON := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/json=%t", fault, asJSON), func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + barrier.storeCalls.Store(0) + barrier.expungeCalls.Store(0) + var faultErr error + barrier.onAppend = func() { + if fault == "message" { + _, faultErr = fixture.store.DB().Exec("ALTER TABLE message_bodies RENAME TO fault_message_bodies") + } else { + _, faultErr = fixture.store.DB().Exec("ALTER TABLE messages RENAME COLUMN reply_to_message_id TO fault_reply_to_message_id") + } + } + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + args := []string{"draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "saved candidate"} + if asJSON { + args = append(args, "--json") + } + root.SetArgs(args) + err := root.ExecuteContext(t.Context()) + requirements.NoError(faultErr) + requirements.ErrorContains(err, "accepted_local_failed") + assertions.Empty(stdout.String()) + if fault == "message" { + _, err = fixture.store.DB().Exec("ALTER TABLE fault_message_bodies RENAME TO message_bodies") + } else { + _, err = fixture.store.DB().Exec("ALTER TABLE messages RENAME COLUMN fault_reply_to_message_id TO reply_to_message_id") + } + requirements.NoError(err) + if asJSON { + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal(stderr.Bytes(), &output)) + assertions.Equal("accepted_local_failed", output.Status) + assertions.Equal(fixture.draft.DraftID, output.DraftID) + assertions.Contains(output.CandidateContent, "saved candidate") + requirements.NotNil(output.PendingReceipt) + assertions.Equal(uint32(2), output.PendingReceipt.UID) + assertions.True(output.ManualReconciliation) + } else { + assertions.Contains(stderr.String(), "status: accepted_local_failed") + assertions.Contains(stderr.String(), "saved candidate") + assertions.Contains(stderr.String(), "pending receipt (revision 1): Drafts uidvalidity=1 uid=2") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + } + pending, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(pending.Pending) + requirements.NotNil(pending.Pending.ReplacementReceipt) + assertions.Equal(uint32(2), pending.Pending.ReplacementReceipt.UID) + assertions.Equal(int64(1), pending.Revision) + assertions.Contains(string(pending.Pending.Raw), "saved candidate") + assertions.Equal(int32(1), barrier.appendCalls.Load()) + assertions.Zero(barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + assertions.Equal(uint32(2), reviewDraftMailboxCount(t, addr)) + }) + } + } +} + +func TestDraftLifecyclePublicationFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + conversationID, err := fixture.store.EnsureConversation(fixture.source.ID, "publication-conflict", "Publication conflict") + requirements.NoError(err) + _, err = fixture.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: "Drafts|2", ConversationID: conversationID, + MessageType: store.MessageTypeEmail, + }, + BodyText: sql.NullString{String: "conflict", Valid: true}, + RawMIME: []byte("From: conflict@example.com\r\n\r\nconflict\r\n"), + }) + requirements.NoError(err) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "candidate", "--json"}) + err = root.ExecuteContext(t.Context()) + assertions.Empty(stdout.String()) + events := []api.CLIRunEvent{{Type: cliStreamStderr, Data: stderr.String()}} + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + var output struct { + Status string `json:"status"` + Revision int64 `json:"revision"` + CandidateContent string `json:"candidate_content"` + PendingOperation string `json:"pending_operation"` + PendingReceipt struct { + UID uint32 `json:"uid"` + } `json:"pending_receipt"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("accepted_local_failed", output.Status) + assertions.Equal(int64(1), output.Revision) + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal("edit", output.PendingOperation) + assertions.Equal(uint32(2), output.PendingReceipt.UID) +} + +func TestDraftLifecycleHumanAcknowledgedReceiptWhenRecordFails(t *testing.T) { + testutil.SkipIfPostgres(t, "receipt write fault injection uses a SQLite trigger") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_draft_replacement_receipt + BEFORE UPDATE OF pending_replacement_uid ON imap_drafts + WHEN NEW.pending_replacement_uid IS NOT NULL + AND OLD.pending_replacement_uid IS NULL + BEGIN + SELECT RAISE(FAIL, 'injected replacement receipt failure'); + END + `) + requirements.NoError(err) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, created.DraftID, + "--revision", "1", "--body", "candidate"}) + err = root.ExecuteContext(t.Context()) + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: accepted_local_failed") + assertions.Contains(stderr.String(), "pending operation: edit") + assertions.Contains(stderr.String(), "provider outcome: append_uidplus") + assertions.Contains(stderr.String(), "acknowledged replacement receipt: Drafts uidvalidity=1 uid=2") + assertions.Contains(stderr.String(), "receipt (revision 1): Drafts uidvalidity=1 uid=1") + assertions.Contains(stderr.String(), "candidate content:") + assertions.Contains(stderr.String(), "candidate") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "pending receipt:") + assertions.NotContains(stderr.String(), "Error: accepted_local_failed") + + pending, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + requirements.Equal(int64(1), pending.Revision) + requirements.NotNil(pending.Pending) + requirements.Nil(pending.Pending.ReplacementReceipt) + requirements.Equal(uint32(1), pending.Pending.OriginalReceipt.UID) + requirements.Contains(string(pending.Pending.Raw), "candidate") + provider := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + oldObservation, err := provider.InspectDraft(t.Context(), imaplib.DraftReceipt{ + Mailbox: "Drafts", UIDValidity: 1, UID: 1, + }) + requirements.NoError(err) + newObservation, err := provider.InspectDraft(t.Context(), imaplib.DraftReceipt{ + Mailbox: "Drafts", UIDValidity: 1, UID: 2, + }) + requirements.NoError(err) + assertions.True(oldObservation.Present) + assertions.True(newObservation.Present) + requirements.NoError(provider.Close()) +} + +func TestDraftLifecycleUnknownAppend(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewDropAppendServer(t) + dbPath := filepath.Join(t.TempDir(), "lifecycle.db") + fixture := newReviewManagedLifecycleFixtureOnServerWithDBPath(t, addr, dbPath, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + var providerCalls atomic.Int32 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + } + + events, err := runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftEditCommand, fixture.draft.DraftID, "--revision", "1", "--body", "candidate", "--json") + requirements.Error(err) + assertions.Equal("remote_unknown", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"candidate_content"`) + assertions.Contains(events[0].Data, "candidate") + assertions.Equal(uint32(2), reviewDraftMailboxCount(t, fixture.config.Addr())) + pending, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(pending.Pending) + assertions.Equal("remote_unknown", pending.Pending.Code) + assertions.Equal(int32(1), providerCalls.Load()) + + requirements.NoError(fixture.store.Close()) + reopened, err := store.OpenForTest(dbPath) + requirements.NoError(err) + t.Cleanup(func() { _ = reopened.Close() }) + reopenedDraft, err := reopened.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(reopenedDraft.Pending) + assertions.Equal("remote_unknown", reopenedDraft.Pending.Code) + assertions.Contains(string(reopenedDraft.Pending.Raw), "candidate") + freshAdapter := &storeAPIAdapter{ + store: reopened, + draftPolicy: []config.IMAPDraftSource{{SourceID: fixture.source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + }, + } + events, err = runReviewLifecycle(t, freshAdapter, api.CLIRunDraftEditCommand, fixture.draft.DraftID, "--revision", "1", "--body", "retry", "--json") + requirements.Error(err) + assertions.Equal("pending_operation", err.Error()) + assertions.Empty(events) + assertions.Equal(int32(1), providerCalls.Load()) +} + +func TestDraftLifecycleHumanUnknownAppendThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewDropAppendServer(t) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, fixture.draft.DraftID, + "--revision", "1", "--body", "candidate"}) + err := root.ExecuteContext(t.Context()) + requirements.Error(err) + assertions.Equal("remote_unknown", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: pending") + assertions.Contains(stderr.String(), "pending operation: edit") + assertions.Contains(stderr.String(), "provider outcome: remote_unknown") + assertions.Contains(stderr.String(), "candidate content:") + assertions.Contains(stderr.String(), "candidate") + assertions.Contains(stderr.String(), "receipt (revision 1): Drafts") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "Error: remote_unknown") + assertions.Equal(1, strings.Count(stderr.String(), "provider outcome:")) +} + +func TestDraftLifecycleRetainedGet(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "retained") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("draft-get must not connect to IMAP") + } + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ? + `), created.MessageID) + requirements.NoError(err) + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "retained") + assertions.Contains(events[0].Data, `"provider_observation":{"state":"not_checked","code":"not_checked"`) + assertions.Zero(providerCalls.Load()) + + _, err = fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationEdit, []byte("candidate raw")) + requirements.NoError(err) + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + assertions.Contains(events[0].Data, `"candidate_content":"candidate raw"`) +} + +func TestDraftLifecycleRepeatedDelete(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "discard me") + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var deleted struct { + Revision int64 `json:"revision"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &deleted)) + assertions.Equal(int64(2), deleted.Revision) + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("discarded retry must be offline") + } + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"already_discarded"`) + assertions.Zero(providerCalls.Load()) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("revision_mismatch", err.Error()) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "2", "--body", "new", "--json") + requirements.Error(err) + assertions.Equal("draft_discarded", err.Error()) +} + +func TestDraftLifecycleExactCopy(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "copy") + copyClient := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + copyReceipt, err := copyClient.AppendDraft(t.Context(), "Drafts", []byte("From: alice@example.com\r\nTo: "+testutil.IMAPTestUsername+"\r\nSubject: Question\r\n\r\ncopy\r\n")) + requirements.NoError(err) + requirements.NoError(copyClient.Close()) + requirements.NotEqual(uint32(0), copyReceipt.UID) + testutil.ExpungeIMAPMessage(t, fixture.config.Addr(), "Drafts", emersionimap.UID(1)) + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "must refuse", "--json") + requirements.Error(err) + assertions.Equal("absent", err.Error()) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"code":"absent"`) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Nil(draft.Pending) + + second := createReviewDraft(t, fixture, adapter, "valid second draft") + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, second.DraftID, "--revision", "1", "--body", "works", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"revision":2`) + inspection := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := inspection.InspectDraft(t.Context(), imaplib.DraftReceipt{Mailbox: "Drafts", UIDValidity: copyReceipt.UIDValidity, UID: copyReceipt.UID}) + requirements.NoError(err) + assertions.True(observation.Present) +} + +func TestDraftLifecyclePolicy(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "policy") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("policy refusal must precede provider") + } + adapter.draftPolicy[0].Mailbox = "Other" + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "blocked", "--json") + requirements.Error(err) + assertions.Equal("invalid_mailbox", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls.Load()) + adapter.draftPolicy = nil + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "policy") + assertions.Zero(providerCalls.Load()) + + adapter.draftPolicy = []config.IMAPDraftSource{{SourceID: fixture.source.ID, Enabled: true, Mailbox: "Drafts"}} + badConfig, err := (&imaplib.Config{Host: "127.0.0.1", Port: 1, Username: testutil.IMAPTestUsername}).ToJSON() + requirements.NoError(err) + requirements.NoError(fixture.store.UpdateSourceSyncConfig(fixture.source.ID, badConfig)) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("invalid_source", err.Error()) + assertions.Zero(providerCalls.Load()) +} + +func TestDraftLifecycleBoundaries(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "boundaries") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("local draft-get must not open provider") + } + for _, state := range []string{"active", "pending", "discarded"} { + t.Run(state, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + switch state { + case "pending": + _, err := fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationEdit, []byte("pending")) + requirements.NoError(err) + case "discarded": + _, err := fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(context.Background(), created.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + _, err = fixture.store.FinishIMAPDraftRemovalContext(context.Background(), created.DraftID, 1) + requirements.NoError(err) + } + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "boundaries") + assertions.Zero(providerCalls.Load()) + }) + if state == "pending" { + _, err := fixture.store.DB().Exec(fixture.store.Rebind(`UPDATE imap_drafts SET pending_operation = NULL, pending_original_message_id = NULL, pending_original_mailbox = NULL, pending_original_uidvalidity = NULL, pending_original_uid = NULL, pending_raw = NULL, pending_code = NULL WHERE draft_id = ?`), created.DraftID) + requirements.NoError(err) + } + } +} + +func TestDraftLifecycleCleanupOutcomeEvidence(t *testing.T) { + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + for _, fault := range []string{"none", "record", "read"} { + t.Run(fmt.Sprintf("%s/fault=%s", operation, fault), func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.armed.Store(true) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate") + } + eventsCh := make(chan []api.CLIRunEvent, 1) + errCh := make(chan error, 1) + go func() { + events, err := runReviewLifecycle(t, fixture.adapter, args...) + eventsCh <- events + errCh <- err + }() + <-barrier.stored + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsDel, emersionimap.FlagDeleted) + if fault == "record" { + _, err := fixture.store.DB().Exec(fixture.store.Rebind("UPDATE imap_drafts SET revision = revision + 1 WHERE draft_id = ?"), fixture.draft.DraftID) + requirements.NoError(err) + } + if fault == "read" { + _, err := fixture.store.DB().Exec("ALTER TABLE imap_drafts RENAME TO fault_imap_drafts") + requirements.NoError(err) + } + close(barrier.release) + events, err := <-eventsCh, <-errCh + requirements.Error(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + assertions.Equal("store_conflict", output.PendingCode) + requirements.NotNil(output.Observation) + assertions.Equal("store_conflict", output.Observation.Code) + assertions.Equal(uint32(1), output.Observation.UID) + assertions.True(output.Observation.Present) + if fault == "read" { + _, err := fixture.store.DB().Exec("ALTER TABLE fault_imap_drafts RENAME TO imap_drafts") + requirements.NoError(err) + } + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(latest.Revision, output.Revision) + if fault != "none" { + assertions.Equal("local_persistence_failed", err.Error()) + coded, ok := errors.AsType[*api.CLIRunCodedError](err) + requirements.True(ok) + if fault == "record" { + requirements.ErrorIs(coded.Err, store.ErrIMAPDraftRevision) + } else { + requirements.ErrorContains(coded.Err, "imap_drafts") + } + assertions.NotEqual("store_conflict", latest.Pending.Code) + } else { + assertions.Equal("store_conflict", err.Error()) + assertions.Equal("store_conflict", latest.Pending.Code) + } + if operation == api.CLIRunDraftEditCommand { + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal(uint32(2), output.Receipt.UID) + requirements.NotNil(output.PendingReceipt) + assertions.Equal(uint32(2), output.PendingReceipt.UID) + assertions.Equal(uint32(2), latest.CurrentReceipt.UID) + assertions.Equal(uint32(1), latest.Pending.OriginalReceipt.UID) + } else { + assertions.Nil(latest.DiscardedAt) + } + }) + } + } +} + +func TestDraftLifecycleCleanup(t *testing.T) { + testutil.SkipIfPostgres(t, "finish failure injection uses a SQLite trigger") + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_draft_finish + AFTER UPDATE OF pending_code ON imap_drafts + WHEN NEW.pending_code = 'removed' + BEGIN + UPDATE imap_drafts SET revision = NEW.revision + 1 WHERE draft_id = NEW.draft_id; + END + `) + requirements.NoError(err) + barrier.armed.Store(true) + eventsCh := make(chan []api.CLIRunEvent, 1) + errCh := make(chan error, 1) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate\nline two") + } + go func() { + events, err := runReviewLifecycle(t, fixture.adapter, args...) + eventsCh <- events + errCh <- err + }() + <-barrier.stored + close(barrier.release) + events, err := <-eventsCh, <-errCh + requirements.Error(err) + assertions.Equal("cleanup_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"pending"`) + assertions.Contains(events[0].Data, `"manual_reconciliation":true`) + assertions.Contains(events[0].Data, `"pending_operation":"`+strings.TrimPrefix(operation, "draft-")+`"`) + assertions.Contains(events[0].Data, `"pending_code":"removed"`) + assertions.Contains(events[0].Data, `"code":"removed"`) + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(store.IMAPDraftCodeRemoved, latest.Pending.Code) + assertions.Equal(operation, "draft-"+latest.Pending.Operation) + + _, err = fixture.store.DB().Exec("DROP TRIGGER fail_draft_finish") + requirements.NoError(err) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("local completion must not connect to IMAP") + } + retryArgs := []string{operation, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10), "--json"} + if operation == api.CLIRunDraftEditCommand { + _, err = runReviewLifecycle(t, fixture.adapter, append(retryArgs, "--body", "different")...) + requirements.ErrorContains(err, "pending_operation") + retryArgs = append(retryArgs, "--body", "candidate\r\nline two") + } + staleArgs := append([]string(nil), retryArgs...) + staleArgs[3] = "1" + _, err = runReviewLifecycle(t, fixture.adapter, staleArgs...) + requirements.ErrorContains(err, "revision_mismatch") + if operation == api.CLIRunDraftDeleteCommand { + _, err = runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftEditCommand, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10), "--body", "candidate") + } else { + _, err = runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftDeleteCommand, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10)) + } + requirements.ErrorContains(err, "pending_operation") + events, err = runReviewLifecycle(t, fixture.adapter, retryArgs...) + requirements.NoError(err) + requirements.Len(events, 1) + finished, err := fixture.store.GetIMAPDraftContext(t.Context(), latest.DraftID) + requirements.NoError(err) + assertions.Nil(finished.Pending) + if operation == api.CLIRunDraftDeleteCommand { + assertions.NotNil(finished.DiscardedAt) + assertions.Equal(latest.Revision+1, finished.Revision) + } else { + assertions.Equal(latest.Revision, finished.Revision) + assertions.Contains(events[0].Data, `"status":"edited"`) + } + }) + } +} + +func TestDraftLifecycleRemovedOutcomePersistenceFailure(t *testing.T) { + testutil.SkipIfPostgres(t, "removed outcome failure injection uses a SQLite trigger") + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewCmdStoreBarrierServer(t, &reviewCmdStoreBarrier{ + stored: make(chan struct{}), release: make(chan struct{}), + }) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_removed_outcome + BEFORE UPDATE OF pending_code ON imap_drafts + WHEN NEW.pending_code = 'removed' + BEGIN + SELECT RAISE(FAIL, 'injected removed outcome persistence failure'); + END + `) + requirements.NoError(err) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate") + } + events, err := runReviewLifecycle(t, fixture.adapter, args...) + requirements.Error(err) + assertions.Equal("local_persistence_failed", err.Error()) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + requirements.NotNil(output.Observation) + assertions.Equal(store.IMAPDraftCodeRemoved, output.Observation.Code) + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(operation, "draft-"+latest.Pending.Operation) + assertions.NotEqual(store.IMAPDraftCodeRemoved, latest.Pending.Code) + assertions.Nil(latest.DiscardedAt) + }) + } +} + +func TestDraftLifecycleHumanCleanupPartialThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.armed.Store(true) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftDeleteCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftDeleteCommand, fixture.draft.DraftID, + "--revision", "1"}) + + errCh := make(chan error, 1) + go func() { errCh <- root.ExecuteContext(t.Context()) }() + <-barrier.stored + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts SET revision = 2 WHERE draft_id = ? + `), fixture.draft.DraftID) + requirements.NoError(err) + close(barrier.release) + err = <-errCh + requirements.Error(err) + assertions.Equal("local_persistence_failed", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: pending") + assertions.Contains(stderr.String(), "pending operation: delete") + assertions.Contains(stderr.String(), "provider outcome: removed") + assertions.Contains(stderr.String(), "old provider receipt: Drafts") + assertions.Contains(stderr.String(), "receipt (revision 2): Drafts") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 2") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "Error: cleanup_local_failed") +} + +func TestDraftLifecycleSyncProjection(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "sync projection") + var analyticsDir string + if !fixture.store.IsPostgreSQL() { + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir = filepath.Join(cacheRoot, "analytics") + refreshCache := func() error { + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(refreshCache()) + adapter.draftCacheRefresh = func(context.Context, string) error { return refreshCache() } + } + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "projected", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output struct { + MessageID int64 `json:"message_id"` + Revision int64 `json:"revision"` + Receipt struct { + UIDValidity uint32 `json:"uidvalidity"` + UID uint32 `json:"uid"` + } `json:"receipt"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(int64(2), output.Revision) + raw, err := fixture.store.GetMessageRawContext(context.Background(), output.MessageID) + requirements.NoError(err) + assertions.Contains(string(raw), "projected") + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Equal(output.MessageID, draft.CurrentMessageID) + assertions.Equal(output.Receipt.UID, draft.CurrentReceipt.UID) + assertions.Nil(draft.Pending) + if !fixture.store.IsPostgreSQL() { + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("projected"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + assertions.Equal(output.MessageID, results[0].ID) + assertions.Contains(results[0].Snippet, "projected") + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftLabel query.AggregateRow + for _, row := range rows { + if row.Key == "Drafts" { + draftLabel = row + break + } + } + assertions.Equal(int64(1), draftLabel.Count) + } + adminClient, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + requirements.NoError(err) + requirements.NoError(adminClient.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + requirements.NoError(adminClient.Create("INBOX", nil).Wait()) + requirements.NoError(adminClient.Close()) + syncClient := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + syncOptions := msgsync.DefaultOptions() + syncOptions.SourceType = "imap" + syncOptions.NoResume = true + summary, err := newMessageSyncer(syncClient, fixture.store, syncOptions). + WithLogger(slog.New(slog.DiscardHandler)). + FullWithFinalizer(t.Context(), fixture.source, func(summary *gmail.SyncSummary) error { + return saveIMAPFolderStates(t.Context(), fixture.store, fixture.source, syncClient, summary, syncOptions.Limit) + }) + requirements.NoError(err) + requirements.Zero(summary.Errors) + requirements.NoError(syncClient.Close()) + draft, err = fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Equal(output.MessageID, draft.CurrentMessageID) + assertions.Equal(output.Receipt.UIDValidity, draft.CurrentReceipt.UIDValidity) + assertions.Equal(output.Receipt.UID, draft.CurrentReceipt.UID) + assertions.Nil(draft.Pending) + states, err := fixture.store.GetIMAPFolderStates(fixture.source.ID) + requirements.NoError(err) + var draftsState store.IMAPFolderState + for _, state := range states { + if state.Mailbox == "Drafts" { + draftsState = state + break + } + } + assertions.Equal("Drafts", draftsState.Mailbox) + assertions.Equal(output.Receipt.UIDValidity, draftsState.UIDValidity) + assertions.Equal(output.Receipt.UID+1, draftsState.UIDNext) + provider := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := provider.InspectDraft(t.Context(), imaplib.DraftReceipt{Mailbox: "Drafts", UIDValidity: output.Receipt.UIDValidity, UID: output.Receipt.UID}) + requirements.NoError(err) + assertions.True(observation.Present) +} + +func TestDraftLifecycleCancellation(t *testing.T) { + requirements := require.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + baseCtx, cancel := context.WithCancel(t.Context()) + ctx := &reviewLateCancelContext{Context: baseCtx, cancel: cancel} + barrier.onAppend = ctx.arm + addr, _ := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewDraftReplyFixtureAtAddr(t, addr) + adapter := fixture.grantedAdapter() + adapter.draftCacheRefresh = func(context.Context, string) error { return nil } + args := []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), + "--from", testutil.IMAPTestUsername, "--body", "known cancellation", "--json"} + var events []api.CLIRunEvent + err := adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.True(ctx.cancelled.Load()) + requirements.Len(events, 1) + var created draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &created)) + requirements.Equal(draftReplyStatusCreated, created.Status) + requirements.Equal(int64(1), created.Revision) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + requirements.Equal(created.MessageID, draft.CurrentMessageID) + requirements.Equal(created.UID, draft.CurrentReceipt.UID) + requirements.Nil(draft.Pending) + requirements.Equal(int32(1), barrier.appendCalls.Load()) + requirements.Zero(barrier.storeCalls.Load()) +} + +func TestDraftLifecycleCleanupCompletesAfterLateCancellation(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + baseCtx, cancel := context.WithCancel(t.Context()) + ctx := &reviewLateCancelContext{Context: baseCtx, cancel: cancel} + barrier.onFetch = func(n int32) { + if n == 4 { + ctx.arm() + } + } + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftDeleteCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.True(ctx.cancelled.Load()) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"deleted"`) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.DiscardedAt) + requirements.Nil(draft.Pending) + assertions.Equal(int32(4), barrier.fetchCalls.Load()) + assertions.Equal(int32(2), barrier.storeCalls.Load()) + assertions.Equal(int32(1), barrier.expungeCalls.Load()) +} + +func TestDraftLifecycleRejectsEnvAndCwd(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + adapter := &storeAPIAdapter{} + for _, request := range []api.CLIRunRequest{ + {Args: []string{api.CLIRunDraftGetCommand, "draft-test"}, Env: map[string]string{"HOME": "blocked"}}, + {Args: []string{api.CLIRunDraftGetCommand, "draft-test"}, Cwd: "C:\\blocked"}, + } { + err := adapter.runCLIDraftLifecycle(t.Context(), request, nil) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } +} + +type reviewCreatedDraft struct { + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + MessageID int64 `json:"message_id"` +} + +type reviewLateCancelContext struct { + context.Context + + cancel context.CancelFunc + armed atomic.Bool + cancelled atomic.Bool +} + +func (c *reviewLateCancelContext) arm() { + c.armed.Store(true) +} + +func (c *reviewLateCancelContext) Err() error { + if c.armed.Load() && c.cancelled.CompareAndSwap(false, true) { + c.cancel() + } + return c.Context.Err() +} + +func newReviewDraftReplyFixtureAtAddr(t *testing.T, addr string) draftReplyFixture { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portText) + require.NoError(t, err) + imapConfig := &imaplib.Config{Host: host, Port: port, Username: testutil.IMAPTestUsername} + + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + configJSON = strings.TrimSuffix(configJSON, "}") + `,"draft_enabled":false,"drafts_mailbox":"AttackerMailbox"}` + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + require.NoError(t, st.AddAccountIdentity(source.ID, testutil.IMAPTestUsername, "manual")) + conversationID, err := st.EnsureConversation(source.ID, "thread-666", "Question") + require.NoError(t, err) + senderID, err := st.EnsureParticipant("sender@example.com", "Sender", "example.com") + require.NoError(t, err) + ownerID, err := st.EnsureParticipant(testutil.IMAPTestUsername, "", "example.com") + require.NoError(t, err) + parentRaw := []byte("From: Sender \r\n" + + "To: " + testutil.IMAPTestUsername + "\r\n" + + "Subject: Question\r\n" + + "Message-ID: \r\n\r\n" + + "Parent body\r\n") + parentID, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: "INBOX|9", + ConversationID: conversationID, RFC822MessageID: sql.NullString{String: "parent@example.com", Valid: true}, + MessageType: store.MessageTypeEmail, SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + Subject: sql.NullString{String: "Question", Valid: true}, + SentAt: sql.NullTime{Time: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), Valid: true}, + SizeEstimate: int64(len(parentRaw)), + }, + BodyText: sql.NullString{String: "Parent body", Valid: true}, RawMIME: parentRaw, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: []int64{senderID}, EmailAddresses: []string{"sender@example.com"}}, + {Type: "to", ParticipantIDs: []int64{ownerID}, EmailAddresses: []string{testutil.IMAPTestUsername}}, + }, + }) + require.NoError(t, err) + return draftReplyFixture{store: st, source: source, parentID: parentID, config: imapConfig, refreshed: new([]string)} +} + +func createReviewDraft(t *testing.T, fixture draftReplyFixture, adapter *storeAPIAdapter, body string) reviewCreatedDraft { + t.Helper() + events, err := fixture.run(t, adapter, "--body", body, "--json") + require.NoError(t, err) + require.Len(t, events, 1) + var created reviewCreatedDraft + require.NoError(t, json.Unmarshal([]byte(events[0].Data), &created)) + require.NotEmpty(t, created.DraftID) + require.Equal(t, int64(1), created.Revision) + return created +} + +func runReviewLifecycle(t *testing.T, adapter *storeAPIAdapter, args ...string) ([]api.CLIRunEvent, error) { + t.Helper() + var events []api.CLIRunEvent + err := adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + return events, err +} + +type reviewManagedLifecycleFixture struct { + store *store.Store + source *store.Source + config *imaplib.Config + draft store.IMAPDraft + adapter *storeAPIAdapter +} + +func newReviewManagedLifecycleFixtureOnServer(t *testing.T, addr string, appendInitial func()) reviewManagedLifecycleFixture { + t.Helper() + imapConfig := reviewIMAPConfig(t, addr) + appendInitial() + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + conversationID, err := st.EnsureConversation(source.ID, "lifecycle-review", "Lifecycle review") + require.NoError(t, err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + raw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n") + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), MessageType: store.MessageTypeEmail, ConversationID: conversationID}, + BodyText: sql.NullString{String: "original", Valid: true}, RawMIME: raw, + } + }) + require.NoError(t, err) + adapter := &storeAPIAdapter{ + store: st, + draftPolicy: []config.IMAPDraftSource{{SourceID: source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(imapConfig, testutil.IMAPTestPassword), nil + }, + } + return reviewManagedLifecycleFixture{store: st, source: source, config: imapConfig, draft: draft, adapter: adapter} +} + +func newReviewManagedLifecycleFixtureOnServerWithDBPath( + t *testing.T, addr, dbPath string, appendInitial func(), +) reviewManagedLifecycleFixture { + t.Helper() + imapConfig := reviewIMAPConfig(t, addr) + appendInitial() + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + st, err := store.OpenForTest(dbPath) + require.NoError(t, err) + require.NoError(t, st.InitSchema()) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + conversationID, err := st.EnsureConversation(source.ID, "lifecycle-review", "Lifecycle review") + require.NoError(t, err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + raw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n") + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), MessageType: store.MessageTypeEmail, ConversationID: conversationID}, + BodyText: sql.NullString{String: "original", Valid: true}, RawMIME: raw, + } + }) + require.NoError(t, err) + adapter := &storeAPIAdapter{ + store: st, + draftPolicy: []config.IMAPDraftSource{{SourceID: source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(imapConfig, testutil.IMAPTestPassword), nil + }, + } + return reviewManagedLifecycleFixture{store: st, source: source, config: imapConfig, draft: draft, adapter: adapter} +} + +type reviewDropAppendSession struct { + imapserver.Session + + conn *imapserver.Conn +} + +func (s *reviewDropAppendSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + data, err := s.Session.Append(mailbox, r, options) + if err == nil { + _ = s.conn.NetConn().Close() + } + if err != nil { + return nil, fmt.Errorf("drop append session: %w", err) + } + return data, nil +} + +func startReviewDropAppendServer(t *testing.T) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(conn *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return &reviewDropAppendSession{Session: memServer.NewSession(), conn: conn}, nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +type reviewCmdStoreBarrier struct { + stored chan struct{} + release chan struct{} + armed atomic.Bool + blocked atomic.Bool + appendCalls atomic.Int32 + fetchCalls atomic.Int32 + storeCalls atomic.Int32 + expungeCalls atomic.Int32 + appendErr error + onAppend func() + onFetch func(int32) +} + +type reviewCmdStoreBarrierSession struct { + imapserver.Session + + barrier *reviewCmdStoreBarrier +} + +func (s *reviewCmdStoreBarrierSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + if s.barrier.appendErr != nil { + s.barrier.appendCalls.Add(1) + return nil, s.barrier.appendErr + } + data, err := s.Session.Append(mailbox, r, options) + s.barrier.appendCalls.Add(1) + if s.barrier.onAppend != nil { + s.barrier.onAppend() + } + if err != nil { + return nil, fmt.Errorf("append barrier session: %w", err) + } + return data, nil +} + +func (s *reviewCmdStoreBarrierSession) Fetch( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + options *emersionimap.FetchOptions, +) error { + err := s.Session.Fetch(w, numSet, options) + n := s.barrier.fetchCalls.Add(1) + if s.barrier.onFetch != nil { + s.barrier.onFetch(n) + } + if err != nil { + return fmt.Errorf("fetch barrier session: %w", err) + } + return nil +} + +func (s *reviewCmdStoreBarrierSession) Store( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + flags *emersionimap.StoreFlags, + options *emersionimap.StoreOptions, +) error { + err := s.Session.Store(w, numSet, flags, options) + s.barrier.storeCalls.Add(1) + if s.barrier.armed.Load() && s.barrier.blocked.CompareAndSwap(false, true) { + close(s.barrier.stored) + <-s.barrier.release + } + if err != nil { + return fmt.Errorf("store barrier session: %w", err) + } + return nil +} + +func (s *reviewCmdStoreBarrierSession) Expunge( + w *imapserver.ExpungeWriter, + uids *emersionimap.UIDSet, +) error { + err := s.Session.Expunge(w, uids) + s.barrier.expungeCalls.Add(1) + if err != nil { + return fmt.Errorf("expunge barrier session: %w", err) + } + return nil +} + +func startReviewCmdStoreBarrierServer(t *testing.T, barrier *reviewCmdStoreBarrier) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(conn *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return &reviewCmdStoreBarrierSession{Session: memServer.NewSession(), barrier: barrier}, nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +func reviewIMAPConfig(t *testing.T, addr string) *imaplib.Config { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portText) + require.NoError(t, err) + return &imaplib.Config{Host: host, Port: port, Username: testutil.IMAPTestUsername} +} + +func reviewStoreFlagsForLifecycle(t *testing.T, addr string, uid uint32, op emersionimap.StoreFlagsOp, flag emersionimap.Flag) { + t.Helper() + client, err := imapclient.DialInsecure(addr, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select("Drafts", nil).Wait() + require.NoError(t, err) + var uids emersionimap.UIDSet + uids.AddNum(emersionimap.UID(uid)) + require.NoError(t, client.Store(uids, &emersionimap.StoreFlags{Op: op, Flags: []emersionimap.Flag{flag}}, nil).Close()) +} + +func reviewDraftMailboxCount(t *testing.T, addr string) uint32 { + t.Helper() + client, err := imapclient.DialInsecure(addr, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + data, err := client.Status("Drafts", &emersionimap.StatusOptions{NumMessages: true}).Wait() + require.NoError(t, err) + require.NotNil(t, data.NumMessages) + return *data.NumMessages +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_test.go new file mode 100644 index 000000000..f59e748ef --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_test.go @@ -0,0 +1,64 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "log/slog" + "net/http/httptest" + "strconv" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftLifecycleEndToEnd(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + configureRemoteDaemonForTest(t, server.URL) + run := func(args ...string) (string, error) { + root := &cobra.Command{Use: "msgvault"} + root.AddCommand(newDraftReplyCommand(), newDraftGetCommand(), newDraftEditCommand(), newDraftDeleteCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs(args) + err := root.ExecuteContext(t.Context()) + requirements.Empty(stderr.String()) + return stdout.String(), err + } + createdJSON, err := run("draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "initial body", "--json") + requirements.NoError(err) + var created struct { + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + } + requirements.NoError(json.Unmarshal([]byte(createdJSON), &created)) + requirements.NotEmpty(created.DraftID) + requirements.Equal(int64(1), created.Revision) + + getEvent, err := run(api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Contains(getEvent, "initial body") + humanGet, err := run(api.CLIRunDraftGetCommand, created.DraftID) + requirements.NoError(err) + requirements.Contains(humanGet, "content:\ninitial body") + requirements.Contains(humanGet, "receipt (revision 1): Drafts") + + editEvent, err := run(api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "edited body", "--json") + requirements.NoError(err) + requirements.Contains(editEvent, "\"revision\":2") + + deleteEvent, err := run(api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Contains(deleteEvent, "\"lifecycle\":\"discarded\"") + _, err = run(api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) +} diff --git a/cmd/msgvault/cmd/serve_reply_draft.go b/cmd/msgvault/cmd/serve_reply_draft.go index 9264e90b3..70d127dbc 100644 --- a/cmd/msgvault/cmd/serve_reply_draft.go +++ b/cmd/msgvault/cmd/serve_reply_draft.go @@ -46,6 +46,8 @@ type draftReplyTarget struct { type draftReplyOutput struct { Status string `json:"status"` + DraftID string `json:"draft_id,omitempty"` + Revision int64 `json:"revision,omitzero"` MessageID int64 `json:"message_id,omitzero"` OperationRef string `json:"operation_ref"` RFC822MessageID string `json:"rfc822_message_id"` @@ -210,7 +212,16 @@ func (a *storeAPIAdapter) runCLIReplyDraft( } defer func() { _ = execution.Release() }() - receipt, err := a.appendDraftReply(ctx, target, reply.Raw, emit) + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory + } + client, err := clientFactory(ctx, target.source) + if err != nil { + return draftReplyError("invalid_source", fmt.Errorf("build IMAP client for source %d: %w", target.source.ID, err)) + } + defer func() { _ = client.Close() }() + receipt, err := a.appendDraftReplyWithClient(ctx, client, target, reply.Raw, emit) if err != nil { return err } @@ -227,7 +238,9 @@ func (a *storeAPIAdapter) runCLIReplyDraft( UID: receipt.UID, UIDValidity: receipt.UIDValidity, } - localID, err := a.store.PersistIMAPDraftContext(ctx, receiptModel, draftReplyParticipants(reply.Parsed), func(ids []int64) *store.MessagePersistData { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + draft, err := a.store.PersistIMAPDraftContext(evidenceCtx, receiptModel, draftReplyParticipants(reply.Parsed), func(ids []int64) *store.MessagePersistData { return draftReplyPersistData(target, reply, receiptModel, messageIDValue, ids) }) if err != nil { @@ -235,16 +248,14 @@ func (a *storeAPIAdapter) runCLIReplyDraft( _ = emitDraftReplyOutput(emit, cliStreamStderr, intent.JSON, result) return draftReplyError(draftReplyStatusLocalFailed, err) } - result.MessageID = localID + defer a.releaseDraftSourceAndRefreshCache(ctx, target.source, execution) + result.DraftID = draft.DraftID + result.Revision = draft.Revision + result.MessageID = draft.CurrentMessageID if err := emitDraftReplyOutput(emit, cliStreamStdout, intent.JSON, result); err != nil { return draftReplyError("output_failed", err) } - // The draft is durable and reported. Free the source for syncs before the - // cache rebuild, which can take a while and needs no lock. - if err := execution.Release(); err != nil { - logger.Error("release source after draft", "source_id", target.source.ID, "error", err) - } - a.refreshDraftCache(ctx, target.source) + _ = client.Close() return nil } @@ -339,23 +350,13 @@ func defaultDraftClientFactory(ctx context.Context, source *store.Source) (*imap return imapClient, nil } -// appendDraftReply sends the single APPEND. Any failure after this point -// leaves a state the operator must inspect before retrying. -func (a *storeAPIAdapter) appendDraftReply( +func (a *storeAPIAdapter) appendDraftReplyWithClient( ctx context.Context, + client *imaplib.Client, target draftReplyTarget, raw []byte, emit func(api.CLIRunEvent) error, ) (imaplib.DraftAppendResult, error) { - clientFactory := a.draftClientFactory - if clientFactory == nil { - clientFactory = defaultDraftClientFactory - } - client, err := clientFactory(ctx, target.source) - if err != nil { - return imaplib.DraftAppendResult{}, draftReplyError("invalid_source", fmt.Errorf("build IMAP client for source %d: %w", target.source.ID, err)) - } - defer func() { _ = client.Close() }() receipt, err := client.AppendDraft(ctx, target.mailbox, raw) if err != nil { if emit != nil { @@ -437,8 +438,8 @@ func emitDraftReplyOutput(emit func(api.CLIRunEvent) error, stream string, asJSO case asJSON: text = string(marshalDraftReplyOutput(result)) + "\n" case result.Status == draftReplyStatusCreated: - text = fmt.Sprintf("created draft message %d (%s|%d|%d), operation %s\n", - result.MessageID, result.Mailbox, result.UIDValidity, result.UID, result.OperationRef) + text = fmt.Sprintf("created draft message %d (%s|%d|%d), operation %s, draft %s revision %d\n", + result.MessageID, result.Mailbox, result.UIDValidity, result.UID, result.OperationRef, result.DraftID, result.Revision) default: text = fmt.Sprintf("remote accepted; local persistence failed, inspect operation %s\n", result.OperationRef) } diff --git a/cmd/msgvault/cmd/serve_reply_draft_test.go b/cmd/msgvault/cmd/serve_reply_draft_test.go index a41581a39..1a93be87c 100644 --- a/cmd/msgvault/cmd/serve_reply_draft_test.go +++ b/cmd/msgvault/cmd/serve_reply_draft_test.go @@ -451,7 +451,10 @@ func TestAppendDraftReplyRetainsFailureCause(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) var events []api.CLIRunEvent - _, err := adapter.appendDraftReply(t.Context(), draftReplyTarget{source: &store.Source{}, mailbox: tc.mailbox}, []byte("Subject: Reply\r\n\r\nreply body\r\n"), func(event api.CLIRunEvent) error { + client, err := adapter.draftClientFactory(t.Context(), &store.Source{}) + requirements.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + _, err = adapter.appendDraftReplyWithClient(t.Context(), client, draftReplyTarget{source: &store.Source{}, mailbox: tc.mailbox}, []byte("Subject: Reply\r\n\r\nreply body\r\n"), func(event api.CLIRunEvent) error { events = append(events, event) return nil }) diff --git a/docs/changelog.md b/docs/changelog.md index 750e4c4a9..098af02db 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -215,6 +215,10 @@ See [Web UI](web-ui.md), [TUI](usage/tui.md), Enable `[server] agent_access = true` with a non-empty `api_key`; see [agent-token](cli-reference.md#agent-token) and [configuration](configuration.md#server). +- Read, edit, and delete managed IMAP drafts with `draft-get`, `draft-edit`, and + `draft-delete`. Reads use retained archive content; edits and deletes require + the reported revision and exact provider receipt. Uncertain provider results + keep candidate evidence and block the draft until recovery support is added. - Refresh the archived body, recipients, and attachments when a trusted outgoing IMAP copy is edited or moves from Drafts to Sent. Ordinary received-mail and All Mail copies cannot replace that content. Historical rows that already lost diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 0e974a814..96539e68a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -165,6 +165,31 @@ changes. Draft creation never moves an IMAP cursor. --- +## draft-get, draft-edit, and draft-delete + +Read, edit, or delete an IMAP draft created by `draft-reply`: + +```bash +msgvault draft-get [--json] +msgvault draft-edit --revision --body [--json] +msgvault draft-delete --revision [--json] +``` + +The creation result supplies the opaque `draft_id` and initial revision. + +- `--revision` is required for edit and delete; use the current positive revision. +- `--body` is required for edit; `--body=` sets an empty plain-text body. +- `--json` emits one JSON result. + +`draft-get` reads retained archive content, including discarded drafts, without +connecting to IMAP or requiring the source's draft mutation grant. Edit and +delete require the same source policy as `draft-reply`. Delete removes the +provider draft and retains its archived content. These commands never send mail. +See [Manage a created draft](usage/imap.md#manage-a-created-draft) for revision, +provider checks, retention, and retry behavior. + +--- + ## list-folders List the selectable folders in one or all configured IMAP accounts, including diff --git a/docs/usage/imap.md b/docs/usage/imap.md index e9ca02c55..87916de04 100644 --- a/docs/usage/imap.md +++ b/docs/usage/imap.md @@ -245,6 +245,80 @@ outcome to decide what to do next: | `remote_unknown` or `accepted_unidentified` | Inspect the Drafts folder before retrying; the draft may already exist. | | `remote_accepted_local_failed` | The server accepted the draft, but the local save failed. Use the reported `operation_ref` and mailbox receipt to inspect it before another request. | +## Manage a created draft + +The creation result includes an opaque `draft_id` and revision `1`. Read the +archived draft and its current revision through the daemon: + +```bash +msgvault draft-get --json +``` + +`draft-get` never connects to IMAP or changes pending operations. It also reads +discarded drafts and works when the source's draft mutation grant is disabled +or its provider configuration is unavailable. + +### Edit or delete the provider draft + +Use the current revision from `draft-get` or the last successful operation: + +```bash +msgvault draft-edit --revision 1 --body 'Updated text' --json +msgvault draft-delete --revision 2 --json +``` + +Edit and delete require the same daemon-host `[[imap.drafts]]` grant and provider +configuration as draft creation. Policy changes take effect after a daemon +restart. Neither command sends mail. + +Editing supports plain-text drafts without attachments. `--body=` sets an empty +body. The edit preserves the From, To, Cc, Bcc, Reply-To, Subject, In-Reply-To, and +References headers. Msgvault appends one replacement, records the new revision, +removes the exact old UID, and confirms that it is absent. + +Delete removes the exact provider draft and marks the managed draft +`discarded` after confirming its absence. The local archived content remains +readable with `draft-get`; ordinary archive garbage collection retains it. +A discarded draft cannot be edited. Repeating delete with its current revision +returns `already_discarded`. + +### Provider checks before a change + +Msgvault refuses stale revisions, UIDVALIDITY changes, external moves, a missing +`\Draft` flag, an existing `\Deleted` flag, or missing UIDPLUS before writing. +Edits also reject multipart drafts. An active source sync returns `sync_active`; +retry after it finishes. + +When the mailbox supplies CONDSTORE metadata, msgvault uses the exact UID's +positive MODSEQ to guard its singleton `UID STORE`, then verifies that UID's +flags. If the server does not advertise CONDSTORE, msgvault runs a fresh `SELECT` +and exact-UID `FETCH` immediately before the nonconditional `UID STORE`. +When CONDSTORE is advertised, missing or zero mailbox or message MODSEQ metadata +refuses the change before writing with `modseq_unusable`. This also applies to +`NOMODSEQ` mailboxes: the current IMAP parser cannot distinguish that response +from missing metadata. + +Every removal path checks the mailbox generation and requires both `\Draft` and +`\Deleted` again before `UID EXPUNGE`, then confirms exact-UID absence. +`UID EXPUNGE` itself is not conditional, so another client can still change flags +after the last check. + +### If an edit or delete does not finish + +- A delete failure before any remote write clears the pending claim. After + resolving the reported problem, retry with the same revision. +- An uncertain APPEND or removal returns a nonzero result with the saved + candidate and available receipt evidence. The operation stays pending and + blocks further changes. Inspect the provider state before any manual + reconciliation; general recovery for uncertain writes is not available. +- If `draft-get --json` reports `pending_code: "removed"`, removal was + confirmed and saved, but local completion is still pending. Repeat the matching + `draft-edit` or `draft-delete` command with the revision from that read to finish + locally without another remote write. The source policy still applies. For an + edit, `--body` must match the already published replacement after MIME + normalization. A `removed` observation in an error response alone is not enough; + `draft-get` must report the saved pending code. + ## Keep edited outgoing mail current After you edit or send a draft in your mail application, IMAP sync can update diff --git a/go.mod b/go.mod index 7f2adbd5b..fac972042 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.kenn.io/msgvault go 1.27.0 -replace github.com/emersion/go-imap/v2 => github.com/hstern/go-imap/v2 v2.0.0-beta.8.0.20260621192506-dabdeca47dc7 +replace github.com/emersion/go-imap/v2 => github.com/rodboev/go-imap/v2 v2.0.0-beta.8.0.20260916140841-7dc6eaf3b23f require ( charm.land/bubbles/v2 v2.1.1 diff --git a/go.sum b/go.sum index 39acaff28..7ba36e501 100644 --- a/go.sum +++ b/go.sum @@ -176,8 +176,6 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hhrutter/tiff v1.0.6 h1:p5I4Oi20jit3uWIBBaAoMDqrKztw/1JQCQC2TgqK1qU= github.com/hhrutter/tiff v1.0.6/go.mod h1:9+PDcnTBkMrJ8fWXkN1ZPv5ZNcKsFuTGVQU3ysaQbco= -github.com/hstern/go-imap/v2 v2.0.0-beta.8.0.20260621192506-dabdeca47dc7 h1:nwt9R8tzoiZXAo3jcoMQbLNxEJzTTCOIaKa90emTJE4= -github.com/hstern/go-imap/v2 v2.0.0-beta.8.0.20260621192506-dabdeca47dc7/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48= github.com/inbucket/html2text v1.0.0 h1:N5kza++4uBBDJ2Z3KUnTRyPNoBcW+YfOgNiNmNB+sgs= github.com/inbucket/html2text v1.0.0/go.mod h1:5TrhXQKGU+LXurODaSm55Y9eXoPBRnYiOz4x2XfUoJU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -263,6 +261,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rodboev/go-imap/v2 v2.0.0-beta.8.0.20260916140841-7dc6eaf3b23f h1:09wQGbWNjzsbRIQ2osxFyxVFWJjifkhG8k7vUJUTqGY= +github.com/rodboev/go-imap/v2 v2.0.0-beta.8.0.20260916140841-7dc6eaf3b23f/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48= github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rotisserie/eris v0.5.4 h1:Il6IvLdAapsMhvuOahHWiBnl1G++Q0/L5UIkI5mARSk= diff --git a/internal/api/cli_allowlist_drafts_test.go b/internal/api/cli_allowlist_drafts_test.go index 6b8a7f3e4..42a6ffcaa 100644 --- a/internal/api/cli_allowlist_drafts_test.go +++ b/internal/api/cli_allowlist_drafts_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,13 +19,23 @@ import ( ) func TestCLIRunDraftAllowlist(t *testing.T) { - assert.True(t, cliRunCommandAllowed([]string{"draft-reply", "42", "--from=alice@example.com", "--body=body"})) - assert.False(t, cliRunCommandAllowed([]string{"configure-imap-drafts"})) - assert.False(t, cliRunCommandAllowed([]string{"draft-reply"})) + assertions := assert.New(t) + assertions.True(cliRunCommandAllowed([]string{"draft-reply", "42", "--from=alice@example.com", "--body=body"})) + assertions.True(IsCLIRunDraftLifecycle([]string{"draft-get", "draft-abc"})) + assertions.True(cliRunCommandAllowed([]string{"draft-edit", "draft-abc", "--revision=1", "--body=body"})) + assertions.True(cliRunCommandAllowed([]string{"draft-delete", "draft-abc", "--revision=1"})) + assertions.False(cliRunCommandAllowed([]string{"configure-imap-drafts"})) + assertions.False(cliRunCommandAllowed([]string{"draft-reply"})) + assertions.False(cliRunCommandAllowed([]string{"draft-get"})) } // newDelegatedTestServer creates a server with agentGrants enabled and issues a grant. func newDelegatedTestServer(t *testing.T) (*Server, string) { + t.Helper() + return newDelegatedTestServerWithGate(t, nil) +} + +func newDelegatedTestServerWithGate(t *testing.T, gate OperationGate) (*Server, string) { t.Helper() reg := agentgrant.NewRegistry() stub := &stubSourceStore{ @@ -41,10 +52,11 @@ func newDelegatedTestServer(t *testing.T) (*Server, string) { }, } srv := NewServerWithOptions(ServerOptions{ - Config: cfg, - Store: stub, - Logger: testLogger(), - Scheduler: newMockScheduler(), + Config: cfg, + Store: stub, + Logger: testLogger(), + Scheduler: newMockScheduler(), + OperationGate: gate, }) srv.agentGrants = reg @@ -181,6 +193,120 @@ func TestDelegatedCLIRunRequiresGrantedPermission(t *testing.T) { } } +func TestDelegatedDraftLifecycleCommandsSkipBusyOperationGate(t *testing.T) { + gate := NewSerialOperationGate() + srv, secret := newDelegatedTestServerWithGate(t, gate) + serverStore, ok := srv.store.(*stubSourceStore) + setupRequirements := require.New(t) + setupRequirements.True(ok, "delegated fixture must expose its stub store") + + runnerCalls := 0 + serverStore.runFunc = func(_ context.Context, _ CLIRunRequest, _ func(CLIRunEvent) error) error { + runnerCalls++ + return nil + } + + oldWaitLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldWaitLimit }) + + release, ok := gate.BeginLabeledWorkContext(context.Background(), "owner draft operation") + setupRequirements.True(ok, "hold operation gate for delegated requests") + defer release() + + commands := []struct { + name string + args []string + }{ + {name: "get", args: []string{"draft-get", "draft-abc"}}, + {name: "edit", args: []string{"draft-edit", "draft-abc", "--revision=1", "--body=updated"}}, + {name: "delete", args: []string{"draft-delete", "draft-abc", "--revision=1"}}, + } + + for _, command := range commands { + t.Run(command.name, func(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + body, err := json.Marshal(CLIRunRequest{Args: command.args}) + requirements.NoError(err) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/cli/run", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(apiprotocol.AgentTokenHeader, secret) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + srv.Router().ServeHTTP(response, req) + close(done) + }() + + select { + case <-done: + case <-time.After(200 * time.Millisecond): + requirements.FailNow("delegated lifecycle rejection must not wait on a held operation gate") + } + + assertions.Equal(http.StatusBadRequest, response.Code) + var apiError ErrorResponse + requirements.NoError(json.NewDecoder(response.Body).Decode(&apiError)) + assertions.Equal("command_not_allowed", apiError.Error) + assertions.Equal(0, runnerCalls) + assertions.False(gate.HasRequestWaiters()) + }) + } +} + +func TestOwnerDraftLifecycleCommandsUseOperationGateAndRunner(t *testing.T) { + gate := &recordingOperationGate{allow: true} + srv, _ := newDelegatedTestServerWithGate(t, gate) + serverStore, ok := srv.store.(*stubSourceStore) + setupRequirements := require.New(t) + setupRequirements.True(ok, "owner fixture must expose its stub store") + + runnerCalls := 0 + serverStore.runFunc = func(_ context.Context, _ CLIRunRequest, _ func(CLIRunEvent) error) error { + runnerCalls++ + return nil + } + + commands := []struct { + name string + args []string + }{ + {name: "get", args: []string{"draft-get", "draft-abc"}}, + {name: "edit", args: []string{"draft-edit", "draft-abc", "--revision=1", "--body=updated"}}, + {name: "delete", args: []string{"draft-delete", "draft-abc", "--revision=1"}}, + } + + for _, command := range commands { + t.Run(command.name, func(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + body, err := json.Marshal(CLIRunRequest{Args: command.args}) + requirements.NoError(err) + beforeRuns := runnerCalls + beforeBegins, beforeDone := gate.counts() + + req := httptest.NewRequest(http.MethodPost, "/api/v1/cli/run", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "test-owner-key") + response := httptest.NewRecorder() + srv.Router().ServeHTTP(response, req) + + assertions.Equal(http.StatusOK, response.Code) + assertions.Equal(beforeRuns+1, runnerCalls) + begins, done := gate.counts() + if command.name == "get" { + assertions.Equal(beforeBegins, begins) + assertions.Equal(beforeDone, done) + } else { + assertions.Equal(beforeBegins+1, begins) + assertions.Equal(beforeDone+1, done) + } + }) + } +} + // TestDelegatedGrantScopesSource is the mutation probe for cli_handlers.go:1315. // It drives a delegated draft-reply through the real handler against a source // that is not in the grant, and asserts the request is refused. diff --git a/internal/api/cli_handlers.go b/internal/api/cli_handlers.go index 3d64f7ee6..286d8e706 100644 --- a/internal/api/cli_handlers.go +++ b/internal/api/cli_handlers.go @@ -1353,7 +1353,7 @@ func (s *Server) handleCLIRun(w http.ResponseWriter, r *http.Request) { } func (s *Server) cliRunEnvAllowedForCommand(args []string, name string) bool { - if IsCLIRunDraftReply(args) { + if IsCLIRunDraftReply(args) || IsCLIRunDraftLifecycle(args) { return false } if len(args) >= 3 && args[0] == cliRunPersonCommand { @@ -1597,6 +1597,9 @@ func cliRunCommandAllowed(args []string) bool { if IsCLIRunDraftReply(args) { return len(args) >= 2 } + if IsCLIRunDraftLifecycle(args) { + return len(args) >= 2 + } if args[0] == "backup" { return len(args) >= 2 && args[1] == "create" } diff --git a/internal/api/cli_run_draft.go b/internal/api/cli_run_draft.go index a47ec6cb2..1ac6d20b9 100644 --- a/internal/api/cli_run_draft.go +++ b/internal/api/cli_run_draft.go @@ -6,6 +6,12 @@ import "go.kenn.io/msgvault/internal/agentgrant" // in-process instead of spawning a subprocess. const CLIRunDraftReplyCommand = "draft-reply" +const ( + CLIRunDraftGetCommand = "draft-get" + CLIRunDraftEditCommand = "draft-edit" + CLIRunDraftDeleteCommand = "draft-delete" +) + // IsCLIRunDraftReply reports whether args invoke the in-process draft-reply // route. func IsCLIRunDraftReply(args []string) bool { @@ -16,6 +22,20 @@ func delegatedCLIRunAdmitted(args []string, grant *agentgrant.Grant) bool { return grant != nil && IsCLIRunDraftReply(args) && grant.HasPermission(agentgrant.PermissionDraftCreate) } +// IsCLIRunDraftLifecycle reports whether args invoke one of the managed draft +// lifecycle routes that the daemon executes in-process. +func IsCLIRunDraftLifecycle(args []string) bool { + if len(args) == 0 { + return false + } + switch args[0] { + case CLIRunDraftGetCommand, CLIRunDraftEditCommand, CLIRunDraftDeleteCommand: + return true + default: + return false + } +} + // CLIRunCodedError carries a fixed code for the client and the underlying // cause for the daemon log. Clients only ever see Code. type CLIRunCodedError struct { diff --git a/internal/api/operation_gate.go b/internal/api/operation_gate.go index 6dc249ecf..4f5c922d4 100644 --- a/internal/api/operation_gate.go +++ b/internal/api/operation_gate.go @@ -516,6 +516,7 @@ func cliRepairMessageGateDecision(r *http.Request) (label string, skip bool, err // cliRunReadOnlyCommands are proxied CLI commands that only read. Keys are // the leading command-path words of CLIRunRequest args (flags follow them). var cliRunReadOnlyCommands = map[string]bool{ + "draft-get": true, "logs": true, "list-deletions": true, "show-deletion": true, diff --git a/internal/imap/client.go b/internal/imap/client.go index 2fac3d9cf..850370c5e 100644 --- a/internal/imap/client.go +++ b/internal/imap/client.go @@ -550,18 +550,47 @@ func (c *Client) withConn(ctx context.Context, fn func(*imapclient.Client) error } err := fn(c.conn) if err != nil && isNetworkError(err) { - if c.conn != nil { - _ = c.conn.Close() - } - c.conn = nil - c.selectedMailbox = "" - c.selectedUIDValidity = 0 - c.qresyncEnabled = false - c.clearQresyncCapture() + c.invalidateConnLocked(c.conn) } return err } +// withDraftConn keeps a draft operation's transport tied to its context. +// The callback may wait for a server response, so cancellation closes the +// captured connection and prevents it from being reused. +func (c *Client) withDraftConn(ctx context.Context, fn func(*imapclient.Client) error) error { + return c.withConn(ctx, func(conn *imapclient.Client) error { + cancelDone := make(chan struct{}) + stopCancel := context.AfterFunc(ctx, func() { + _ = conn.Close() + close(cancelDone) + }) + defer func() { + if !stopCancel() { + <-cancelDone + c.invalidateConnLocked(conn) + return + } + if ctx.Err() != nil { + c.invalidateConnLocked(conn) + } + }() + return fn(conn) + }) +} + +func (c *Client) invalidateConnLocked(conn *imapclient.Client) { + if conn == nil || c.conn != conn { + return + } + _ = conn.Close() + c.conn = nil + c.selectedMailbox = "" + c.selectedUIDValidity = 0 + c.qresyncEnabled = false + c.clearQresyncCapture() +} + // selectMailbox selects a mailbox if not already selected. Caller must hold mu. func (c *Client) selectMailbox(mailbox string) error { if c.selectedMailbox == mailbox { diff --git a/internal/imap/draft.go b/internal/imap/draft.go index 5ac0956d4..11e78112c 100644 --- a/internal/imap/draft.go +++ b/internal/imap/draft.go @@ -64,16 +64,16 @@ func (c *Client) AppendDraft(ctx context.Context, mailbox string, raw []byte) (D return DraftAppendResult{State: DraftStateRejected, Code: "invalid_message"}, errors.New("draft message is empty") } if err := ctx.Err(); err != nil { - return DraftAppendResult{State: DraftStateCancelled, Code: "cancelled"}, err + return DraftAppendResult{State: DraftStateCancelled, Code: DraftStateCancelled}, err } var result DraftAppendResult - err := c.withConn(ctx, func(conn *imapclient.Client) error { + err := c.withDraftConn(ctx, func(conn *imapclient.Client) error { if !conn.Caps().Has(imaplib.CapUIDPlus) { result = DraftAppendResult{State: DraftStateRejected, Code: "uidplus_required"} return &DraftAppendError{State: result.State, Code: result.Code, Err: errors.New("IMAP server does not advertise UIDPLUS")} } if err := ctx.Err(); err != nil { - result = DraftAppendResult{State: DraftStateCancelled, Code: "cancelled"} + result = DraftAppendResult{State: DraftStateCancelled, Code: DraftStateCancelled} return err } command := conn.Append(mailbox, int64(len(raw)), &imaplib.AppendOptions{ @@ -132,8 +132,11 @@ func (c *Client) AppendDraft(ctx context.Context, mailbox string, raw []byte) (D if appendErr, ok := errors.AsType[*DraftAppendError](err); ok { return result, appendErr } + if result.State != "" { + return result, &DraftAppendError{State: result.State, Code: result.Code, Err: err} + } if ctx.Err() != nil { - result = DraftAppendResult{State: DraftStateCancelled, Code: "cancelled"} + result = DraftAppendResult{State: DraftStateCancelled, Code: DraftStateCancelled} return result, &DraftAppendError{State: result.State, Code: result.Code, Err: ctx.Err()} } if result.State == "" { diff --git a/internal/imap/draft_inspect.go b/internal/imap/draft_inspect.go new file mode 100644 index 000000000..e900b1c49 --- /dev/null +++ b/internal/imap/draft_inspect.go @@ -0,0 +1,198 @@ +package imap + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + imaplib "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" +) + +// DraftReceipt is the immutable provider identity recorded by APPEND. +type DraftReceipt struct { + Mailbox string + UIDValidity uint32 + UID uint32 +} + +// DraftObservation describes one fresh exact-UID check. +type DraftObservation struct { + State string `json:"state"` + Code string `json:"code,omitempty"` + Mailbox string `json:"mailbox"` + UIDValidity uint32 `json:"uidvalidity"` + UID uint32 `json:"uid"` + Flags []imaplib.Flag `json:"flags,omitempty"` + Present bool `json:"present"` + Draft bool `json:"draft"` + Deleted bool `json:"deleted"` + Complete bool `json:"complete"` + UIDPlus bool `json:"uidplus"` + // WriteAttempted records whether RemoveDraft attempted UID STORE, even when + // a later inspection replaces the provider observation. + WriteAttempted bool `json:"-"` +} + +const draftObservationStateIncomplete = "incomplete" + +func validateDraftReceipt(receipt DraftReceipt) error { + if strings.TrimSpace(receipt.Mailbox) == "" || receipt.UIDValidity == 0 || receipt.UID == 0 { + return errors.New("invalid IMAP draft receipt") + } + return nil +} + +func newDraftObservation(receipt DraftReceipt) DraftObservation { + return DraftObservation{ + State: "unknown", Mailbox: receipt.Mailbox, + UIDValidity: receipt.UIDValidity, UID: receipt.UID, + } +} + +func (c *Client) selectDraftMailbox( + conn *imapclient.Client, + receipt DraftReceipt, + readOnly, condStore bool, +) (DraftObservation, bool, error) { + options := &imaplib.SelectOptions{ReadOnly: readOnly, CondStore: condStore} + selected, err := conn.Select(receipt.Mailbox, options).Wait() + observation := newDraftObservation(receipt) + observation.UIDPlus = conn.Caps().Has(imaplib.CapUIDPlus) + if err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = "select_failed" + return observation, false, fmt.Errorf("SELECT %q: %w", receipt.Mailbox, err) + } + c.selectedMailbox = receipt.Mailbox + c.selectedUIDValidity = selected.UIDValidity + c.selectedNumMessages = selected.NumMessages + if selected.UIDValidity != receipt.UIDValidity { + observation.State = draftObservationStateIncomplete + observation.Code = "uidvalidity_mismatch" + return observation, false, fmt.Errorf( + "UIDVALIDITY mismatch for %q: expected %d, found %d", + receipt.Mailbox, receipt.UIDValidity, selected.UIDValidity, + ) + } + if condStore && selected.HighestModSeq == 0 { + // The parser discards NOMODSEQ, so zero cannot distinguish it from + // missing metadata. Refuse both before any draft write. + observation.State = draftObservationStateIncomplete + observation.Code = "modseq_unusable" + return observation, false, errors.New("draft mailbox has no usable HIGHESTMODSEQ") + } + return observation, condStore, nil +} + +func inspectDraftUID(conn *imapclient.Client, receipt DraftReceipt, observation DraftObservation) (DraftObservation, error) { + updated, _, err := inspectDraftUIDWithModSeq(conn, receipt, observation, false) + return updated, err +} + +func inspectDraftUIDWithModSeq( + conn *imapclient.Client, + receipt DraftReceipt, + observation DraftObservation, + includeModSeq bool, +) (DraftObservation, uint64, error) { + observation.Present = false + observation.Draft = false + observation.Deleted = false + observation.Complete = false + observation.Code = "" + observation.Flags = nil + var uids imaplib.UIDSet + uids.AddNum(imaplib.UID(receipt.UID)) + messages, err := conn.Fetch(uids, &imaplib.FetchOptions{ + UID: true, Flags: true, ModSeq: includeModSeq, + }).Collect() + if err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = "fetch_failed" + return observation, 0, fmt.Errorf("FETCH draft UID %d: %w", receipt.UID, err) + } + for _, message := range messages { + if message == nil || uint32(message.UID) != receipt.UID { + continue + } + observation.Present = true + observation.State = "present" + observation.Flags = slices.Clone(message.Flags) + observation.Draft = hasFlag(message.Flags, imaplib.FlagDraft) + observation.Deleted = hasFlag(message.Flags, imaplib.FlagDeleted) + if !observation.Draft { + observation.Code = "not_draft" + return observation, message.ModSeq, errors.New("target message is not marked \\Draft") + } + if includeModSeq && message.ModSeq == 0 { + observation.State = draftObservationStateIncomplete + observation.Code = "modseq_unusable" + return observation, 0, errors.New("draft target has no usable MODSEQ") + } + return observation, message.ModSeq, nil + } + observation.State = "absent" + observation.Code = "absent" + return observation, 0, nil +} + +func hasFlag(flags []imaplib.Flag, want imaplib.Flag) bool { + for _, flag := range flags { + if strings.EqualFold(string(flag), string(want)) { + return true + } + } + return false +} + +// InspectDraft performs a fresh read-only SELECT and exact UID FETCH. +func (c *Client) InspectDraft(ctx context.Context, receipt DraftReceipt) (DraftObservation, error) { + if err := validateDraftReceipt(receipt); err != nil { + return newDraftObservation(receipt), err + } + observation := newDraftObservation(receipt) + err := c.withDraftConn(ctx, func(conn *imapclient.Client) error { + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + observation.UIDPlus = conn.Caps().Has(imaplib.CapUIDPlus) + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + selectedObservation, useCondStore, selectErr := c.selectDraftMailbox(conn, receipt, true, conn.Caps().Has(imaplib.CapCondStore)) + observation = selectedObservation + if selectErr != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(selectErr) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return selectErr + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + var inspectErr error + observation, _, inspectErr = inspectDraftUIDWithModSeq(conn, receipt, observation, useCondStore) + if inspectErr != nil && ctx.Err() != nil && isNetworkError(inspectErr) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctx.Err() + } + return inspectErr + }) + if err != nil && ctx.Err() != nil && observation.Code == "" { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + } + return observation, err +} diff --git a/internal/imap/draft_remove.go b/internal/imap/draft_remove.go new file mode 100644 index 000000000..b6ebab58d --- /dev/null +++ b/internal/imap/draft_remove.go @@ -0,0 +1,209 @@ +package imap + +import ( + "context" + "errors" + "fmt" + + imaplib "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" +) + +// RemoveDraft conditionally marks one exact UID deleted, UID EXPUNGEs it, and +// confirms that the same UID is absent before reporting completion. UID +// EXPUNGE remains a separate, non-conditional step. +func (c *Client) RemoveDraft(ctx context.Context, receipt DraftReceipt) (DraftObservation, error) { + if err := validateDraftReceipt(receipt); err != nil { + return newDraftObservation(receipt), err + } + observation := newDraftObservation(receipt) + writeAttempted := false + err := c.withDraftConn(ctx, func(conn *imapclient.Client) error { + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + if !conn.Caps().Has(imaplib.CapUIDPlus) { + observation.State = draftObservationStateIncomplete + observation.Code = "uidplus_required" + return errors.New("server does not support UIDPLUS; exact draft removal is unavailable") + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + selectedObservation, useCondStore, err := c.selectDraftMailbox(conn, receipt, false, conn.Caps().Has(imaplib.CapCondStore)) + observation = selectedObservation + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return err + } + var modSeq uint64 + observation, modSeq, err = inspectDraftUIDWithModSeq(conn, receipt, observation, useCondStore) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return err + } + if !observation.Present { + observation.Code = "not_found" + return errors.New("draft target is absent") + } + if observation.Deleted { + observation.Code = "already_deleted" + return errors.New("draft target already has the \\Deleted flag") + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + var uids imaplib.UIDSet + uids.AddNum(imaplib.UID(receipt.UID)) + storeOptions := (*imaplib.StoreOptions)(nil) + if useCondStore { + storeOptions = &imaplib.StoreOptions{UnchangedSince: modSeq} + } + writeAttempted = true + storeCommand := conn.Store(uids, &imaplib.StoreFlags{ + Op: imaplib.StoreFlagsAdd, Silent: true, + Flags: []imaplib.Flag{imaplib.FlagDeleted}, + }, storeOptions) + storeErr := storeCommand.Close() + if storeErr == nil && storeCommand.ModifiedUIDs().Contains(imaplib.UID(receipt.UID)) { + storeErr = errors.New("draft target changed before UID STORE") + } + if storeErr != nil && isNetworkError(storeErr) { + if ctxErr := ctx.Err(); ctxErr != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + observation.State = draftObservationStateIncomplete + observation.Code = "store_failed" + return fmt.Errorf("UID STORE \\Deleted: %w", storeErr) + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + selectedObservation, _, err = c.selectDraftMailbox(conn, receipt, false, false) + observation = selectedObservation + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return err + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + observation, err = inspectDraftUID(conn, receipt, observation) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return err + } + if storeErr != nil { + observation.State = draftObservationStateIncomplete + observation.Code = "store_conflict" + return fmt.Errorf("UID STORE \\Deleted: %w", storeErr) + } + if !observation.Present { + observation.State = draftObservationStateIncomplete + observation.Code = "store_conflict" + return errors.New("draft target disappeared after UID STORE") + } + if !observation.Draft { + observation.State = draftObservationStateIncomplete + observation.Code = "store_conflict" + return errors.New("draft target lost \\Draft after UID STORE") + } + if !observation.Deleted { + observation.State = draftObservationStateIncomplete + observation.Code = "store_conflict" + return errors.New("UID STORE did not set \\Deleted") + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + // ponytail: UID EXPUNGE has no conditional form; fresh checks are the upgrade point. + if err := conn.UIDExpunge(uids).Close(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + observation.State = draftObservationStateIncomplete + observation.Code = "expunge_failed" + return fmt.Errorf("UID EXPUNGE: %w", err) + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + selectedObservation, _, err = c.selectDraftMailbox(conn, receipt, true, false) + observation = selectedObservation + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + return err + } + if err := ctx.Err(); err != nil { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return err + } + observation, err = inspectDraftUID(conn, receipt, observation) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil && isNetworkError(err) { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + return ctxErr + } + observation.State = draftObservationStateIncomplete + if observation.Code == "" { + observation.Code = "confirmation_failed" + } + return err + } + if observation.Present { + observation.State = draftObservationStateIncomplete + observation.Code = "survivor" + return errors.New("draft target survived UID EXPUNGE") + } + observation.State = "absent" + observation.Code = "removed" + observation.Complete = true + return nil + }) + if err != nil && ctx.Err() != nil && observation.Code == "" { + observation.State = draftObservationStateIncomplete + observation.Code = DraftStateCancelled + } + observation.WriteAttempted = writeAttempted + return observation, err +} diff --git a/internal/imap/draft_remove_test.go b/internal/imap/draft_remove_test.go new file mode 100644 index 000000000..266c828be --- /dev/null +++ b/internal/imap/draft_remove_test.go @@ -0,0 +1,36 @@ +package imap + +import ( + "net" + "strconv" + "testing" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftRemoveExactUID(t *testing.T) { + requirements := require.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + host, portText, err := net.SplitHostPort(addr) + requirements.NoError(err) + port, err := strconv.Atoi(portText) + requirements.NoError(err) + client := NewClient(&Config{Host: host, Port: port, Username: testutil.IMAPTestUsername}, testutil.IMAPTestPassword) + defer func() { _ = client.Close() }() + receipt, err := client.AppendDraft(t.Context(), "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\nbody\r\n")) + requirements.NoError(err) + requirements.NotZero(receipt.UID) + observation, err := client.InspectDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + requirements.NoError(err) + requirements.True(observation.Present) + requirements.True(observation.Draft) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + requirements.NoError(err) + requirements.True(removed.Complete) + requirements.False(removed.Present) +} diff --git a/internal/imap/draft_replacement_test.go b/internal/imap/draft_replacement_test.go new file mode 100644 index 000000000..be735631e --- /dev/null +++ b/internal/imap/draft_replacement_test.go @@ -0,0 +1,75 @@ +package imap + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildDraftReplacement(t *testing.T) { + requirements := require.New(t) + now := time.Date(2026, time.September, 15, 20, 0, 0, 0, time.UTC) + result, err := BuildDraftReplacement([]byte("From: Alice \r\nTo: Bob \r\nCc: Carol \r\nBcc: Secret \r\nSubject: Question\r\nMessage-ID: \r\nIn-Reply-To: \r\nReferences: \r\nContent-Type: text/plain; charset=utf-8\r\n\r\nold\r\n"), "new", now, "new@example.com") + requirements.NoError(err) + text := string(result.Raw) + requirements.Contains(text, "From:") + requirements.Contains(text, "alice@example.com") + requirements.Contains(text, "To:") + requirements.Contains(text, "bob@example.com") + requirements.Contains(text, "Cc:") + requirements.Contains(text, "carol@example.com") + requirements.Contains(text, "Bcc:") + requirements.Contains(text, "secret@example.com") + requirements.Contains(text, "Message-ID: ") + requirements.NotContains(text, "Message-ID: ") + requirements.Contains(text, "In-Reply-To: ") + requirements.Contains(text, "References: ") + requirements.Contains(text, "Date: Tue, 15 Sep 2026 20:00:00 +0000") + requirements.Contains(text, "new") + requirements.Equal("", result.Parsed.MessageID) +} + +func TestBuildDraftReplacementPreservesReplyTo(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + raw := []byte("From: owner@example.com\r\nTo: recipient@example.com\r\nReply-To: Replies \r\n\r\nold body\r\n") + replacement, err := BuildDraftReplacement(raw, "new body", time.Now(), "replacement@example.com") + require.NoError(err) + require.Len(replacement.Parsed.ReplyTo, 1) + assert.Equal("Replies", replacement.Parsed.ReplyTo[0].Name) + assert.Equal("replies@example.com", replacement.Parsed.ReplyTo[0].Email) + + reply, err := BuildReply(replacement.Raw, "recipient@example.com", "response", time.Now(), "reply@example.com") + require.NoError(err) + require.Len(reply.Parsed.To, 1) + assert.Equal("replies@example.com", reply.Parsed.To[0].Email) +} + +func TestBuildDraftReplacementRejectsMultipart(t *testing.T) { + requirements := require.New(t) + _, err := BuildDraftReplacement([]byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: multipart/mixed; boundary=x\r\n\r\n--x\r\n"), "new", time.Now(), "") + requirements.Error(err) + requirements.ErrorContains(err, "plain-text") +} + +func TestBuildDraftReplacementAcceptsEmptyBody(t *testing.T) { + requirements := require.New(t) + result, err := BuildDraftReplacement([]byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\nold\r\n"), "", time.Now(), "empty@example.com") + requirements.NoError(err) + requirements.Empty(result.Parsed.BodyText) +} + +func TestBuildDraftReplacementRejectsMalformedOrRichDraft(t *testing.T) { + requirements := require.New(t) + for _, raw := range []string{ + "From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: broken;\r\n\r\nold", + "From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/html\r\n\r\nold", + } { + _, err := BuildDraftReplacement([]byte(raw), "new", time.Now(), "") + requirements.Error(err) + } + _, err := BuildDraftReplacement([]byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\nold"), "bad\x00body", time.Now(), "") + requirements.Error(err) +} diff --git a/internal/imap/draft_review_test.go b/internal/imap/draft_review_test.go new file mode 100644 index 000000000..70e50bf71 --- /dev/null +++ b/internal/imap/draft_review_test.go @@ -0,0 +1,1058 @@ +package imap + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/emersion/go-imap/v2/imapserver" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftRemoveExactUIDCondStoreWire(t *testing.T) { + for _, scenario := range []struct { + name string + conflictStatus string + externalDeleted bool + }{ + {name: "success"}, + {name: "NO conflict", conflictStatus: "NO"}, + {name: "OK conflict", conflictStatus: "OK"}, + {name: "OK conflict with external Deleted", conflictStatus: "OK", externalDeleted: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, server := startDraftRemovalWireServer(t, draftRemovalWireOptions{ + conflictStatus: scenario.conflictStatus, externalDeleted: scenario.externalDeleted, fetchModSeq: "41", + }) + client := reviewIMAPClient(t, addr) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{ + Mailbox: "Drafts", UIDValidity: 77, UID: 7, + }) + assertions.True(removed.WriteAttempted) + if scenario.conflictStatus != "" { + requirements.Error(err) + assertions.Equal("store_conflict", removed.Code) + assertions.False(removed.Complete) + assertions.True(removed.Present) + assertions.Equal(scenario.externalDeleted, removed.Deleted) + } else { + requirements.NoError(err) + assertions.True(removed.Complete) + assertions.False(removed.Present) + } + + commands := server.commandTexts() + requirements.GreaterOrEqual(len(commands), 5) + joined := strings.Join(commands, "\n") + assertions.Contains(joined, "SELECT \"Drafts\" (CONDSTORE)") + assertions.Contains(joined, "UID FETCH 7 (") + assertions.Contains(joined, "MODSEQ") + assertions.Contains(joined, "FLAGS") + assertions.Contains(joined, "UID STORE 7 (UNCHANGEDSINCE 41) +FLAGS.SILENT (\\Deleted)") + assertions.Contains(joined, "SELECT \"Drafts\"") + assertions.Contains(joined, "UID FETCH 7 (UID FLAGS)") + if scenario.conflictStatus != "" { + assertions.NotContains(joined, "UID EXPUNGE") + return + } + assertions.Contains(joined, "UID EXPUNGE 7") + assertions.Contains(joined, "EXAMINE \"Drafts\"") + }) + } +} + +func TestDraftMailboxModSeq(t *testing.T) { + for _, scenario := range []struct { + name string + options draftRemovalWireOptions + code string + }{ + {name: "persistent", options: draftRemovalWireOptions{fetchModSeq: "41"}}, + {name: "NOMODSEQ", options: draftRemovalWireOptions{noModSeq: true}, code: "modseq_unusable"}, + {name: "missing SELECT MODSEQ", options: draftRemovalWireOptions{omitSelectModSeq: true}, code: "modseq_unusable"}, + {name: "without CONDSTORE", options: draftRemovalWireOptions{noCondStore: true, omitSelectModSeq: true}}, + {name: "missing message MODSEQ", code: "modseq_unusable"}, + {name: "zero message MODSEQ", options: draftRemovalWireOptions{fetchModSeq: "0"}, code: "modseq_unusable"}, + {name: "FETCH failure", options: draftRemovalWireOptions{fetchError: true}, code: "fetch_failed"}, + } { + t.Run(scenario.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, server := startDraftRemovalWireServer(t, scenario.options) + client := reviewIMAPClient(t, addr) + receipt := DraftReceipt{Mailbox: "Drafts", UIDValidity: 77, UID: 7} + observation, err := client.InspectDraft(t.Context(), receipt) + if scenario.code == "" { + requirements.NoError(err) + assertions.True(observation.Present) + } else { + requirements.Error(err) + assertions.Equal(scenario.code, observation.Code) + } + assertions.False(observation.WriteAttempted) + commands := strings.Join(server.commandTexts(), "\n") + if scenario.options.noCondStore { + assertions.NotContains(commands, "(CONDSTORE)") + } else { + assertions.Contains(commands, "EXAMINE \"Drafts\" (CONDSTORE)") + } + assertions.NotContains(commands, "UID STORE") + + removed, err := client.RemoveDraft(t.Context(), receipt) + commands = strings.Join(server.commandTexts(), "\n") + if scenario.code != "" { + requirements.Error(err) + assertions.Equal(scenario.code, removed.Code) + assertions.False(removed.WriteAttempted) + assertions.NotContains(commands, "UID STORE") + assertions.NotContains(commands, "UID EXPUNGE") + return + } + requirements.NoError(err) + assertions.True(removed.Complete) + assertions.True(removed.WriteAttempted) + assertions.Contains(commands, "UID EXPUNGE 7") + if scenario.options.noCondStore { + assertions.NotContains(commands, "MODSEQ") + assertions.NotContains(commands, "UNCHANGEDSINCE") + assertions.Contains(commands, "UID STORE 7 +FLAGS.SILENT (\\Deleted)") + } else { + assertions.Contains(commands, "UID STORE 7 (UNCHANGEDSINCE 41) +FLAGS.SILENT (\\Deleted)") + } + }) + } +} + +type draftRemovalWireOptions struct { + conflictStatus string + externalDeleted bool + noModSeq bool + noCondStore bool + omitSelectModSeq bool + fetchModSeq string + fetchError bool +} + +type draftRemovalWireServer struct { + mu sync.Mutex + commands []string +} + +func (s *draftRemovalWireServer) record(command string) { + s.mu.Lock() + defer s.mu.Unlock() + s.commands = append(s.commands, command) +} + +func (s *draftRemovalWireServer) commandTexts() []string { + s.mu.Lock() + defer s.mu.Unlock() + commands := make([]string, 0, len(s.commands)) + for _, line := range s.commands { + _, command, ok := strings.Cut(line, " ") + if ok { + commands = append(commands, command) + } + } + return commands +} + +func startDraftRemovalWireServer(t *testing.T, options draftRemovalWireOptions) (string, *draftRemovalWireServer) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := &draftRemovalWireServer{} + go serveDraftRemovalWire(listener, server, options) + t.Cleanup(func() { _ = listener.Close() }) + return listener.Addr().String(), server +} + +func serveDraftRemovalWire(listener net.Listener, server *draftRemovalWireServer, options draftRemovalWireOptions) { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go serveDraftRemovalWireConn(conn, server, options) + } +} + +func serveDraftRemovalWireConn(conn net.Conn, server *draftRemovalWireServer, options draftRemovalWireOptions) { + defer func() { _ = conn.Close() }() + _, _ = io.WriteString(conn, "* OK synthetic IMAP ready\r\n") + reader := bufio.NewReader(conn) + deleted, expunged := false, false + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + line = strings.TrimSpace(line) + server.record(line) + tag, command, ok := strings.Cut(line, " ") + if !ok { + return + } + upper := strings.ToUpper(command) + switch { + case upper == "CAPABILITY": + caps := "IMAP4rev1 UIDPLUS" + if !options.noCondStore { + caps += " CONDSTORE" + } + _, _ = fmt.Fprintf(conn, "* CAPABILITY %s\r\n%s OK CAPABILITY completed\r\n", caps, tag) + case strings.HasPrefix(upper, "LOGIN "): + _, _ = fmt.Fprintf(conn, "%s OK LOGIN completed\r\n", tag) + case strings.HasPrefix(upper, "SELECT ") || strings.HasPrefix(upper, "EXAMINE "): + exists := 1 + if expunged { + exists = 0 + } + modSeqCode := "HIGHESTMODSEQ 42" + if options.noModSeq { + modSeqCode = "NOMODSEQ" + } + modSeqResponse := "* OK [" + modSeqCode + "]\r\n" + if options.omitSelectModSeq { + modSeqResponse = "" + } + _, _ = fmt.Fprintf(conn, + "* FLAGS (\\Draft \\Deleted)\r\n* %d EXISTS\r\n* OK [UIDVALIDITY 77]\r\n* OK [UIDNEXT 8]\r\n%s%s OK SELECT completed\r\n", + exists, modSeqResponse, tag) + case strings.HasPrefix(upper, "UID FETCH "): + if options.fetchError || (options.noModSeq && strings.Contains(upper, "MODSEQ")) { + _, _ = fmt.Fprintf(conn, "%s BAD FETCH rejected\r\n", tag) + continue + } + if !expunged { + flags := "\\Draft" + if deleted { + flags += " \\Deleted" + } + modSeq := "" + if options.fetchModSeq != "" { + modSeq = " MODSEQ (" + options.fetchModSeq + ")" + } + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 7 FLAGS (%s)%s)\r\n", flags, modSeq) + } + _, _ = fmt.Fprintf(conn, "%s OK UID FETCH completed\r\n", tag) + case strings.HasPrefix(upper, "UID STORE "): + if options.noModSeq && strings.Contains(upper, "UNCHANGEDSINCE") { + _, _ = fmt.Fprintf(conn, "%s BAD STORE rejected\r\n", tag) + } else if options.conflictStatus != "" { + if options.externalDeleted { + deleted = true + options.fetchModSeq = "42" + } + _, _ = fmt.Fprintf(conn, "%s %s [MODIFIED 7] conditional conflict\r\n", tag, options.conflictStatus) + } else { + deleted = true + _, _ = fmt.Fprintf(conn, "%s OK UID STORE completed\r\n", tag) + } + case strings.HasPrefix(upper, "UID EXPUNGE "): + expunged = deleted + _, _ = fmt.Fprintf(conn, "%s OK UID EXPUNGE completed\r\n", tag) + case upper == "LOGOUT": + _, _ = fmt.Fprintf(conn, "* BYE closing\r\n%s OK LOGOUT completed\r\n", tag) + return + default: + _, _ = fmt.Fprintf(conn, "%s BAD unsupported synthetic command\r\n", tag) + } + } +} + +func TestDraftRemoveExactUIDRefusals(t *testing.T) { + t.Run("initially absent", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(client.Close()) + testutil.ExpungeIMAPMessage(t, addr, "Drafts", emersionimap.UID(receipt.UID)) + client = reviewIMAPClient(t, addr) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + requirements.Error(err) + assertions.Equal("not_found", removed.Code) + assertions.False(removed.WriteAttempted) + }) + + t.Run("UIDVALIDITY mismatch", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 1}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + client := reviewIMAPClient(t, addr) + observation, err := client.InspectDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: 999, UID: 1}) + requirements.Error(err) + assertions.Equal("uidvalidity_mismatch", observation.Code) + }) + + t.Run("folder generation changed", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("old generation")) + requirements.NoError(err) + requirements.NoError(user.Delete("Drafts")) + requirements.NoError(user.Create("Drafts", nil)) + observation, err := client.InspectDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + requirements.Error(err) + assertions.Equal("uidvalidity_mismatch", observation.Code) + }) + + t.Run("already deleted", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(client.Close()) + reviewStoreFlags(t, addr, receipt.UID, emersionimap.StoreFlagsAdd) + client = reviewIMAPClient(t, addr) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + requirements.Error(err) + assertions.Equal("already_deleted", removed.Code) + assertions.False(removed.WriteAttempted) + }) + + t.Run("missing Draft flag", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + testutil.AppendIMAPRawMessage(t, user, "Drafts", reviewDraftRaw("ordinary")) + client := reviewIMAPClient(t, addr) + observation, err := client.InspectDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: 1, UID: 1}) + requirements.Error(err) + assertions.Equal("not_draft", observation.Code) + }) + + t.Run("UIDPLUS required", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}}, + }) + client := reviewIMAPClient(t, addr) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: 1, UID: 1}) + requirements.Error(err) + assertions.Equal("uidplus_required", removed.Code) + assertions.False(removed.WriteAttempted) + }) +} + +func TestDraftRemoveExactUIDKeepsUnrelatedDeletedUID(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, _ := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + client := reviewIMAPClient(t, addr) + target, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + unrelated, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("unrelated")) + requirements.NoError(err) + requirements.NoError(client.Close()) + reviewStoreFlags(t, addr, unrelated.UID, emersionimap.StoreFlagsAdd) + client = reviewIMAPClient(t, addr) + removed, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: target.UIDValidity, UID: target.UID}) + requirements.NoError(err) + assertions.True(removed.Complete) + remaining := reviewIMAPClient(t, addr) + observation, err := remaining.InspectDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: unrelated.UIDValidity, UID: unrelated.UID}) + requirements.NoError(err) + assertions.True(observation.Present) + assertions.True(observation.Deleted) +} + +func TestDraftRemoveExactUIDReportsStoreConflictAfterDeletedFlagRace(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, _ *imapserver.Conn) imapserver.Session { + return &reviewStoreBarrierSession{Session: session, barrier: barrier} + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(client.Close()) + + type result struct { + observation DraftObservation + err error + } + done := make(chan result, 1) + client = reviewIMAPClient(t, addr) + go func() { + observation, err := client.RemoveDraft(t.Context(), DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + done <- result{observation: observation, err: err} + }() + <-barrier.stored + reviewStoreFlags(t, addr, receipt.UID, emersionimap.StoreFlagsDel) + close(barrier.release) + removed := <-done + requirements.Error(removed.err) + assertions.Equal("store_conflict", removed.observation.Code) + assertions.False(removed.observation.Complete) + assertions.True(removed.observation.Present) + assertions.Zero(barrier.expungeCalls.Load()) +} + +func TestDraftRemoveExactUIDReportsSurvivorAfterExpunge(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, _ *imapserver.Conn) imapserver.Session { + return &reviewNoopExpungeSession{Session: session} + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + removed, err := client.RemoveDraft(t.Context(), receiptToDraftReceipt(receipt)) + requirements.Error(err) + assertions.Equal("survivor", removed.Code) + assertions.False(removed.Complete) + assertions.True(removed.Present) +} + +func TestDraftRemoveExactUIDRefusesDraftFlagChangeBeforeExpunge(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, _ *imapserver.Conn) imapserver.Session { + return &reviewStoreBarrierSession{Session: session, barrier: barrier} + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(client.Close()) + + type result struct { + observation DraftObservation + err error + } + done := make(chan result, 1) + client = reviewIMAPClient(t, addr) + go func() { + observation, err := client.RemoveDraft(t.Context(), receiptToDraftReceipt(receipt)) + done <- result{observation: observation, err: err} + }() + <-barrier.stored + reviewStoreFlag(t, addr, receipt.UID, emersionimap.StoreFlagsDel, emersionimap.FlagDraft) + close(barrier.release) + removed := <-done + requirements.Error(removed.err) + assertions.Equal("not_draft", removed.observation.Code) + assertions.True(removed.observation.WriteAttempted) + assertions.False(removed.observation.Complete) + assertions.Zero(barrier.expungeCalls.Load()) +} + +func TestDraftRemoveExactUIDRefusesGenerationChangeBeforeExpunge(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, _ *imapserver.Conn) imapserver.Session { + return &reviewStoreBarrierSession{Session: session, barrier: barrier} + }) + client := reviewIMAPClient(t, addr) + receipt, err := client.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(client.Close()) + + generationChanged := make(chan error, 1) + barrier.onStore = func() { + external, dialErr := imapclient.DialInsecure(addr, nil) + if dialErr != nil { + generationChanged <- dialErr + return + } + defer func() { _ = external.Close() }() + if loginErr := external.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait(); loginErr != nil { + generationChanged <- loginErr + return + } + if deleteErr := external.Delete("Drafts").Wait(); deleteErr != nil { + generationChanged <- deleteErr + return + } + generationChanged <- external.Create("Drafts", nil).Wait() + } + + type result struct { + observation DraftObservation + err error + } + done := make(chan result, 1) + client = reviewIMAPClient(t, addr) + go func() { + observation, err := client.RemoveDraft(t.Context(), receiptToDraftReceipt(receipt)) + done <- result{observation: observation, err: err} + }() + <-barrier.stored + requirements.NoError(<-generationChanged) + close(barrier.release) + removed := <-done + requirements.Error(removed.err) + assertions.Equal("uidvalidity_mismatch", removed.observation.Code) + assertions.True(removed.observation.WriteAttempted) + assertions.False(removed.observation.Complete) + assertions.Zero(barrier.expungeCalls.Load()) +} + +func TestDraftLifecycleCancellationBarriers(t *testing.T) { + t.Run("append cancellation leaves uncertain result", func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := newReviewWireBarrier("APPEND") + defer barrier.releaseNow() + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, conn *imapserver.Conn) imapserver.Session { + return &reviewWireBarrierSession{Session: session, barrier: barrier} + }) + seedClient := reviewIMAPClient(t, addr) + _, err := seedClient.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("seed")) + requirements.NoError(err) + requirements.NoError(seedClient.Close()) + barrier.armed.Store(true) + client := reviewIMAPClient(t, addr) + oldConn := reviewFreshConn(t, client) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + resultCh := make(chan DraftAppendResult, 1) + errCh := make(chan error, 1) + go func() { + result, err := client.AppendDraft(ctx, "Drafts", reviewDraftRaw("candidate")) + resultCh <- result + errCh <- err + }() + <-barrier.entered + cancel() + result, err := awaitReviewAppendResult(t, resultCh, errCh) + requirements.Error(err) + assertions.Equal(DraftStateRemoteUnknown, result.State) + assertions.Equal("remote_unknown", result.Code) + assertions.Equal(int32(2), barrier.appendCalls.Load()) + assertions.Zero(barrier.selectCalls.Load()) + assertions.Zero(barrier.fetchCalls.Load()) + assertions.Zero(barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + requirements.Nil(reviewCurrentConn(t, client)) + barrier.releaseNow() + fresh := reviewFreshConn(t, client) + assertions.NotSame(oldConn, fresh) + }) + + for _, command := range []string{"SELECT", "FETCH", "STORE", "EXPUNGE"} { + t.Run("removal cancellation stops before "+command, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := newReviewWireBarrier(command) + defer barrier.releaseNow() + addr := startReviewIMAPServer(t, emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, func(session imapserver.Session, _ *imapserver.Conn) imapserver.Session { + return &reviewWireBarrierSession{Session: session, barrier: barrier} + }) + seedClient := reviewIMAPClient(t, addr) + receipt, err := seedClient.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(seedClient.Close()) + barrier.armed.Store(true) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + resultCh := make(chan DraftObservation, 1) + errCh := make(chan error, 1) + client := reviewIMAPClient(t, addr) + oldConn := reviewFreshConn(t, client) + go func() { + observation, err := client.RemoveDraft(ctx, DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID}) + resultCh <- observation + errCh <- err + }() + <-barrier.entered + cancel() + observation, err := awaitReviewRemovalResult(t, resultCh, errCh) + requirements.Error(err) + assertions.Equal("cancelled", observation.Code) + assertions.Equal(command == "STORE" || command == "EXPUNGE", observation.WriteAttempted) + assertions.False(observation.Complete) + switch command { + case "SELECT": + assertions.Equal(int32(1), barrier.selectCalls.Load()) + assertions.Zero(barrier.fetchCalls.Load()) + assertions.Zero(barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + case "FETCH": + assertions.Equal(int32(1), barrier.selectCalls.Load()) + assertions.Equal(int32(1), barrier.fetchCalls.Load()) + assertions.Zero(barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + case "STORE": + assertions.Equal(int32(1), barrier.selectCalls.Load()) + assertions.Equal(int32(1), barrier.fetchCalls.Load()) + assertions.Equal(int32(1), barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + case "EXPUNGE": + assertions.Equal(int32(2), barrier.selectCalls.Load()) + assertions.Equal(int32(2), barrier.fetchCalls.Load()) + assertions.Equal(int32(1), barrier.storeCalls.Load()) + assertions.Equal(int32(1), barrier.expungeCalls.Load()) + } + requirements.Nil(reviewCurrentConn(t, client)) + barrier.releaseNow() + fresh := reviewFreshConn(t, client) + assertions.NotSame(oldConn, fresh) + }) + } +} + +func TestDraftNetworkFailureInvalidatesTransport(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + var dropped atomic.Bool + addr := startReviewIMAPServer(t, emersionimap.CapSet{ + emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}, + }, func(session imapserver.Session, conn *imapserver.Conn) imapserver.Session { + return &reviewDropConnectionSession{ + Session: session, conn: conn, dropped: &dropped, + } + }) + seedClient := reviewIMAPClient(t, addr) + receipt, err := seedClient.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("target")) + requirements.NoError(err) + requirements.NoError(seedClient.Close()) + client := reviewIMAPClient(t, addr) + oldConn := reviewFreshConn(t, client) + observation, err := client.InspectDraft(t.Context(), receiptToDraftReceipt(receipt)) + requirements.Error(err) + assertions.True(dropped.Load()) + assertions.Equal("select_failed", observation.Code) + requirements.Nil(reviewCurrentConn(t, client)) + observation, err = client.InspectDraft(t.Context(), receiptToDraftReceipt(receipt)) + requirements.NoError(err) + assertions.True(observation.Present) + fresh := reviewCurrentConn(t, client) + assertions.NotSame(oldConn, fresh) +} + +func TestAppendDraftPreservesTransportResultWhenCancellationRaces(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + baseCtx, cancel := context.WithCancel(t.Context()) + defer cancel() + tracingCtx := &reviewAppendRaceContext{Context: baseCtx, cancel: cancel} + var enabled atomic.Bool + addr := startReviewIMAPServer(t, emersionimap.CapSet{ + emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}, + }, func(session imapserver.Session, conn *imapserver.Conn) imapserver.Session { + return &reviewDropAppendSession{ + Session: session, conn: conn, enabled: &enabled, arm: tracingCtx.arm, + } + }) + seedClient := reviewIMAPClient(t, addr) + _, err := seedClient.AppendDraft(t.Context(), "Drafts", reviewDraftRaw("seed")) + requirements.NoError(err) + requirements.NoError(seedClient.Close()) + enabled.Store(true) + client := reviewIMAPClient(t, addr) + result, err := client.AppendDraft(tracingCtx, "Drafts", reviewDraftRaw("candidate")) + requirements.Error(err) + assertions.Equal(DraftStateRemoteUnknown, result.State) + assertions.Equal("remote_unknown", result.Code) + assertions.Equal("remote_unknown", err.Error()) + assertions.True(tracingCtx.cancelled.Load()) +} + +func receiptToDraftReceipt(receipt DraftAppendResult) DraftReceipt { + return DraftReceipt{Mailbox: "Drafts", UIDValidity: receipt.UIDValidity, UID: receipt.UID} +} + +type reviewWireBarrier struct { + command string + entered chan struct{} + release chan struct{} + armed atomic.Bool + blocked atomic.Bool + releaseMu sync.Once + + appendCalls atomic.Int32 + selectCalls atomic.Int32 + fetchCalls atomic.Int32 + storeCalls atomic.Int32 + expungeCalls atomic.Int32 +} + +func newReviewWireBarrier(command string) *reviewWireBarrier { + return &reviewWireBarrier{ + command: command, + entered: make(chan struct{}), + release: make(chan struct{}), + } +} + +func (b *reviewWireBarrier) record(command string) { + switch command { + case "APPEND": + b.appendCalls.Add(1) + case "SELECT": + b.selectCalls.Add(1) + case "FETCH": + b.fetchCalls.Add(1) + case "STORE": + b.storeCalls.Add(1) + case "EXPUNGE": + b.expungeCalls.Add(1) + } + if !b.armed.Load() || command != b.command || !b.blocked.CompareAndSwap(false, true) { + return + } + close(b.entered) + <-b.release +} + +func (b *reviewWireBarrier) releaseNow() { + b.releaseMu.Do(func() { close(b.release) }) +} + +type reviewWireBarrierSession struct { + imapserver.Session + + barrier *reviewWireBarrier +} + +type reviewDropConnectionSession struct { + imapserver.Session + + conn *imapserver.Conn + dropped *atomic.Bool +} + +type reviewAppendRaceContext struct { + context.Context + + cancel context.CancelFunc + armed atomic.Bool + cancelled atomic.Bool +} + +func (c *reviewAppendRaceContext) arm() { + c.armed.Store(true) +} + +func (c *reviewAppendRaceContext) Err() error { + if c.armed.Load() && c.cancelled.CompareAndSwap(false, true) { + c.cancel() + } + return c.Context.Err() +} + +type reviewDropAppendSession struct { + imapserver.Session + + conn *imapserver.Conn + enabled *atomic.Bool + arm func() +} + +func (s *reviewDropAppendSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + data, err := s.Session.Append(mailbox, r, options) + if s.enabled.Load() { + s.arm() + _ = s.conn.NetConn().Close() + } + if err != nil { + return data, fmt.Errorf("drop append session: %w", err) + } + return data, nil +} + +func (s *reviewDropConnectionSession) Select( + mailbox string, + options *emersionimap.SelectOptions, +) (*emersionimap.SelectData, error) { + data, err := s.Session.Select(mailbox, options) + if s.dropped.CompareAndSwap(false, true) { + _ = s.conn.NetConn().Close() + } + if err != nil { + return data, fmt.Errorf("drop connection session SELECT: %w", err) + } + return data, nil +} + +func (s *reviewWireBarrierSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + data, err := s.Session.Append(mailbox, r, options) + s.barrier.record("APPEND") + if err != nil { + return data, fmt.Errorf("wire barrier session APPEND: %w", err) + } + return data, nil +} + +func (s *reviewWireBarrierSession) Select( + mailbox string, + options *emersionimap.SelectOptions, +) (*emersionimap.SelectData, error) { + data, err := s.Session.Select(mailbox, options) + s.barrier.record("SELECT") + if err != nil { + return data, fmt.Errorf("wire barrier session SELECT: %w", err) + } + return data, nil +} + +func (s *reviewWireBarrierSession) Fetch( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + options *emersionimap.FetchOptions, +) error { + err := s.Session.Fetch(w, numSet, options) + s.barrier.record("FETCH") + if err != nil { + return fmt.Errorf("wire barrier session FETCH: %w", err) + } + return nil +} + +func (s *reviewWireBarrierSession) Store( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + flags *emersionimap.StoreFlags, + options *emersionimap.StoreOptions, +) error { + err := s.Session.Store(w, numSet, flags, options) + s.barrier.record("STORE") + if err != nil { + return fmt.Errorf("wire barrier session STORE: %w", err) + } + return nil +} + +func (s *reviewWireBarrierSession) Expunge( + w *imapserver.ExpungeWriter, + uids *emersionimap.UIDSet, +) error { + err := s.Session.Expunge(w, uids) + s.barrier.record("EXPUNGE") + if err != nil { + return fmt.Errorf("wire barrier session EXPUNGE: %w", err) + } + return nil +} + +func reviewCurrentConn(t *testing.T, client *Client) *imapclient.Client { + t.Helper() + client.mu.Lock() + defer client.mu.Unlock() + return client.conn +} + +func reviewFreshConn(t *testing.T, client *Client) *imapclient.Client { + t.Helper() + var fresh *imapclient.Client + require.NoError(t, client.withDraftConn(t.Context(), func(conn *imapclient.Client) error { + fresh = conn + return nil + })) + return fresh +} + +func awaitReviewAppendResult( + t *testing.T, + results <-chan DraftAppendResult, + errs <-chan error, +) (DraftAppendResult, error) { + t.Helper() + var result DraftAppendResult + var err error + select { + case result = <-results: + case <-time.After(2 * time.Second): + require.FailNow(t, "draft append did not return while the response barrier was held") + } + select { + case err = <-errs: + case <-time.After(2 * time.Second): + require.FailNow(t, "draft append error did not return while the response barrier was held") + } + return result, err +} + +func awaitReviewRemovalResult( + t *testing.T, + results <-chan DraftObservation, + errs <-chan error, +) (DraftObservation, error) { + t.Helper() + var observation DraftObservation + var err error + select { + case observation = <-results: + case <-time.After(2 * time.Second): + require.FailNow(t, "draft removal did not return while the response barrier was held") + } + select { + case err = <-errs: + case <-time.After(2 * time.Second): + require.FailNow(t, "draft removal error did not return while the response barrier was held") + } + return observation, err +} + +type reviewStoreBarrier struct { + stored chan struct{} + release chan struct{} + blocked atomic.Bool + expungeCalls atomic.Int32 + onStore func() +} + +type reviewStoreBarrierSession struct { + imapserver.Session + + barrier *reviewStoreBarrier +} + +type reviewNoopExpungeSession struct { + imapserver.Session +} + +func (s *reviewStoreBarrierSession) Store( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + flags *emersionimap.StoreFlags, + options *emersionimap.StoreOptions, +) error { + err := s.Session.Store(w, numSet, flags, options) + if s.barrier.blocked.CompareAndSwap(false, true) { + close(s.barrier.stored) + if s.barrier.onStore != nil { + s.barrier.onStore() + } + <-s.barrier.release + } + if err != nil { + return fmt.Errorf("store barrier session: %w", err) + } + return nil +} + +func (s *reviewStoreBarrierSession) Expunge( + w *imapserver.ExpungeWriter, + uids *emersionimap.UIDSet, +) error { + err := s.Session.Expunge(w, uids) + s.barrier.expungeCalls.Add(1) + if err != nil { + return fmt.Errorf("expunge barrier session: %w", err) + } + return nil +} + +func (s *reviewNoopExpungeSession) Expunge( + *imapserver.ExpungeWriter, + *emersionimap.UIDSet, +) error { + return nil +} + +func startReviewIMAPServer( + t *testing.T, + caps emersionimap.CapSet, + wrap func(imapserver.Session, *imapserver.Conn) imapserver.Session, +) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: caps, + InsecureAuth: true, + NewSession: func(conn *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + if wrap != nil { + return wrap(memServer.NewSession(), conn), nil, nil + } + return memServer.NewSession(), nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String() +} + +func reviewIMAPClient(t *testing.T, addr string) *Client { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portText) + require.NoError(t, err) + client := NewClient(&Config{Host: host, Port: port, Username: testutil.IMAPTestUsername}, testutil.IMAPTestPassword) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func reviewStoreFlags(t *testing.T, addr string, uid uint32, op emersionimap.StoreFlagsOp) { + t.Helper() + reviewStoreFlag(t, addr, uid, op, emersionimap.FlagDeleted) +} + +func reviewStoreFlag(t *testing.T, addr string, uid uint32, op emersionimap.StoreFlagsOp, flag emersionimap.Flag) { + t.Helper() + client, err := imapclient.DialInsecure(addr, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select("Drafts", nil).Wait() + require.NoError(t, err) + var uids emersionimap.UIDSet + uids.AddNum(emersionimap.UID(uid)) + require.NoError(t, client.Store(uids, &emersionimap.StoreFlags{ + Op: op, Flags: []emersionimap.Flag{flag}, + }, nil).Close()) +} + +func reviewDraftRaw(body string) []byte { + return []byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\n" + body + "\r\n") +} diff --git a/internal/imap/reply.go b/internal/imap/reply.go index 4f7fd673d..82c33756f 100644 --- a/internal/imap/reply.go +++ b/internal/imap/reply.go @@ -330,3 +330,160 @@ func writeFoldedHeader(raw *bytes.Buffer, name, value string) { } raw.WriteString(line + "\r\n") } + +// BuildDraftReplacement keeps the envelope and thread headers of a plain-text +// draft while replacing its body and assigning a new Date and Message-ID. +func BuildDraftReplacement(currentRaw []byte, body string, now time.Time, messageID string) (ReplyDraft, error) { + if len(currentRaw) == 0 { + return ReplyDraft{}, errors.New("draft message is empty") + } + if !utf8.ValidString(body) || strings.ContainsAny(body, "\x00") { + return ReplyDraft{}, errors.New("invalid draft body") + } + message, err := mail.ReadMessage(bytes.NewReader(currentRaw)) + if err != nil { + return ReplyDraft{}, fmt.Errorf("read draft headers: %w", err) + } + if err := validateSingletonHeaders(message.Header); err != nil { + return ReplyDraft{}, err + } + contentType := message.Header.Get("Content-Type") + if contentType != "" { + mediaType, _, err := stdmime.ParseMediaType(contentType) + if err != nil { + return ReplyDraft{}, fmt.Errorf("invalid draft Content-Type: %w", err) + } + if !strings.EqualFold(mediaType, "text/plain") { + return ReplyDraft{}, errors.New("draft replacement requires a plain-text message") + } + } + parsedCurrent, err := msgmime.Parse(currentRaw) + if err != nil { + return ReplyDraft{}, fmt.Errorf("parse draft MIME: %w", err) + } + if len(parsedCurrent.Attachments) != 0 || parsedCurrent.BodyHTML != "" { + return ReplyDraft{}, errors.New("draft replacement does not support multipart or attachments") + } + + from, err := parseDraftHeaderAddresses(message.Header, "From", true) + if err != nil { + return ReplyDraft{}, err + } + to, err := parseDraftHeaderAddresses(message.Header, "To", true) + if err != nil { + return ReplyDraft{}, err + } + cc, err := parseDraftHeaderAddresses(message.Header, "Cc", false) + if err != nil { + return ReplyDraft{}, err + } + bcc, err := parseDraftHeaderAddresses(message.Header, "Bcc", false) + if err != nil { + return ReplyDraft{}, err + } + replyTo, err := parseDraftHeaderAddresses(message.Header, "Reply-To", false) + if err != nil { + return ReplyDraft{}, err + } + if len(from) != 1 || len(to) == 0 { + return ReplyDraft{}, errors.New("draft must contain one From and at least one To address") + } + subject := decodeHeader(message.Header.Get("Subject")) + if !validHeaderValue(subject) { + return ReplyDraft{}, errors.New("invalid draft Subject header") + } + inReplyTo, err := parseMessageIDHeader(message.Header.Get("In-Reply-To")) + if err != nil { + return ReplyDraft{}, fmt.Errorf("invalid draft In-Reply-To: %w", err) + } + references, err := parseMessageIDHeader(message.Header.Get("References")) + if err != nil { + return ReplyDraft{}, fmt.Errorf("invalid draft References: %w", err) + } + if messageID == "" { + messageID, err = newReplyMessageID() + if err != nil { + return ReplyDraft{}, err + } + } + messageID, err = normalizeWireMessageID(messageID) + if err != nil { + return ReplyDraft{}, err + } + + fromValue := formatAddresses(from) + toValue := formatAddresses(to) + ccValue := formatAddresses(cc) + bccValue := formatAddresses(bcc) + replyToValue := formatAddresses(replyTo) + for _, value := range []string{fromValue, toValue, ccValue, bccValue, replyToValue} { + if value != "" && !validHeaderValue(value) { + return ReplyDraft{}, errors.New("invalid draft address header") + } + } + var raw bytes.Buffer + writeHeader := func(name, value string) { + _, _ = fmt.Fprintf(&raw, "%s: %s\r\n", name, value) + } + writeHeader("Date", now.UTC().Format(time.RFC1123Z)) + writeHeader("From", fromValue) + writeHeader("To", toValue) + if ccValue != "" { + writeHeader("Cc", ccValue) + } + if bccValue != "" { + writeHeader("Bcc", bccValue) + } + if replyToValue != "" { + writeHeader("Reply-To", replyToValue) + } + if subject != "" { + if !isASCII(subject) { + subject = stdmime.QEncoding.Encode("UTF-8", subject) + } + writeHeader("Subject", subject) + } + writeHeader("Message-ID", "<"+messageID+">") + if len(inReplyTo) > 0 { + writeHeader("In-Reply-To", formatMessageIDs(inReplyTo)) + } + if len(references) > 0 { + writeFoldedHeader(&raw, "References", formatMessageIDs(references)) + } + writeHeader("MIME-Version", "1.0") + writeHeader("Content-Type", `text/plain; charset="utf-8"`) + writeHeader("Content-Transfer-Encoding", "quoted-printable") + raw.WriteString("\r\n") + qp := quotedprintable.NewWriter(&raw) + if _, err := io.WriteString(qp, body); err != nil { + return ReplyDraft{}, fmt.Errorf("encode draft body: %w", err) + } + if err := qp.Close(); err != nil { + return ReplyDraft{}, fmt.Errorf("close draft body: %w", err) + } + parsed, err := msgmime.Parse(raw.Bytes()) + if err != nil { + return ReplyDraft{}, fmt.Errorf("parse composed draft: %w", err) + } + return ReplyDraft{Raw: raw.Bytes(), Parsed: parsed}, nil +} + +func parseDraftHeaderAddresses(header mail.Header, name string, required bool) ([]*mail.Address, error) { + values := headerValues(header, name) + if len(values) == 0 { + if required { + return nil, fmt.Errorf("draft is missing %s header", name) + } + return nil, nil + } + addresses, err := mail.ParseAddressList(strings.Join(values, ", ")) + if err != nil { + return nil, fmt.Errorf("invalid draft %s header: %w", name, err) + } + for _, address := range addresses { + if err := validateAddress(address); err != nil { + return nil, fmt.Errorf("invalid draft %s address: %w", name, err) + } + } + return addresses, nil +} diff --git a/internal/store/dialect_pg.go b/internal/store/dialect_pg.go index f4401162d..23b300e01 100644 --- a/internal/store/dialect_pg.go +++ b/internal/store/dialect_pg.go @@ -2331,7 +2331,7 @@ var exclusiveLockTables = []string{ "collections", "collection_sources", "account_identities", "applied_migrations", "sync_operations", "source_import_items", "sync_run_items", "sync_checkpoints", - "imap_folder_state", "imap_message_memberships", + "imap_folder_state", "imap_message_memberships", "imap_drafts", } // BeginExclusive opens a transaction on conn and locks every table the diff --git a/internal/store/gc.go b/internal/store/gc.go index 484f886d1..c5d9ffa6c 100644 --- a/internal/store/gc.go +++ b/internal/store/gc.go @@ -9,6 +9,15 @@ import ( "strings" ) +// imapDraftRetainedMessageSQL identifies message rows still owned by a +// managed draft. Ordinary archive GC must leave both the current content and +// an interrupted operation's original content available for local reads. +const imapDraftRetainedMessageSQL = `EXISTS ( + SELECT 1 FROM imap_drafts draft_owner + WHERE draft_owner.current_message_id = messages.id + OR draft_owner.pending_original_message_id = messages.id + )` + // ErrGCUnsupported is returned before mutation when archive GC is requested // against PostgreSQL. PostgreSQL retention and compaction require an // operator-managed backup and VACUUM policy outside this SQLite command. @@ -36,7 +45,7 @@ func planGCWith(q querier) (GCPlan, error) { var plan GCPlan if err := q.QueryRow(` SELECT - COUNT(*) FILTER (WHERE deleted_from_source_at IS NOT NULL), + COUNT(*) FILTER (WHERE deleted_from_source_at IS NOT NULL AND NOT (`+imapDraftRetainedMessageSQL+`)), COUNT(*) FILTER ( WHERE deleted_at IS NOT NULL AND deleted_from_source_at IS NULL @@ -50,6 +59,7 @@ func planGCWith(q querier) (GCPlan, error) { FROM ( SELECT id FROM messages WHERE deleted_from_source_at IS NOT NULL + AND NOT (`+imapDraftRetainedMessageSQL+`) ORDER BY id ) `) @@ -117,6 +127,7 @@ func (s *Store) ExecuteGCContext( SELECT DISTINCT conversation_id AS id FROM messages WHERE deleted_from_source_at IS NOT NULL + AND NOT (`+imapDraftRetainedMessageSQL+`) AND conversation_id IS NOT NULL ORDER BY conversation_id ) @@ -131,6 +142,7 @@ func (s *Store) ExecuteGCContext( WHERE rowid IN ( SELECT id FROM messages WHERE deleted_from_source_at IS NOT NULL + AND NOT (` + imapDraftRetainedMessageSQL + `) ) `); err != nil { return fmt.Errorf("delete source-deleted FTS rows: %w", err) @@ -143,14 +155,16 @@ func (s *Store) ExecuteGCContext( WHERE reply_to_message_id IN ( SELECT id FROM messages WHERE deleted_from_source_at IS NOT NULL + AND NOT (` + imapDraftRetainedMessageSQL + `) ) `); err != nil { return fmt.Errorf("clear replies to source-deleted messages: %w", err) } result, err := q.Exec(` - DELETE FROM messages - WHERE deleted_from_source_at IS NOT NULL + DELETE FROM messages + WHERE deleted_from_source_at IS NOT NULL + AND NOT (` + imapDraftRetainedMessageSQL + `) `) if err != nil { return fmt.Errorf("delete source-deleted messages: %w", err) diff --git a/internal/store/imap_draft_lifecycle.go b/internal/store/imap_draft_lifecycle.go new file mode 100644 index 000000000..018ea5ba1 --- /dev/null +++ b/internal/store/imap_draft_lifecycle.go @@ -0,0 +1,626 @@ +package store + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +const ( + IMAPDraftOperationEdit = "edit" + IMAPDraftOperationDelete = "delete" + IMAPDraftCodeRejected = "append_rejected" + IMAPDraftCodeCleanup = "cleanup_pending" + IMAPDraftCodeRemoved = "removed" +) + +// GetMessageReplyToMessageIDContext reads the archived reply link needed when +// an owned draft is replaced. It is a direct message-row lookup. +func (s *Store) GetMessageReplyToMessageIDContext(ctx context.Context, messageID int64) (sql.NullInt64, error) { + var replyTo sql.NullInt64 + err := s.db.QueryRowContext(ctx, + s.Rebind(`SELECT reply_to_message_id FROM messages WHERE id = ?`), messageID, + ).Scan(&replyTo) + return replyTo, err +} + +// GetIMAPDraft returns one managed draft and its durable pending evidence. +func (s *Store) GetIMAPDraft(draftID string) (IMAPDraft, error) { + return s.GetIMAPDraftContext(context.Background(), draftID) +} + +// GetIMAPDraftContext reads ownership by draft ID. It never consults live +// mailbox memberships, so a sync observation cannot change mutation authority. +func (s *Store) GetIMAPDraftContext(ctx context.Context, draftID string) (IMAPDraft, error) { + if err := validateIMAPDraftID(draftID); err != nil { + return IMAPDraft{}, err + } + return loadIMAPDraft(ctx, s.db, "", draftID) +} + +func validateIMAPDraftID(draftID string) error { + if strings.TrimSpace(draftID) == "" || strings.ContainsAny(draftID, "\x00\r\n") { + return errors.New("invalid IMAP draft ID") + } + return nil +} + +func loadIMAPDraft( + ctx context.Context, + q interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row + }, + lockClause string, + draftID string, +) (IMAPDraft, error) { + var ( + draft IMAPDraft + discardedAt nullableTimestamp + pendingOperation sql.NullString + pendingOriginalMessageID sql.NullInt64 + pendingOriginalMailbox sql.NullString + pendingOriginalUIDValidity, pendingUID sql.NullInt64 + pendingRaw []byte + pendingReplacementMailbox sql.NullString + pendingReplacementUIDValidity sql.NullInt64 + pendingReplacementUID sql.NullInt64 + pendingCode sql.NullString + ) + err := q.QueryRowContext(ctx, ` + SELECT draft_id, source_id, current_message_id, current_mailbox, + current_uidvalidity, current_uid, revision, discarded_at, + pending_operation, pending_original_message_id, + pending_original_mailbox, pending_original_uidvalidity, + pending_original_uid, pending_raw, pending_replacement_mailbox, + pending_replacement_uidvalidity, pending_replacement_uid, + pending_code + FROM imap_drafts + WHERE draft_id = ?`+lockClause, draftID).Scan( + &draft.DraftID, &draft.SourceID, &draft.CurrentMessageID, + &draft.CurrentReceipt.Mailbox, &draft.CurrentReceipt.UIDValidity, + &draft.CurrentReceipt.UID, &draft.Revision, &discardedAt, + &pendingOperation, &pendingOriginalMessageID, &pendingOriginalMailbox, + &pendingOriginalUIDValidity, &pendingUID, &pendingRaw, + &pendingReplacementMailbox, &pendingReplacementUIDValidity, + &pendingReplacementUID, &pendingCode, + ) + if errors.Is(err, sql.ErrNoRows) { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftNotFound) + } + if err != nil { + return IMAPDraft{}, fmt.Errorf("load IMAP draft %q: %w", draftID, err) + } + draft.CurrentReceipt.SourceID = draft.SourceID + if discardedAt.Valid { + t := discardedAt.Time + draft.DiscardedAt = &t + } + if pendingOperation.Valid { + if !pendingOriginalMessageID.Valid || !pendingOriginalMailbox.Valid || + !pendingOriginalUIDValidity.Valid || !pendingUID.Valid { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + originalUIDValidity, err := checkedIMAPDraftUint32(pendingOriginalUIDValidity.Int64) + if err != nil { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + originalUID, err := checkedIMAPDraftUint32(pendingUID.Int64) + if err != nil { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + pending := &IMAPDraftPending{ + Operation: pendingOperation.String, + OriginalMessageID: pendingOriginalMessageID.Int64, + OriginalReceipt: IMAPDraftReceipt{ + SourceID: draft.SourceID, Mailbox: pendingOriginalMailbox.String, + UIDValidity: originalUIDValidity, UID: originalUID, + }, + Raw: append([]byte(nil), pendingRaw...), + Code: pendingCode.String, + } + if pendingReplacementMailbox.Valid { + if !pendingReplacementUIDValidity.Valid || !pendingReplacementUID.Valid { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + replacementUIDValidity, err := checkedIMAPDraftUint32(pendingReplacementUIDValidity.Int64) + if err != nil { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + replacementUID, err := checkedIMAPDraftUint32(pendingReplacementUID.Int64) + if err != nil { + return IMAPDraft{}, fmt.Errorf("draft %q: %w", draftID, ErrIMAPDraftState) + } + pending.ReplacementReceipt = &IMAPDraftReceipt{ + SourceID: draft.SourceID, Mailbox: pendingReplacementMailbox.String, + UIDValidity: replacementUIDValidity, UID: replacementUID, + } + } + draft.Pending = pending + } + return draft, nil +} + +func checkedIMAPDraftUint32(value int64) (uint32, error) { + if value <= 0 || value > int64(^uint32(0)) { + return 0, errors.New("draft receipt value is outside uint32 range") + } + return uint32(value), nil +} + +func (s *Store) loadIMAPDraftTx(ctx context.Context, tx *loggedTx, draftID string) (IMAPDraft, error) { + return loadIMAPDraft(ctx, tx, s.dialect.SelectForUpdate(), draftID) +} + +func (s *Store) lockIMAPDraftTx(ctx context.Context, tx *loggedTx, draftID string) error { + if lockSQL := s.dialect.RowWriterLockSQL("imap_drafts", "updated_at"); lockSQL != "" { + // Managed drafts use draft_id instead of the dialect helper's id key. + lockSQL = strings.Replace(lockSQL, "WHERE id = ?", "WHERE draft_id = ?", 1) + if _, err := tx.ExecContext(ctx, lockSQL, draftID); err != nil { + return fmt.Errorf("lock IMAP draft %q: %w", draftID, err) + } + } + return nil +} + +// ClaimIMAPDraftContext durably records the original receipt and candidate +// bytes before a provider mutation. The revision remains unchanged. +func (s *Store) ClaimIMAPDraftContext( + ctx context.Context, + draftID string, + revision int64, + operation string, + replacementRaw []byte, +) (IMAPDraft, error) { + if err := validateIMAPDraftID(draftID); err != nil { + return IMAPDraft{}, err + } + if revision <= 0 { + return IMAPDraft{}, fmt.Errorf("%w: expected positive revision", ErrIMAPDraftRevision) + } + if operation != IMAPDraftOperationEdit && operation != IMAPDraftOperationDelete { + return IMAPDraft{}, fmt.Errorf("%w: unknown operation %q", ErrIMAPDraftState, operation) + } + if operation == IMAPDraftOperationEdit && len(replacementRaw) == 0 { + return IMAPDraft{}, errors.New("edit candidate must not be empty") + } + var claimed IMAPDraft + err := s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIMAPDraftTx(ctx, tx, draftID); err != nil { + return err + } + draft, err := s.loadIMAPDraftTx(ctx, tx, draftID) + if err != nil { + return err + } + if draft.Revision != revision { + return fmt.Errorf("%w: expected %d, found %d", ErrIMAPDraftRevision, revision, draft.Revision) + } + if draft.DiscardedAt != nil { + return fmt.Errorf("%w: draft is discarded", ErrIMAPDraftState) + } + if draft.Pending != nil { + return ErrIMAPDraftPending + } + args := []any{ + operation, draft.CurrentMessageID, draft.CurrentReceipt.Mailbox, + draft.CurrentReceipt.UIDValidity, draft.CurrentReceipt.UID, + } + var raw any + if operation == IMAPDraftOperationEdit { + raw = append([]byte(nil), replacementRaw...) + } + args = append(args, raw) + result, err := tx.ExecContext(ctx, ` + UPDATE imap_drafts + SET pending_operation = ?, + pending_original_message_id = ?, + pending_original_mailbox = ?, + pending_original_uidvalidity = ?, + pending_original_uid = ?, + pending_raw = ?, + pending_replacement_mailbox = NULL, + pending_replacement_uidvalidity = NULL, + pending_replacement_uid = NULL, + pending_code = NULL, + updated_at = `+s.dialect.Now()+` + WHERE draft_id = ? AND revision = ? AND pending_operation IS NULL + `, append(args, draftID, revision)...) + if err != nil { + return fmt.Errorf("claim IMAP draft %q: %w", draftID, err) + } + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("check IMAP draft claim %q: %w", draftID, err) + } + if affected != 1 { + return ErrIMAPDraftPending + } + claimed = draft + claimed.Pending = &IMAPDraftPending{ + Operation: operation, OriginalMessageID: draft.CurrentMessageID, + OriginalReceipt: draft.CurrentReceipt, Raw: append([]byte(nil), replacementRaw...), + } + if operation == IMAPDraftOperationDelete { + claimed.Pending.Raw = nil + } + return nil + }) + if err != nil { + return IMAPDraft{}, err + } + return claimed, nil +} + +// RecordIMAPDraftOutcomeContext records a provider result against the current +// pending attempt. It never publishes a replacement or clears the claim. +func (s *Store) RecordIMAPDraftOutcomeContext( + ctx context.Context, + draftID string, + revision int64, + code string, + replacement *IMAPDraftReceipt, +) error { + if err := validateIMAPDraftID(draftID); err != nil { + return err + } + if revision <= 0 || strings.TrimSpace(code) == "" { + return fmt.Errorf("%w: outcome requires positive revision and code", ErrIMAPDraftState) + } + return s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIMAPDraftTx(ctx, tx, draftID); err != nil { + return err + } + draft, err := s.loadIMAPDraftTx(ctx, tx, draftID) + if err != nil { + return err + } + if draft.Revision != revision { + return fmt.Errorf("%w: expected %d, found %d", ErrIMAPDraftRevision, revision, draft.Revision) + } + if draft.Pending == nil { + return ErrIMAPDraftState + } + if replacement != nil { + if replacement.SourceID == 0 { + replacementCopy := *replacement + replacementCopy.SourceID = draft.SourceID + replacement = &replacementCopy + } + if replacement.SourceID != draft.SourceID { + return errors.New("replacement receipt source does not match draft") + } + if err := validateIMAPDraftReceipt(*replacement); err != nil { + return err + } + if draft.Pending.Operation != IMAPDraftOperationEdit { + return errors.New("delete outcome cannot carry a replacement receipt") + } + } + var result sql.Result + if replacement == nil { + result, err = tx.ExecContext(ctx, ` + UPDATE imap_drafts + SET pending_code = ?, updated_at = `+s.dialect.Now()+` + WHERE draft_id = ? AND revision = ? AND pending_operation IS NOT NULL + `, code, draftID, revision) + } else { + result, err = tx.ExecContext(ctx, ` + UPDATE imap_drafts + SET pending_code = ?, pending_replacement_mailbox = ?, + pending_replacement_uidvalidity = ?, pending_replacement_uid = ?, + updated_at = `+s.dialect.Now()+` + WHERE draft_id = ? AND revision = ? AND pending_operation IS NOT NULL + `, code, replacement.Mailbox, replacement.UIDValidity, replacement.UID, draftID, revision) + } + if err != nil { + return fmt.Errorf("record IMAP draft outcome %q: %w", draftID, err) + } + if affected, affectedErr := result.RowsAffected(); affectedErr != nil { + return affectedErr + } else if affected != 1 { + return ErrIMAPDraftState + } + return nil + }) +} + +// AbortIMAPDraftContext clears an unadvanced operation after a proven no-effect +// APPEND or a deletion for which no provider write was attempted. +func (s *Store) AbortIMAPDraftContext(ctx context.Context, draftID string, revision int64, outcome string) (IMAPDraft, error) { + if err := validateIMAPDraftID(draftID); err != nil { + return IMAPDraft{}, err + } + if outcome != "rejected" && outcome != "cancelled" && outcome != "not_attempted" { + return IMAPDraft{}, errors.New("IMAP draft abort requires a definitive no-effect outcome") + } + var active IMAPDraft + err := s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIMAPDraftTx(ctx, tx, draftID); err != nil { + return err + } + draft, err := s.loadIMAPDraftTx(ctx, tx, draftID) + if err != nil { + return err + } + if draft.Revision != revision { + return ErrIMAPDraftRevision + } + if draft.DiscardedAt != nil || draft.Pending == nil || + draft.Pending.ReplacementReceipt != nil || draft.CurrentMessageID != draft.Pending.OriginalMessageID || + draft.CurrentReceipt != draft.Pending.OriginalReceipt { + return errors.New("IMAP draft abort requires an unadvanced operation without an accepted replacement") + } + if (draft.Pending.Operation == IMAPDraftOperationDelete) != (outcome == "not_attempted") { + return errors.New("IMAP draft abort outcome does not match the pending operation") + } + if err := s.clearIMAPDraftPendingTx(ctx, tx, draftID, revision); err != nil { + return err + } + active = draft + active.Pending = nil + return nil + }) + if err != nil { + return IMAPDraft{}, err + } + return active, nil +} + +// PublishIMAPDraftReplacementContext stores the replacement message and +// advances the public revision while retaining the original cleanup evidence. +func (s *Store) PublishIMAPDraftReplacementContext( + ctx context.Context, + draftID string, + revision int64, + participants []ParticipantPersistData, + build func([]int64) *MessagePersistData, +) (IMAPDraft, error) { + if err := validateIMAPDraftID(draftID); err != nil { + return IMAPDraft{}, err + } + if revision <= 0 || build == nil { + return IMAPDraft{}, errors.New("invalid IMAP draft publication") + } + var published IMAPDraft + err := s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIMAPDraftTx(ctx, tx, draftID); err != nil { + return err + } + draft, err := s.loadIMAPDraftTx(ctx, tx, draftID) + if err != nil { + return err + } + if draft.Revision != revision { + return ErrIMAPDraftRevision + } + if draft.Pending == nil || draft.Pending.Operation != IMAPDraftOperationEdit || draft.Pending.ReplacementReceipt == nil { + return errors.New("IMAP draft replacement receipt is not recorded") + } + receipt := *draft.Pending.ReplacementReceipt + if err := invalidatePreviousIMAPDraftSourceKey(ctx, tx, receipt); err != nil { + return err + } + var existingMessageID int64 + if err := tx.QueryRowContext(ctx, ` + SELECT id FROM messages WHERE source_id = ? AND source_message_id = ? + `, draft.SourceID, IMAPDraftSourceMessageID(receipt)).Scan(&existingMessageID); err == nil { + return fmt.Errorf("replacement source key already belongs to message %d", existingMessageID) + } else if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("check replacement source key: %w", err) + } + var existingMembershipID int64 + if err := tx.QueryRowContext(ctx, ` + SELECT message_id FROM imap_message_memberships + WHERE source_id = ? AND mailbox = ? AND uidvalidity = ? AND uid = ? + `, receipt.SourceID, receipt.Mailbox, receipt.UIDValidity, receipt.UID).Scan(&existingMembershipID); err == nil { + return fmt.Errorf("replacement receipt already belongs to message %d", existingMembershipID) + } else if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("check replacement receipt: %w", err) + } + prepare := func(_ context.Context, _ *loggedTx, data *MessagePersistData) (*MessagePersistData, error) { + if data == nil || data.Message == nil { + return nil, errors.New("persist IMAP draft replacement requires a message") + } + if data.Message.SourceID != draft.SourceID || data.Message.SourceMessageID != IMAPDraftSourceMessageID(receipt) { + return nil, errors.New("replacement message identity does not match receipt") + } + if data.MIMEAttachmentReplacement != nil { + return nil, errors.New("IMAP draft replacements cannot contain attachments") + } + if !bytes.Equal(data.RawMIME, draft.Pending.Raw) { + return nil, errors.New("replacement MIME does not match the claimed candidate") + } + return data, nil + } + after := func(ctx context.Context, tx *loggedTx, _ *MessagePersistData, messageID int64) error { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO imap_message_memberships + (source_id, mailbox, uidvalidity, uid, message_id, flags, updated_at) + VALUES (?, ?, ?, ?, ?, %s, %s) + `, s.dialect.JSONBindExpr(), s.dialect.Now()), receipt.SourceID, receipt.Mailbox, receipt.UIDValidity, receipt.UID, messageID, imapDraftFlagsJSON); err != nil { + return fmt.Errorf("persist replacement IMAP membership: %w", err) + } + labelID, err := ensureIMAPMailboxLabel(ctx, tx, receipt.SourceID, receipt.Mailbox) + if err != nil { + return err + } + if err := replaceMessageLabelsTx(boundQuerier{ctx: ctx, q: tx}, messageID, []int64{labelID}); err != nil { + return fmt.Errorf("persist replacement IMAP label: %w", err) + } + result, err := tx.ExecContext(ctx, fmt.Sprintf(` + UPDATE imap_drafts + SET current_message_id = ?, current_mailbox = ?, + current_uidvalidity = ?, current_uid = ?, + revision = revision + 1, pending_code = ?, updated_at = %s + WHERE draft_id = ? AND revision = ? AND pending_operation = 'edit' + `, s.dialect.Now()), messageID, receipt.Mailbox, receipt.UIDValidity, receipt.UID, + IMAPDraftCodeCleanup, draftID, revision) + if err != nil { + return fmt.Errorf("publish IMAP draft replacement %q: %w", draftID, err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ErrIMAPDraftRevision + } + published = draft + published.CurrentMessageID = messageID + published.CurrentReceipt = receipt + published.Revision++ + published.Pending = &IMAPDraftPending{ + Operation: draft.Pending.Operation, OriginalMessageID: draft.Pending.OriginalMessageID, + OriginalReceipt: draft.Pending.OriginalReceipt, Raw: append([]byte(nil), draft.Pending.Raw...), + ReplacementReceipt: &receipt, Code: IMAPDraftCodeCleanup, + } + return nil + } + _, err = s.persistMessageWithParticipantsTx(ctx, tx, nil, participants, build, prepare, after) + return err + }) + if err != nil { + return IMAPDraft{}, err + } + return published, nil +} + +// FinishIMAPDraftRemovalContext records confirmed exact absence. For an edit +// it clears the old cleanup evidence without another revision advance. For a +// delete it marks the current message discarded and advances the revision. +func (s *Store) FinishIMAPDraftRemovalContext(ctx context.Context, draftID string, revision int64) (IMAPDraft, error) { + if err := validateIMAPDraftID(draftID); err != nil { + return IMAPDraft{}, err + } + var finished IMAPDraft + err := s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIMAPDraftTx(ctx, tx, draftID); err != nil { + return err + } + draft, err := s.loadIMAPDraftTx(ctx, tx, draftID) + if err != nil { + return err + } + if draft.Revision != revision { + return ErrIMAPDraftRevision + } + if draft.Pending == nil { + return ErrIMAPDraftState + } + if draft.Pending.Code != IMAPDraftCodeRemoved { + return fmt.Errorf("%w: pending removal code is %q, want %q", ErrIMAPDraftState, draft.Pending.Code, IMAPDraftCodeRemoved) + } + if draft.Pending.Operation == IMAPDraftOperationEdit && draft.Pending.ReplacementReceipt == nil { + return errors.New("cannot finish edit before replacement publication") + } + if err := s.retireIMAPDraftMembershipTx(ctx, tx, draft.Pending.OriginalMessageID, draft.Pending.OriginalReceipt); err != nil { + return err + } + if draft.Pending.Operation == IMAPDraftOperationDelete { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` + UPDATE imap_drafts SET discarded_at = %s, revision = revision + 1, + pending_operation = NULL, pending_original_message_id = NULL, + pending_original_mailbox = NULL, pending_original_uidvalidity = NULL, + pending_original_uid = NULL, pending_raw = NULL, + pending_replacement_mailbox = NULL, + pending_replacement_uidvalidity = NULL, pending_replacement_uid = NULL, + pending_code = NULL, updated_at = %s + WHERE draft_id = ? AND revision = ? AND pending_operation = 'delete' + `, s.dialect.Now(), s.dialect.Now()), draftID, revision); err != nil { + return fmt.Errorf("discard IMAP draft %q: %w", draftID, err) + } + finished = draft + finished.Revision++ + finished.DiscardedAt = ptrTimeNow() + finished.Pending = nil + return nil + } + if err := s.clearIMAPDraftPendingTx(ctx, tx, draftID, revision); err != nil { + return err + } + finished = draft + finished.Pending = nil + return nil + }) + if err != nil { + return IMAPDraft{}, err + } + return finished, nil +} + +func ptrTimeNow() *time.Time { + t := time.Now() + return &t +} + +func (s *Store) clearIMAPDraftPendingTx(ctx context.Context, tx *loggedTx, draftID string, revision int64) error { + result, err := tx.ExecContext(ctx, ` + UPDATE imap_drafts + SET pending_operation = NULL, pending_original_message_id = NULL, + pending_original_mailbox = NULL, pending_original_uidvalidity = NULL, + pending_original_uid = NULL, pending_raw = NULL, + pending_replacement_mailbox = NULL, + pending_replacement_uidvalidity = NULL, pending_replacement_uid = NULL, + pending_code = NULL, updated_at = `+s.dialect.Now()+` + WHERE draft_id = ? AND revision = ? AND pending_operation IS NOT NULL + `, draftID, revision) + if err != nil { + return fmt.Errorf("clear IMAP draft pending evidence %q: %w", draftID, err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return ErrIMAPDraftState + } + return nil +} + +func (s *Store) retireIMAPDraftMembershipTx( + ctx context.Context, + tx *loggedTx, + messageID int64, + receipt IMAPDraftReceipt, +) error { + if messageID <= 0 || receipt.SourceID <= 0 { + return errors.New("invalid IMAP draft cleanup identity") + } + if _, err := tx.ExecContext(ctx, ` + DELETE FROM imap_message_memberships + WHERE source_id = ? AND mailbox = ? AND uidvalidity = ? AND uid = ? AND message_id = ? + `, receipt.SourceID, receipt.Mailbox, receipt.UIDValidity, receipt.UID, messageID); err != nil { + return fmt.Errorf("retire IMAP draft membership: %w", err) + } + mailboxes, err := imapMembershipMailboxes(ctx, tx, receipt.SourceID, messageID) + if err != nil { + return err + } + labelIDs := make([]int64, 0, len(mailboxes)) + for _, mailbox := range mailboxes { + labelID, err := ensureIMAPMailboxLabel(ctx, tx, receipt.SourceID, mailbox) + if err != nil { + return err + } + labelIDs = append(labelIDs, labelID) + } + if err := replaceMessageLabelsTx(boundQuerier{ctx: ctx, q: tx}, messageID, labelIDs); err != nil { + return fmt.Errorf("rebuild IMAP draft labels: %w", err) + } + if len(mailboxes) == 0 { + if _, err := tx.ExecContext(ctx, ` + UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP + WHERE id = ? AND source_id = ? AND deleted_from_source_at IS NULL + `, messageID, receipt.SourceID); err != nil { + return fmt.Errorf("tombstone retired IMAP draft: %w", err) + } + } + if err := s.bumpDerivedDataRevision(tx); err != nil { + return fmt.Errorf("bump derived-data revision for retired IMAP draft: %w", err) + } + return nil +} diff --git a/internal/store/imap_draft_lifecycle_test.go b/internal/store/imap_draft_lifecycle_test.go new file mode 100644 index 000000000..96031e53e --- /dev/null +++ b/internal/store/imap_draft_lifecycle_test.go @@ -0,0 +1,194 @@ +package store_test + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestManagedIMAPDraftLifecycleAndRetention(t *testing.T) { + requirements := require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", "imap://alice@example.com:143") + requirements.NoError(err) + conversationID, err := st.EnsureConversation(source.ID, "draft-lifecycle", "Draft lifecycle") + requirements.NoError(err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + draft, err := st.PersistIMAPDraftContext(t.Context(), receipt, nil, func(_ []int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + BodyText: sql.NullString{String: "old", Valid: true}, + RawMIME: []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\nold\r\n"), + } + }) + requirements.NoError(err) + requirements.NotEmpty(draft.DraftID) + requirements.Equal(int64(1), draft.Revision) + + candidateRaw := []byte("candidate") + claimed, err := st.ClaimIMAPDraftContext(t.Context(), draft.DraftID, 1, store.IMAPDraftOperationEdit, candidateRaw) + requirements.NoError(err) + requirements.Equal(candidateRaw, claimed.Pending.Raw) + _, err = st.ClaimIMAPDraftContext(t.Context(), draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.ErrorIs(err, store.ErrIMAPDraftPending) + + replacement := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, 1, "append_uidplus", &replacement)) + published, err := st.PublishIMAPDraftReplacementContext(t.Context(), draft.DraftID, 1, nil, func(_ []int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(replacement), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + BodyText: sql.NullString{String: "candidate", Valid: true}, + RawMIME: candidateRaw, + } + }) + requirements.NoError(err) + requirements.Equal(int64(2), published.Revision) + requirements.Equal(replacement.UID, published.CurrentReceipt.UID) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, 2, "survivor", nil)) + + _, err = st.FinishIMAPDraftRemovalContext(t.Context(), draft.DraftID, 2) + requirements.ErrorIs(err, store.ErrIMAPDraftState) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, 2, store.IMAPDraftCodeRemoved, nil)) + finished, err := st.FinishIMAPDraftRemovalContext(t.Context(), draft.DraftID, 2) + requirements.NoError(err) + requirements.Nil(finished.Pending) + requirements.Equal(int64(2), finished.Revision) + + var oldDeleted sql.NullTime + requirements.NoError(st.DB().QueryRow(st.Rebind(` + SELECT deleted_from_source_at FROM messages WHERE id = ? + `), draft.CurrentMessageID).Scan(&oldDeleted)) + requirements.True(oldDeleted.Valid) +} + +func TestManagedIMAPDraftReplacementUIDReuse(t *testing.T) { + for _, scenario := range []struct { + name string + uidValidity uint32 + conflict bool + }{ + {name: "previous generation", uidValidity: 1}, + {name: "same generation", uidValidity: 2, conflict: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", "imap://alice@example.com:143") + requirements.NoError(err) + conversationID, err := st.EnsureConversation(source.ID, "draft-uid-reuse", "Draft UID reuse") + requirements.NoError(err) + build := func(receipt store.IMAPDraftReceipt, raw string) func([]int64) *store.MessagePersistData { + return func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + RawMIME: []byte(raw), + } + } + } + oldReceipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: scenario.uidValidity, UID: 2} + archived, err := st.PersistIMAPDraftContext(t.Context(), oldReceipt, nil, build(oldReceipt, "archived")) + requirements.NoError(err) + current := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 2, UID: 1} + draft, err := st.PersistIMAPDraftContext(t.Context(), current, nil, build(current, "current")) + requirements.NoError(err) + _, err = st.ClaimIMAPDraftContext(t.Context(), draft.DraftID, 1, store.IMAPDraftOperationEdit, []byte("replacement")) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 2, UID: 2} + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, 1, "append_uidplus", &replacement)) + published, err := st.PublishIMAPDraftReplacementContext(t.Context(), draft.DraftID, 1, nil, build(replacement, "replacement")) + if scenario.conflict { + requirements.ErrorContains(err, "replacement source key already belongs") + } else { + requirements.NoError(err) + assertions.Equal(int64(2), published.Revision) + assertions.Equal(replacement, published.CurrentReceipt) + raw, err := st.GetMessageRaw(published.CurrentMessageID) + requirements.NoError(err) + assertions.Equal("replacement", string(raw)) + } + raw, err := st.GetMessageRaw(archived.CurrentMessageID) + requirements.NoError(err) + assertions.Equal("archived", string(raw)) + }) + } +} + +func TestManagedIMAPDraftRetainedByGCWhileCurrent(t *testing.T) { + testutil.SkipIfPostgres(t, "archive GC is SQLite-only") + requirements := require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", "imap://gc@example.com:143") + requirements.NoError(err) + conversationID, err := st.EnsureConversation(source.ID, "draft-gc", "Draft GC") + requirements.NoError(err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + draft, err := st.PersistIMAPDraftContext(t.Context(), receipt, nil, func(_ []int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), MessageType: store.MessageTypeEmail, ConversationID: conversationID}, + RawMIME: []byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\nbody\r\n"), + } + }) + requirements.NoError(err) + _, err = st.DB().Exec(st.Rebind(`UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ?`), draft.CurrentMessageID) + requirements.NoError(err) + plan, err := st.PlanGCContext(t.Context()) + requirements.NoError(err) + requirements.Equal(int64(0), plan.SourceDeleted) + requirements.Empty(plan.SourceDeletedIDs) +} + +func TestManagedIMAPDraftRemovalAdvancesDerivedRevisionWithSurvivingMembership(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st, source, draft, _ := newReviewManagedDraft(t, "surviving-membership", 61, "body") + + archiveLabelID, err := st.EnsureLabel(source.ID, "Archive", "Archive", "user") + requirements.NoError(err) + _, err = st.DB().Exec(st.Rebind(` + INSERT INTO imap_message_memberships + (source_id, mailbox, uidvalidity, uid, message_id) + VALUES (?, ?, ?, ?, ?) + `), source.ID, "Archive", 1, 62, draft.CurrentMessageID) + requirements.NoError(err) + requirements.NoError(st.AddMessageLabels(draft.CurrentMessageID, []int64{archiveLabelID})) + + beforeRevision, err := st.DerivedDataRevision() + requirements.NoError(err) + _, err = st.ClaimIMAPDraftContext( + t.Context(), draft.DraftID, draft.Revision, store.IMAPDraftOperationDelete, nil, + ) + requirements.NoError(err) + _, err = st.FinishIMAPDraftRemovalContext(t.Context(), draft.DraftID, draft.Revision) + requirements.ErrorIs(err, store.ErrIMAPDraftState) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, draft.Revision, store.IMAPDraftCodeRemoved, nil)) + finished, err := st.FinishIMAPDraftRemovalContext(t.Context(), draft.DraftID, draft.Revision) + requirements.NoError(err) + + afterRevision, err := st.DerivedDataRevision() + requirements.NoError(err) + assertions.Equal(beforeRevision+1, afterRevision) + assertions.Equal([]string{"Archive"}, messageLabels(t, st, draft.CurrentMessageID)) + assertions.False(messageTombstoned(t, st, draft.CurrentMessageID)) + assertions.Nil(finished.Pending) + + var retainedMailbox string + requirements.NoError(st.DB().QueryRow(st.Rebind(` + SELECT mailbox FROM imap_message_memberships + WHERE source_id = ? AND message_id = ? + `), source.ID, draft.CurrentMessageID).Scan(&retainedMailbox)) + assertions.Equal("Archive", retainedMailbox) +} diff --git a/internal/store/imap_draft_review_test.go b/internal/store/imap_draft_review_test.go new file mode 100644 index 000000000..a1b79fe44 --- /dev/null +++ b/internal/store/imap_draft_review_test.go @@ -0,0 +1,315 @@ +package store_test + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestManagedIMAPDraftPendingReopen(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st, _, draft, _ := newReviewManagedDraft(t, "pending-reopen", 11, "original") + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\ncandidate\r\n") + + claimed, err := st.ClaimIMAPDraftContext(context.Background(), draft.DraftID, draft.Revision, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + requirements.NotNil(claimed.Pending) + assertions.Equal(store.IMAPDraftOperationEdit, claimed.Pending.Operation) + assertions.Equal(draft.CurrentMessageID, claimed.Pending.OriginalMessageID) + assertions.Equal(draft.CurrentReceipt, claimed.Pending.OriginalReceipt) + assertions.Equal(candidate, claimed.Pending.Raw) + + dbPath := store.DBPathForTest(st) + requirements.NoError(st.Close()) + reopenedStore, err := store.Open(dbPath) + requirements.NoError(err) + t.Cleanup(func() { _ = reopenedStore.Close() }) + + reopened, err := reopenedStore.GetIMAPDraftContext(context.Background(), draft.DraftID) + requirements.NoError(err) + requirements.NotNil(reopened.Pending) + assertions.Equal(candidate, reopened.Pending.Raw) + assertions.Equal(draft.CurrentReceipt, reopened.Pending.OriginalReceipt) + assertions.Equal(int64(1), reopened.Revision) + _, err = reopenedStore.ClaimIMAPDraftContext(context.Background(), draft.DraftID, draft.Revision, store.IMAPDraftOperationDelete, nil) + requirements.ErrorIs(err, store.ErrIMAPDraftPending) +} + +func TestManagedIMAPDraftRevision(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st, source, draft, conversationID := newReviewManagedDraft(t, "revision", 21, "original") + + _, err := st.ClaimIMAPDraftContext(context.Background(), draft.DraftID, 2, store.IMAPDraftOperationEdit, []byte("candidate")) + requirements.ErrorIs(err, store.ErrIMAPDraftRevision) + second, err := store.Open(store.DBPathForTest(st)) + requirements.NoError(err) + t.Cleanup(func() { _ = second.Close() }) + + start := make(chan struct{}) + type claimResult struct { + claimed store.IMAPDraft + err error + } + results := make(chan claimResult, 2) + claim := func(client *store.Store, candidate []byte) { + <-start + claimed, claimErr := client.ClaimIMAPDraftContext( + context.Background(), draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate, + ) + results <- claimResult{claimed: claimed, err: claimErr} + } + go claim(st, []byte("candidate")) + go claim(second, []byte("other")) + close(start) + firstResult := <-results + secondResult := <-results + var winner claimResult + wins := 0 + for _, result := range []claimResult{firstResult, secondResult} { + if result.err == nil { + wins++ + winner = result + continue + } + requirements.ErrorIs(result.err, store.ErrIMAPDraftPending) + } + requirements.Equal(1, wins) + requirements.NotNil(winner.claimed.Pending) + requirements.Equal(int64(1), winner.claimed.Revision) + _, err = st.AbortIMAPDraftContext(context.Background(), draft.DraftID, 1, "rejected") + requirements.NoError(err) + active, err := st.GetIMAPDraftContext(context.Background(), draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), active.Revision) + requirements.Nil(active.Pending) + + candidateRaw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\n\r\ncandidate\r\n") + claimed, err := st.ClaimIMAPDraftContext(context.Background(), draft.DraftID, 1, store.IMAPDraftOperationEdit, candidateRaw) + requirements.NoError(err) + requirements.NotNil(claimed.Pending) + replacement := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 22} + requirements.NoError(st.RecordIMAPDraftOutcomeContext(context.Background(), draft.DraftID, 1, "append_uidplus", &replacement)) + published, err := st.PublishIMAPDraftReplacementContext(context.Background(), draft.DraftID, 1, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(replacement), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + BodyText: sql.NullString{String: "candidate", Valid: true}, + RawMIME: candidateRaw, + } + }) + requirements.NoError(err) + assertions.Equal(int64(2), published.Revision) + requirements.NotNil(published.Pending) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(context.Background(), draft.DraftID, 2, store.IMAPDraftCodeRemoved, nil)) + finished, err := st.FinishIMAPDraftRemovalContext(context.Background(), draft.DraftID, 2) + requirements.NoError(err) + assertions.Equal(int64(2), finished.Revision) + requirements.Nil(finished.Pending) + assertions.Equal(published.CurrentMessageID, finished.CurrentMessageID) +} + +func TestManagedIMAPDraftAbort(t *testing.T) { + for _, scenario := range []string{"rejected", "cancelled", "remote_unknown", "accepted_unidentified", "created", "delete", "acknowledged", "advanced", "stale"} { + t.Run(scenario, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st, _, draft, _ := newReviewManagedDraft(t, "abort-"+scenario, 71, "original") + operation := store.IMAPDraftOperationEdit + raw := []byte("candidate") + if scenario == "delete" { + operation = store.IMAPDraftOperationDelete + raw = nil + } + _, err := st.ClaimIMAPDraftContext(t.Context(), draft.DraftID, 1, operation, raw) + requirements.NoError(err) + if scenario == "acknowledged" { + receipt := store.IMAPDraftReceipt{SourceID: draft.SourceID, Mailbox: "Drafts", UIDValidity: 1, UID: 72} + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), draft.DraftID, 1, "append_uidplus", &receipt)) + } + if scenario == "advanced" { + _, err = st.DB().Exec(st.Rebind("UPDATE imap_drafts SET current_uid = 72 WHERE draft_id = ?"), draft.DraftID) + requirements.NoError(err) + } + outcome := scenario + revision := int64(1) + if scenario == "delete" || scenario == "acknowledged" || scenario == "advanced" || scenario == "stale" { + outcome = "rejected" + } + if scenario == "stale" { + revision = 2 + } + active, err := st.AbortIMAPDraftContext(t.Context(), draft.DraftID, revision, outcome) + if scenario == "rejected" || scenario == "cancelled" { + requirements.NoError(err) + assertions.Nil(active.Pending) + assertions.Equal(draft.CurrentMessageID, active.CurrentMessageID) + assertions.Equal(draft.CurrentReceipt, active.CurrentReceipt) + } else { + requirements.Error(err) + } + stored, err := st.GetIMAPDraftContext(t.Context(), draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), stored.Revision) + if scenario == "rejected" || scenario == "cancelled" { + assertions.Nil(stored.Pending) + } else { + assertions.NotNil(stored.Pending) + } + }) + } +} + +func TestManagedIMAPDraftConstraints(t *testing.T) { + requirements := require.New(t) + st, _, draft, _ := newReviewManagedDraft(t, "constraints", 31, "original") + update := func(query string, args ...any) error { + _, err := st.DB().Exec(st.Rebind(query), args...) + return err + } + + for _, candidate := range []struct { + name string + extra string + args []any + }{ + {name: "original without operation"}, + {name: "edit bytes without operation", extra: ", pending_raw = ?", args: []any{[]byte("candidate")}}, + {name: "receipt without operation", extra: ", pending_replacement_mailbox = 'Drafts', pending_replacement_uidvalidity = 1, pending_replacement_uid = 32, pending_code = 'append_uidplus'"}, + } { + t.Run(candidate.name, func(t *testing.T) { + requirements := require.New(t) + tx, err := st.DB().Begin() + requirements.NoError(err) + defer func() { _ = tx.Rollback() }() + args := append([]any{draft.CurrentMessageID}, candidate.args...) + args = append(args, draft.DraftID) + _, err = tx.Exec(st.Rebind( + "UPDATE imap_drafts SET pending_operation = NULL, pending_original_message_id = ?, pending_original_mailbox = 'Drafts', pending_original_uidvalidity = 1, pending_original_uid = 31"+candidate.extra+" WHERE draft_id = ?"), args...) + requirements.Error(err) + }) + } + + requirements.Error(update(`UPDATE imap_drafts SET revision = 0 WHERE draft_id = ?`, draft.DraftID)) + requirements.Error(update(` + UPDATE imap_drafts + SET pending_operation = 'edit', pending_original_message_id = ? + WHERE draft_id = ? + `, draft.CurrentMessageID, draft.DraftID)) + requirements.Error(update(` + UPDATE imap_drafts + SET pending_replacement_mailbox = 'Drafts' + WHERE draft_id = ? + `, draft.DraftID)) + requirements.Error(update(`UPDATE imap_drafts SET current_uid = 0 WHERE draft_id = ?`, draft.DraftID)) + fetched, err := st.GetIMAPDraftContext(context.Background(), draft.DraftID) + requirements.NoError(err) + requirements.Equal(int64(1), fetched.Revision) + requirements.Nil(fetched.Pending) +} + +func TestManagedIMAPDraftRetention(t *testing.T) { + testutil.SkipIfPostgres(t, "archive GC is SQLite-only") + requirements := require.New(t) + assertions := assert.New(t) + st, source, draft, conversationID := newReviewManagedDraft(t, "retention", 41, "managed") + _, err := st.DB().Exec(st.Rebind(`UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ?`), draft.CurrentMessageID) + requirements.NoError(err) + controlID, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: "unowned-control", ConversationID: conversationID, + MessageType: store.MessageTypeEmail, + }, + BodyText: sql.NullString{String: "control", Valid: true}, + RawMIME: []byte("From: control@example.com\r\n\r\ncontrol\r\n"), + }) + requirements.NoError(err) + _, err = st.DB().Exec(st.Rebind(`UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ?`), controlID) + requirements.NoError(err) + plan, err := st.PlanGCContext(context.Background()) + requirements.NoError(err) + assertions.Equal(int64(1), plan.SourceDeleted) + assertions.Equal([]int64{controlID}, plan.SourceDeletedIDs) + _, err = st.ExecuteGCContext(context.Background(), plan) + requirements.NoError(err) + _, err = st.GetMessageContext(context.Background(), draft.CurrentMessageID) + requirements.NoError(err) + _, err = st.GetMessageContext(context.Background(), controlID) + requirements.ErrorContains(err, "message not found") + + purge := newReviewManagedDraftOnStore(t, st, "dedup-purge", 42, "purge") + purgeDraft := purge.draft + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages SET deleted_at = CURRENT_TIMESTAMP, delete_batch_id = ? WHERE id = ? + `), "review-batch", purgeDraft.CurrentMessageID) + requirements.NoError(err) + deleted, err := st.DeleteDedupedBatch("review-batch") + requirements.NoError(err) + assertions.Equal(int64(1), deleted) + _, err = st.GetIMAPDraftContext(context.Background(), purgeDraft.DraftID) + requirements.ErrorIs(err, store.ErrIMAPDraftNotFound) +} + +func TestManagedIMAPDraftSourceCascade(t *testing.T) { + requirements := require.New(t) + st, source, draft, _ := newReviewManagedDraft(t, "source-cascade", 51, "source") + requirements.NoError(st.RemoveSource(source.ID)) + _, err := st.GetIMAPDraftContext(context.Background(), draft.DraftID) + requirements.ErrorIs(err, store.ErrIMAPDraftNotFound) +} + +func newReviewManagedDraft( + t *testing.T, + name string, + uid uint32, + body string, +) (*store.Store, *store.Source, store.IMAPDraft, int64) { + t.Helper() + st := testutil.NewTestStore(t) + fixture := newReviewManagedDraftOnStore(t, st, name, uid, body) + return fixture.store, fixture.source, fixture.draft, fixture.conversationID +} + +type reviewManagedDraftFixture struct { + store *store.Store + source *store.Source + draft store.IMAPDraft + conversationID int64 +} + +func newReviewManagedDraftOnStore( + t *testing.T, + st *store.Store, + name string, + uid uint32, + body string, +) reviewManagedDraftFixture { + t.Helper() + requirements := require.New(t) + source, err := st.GetOrCreateSource("imap", fmt.Sprintf("imap://%s@example.com:143", name)) + requirements.NoError(err) + conversationID, err := st.EnsureConversation(source.ID, name, name) + requirements.NoError(err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: uid} + raw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\n" + body + "\r\n") + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + BodyText: sql.NullString{String: body, Valid: true}, RawMIME: raw, + } + }) + requirements.NoError(err) + return reviewManagedDraftFixture{store: st, source: source, draft: draft, conversationID: conversationID} +} diff --git a/internal/store/imap_drafts.go b/internal/store/imap_drafts.go index 1cfb1b1ef..88f936dc4 100644 --- a/internal/store/imap_drafts.go +++ b/internal/store/imap_drafts.go @@ -2,10 +2,13 @@ package store import ( "context" + "crypto/rand" "database/sql" + "encoding/hex" "errors" "fmt" "strings" + "time" ) const imapDraftFlagsJSON = `["\\Draft"]` @@ -18,21 +21,54 @@ type IMAPDraftReceipt struct { UID uint32 } -// PersistIMAPDraftContext commits the local message snapshot and its exact -// mailbox membership in one transaction. It never changes sync cursors. +// IMAPDraftPending contains durable evidence for an edit or delete that has +// not completed its provider cleanup. +type IMAPDraftPending struct { + Operation string + OriginalMessageID int64 + OriginalReceipt IMAPDraftReceipt + Raw []byte + ReplacementReceipt *IMAPDraftReceipt + Code string +} + +// IMAPDraft is the Store-owned lifecycle record for one managed draft. +type IMAPDraft struct { + DraftID string + SourceID int64 + CurrentMessageID int64 + CurrentReceipt IMAPDraftReceipt + Revision int64 + DiscardedAt *time.Time + Pending *IMAPDraftPending +} + +var ( + ErrIMAPDraftNotFound = errors.New("IMAP draft not found") + ErrIMAPDraftRevision = errors.New("IMAP draft revision mismatch") + ErrIMAPDraftPending = errors.New("IMAP draft has a pending operation") + ErrIMAPDraftState = errors.New("invalid IMAP draft state") +) + +// PersistIMAPDraftContext commits the local message snapshot, exact mailbox +// membership, and managed ownership in one transaction. func (s *Store) PersistIMAPDraftContext( ctx context.Context, receipt IMAPDraftReceipt, participants []ParticipantPersistData, build func([]int64) *MessagePersistData, -) (int64, error) { - if receipt.SourceID <= 0 || strings.TrimSpace(receipt.Mailbox) == "" || receipt.UID == 0 || receipt.UIDValidity == 0 { - return 0, errors.New("invalid IMAP draft receipt") +) (IMAPDraft, error) { + if err := validateIMAPDraftReceipt(receipt); err != nil { + return IMAPDraft{}, err } if build == nil { - return 0, errors.New("persist IMAP draft requires a message builder") + return IMAPDraft{}, errors.New("persist IMAP draft requires a message builder") + } + draftID, err := newIMAPDraftID() + if err != nil { + return IMAPDraft{}, err } - var messageID int64 + var draft IMAPDraft before := func(ctx context.Context, tx *loggedTx) error { var sourceType string if err := tx.QueryRowContext(ctx, `SELECT source_type FROM sources WHERE id = ?`, receipt.SourceID).Scan(&sourceType); err != nil { @@ -55,30 +91,11 @@ func (s *Store) PersistIMAPDraftContext( if !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("check IMAP draft membership: %w", err) } - // APPEND can reuse a key held by an old-generation membership at this - // UID, a moved row after the folder changes generation, or a tombstoned - // orphan with no memberships. Preserve that row under an invalidated key; - // sync still owns membership retirement and cursors. - if _, err := tx.ExecContext(ctx, ` - UPDATE messages SET source_message_id = 'msgvault-invalidated:' || CAST(id AS TEXT) - WHERE source_id = ? AND source_message_id = ? AND (EXISTS ( - SELECT 1 FROM imap_message_memberships - WHERE message_id = messages.id AND source_id = messages.source_id - AND mailbox = ? AND uid = ? AND uidvalidity <> ? - ) OR (EXISTS ( - SELECT 1 FROM imap_folder_state WHERE source_id = messages.source_id AND mailbox = ? AND uidvalidity <> ? - ) AND NOT EXISTS ( - SELECT 1 FROM imap_message_memberships WHERE source_id = messages.source_id AND message_id = messages.id AND mailbox = ? AND uid = ? - )) OR (deleted_from_source_at IS NOT NULL AND NOT EXISTS ( - SELECT 1 FROM imap_message_memberships WHERE source_id = messages.source_id AND message_id = messages.id - ))) - `, receipt.SourceID, IMAPDraftSourceMessageID(receipt), receipt.Mailbox, receipt.UID, receipt.UIDValidity, - receipt.Mailbox, receipt.UIDValidity, receipt.Mailbox, receipt.UID); err != nil { - return fmt.Errorf("invalidate previous IMAP draft source key: %w", err) + if err := invalidatePreviousIMAPDraftSourceKey(ctx, tx, receipt); err != nil { + return err } err = tx.QueryRowContext(ctx, ` - SELECT id FROM messages - WHERE source_id = ? AND source_message_id = ? + SELECT id FROM messages WHERE source_id = ? AND source_message_id = ? `, receipt.SourceID, IMAPDraftSourceMessageID(receipt)).Scan(&existing) if err == nil { return errors.New("source_key_conflict") @@ -103,7 +120,7 @@ func (s *Store) PersistIMAPDraftContext( } return data, nil } - after := func(ctx context.Context, tx *loggedTx, _ *MessagePersistData, id int64) error { + after := func(ctx context.Context, tx *loggedTx, data *MessagePersistData, id int64) error { if _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO imap_message_memberships (source_id, mailbox, uidvalidity, uid, message_id, flags, updated_at) @@ -118,13 +135,63 @@ func (s *Store) PersistIMAPDraftContext( if err := replaceMessageLabelsTx(boundQuerier{ctx: ctx, q: tx}, id, []int64{labelID}); err != nil { return fmt.Errorf("persist IMAP draft label: %w", err) } - messageID = id + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO imap_drafts ( + draft_id, source_id, current_message_id, current_mailbox, + current_uidvalidity, current_uid, revision, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 1, %s, %s) + `, s.dialect.Now(), s.dialect.Now()), draftID, receipt.SourceID, id, + receipt.Mailbox, receipt.UIDValidity, receipt.UID); err != nil { + return fmt.Errorf("persist IMAP draft ownership: %w", err) + } + draft = IMAPDraft{ + DraftID: draftID, SourceID: receipt.SourceID, CurrentMessageID: id, + CurrentReceipt: receipt, Revision: 1, + } return nil } if _, err := s.persistMessageWithParticipantsTransaction(ctx, before, participants, build, prepare, after); err != nil { - return 0, err + return IMAPDraft{}, err + } + return draft, nil +} + +// APPEND can reuse a key after a folder epoch reset. Keep the old +// archive row, but free the provider key before inserting the new one. +func invalidatePreviousIMAPDraftSourceKey(ctx context.Context, tx *loggedTx, receipt IMAPDraftReceipt) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE messages SET source_message_id = 'msgvault-invalidated:' || CAST(id AS TEXT) + WHERE source_id = ? AND source_message_id = ? AND (EXISTS ( + SELECT 1 FROM imap_message_memberships + WHERE message_id = messages.id AND source_id = messages.source_id + AND mailbox = ? AND uid = ? AND uidvalidity <> ? + ) OR (EXISTS ( + SELECT 1 FROM imap_folder_state WHERE source_id = messages.source_id AND mailbox = ? AND uidvalidity <> ? + ) AND NOT EXISTS ( + SELECT 1 FROM imap_message_memberships WHERE source_id = messages.source_id AND message_id = messages.id AND mailbox = ? AND uid = ? + )) OR (deleted_from_source_at IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM imap_message_memberships WHERE source_id = messages.source_id AND message_id = messages.id + ))) + `, receipt.SourceID, IMAPDraftSourceMessageID(receipt), receipt.Mailbox, receipt.UID, receipt.UIDValidity, + receipt.Mailbox, receipt.UIDValidity, receipt.Mailbox, receipt.UID); err != nil { + return fmt.Errorf("invalidate previous IMAP draft source key: %w", err) + } + return nil +} + +func validateIMAPDraftReceipt(receipt IMAPDraftReceipt) error { + if receipt.SourceID <= 0 || strings.TrimSpace(receipt.Mailbox) == "" || receipt.UID == 0 || receipt.UIDValidity == 0 { + return errors.New("invalid IMAP draft receipt") + } + return nil +} + +func newIMAPDraftID() (string, error) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", fmt.Errorf("generate IMAP draft ID: %w", err) } - return messageID, nil + return "draft-" + hex.EncodeToString(raw[:]), nil } // IMAPDraftSourceMessageID returns the composite provider key used by sync. diff --git a/internal/store/imap_drafts_test.go b/internal/store/imap_drafts_test.go index e9878cb6a..9ba486fcf 100644 --- a/internal/store/imap_drafts_test.go +++ b/internal/store/imap_drafts_test.go @@ -50,8 +50,9 @@ func TestPersistIMAPDraft(t *testing.T) { {EmailAddress: "alice@example.com", Domain: "example.com"}, {EmailAddress: "user@example.com", Domain: "example.com"}, } - id, err := st.PersistIMAPDraftContext(context.Background(), receipt, participants, build) + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, participants, build) requirements.NoError(err) + id := draft.CurrentMessageID assertions.Positive(id) var draftMembershipCount, cursorCount int requirements.NoError(st.DB().QueryRow(st.Rebind(`SELECT COUNT(*) FROM imap_message_memberships WHERE message_id = ?`), id).Scan(&draftMembershipCount)) @@ -91,8 +92,9 @@ func TestPersistIMAPDraft(t *testing.T) { requirements.NoError(err) assertions.Equal(store.IMAPDraftSourceMessageID(receipt), oldSourceID) - newID, err := st.PersistIMAPDraftContext(context.Background(), receipt, participants, build) + newDraft, err := st.PersistIMAPDraftContext(context.Background(), receipt, participants, build) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(id, newID) oldRaw, err := st.GetMessageRaw(id) requirements.NoError(err) diff --git a/internal/store/imap_identity_test.go b/internal/store/imap_identity_test.go index 1cc92ad55..3dfd87591 100644 --- a/internal/store/imap_identity_test.go +++ b/internal/store/imap_identity_test.go @@ -266,7 +266,7 @@ func TestIMAPIdentity_RekeysRemovedDraftKeyWithOtherMembership(t *testing.T) { newReceipt := store.IMAPDraftReceipt{ SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 30, UID: 1, } - newID, err := f.store.PersistIMAPDraftContext(t.Context(), newReceipt, nil, + newDraft, err := f.store.PersistIMAPDraftContext(t.Context(), newReceipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{ @@ -281,6 +281,7 @@ func TestIMAPIdentity_RekeysRemovedDraftKeyWithOtherMembership(t *testing.T) { } }) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(oldID, newID) } @@ -291,7 +292,7 @@ func TestIMAPIdentity_UIDValidityResetRekeysOrphanedDraftBeforeUpsert(t *testing receipt := store.IMAPDraftReceipt{ SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 10, UID: 1, } - draftID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, + draft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{ @@ -305,6 +306,7 @@ func TestIMAPIdentity_UIDValidityResetRekeysOrphanedDraftBeforeUpsert(t *testing } }) requirements.NoError(err) + draftID := draft.CurrentMessageID requirements.NoError(f.store.UpsertIMAPFolderStates(f.source.ID, []store.IMAPFolderState{{ Mailbox: "Drafts", UIDValidity: 10, UIDNext: 2, }})) @@ -343,7 +345,7 @@ func TestIMAPIdentity_RetiredMailboxRekeysOrphanedDraftSourceKey(t *testing.T) { receipt := store.IMAPDraftReceipt{ SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 10, UID: 1, } - draftID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, + draft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{ @@ -357,6 +359,7 @@ func TestIMAPIdentity_RetiredMailboxRekeysOrphanedDraftSourceKey(t *testing.T) { } }) requirements.NoError(err) + draftID := draft.CurrentMessageID requirements.NoError(f.store.UpsertIMAPFolderStates(f.source.ID, []store.IMAPFolderState{{ Mailbox: "Drafts", UIDValidity: 10, UIDNext: 2, }})) @@ -503,15 +506,18 @@ func TestIMAPIdentity_NewPublishedCopyKeepsKey(t *testing.T) { RawMIME: fmt.Appendf(nil, "Subject: Generation %d\r\n\r\ncontent %d\r\n", receipt.UIDValidity, receipt.UIDValidity), } } - oldID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, build) + oldDraft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, build) requirements.NoError(err) + oldID := oldDraft.CurrentMessageID oldRaw, err := f.store.GetMessageRaw(oldID) requirements.NoError(err) requirements.NoError(f.store.UpsertIMAPFolderStates(f.source.ID, []store.IMAPFolderState{{Mailbox: "Drafts", UIDValidity: 10, UIDNext: 2}})) receipt.UIDValidity = 20 var newID int64 if mode == "append" { - newID, err = f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, build) + newDraft, persistErr := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, build) + err = persistErr + newID = newDraft.CurrentMessageID } else { changed, rekeyErr := f.store.RekeyMessageSourceID(oldID, "Drafts|1", fmt.Sprintf("msgvault-invalidated:%d", oldID)) requirements.NoError(rekeyErr) @@ -700,7 +706,7 @@ func TestIMAPIdentity_SameEpochResetRemovalReleasesSourceKeyForLaterEpoch(t *tes receipt := store.IMAPDraftReceipt{ SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 20, UID: 1, } - newID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, + newDraft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{ @@ -714,6 +720,7 @@ func TestIMAPIdentity_SameEpochResetRemovalReleasesSourceKeyForLaterEpoch(t *tes } }) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(oldID, newID) oldSourceMessageID, err := f.store.GetMessageSourceID(oldID) requirements.NoError(err) @@ -751,7 +758,7 @@ func TestIMAPIdentity_RekeysOrphanedSourceDeletedMessage(t *testing.T) { assertions.Equal("Drafts|1", key) assertions.Zero(membershipCount(t, st, source.ID)) assertions.True(messageTombstoned(t, st, oldID)) - newID, err := st.PersistIMAPDraftContext(t.Context(), receipt, nil, + newDraft, err := st.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{ @@ -765,6 +772,7 @@ func TestIMAPIdentity_RekeysOrphanedSourceDeletedMessage(t *testing.T) { } }) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(oldID, newID) retainedRaw, err := st.GetMessageRaw(oldID) requirements.NoError(err) @@ -798,13 +806,14 @@ func TestIMAPIdentity_MovedCopyAppendBeforeSync(t *testing.T) { requirements.NoError(err) receipt := store.IMAPDraftReceipt{SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 20, UID: 1} newRaw := []byte("Subject: Replacement\r\n\r\nnew content\r\n") - newID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { + newDraft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{ Message: &store.Message{SourceID: f.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), ConversationID: f.convID, MessageType: store.MessageTypeEmail}, RawMIME: newRaw, } }) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(oldID, newID) retainedRaw, err := f.store.GetMessageRaw(oldID) requirements.NoError(err) @@ -857,10 +866,11 @@ func TestIMAPIdentity_OmittedReplacedKeyAllowsAppend(t *testing.T) { archive.Memberships = nil requirements.NoError(f.store.ApplyIMAPMailboxDeltas(f.source.ID, []store.IMAPMailboxDelta{drafts, archive})) receipt := store.IMAPDraftReceipt{SourceID: f.source.ID, Mailbox: "Drafts", UIDValidity: 20, UID: 1} - newID, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { + newDraft, err := f.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func([]int64) *store.MessagePersistData { return &store.MessagePersistData{Message: &store.Message{SourceID: f.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), ConversationID: f.convID, MessageType: store.MessageTypeEmail}} }) requirements.NoError(err) + newID := newDraft.CurrentMessageID assertions.NotEqual(oldID, newID) retainedRaw, err := f.store.GetMessageRaw(oldID) requirements.NoError(err) diff --git a/internal/store/messages.go b/internal/store/messages.go index ef40218cf..0254745ac 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -1888,86 +1888,106 @@ func (s *Store) persistMessageWithParticipantsTransaction( ) (int64, error) { var messageID int64 err := s.withTxContext(ctx, func(tx *loggedTx) error { - if s.dialect.DriverName() != postgresDriverName { - // Reserve SQLite's writer slot before any prior-state or related - // snapshot reads. Otherwise a concurrent commit can leave this - // deferred WAL transaction unable to upgrade to a writer. - if _, err := tx.Exec(`UPDATE embedding_change_clock SET sequence = sequence WHERE singleton = 1`); err != nil { - return fmt.Errorf("lock message persistence: %w", err) - } - } - if len(participants) > 1 { - // Participant merges take the directory lock before rewriting - // message rows. Keep the same order when a repair's preflight - // callback locks its target message for identity revalidation. - if err := s.lockParticipantDirectoryMutationTxContext(ctx, tx); err != nil { - return err - } - } - if beforeParticipants != nil { - if err := beforeParticipants(ctx, tx); err != nil { - return err - } - } - q := boundQuerier{ctx: ctx, q: tx} - participantIDs := make([]int64, len(participants)) - participantInserted := false - for idx, participant := range participants { - if err := ctx.Err(); err != nil { - return err - } - participantID, err := ensureParticipantWith( - q, - s.dialect, - participant.EmailAddress, - participant.DisplayName, - participant.Domain, - func() error { - participantInserted = true - return nil - }, - ) - if err != nil { - return fmt.Errorf("ensure participant %d: %w", idx, err) - } - participantIDs[idx] = participantID + var err error + messageID, err = s.persistMessageWithParticipantsTx( + ctx, tx, beforeParticipants, participants, build, prepare, afterPersist, + ) + return err + }) + return messageID, err +} + +// persistMessageWithParticipantsTx applies the participant and message +// persistence steps to an existing transaction. Lifecycle owners use this to +// publish a message and their ownership row in one commit. +func (s *Store) persistMessageWithParticipantsTx( + ctx context.Context, + tx *loggedTx, + beforeParticipants messagePersistBeforeParticipants, + participants []ParticipantPersistData, + build func([]int64) *MessagePersistData, + prepare messagePersistPrepare, + afterPersist messagePersistAfter, +) (int64, error) { + var messageID int64 + if s.dialect.DriverName() != postgresDriverName { + // Reserve SQLite's writer slot before any prior-state or related + // snapshot reads. Otherwise a concurrent commit can leave this + // deferred WAL transaction unable to upgrade to a writer. + if _, err := tx.ExecContext(ctx, `UPDATE embedding_change_clock SET sequence = sequence WHERE singleton = 1`); err != nil { + return 0, fmt.Errorf("lock message persistence: %w", err) + } + } + if len(participants) > 1 { + // Participant merges take the directory lock before rewriting + // message rows. Keep the same order when a repair's preflight + // callback locks its target message for identity revalidation. + if err := s.lockParticipantDirectoryMutationTxContext(ctx, tx); err != nil { + return 0, err } - if participantInserted { - if err := s.bumpParticipantDisplayNameRevisionContext(ctx, tx); err != nil { - return err - } + } + if beforeParticipants != nil { + if err := beforeParticipants(ctx, tx); err != nil { + return 0, err } - + } + q := boundQuerier{ctx: ctx, q: tx} + participantIDs := make([]int64, len(participants)) + participantInserted := false + for idx, participant := range participants { if err := ctx.Err(); err != nil { - return err - } - data := build(participantIDs) - if data == nil || data.Message == nil { - return errors.New("persist message requires a message") + return 0, err } - if prepare != nil { - var err error - data, err = prepare(ctx, tx, data) - if err != nil { - return err - } + participantID, err := ensureParticipantWith( + q, + s.dialect, + participant.EmailAddress, + participant.DisplayName, + participant.Domain, + func() error { + participantInserted = true + return nil + }, + ) + if err != nil { + return 0, fmt.Errorf("ensure participant %d: %w", idx, err) } - if err := s.requireSyncSource(data.Message.SourceID); err != nil { - return err + participantIDs[idx] = participantID + } + if participantInserted { + if err := s.bumpParticipantDisplayNameRevisionContext(ctx, tx); err != nil { + return 0, err } - id, err := s.persistMessageWith(ctx, tx, data) + } + + if err := ctx.Err(); err != nil { + return 0, err + } + data := build(participantIDs) + if data == nil || data.Message == nil { + return 0, errors.New("persist message requires a message") + } + if prepare != nil { + var err error + data, err = prepare(ctx, tx, data) if err != nil { - return err + return 0, err } - if afterPersist != nil { - if err := afterPersist(ctx, tx, data, id); err != nil { - return err - } + } + if err := s.requireSyncSource(data.Message.SourceID); err != nil { + return 0, err + } + id, err := s.persistMessageWith(ctx, tx, data) + if err != nil { + return 0, err + } + if afterPersist != nil { + if err := afterPersist(ctx, tx, data, id); err != nil { + return 0, err } - messageID = id - return nil - }) - return messageID, err + } + messageID = id + return messageID, nil } func (s *Store) persistMessageWith( diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 11c6c9759..df470d479 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -1914,6 +1914,74 @@ CREATE TABLE IF NOT EXISTS imap_message_memberships ( CREATE INDEX IF NOT EXISTS idx_imap_message_memberships_source_message ON imap_message_memberships(source_id, message_id); +-- Managed outbound drafts retain the exact APPEND receipt and any interrupted +-- replacement bytes. The receipt is the mutation authority; mailbox sync only +-- observes memberships and never changes these fields. +CREATE TABLE IF NOT EXISTS imap_drafts ( + draft_id TEXT PRIMARY KEY, + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + current_message_id INTEGER NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + current_mailbox TEXT NOT NULL, + current_uidvalidity INTEGER NOT NULL, + current_uid INTEGER NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + discarded_at DATETIME, + pending_operation TEXT, + pending_original_message_id INTEGER REFERENCES messages(id) ON DELETE CASCADE, + pending_original_mailbox TEXT, + pending_original_uidvalidity INTEGER, + pending_original_uid INTEGER, + pending_raw BLOB, + pending_replacement_mailbox TEXT, + pending_replacement_uidvalidity INTEGER, + pending_replacement_uid INTEGER, + pending_code TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + CHECK ( + (pending_operation IS NULL + AND pending_original_message_id IS NULL + AND pending_original_mailbox IS NULL + AND pending_original_uidvalidity IS NULL + AND pending_original_uid IS NULL + AND pending_raw IS NULL + AND pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL + AND pending_code IS NULL) + OR (pending_operation IS NOT NULL + AND pending_operation IN ('edit', 'delete') + AND pending_original_message_id IS NOT NULL + AND pending_original_mailbox IS NOT NULL + AND pending_original_uidvalidity IS NOT NULL + AND pending_original_uidvalidity > 0 + AND pending_original_uid IS NOT NULL + AND pending_original_uid > 0 + AND ((pending_operation = 'edit' AND pending_raw IS NOT NULL AND length(pending_raw) > 0) + OR (pending_operation = 'delete' AND pending_raw IS NULL)) + AND (pending_operation = 'edit' + OR (pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL))) + ), + CHECK (discarded_at IS NULL OR pending_operation IS NULL), + CHECK (current_uidvalidity > 0 AND current_uid > 0), + CHECK ((pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL) + OR (pending_replacement_mailbox IS NOT NULL + AND pending_replacement_uidvalidity IS NOT NULL + AND pending_replacement_uidvalidity > 0 + AND pending_replacement_uid IS NOT NULL + AND pending_replacement_uid > 0)) +); + +CREATE INDEX IF NOT EXISTS idx_imap_drafts_current_message + ON imap_drafts(current_message_id); + +CREATE INDEX IF NOT EXISTS idx_imap_drafts_pending_original_message + ON imap_drafts(pending_original_message_id); + -- Imported source items (files/objects already processed for resumable adapters) CREATE TABLE IF NOT EXISTS source_import_items ( id INTEGER PRIMARY KEY, diff --git a/internal/store/schema_pg.sql b/internal/store/schema_pg.sql index bb62165af..1e1b2e6e9 100644 --- a/internal/store/schema_pg.sql +++ b/internal/store/schema_pg.sql @@ -1727,6 +1727,71 @@ CREATE TABLE IF NOT EXISTS imap_message_memberships ( CREATE INDEX IF NOT EXISTS idx_imap_message_memberships_source_message ON imap_message_memberships(source_id, message_id); +CREATE TABLE IF NOT EXISTS imap_drafts ( + draft_id TEXT PRIMARY KEY, + source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + current_message_id BIGINT NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + current_mailbox TEXT NOT NULL, + current_uidvalidity BIGINT NOT NULL, + current_uid BIGINT NOT NULL, + revision BIGINT NOT NULL CHECK (revision > 0), + discarded_at TIMESTAMPTZ, + pending_operation TEXT, + pending_original_message_id BIGINT REFERENCES messages(id) ON DELETE CASCADE, + pending_original_mailbox TEXT, + pending_original_uidvalidity BIGINT, + pending_original_uid BIGINT, + pending_raw BYTEA, + pending_replacement_mailbox TEXT, + pending_replacement_uidvalidity BIGINT, + pending_replacement_uid BIGINT, + pending_code TEXT, + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + CHECK ( + (pending_operation IS NULL + AND pending_original_message_id IS NULL + AND pending_original_mailbox IS NULL + AND pending_original_uidvalidity IS NULL + AND pending_original_uid IS NULL + AND pending_raw IS NULL + AND pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL + AND pending_code IS NULL) + OR (pending_operation IS NOT NULL + AND pending_operation IN ('edit', 'delete') + AND pending_original_message_id IS NOT NULL + AND pending_original_mailbox IS NOT NULL + AND pending_original_uidvalidity IS NOT NULL + AND pending_original_uidvalidity > 0 + AND pending_original_uid IS NOT NULL + AND pending_original_uid > 0 + AND ((pending_operation = 'edit' AND pending_raw IS NOT NULL AND length(pending_raw) > 0) + OR (pending_operation = 'delete' AND pending_raw IS NULL)) + AND (pending_operation = 'edit' + OR (pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL))) + ), + CHECK (discarded_at IS NULL OR pending_operation IS NULL), + CHECK (current_uidvalidity > 0 AND current_uid > 0), + CHECK ((pending_replacement_mailbox IS NULL + AND pending_replacement_uidvalidity IS NULL + AND pending_replacement_uid IS NULL) + OR (pending_replacement_mailbox IS NOT NULL + AND pending_replacement_uidvalidity IS NOT NULL + AND pending_replacement_uidvalidity > 0 + AND pending_replacement_uid IS NOT NULL + AND pending_replacement_uid > 0)) +); + +CREATE INDEX IF NOT EXISTS idx_imap_drafts_current_message + ON imap_drafts(current_message_id); + +CREATE INDEX IF NOT EXISTS idx_imap_drafts_pending_original_message + ON imap_drafts(pending_original_message_id); + CREATE TABLE IF NOT EXISTS source_import_items ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE, From 8131389cffd6e5bf4da8bda0ffe2123c61dc1694 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sun, 20 Sep 2026 22:30:19 -0400 Subject: [PATCH 004/125] docs(imap): fix setup links (#874) The IMAP setup and folder filtering links now work from both GitHub and the published documentation. Both pages use relative Markdown paths, which the docs build converts to the existing site routes. Closes #872 Co-authored-by: Rod Boev --- docs/setup.md | 2 +- docs/usage/imap.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/setup.md b/docs/setup.md index 5c4812d28..b2f04ac9c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -195,7 +195,7 @@ msgvault sync-full you@fastmail.com IMAP accounts are stored in the same database as Gmail accounts. All tools (Web UI, TUI, search, MCP, and REST API) work with IMAP messages the same way. To start with only part of a large account, see -[IMAP Folder Sync](/docs/usage/imap/) for `--folder` and `--skip-folder` examples. +[IMAP Folder Sync](usage/imap.md) for `--folder` and `--skip-folder` examples. !!! tip "Microsoft 365 / Outlook.com" For Outlook, Hotmail, Live.com, and Microsoft 365 accounts, `add-o365` provides OAuth-based access without app passwords. It auto-detects the correct IMAP host and configures XOAUTH2 authentication. See the [OAuth Setup guide](/docs/guides/oauth-setup/#microsoft-365-outlook-hotmail) for details. diff --git a/docs/usage/imap.md b/docs/usage/imap.md index 87916de04..de87972b5 100644 --- a/docs/usage/imap.md +++ b/docs/usage/imap.md @@ -5,7 +5,7 @@ description: Archive IMAP mail efficiently, choose folders, and repair stored la --- Archive mail from an IMAP account, then keep it current without downloading -unchanged messages again. Start with [IMAP account setup](/docs/setup/#add-an-imap-account) +unchanged messages again. Start with [IMAP account setup](../setup.md#add-an-imap-account) if you have not connected the account yet. ```bash From 7837532e357af2c49ce9777789f8a220e0cc545f Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sun, 20 Sep 2026 21:30:53 -0500 Subject: [PATCH 005/125] fix(carddav): reject unsupported configured providers (#839) Reject unsupported CardDAV providers in config files. A typo such as `provider = "googl"` previously selected password authentication and could reuse an existing bound credential; only `""` and `"google"` are now accepted, matching the API. Credential reuse also requires an explicit provider match. Follow-up to #820. Co-authored-by: Wes McKinney --- internal/api/carddav_google.go | 3 ++- internal/api/carddav_test.go | 16 ++++++++++++++++ internal/config/config.go | 3 +++ internal/config/config_test.go | 18 ++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/internal/api/carddav_google.go b/internal/api/carddav_google.go index 564c193db..b0ca0f098 100644 --- a/internal/api/carddav_google.go +++ b/internal/api/carddav_google.go @@ -28,7 +28,8 @@ func normalizeCardDAVAccountRequest(req CardDAVAccountRequest) CardDAVAccountReq } func cardDAVCredentialMatchesConfig(credential carddav.Credential, cfg config.CardDAVConfig) bool { - return credential.Google == (cfg.Provider == "google") && credential.OAuthApp == cfg.OAuthApp + return credential.OAuthApp == cfg.OAuthApp && + ((cfg.Provider == "" && !credential.Google) || (cfg.Provider == "google" && credential.Google)) } func (c *CardDAVController) credentialForRequest(ctx context.Context, req CardDAVAccountRequest) (carddav.Credential, error) { diff --git a/internal/api/carddav_test.go b/internal/api/carddav_test.go index 16a038ab6..199b2396c 100644 --- a/internal/api/carddav_test.go +++ b/internal/api/carddav_test.go @@ -113,6 +113,22 @@ func TestNewCardDAVControllerLoadsCredentialFromConfiguredDataDir(t *testing.T) assert.NoFileExists(filepath.Join(home, "tokens", "carddav.json")) } +func TestCardDAVUnsupportedProviderDoesNotReuseCredential(t *testing.T) { + assertions := assert.New(t) + required := require.New(t) + cfg, st, _ := savedCardDAVFixture(t) + controller, err := NewCardDAVController(cfg, st, slog.New(slog.DiscardHandler)) + required.NoError(err) + required.NotNil(controller.Current()) + + cfg.CardDAV.Provider = "googl" + controller, err = NewCardDAVController(cfg, st, slog.New(slog.DiscardHandler)) + required.NoError(err) + assertions.Nil(controller.Current()) + _, err = controller.reusableCredential(t.Context(), cfg.CardDAV.BaseURL, cfg.CardDAV.Username) + required.ErrorIs(err, carddav.ErrCredentialNotBound) +} + type controlledCardDAVCandidate struct { cardDAVListFixture diff --git a/internal/config/config.go b/internal/config/config.go index 735358dc3..2a9c10363 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -932,6 +932,9 @@ func decodeConfig(cfg *Config, path string, explicit, homeOverride bool, content if err := cfg.Web.Validate(); err != nil { return nil, err } + if cfg.CardDAV.Provider != "" && cfg.CardDAV.Provider != "google" { + return nil, errors.New("carddav.provider must be empty or \"google\"") + } cfg.Integrations.Tasks.ApplyDefaults() if err := cfg.Integrations.Tasks.Validate(); err != nil { return nil, err diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3e19c8e2e..aa938d1bd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "os" "path/filepath" "runtime" @@ -37,6 +38,23 @@ enabled = true assert.NotContains(encoded.String(), "password") } +func TestCardDAVConfigProvider(t *testing.T) { + for _, provider := range []string{"", "google", "googl"} { + t.Run(provider, func(t *testing.T) { + required := require.New(t) + path := filepath.Join(t.TempDir(), "config.toml") + required.NoError(os.WriteFile(path, []byte(fmt.Sprintf("[carddav]\nprovider = %q\n", provider)), 0600)) + cfg, err := Load(path, "") + if provider == "googl" { + required.ErrorContains(err, "carddav.provider") + return + } + required.NoError(err) + assert.Equal(t, provider, cfg.CardDAV.Provider) + }) + } +} + func TestIMAPDraftConfig(t *testing.T) { assertions := assert.New(t) requirements := require.New(t) From c6bb331c40d17648f376ee942378f1e60f43237a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 21 Sep 2026 06:52:47 -0500 Subject: [PATCH 006/125] fix(web): explain query failures and offer recovery (#854) Explain when archive queries run out of memory or temporary disk space, with steps to narrow the results or adjust the server's query limits. Everything, Files, and other views using the shared query error handler receive this guidance; other unexpected failures direct the operator to the server logs. Files now offers Retry after an initial failure and hides its count while loading or unavailable, so a failed request does not appear to mean the archive has no files. The configuration guide explains how to check resources, change the limits, and restart the daemon. Co-authored-by: Wes McKinney --- docs/changelog.md | 3 +++ docs/configuration.md | 7 ++++++ internal/api/explore.go | 11 ++++++++- internal/api/explore_e2e_test.go | 24 +++++++++++++++++++ .../components/files/FilesWorkspace.svelte | 13 ++++++---- .../components/files/FilesWorkspace.test.ts | 21 ++++++++++++++++ web/tests/theme-keyboard.spec.ts | 22 +++++++++++++++++ 7 files changed, 96 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 098af02db..94bac4654 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -138,6 +138,9 @@ See [searching](usage/searching.md), [vector search](usage/vector-search.md), - Share shorter Web UI links. Ordinary tabs use readable workspace and search mode parameters. Links omit defaults, keyboard focus, scroll position, and choices from other tabs; browser Back and Forward retain that session context. +- Understand failed archive queries: memory and temporary-disk failures now + explain the relevant settings and recovery steps. Files offers a retry and + shows a file count only after the listing loads successfully. - Use Web Directory to edit profiles, relationships, and employment, review identity matches and facts, merge or split profiles, publish CardDAV contacts, and inspect curated networks and person attachment galleries. diff --git a/docs/configuration.md b/docs/configuration.md index fc9e2c7ab..b8e2317bb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -668,6 +668,13 @@ Settings for daemon-side aggregate query behavior. The Web UI, TUI, MCP server, | `query_threads` | min(CPUs, 4) | DuckDB threads for daemon aggregate queries; zero keeps the default | | `query_temp_limit` | `2GB` | Maximum spill-to-disk size for daemon aggregate queries; a query that spills past it fails with a DuckDB out-of-memory error | +If a Web UI query runs out of memory or temporary disk space, its error names +the query-limit settings above. Try filters that narrow the results. On the +machine running msgvault, check available memory and free disk space before +raising these limits in `config.toml`. Restart the daemon to apply the change, +then retry the query. The limits cap resource use; they do not reserve memory +or disk space. Cache builds have separate `builder_*` limits. + The daemon starts HTTP health and API routing before analytics cache maintenance. With `engine = "duckdb"`, analytics remain unavailable until a usable cache is ready; if the cache cannot be built or opened, `msgvault serve` diff --git a/internal/api/explore.go b/internal/api/explore.go index 96524ebce..87d2ce90b 100644 --- a/internal/api/explore.go +++ b/internal/api/explore.go @@ -19,6 +19,7 @@ import ( "time" "github.com/danielgtaylor/huma/v2" + "github.com/duckdb/duckdb-go/v2" "go.kenn.io/msgvault/internal/explorecatalog" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/search" @@ -1948,7 +1949,15 @@ func (s *Server) writeExploreError(ctx context.Context, w http.ResponseWriter, e return } s.logger.Error("exploration failed", "error", err) - writeError(w, http.StatusInternalServerError, "explore_failed", "Couldn't load results") + if resourceErr, ok := errors.AsType[*duckdb.Error](err); ok && resourceErr.Type == duckdb.ErrorTypeOutOfMemory { + writeError(w, http.StatusServiceUnavailable, "query_resource_exhausted", + "This query ran out of memory or temporary disk space. Try narrowing the results with filters. "+ + "The person running msgvault can check available resources and increase analytics.query_memory_limit "+ + "or analytics.query_temp_limit in config.toml, then restart the server.") + return + } + writeError(w, http.StatusInternalServerError, "explore_failed", + "Couldn't load results. Try again. If this keeps happening, ask the person running msgvault to check the server logs.") } type ExploreCacheUnavailableResponse struct { diff --git a/internal/api/explore_e2e_test.go b/internal/api/explore_e2e_test.go index 5aed5435b..16fd4e20c 100644 --- a/internal/api/explore_e2e_test.go +++ b/internal/api/explore_e2e_test.go @@ -51,6 +51,30 @@ func TestExploreHTTPUsesCommittedDuckDBReadModel(t *testing.T) { assertions.Equal("Newest", row["title"]) } +func TestExploreResourceLimitExplainsRecovery(t *testing.T) { + _, analyticsDir := newExploreDuckDBFixtureWithDir(t) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil, query.DuckDBOptions{ + MemoryLimit: "1MB", + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + srv := newTestServerWithEngine(t, engine) + + for _, path := range []string{"/api/v1/explore", "/api/v1/files/search"} { + t.Run(path, func(t *testing.T) { + assertions := assert.New(t) + response := postExploreJSON(t, srv, path, `{ "limit": 100 }`) + assertions.Equal(http.StatusServiceUnavailable, response.Code) + var body ErrorResponse + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + assertions.Equal("query_resource_exhausted", body.Error, response.Body.String()) + assertions.Contains(body.Message, "analytics.query_memory_limit") + assertions.Contains(body.Message, "analytics.query_temp_limit") + assertions.Contains(body.Message, "restart") + }) + } +} + func TestExploreGroupsAndFilesUseCompleteDuckDBFacts(t *testing.T) { assertions := assert.New(t) requirements := require.New(t) diff --git a/web/src/lib/components/files/FilesWorkspace.svelte b/web/src/lib/components/files/FilesWorkspace.svelte index 2751e1994..a8aaa94bb 100644 --- a/web/src/lib/components/files/FilesWorkspace.svelte +++ b/web/src/lib/components/files/FilesWorkspace.svelte @@ -722,9 +722,11 @@

{personScoped ? 'Attachments' : 'Files'}

- {totalCount.toLocaleString()} {personPresentation === 'media' && personScoped ? 'media items' : 'files'} + {#if !loading && !error && !unavailable} + {totalCount.toLocaleString()} {personPresentation === 'media' && personScoped ? 'media items' : 'files'} + {/if}
@@ -864,7 +866,10 @@ {:else if error && rows.length === 0}
- +
{:else if loading && rows.length === 0} diff --git a/web/src/lib/components/files/FilesWorkspace.test.ts b/web/src/lib/components/files/FilesWorkspace.test.ts index 2cf86eb11..f06a161d3 100644 --- a/web/src/lib/components/files/FilesWorkspace.test.ts +++ b/web/src/lib/components/files/FilesWorkspace.test.ts @@ -88,6 +88,27 @@ describe('FilesWorkspace', () => { const grid = await screen.findByRole('grid', { name: 'Files results' }); await screen.findByText(rendered); expect(grid.getAttribute('aria-rowcount')).toBe(expected); + expect(screen.queryByText('0 files') !== null).toBe(expected !== null); + }); + + it('shows query failure guidance and reloads files after retry', async () => { + let failed = true; + const fetchFn: typeof fetch = async () => failed + ? Response.json({ + error: 'query_resource_exhausted', + message: 'This query needs more memory or temporary disk space.', + }, { status: 503 }) + : Response.json(response()); + render(FilesWorkspace, { + client: createAPIClient(fetchFn), predicate: { filters: [], presentation: 'table' }, + sort: { field: 'occurred_at', direction: 'desc' } + }); + + expect((await screen.findByRole('alert')).textContent).toContain('more memory or temporary disk space'); + failed = false; + await fireEvent.click(screen.getByRole('button', { name: 'Retry request' })); + expect(await screen.findByText('fixture.pdf')).not.toBeNull(); + expect(screen.queryByRole('alert')).toBeNull(); }); it('owns headers and virtual rows in one focusable grid', async () => { diff --git a/web/tests/theme-keyboard.spec.ts b/web/tests/theme-keyboard.spec.ts index 37c02eaf2..e15b3f3d8 100644 --- a/web/tests/theme-keyboard.spec.ts +++ b/web/tests/theme-keyboard.spec.ts @@ -80,6 +80,28 @@ test('compact workspace links preserve browser navigation and reopen the selecte await expect(page.getByText('synthetic.pdf', { exact: true })).toBeVisible(); }); +test('query failures explain recovery in Everything and Files', async ({ page }) => { + const failure = { + error: 'query_resource_exhausted', + message: 'This query ran out of memory or temporary disk space. Try narrowing the results with filters. ' + + 'The person running msgvault can check available resources and increase analytics.query_memory_limit ' + + 'or analytics.query_temp_limit in config.toml, then restart the server.', + }; + await page.route('**/api/v1/explore', (route) => route.fulfill({ status: 503, json: failure })); + const failFiles = (route: import('@playwright/test').Route) => route.fulfill({ status: 503, json: failure }); + await page.route('**/api/v1/files/search', failFiles); + await page.reload(); + await expect(page.getByRole('alert')).toContainText('analytics.query_memory_limit'); + await expect(page.getByRole('alert')).toContainText('restart the server'); + + await selectKitTopBarTab(page, 'Files'); + await expect(page.getByRole('alert')).toContainText('analytics.query_temp_limit'); + await expect(page.getByText('0 files', { exact: true })).toHaveCount(0); + await page.unroute('**/api/v1/files/search', failFiles); + await page.getByRole('button', { name: 'Retry request' }).click(); + await expect(page.getByText('synthetic.pdf', { exact: true })).toBeVisible(); +}); + test('one registry drives selection, searchable help, palette, and editable suspension', async ({ page }) => { const grid = page.getByRole('grid', { name: 'Everything results' }); const renderedRow = page.locator('[data-row-key="message:1"]'); From 70a56c433e26cc748faec4929af37af394878082 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:28:04 -0700 Subject: [PATCH 007/125] fix(ci): install SQLite headers for browser test builds (#873) Install `libsqlite3-dev` in the `frontend` and `web-e2e` Playwright containers so their daemon builds can compile the default `sqlite_vec` dependency. Both jobs currently fail with `sqlite3.h: No such file or directory` before the browser tests run. The PR dispatcher loads this reusable workflow from `main`, so the dependency must land separately before it can unblock the browser checks on #841. The change adds the package to the two existing apt install lists. Refs #841 Co-authored-by: Rusty Shackleford --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f51e7395..823d192ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: rm -f /etc/apt/sources.list.d/nodesource.list for attempt in 1 2 3; do if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ - && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then + && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip libsqlite3-dev; then exit 0 fi echo "apt attempt $attempt failed; retrying" >&2 @@ -242,7 +242,7 @@ jobs: rm -f /etc/apt/sources.list.d/nodesource.list for attempt in 1 2 3; do if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ - && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then + && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip libsqlite3-dev; then exit 0 fi echo "apt attempt $attempt failed; retrying" >&2 From 0748525f762b56312dd66655047bfee513030d1d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 21 Sep 2026 21:50:06 -0500 Subject: [PATCH 008/125] feat(eval): measure search quality against relevance ratings (#879) Adds `msgvault eval` to measure whether search changes help users find relevant messages. It compares keyword (`fts`), semantic (`vector`), and combined (`hybrid`) search against queries and relevance ratings you provide. ```sh msgvault eval --qrels qrels.txt --topics topics.tsv --modes fts,vector,hybrid -n 100 ``` `topics.tsv` contains query IDs and search text, separated by tabs. `qrels.txt` uses `query_id 0 document_id grade`; grades of 1 or higher mean relevant. - Reports relevant results found, their ranking, and query timings as a table or `--json`. - Records model, search settings, and archive/index sizes for comparison. - Scores messages or conversations with `--doc-key`, counting each conversation once. - Reports skipped queries, missing ratings, and incomplete rankings; rejects conflicting ratings and ambiguous IDs. Vector modes require a local SQLite archive, a configured embedding provider, and an existing index. Compare runs using the same query file. Supersedes #649 because maintainers cannot update its fork branch. Preserves Frederic Masi's authorship and rebases the contribution onto main. Follow-up to #367. Co-authored-by: Wes McKinney --- cmd/msgvault/cmd/eval.go | 1499 +++++++++++++++++ cmd/msgvault/cmd/eval_fts_production_test.go | 259 +++ cmd/msgvault/cmd/eval_key_test.go | 91 + cmd/msgvault/cmd/eval_labels_test.go | 86 + cmd/msgvault/cmd/eval_modes_test.go | 112 ++ cmd/msgvault/cmd/eval_provenance_test.go | 200 +++ cmd/msgvault/cmd/eval_qrels_test.go | 125 ++ cmd/msgvault/cmd/eval_rank_test.go | 411 +++++ cmd/msgvault/cmd/eval_source_scope_test.go | 217 +++ cmd/msgvault/cmd/eval_stub.go | 25 + cmd/msgvault/cmd/eval_topics_test.go | 193 +++ cmd/msgvault/cmd/eval_vector_config_test.go | 133 ++ cmd/msgvault/cmd/eval_vector_corpus_test.go | 237 +++ cmd/msgvault/cmd/serve_vector.go | 82 +- .../cmd/serve_vector_query_client_test.go | 134 ++ cmd/msgvault/cmd/store_resolver.go | 10 +- internal/eval/corpus.go | 227 +++ internal/eval/eval_test.go | 389 +++++ internal/eval/fixture_test.go | 255 +++ internal/eval/metrics.go | 224 +++ internal/eval/rank.go | 124 ++ internal/eval/rank_test.go | 87 + internal/eval/report.go | 145 ++ internal/eval/report_test.go | 108 ++ internal/eval/testdata/threaded/mailbox.json | 129 ++ .../testdata/threaded/qrels_conversation.txt | 8 + .../eval/testdata/threaded/qrels_message.txt | 13 + internal/eval/testdata/threaded/topics.tsv | 3 + internal/query/sqlite.go | 65 +- internal/query/sqlite_crud_test.go | 57 + internal/store/api.go | 60 +- internal/store/api_batch_chunk_test.go | 79 + 32 files changed, 5735 insertions(+), 52 deletions(-) create mode 100644 cmd/msgvault/cmd/eval.go create mode 100644 cmd/msgvault/cmd/eval_fts_production_test.go create mode 100644 cmd/msgvault/cmd/eval_key_test.go create mode 100644 cmd/msgvault/cmd/eval_labels_test.go create mode 100644 cmd/msgvault/cmd/eval_modes_test.go create mode 100644 cmd/msgvault/cmd/eval_provenance_test.go create mode 100644 cmd/msgvault/cmd/eval_qrels_test.go create mode 100644 cmd/msgvault/cmd/eval_rank_test.go create mode 100644 cmd/msgvault/cmd/eval_source_scope_test.go create mode 100644 cmd/msgvault/cmd/eval_stub.go create mode 100644 cmd/msgvault/cmd/eval_topics_test.go create mode 100644 cmd/msgvault/cmd/eval_vector_config_test.go create mode 100644 cmd/msgvault/cmd/eval_vector_corpus_test.go create mode 100644 cmd/msgvault/cmd/serve_vector_query_client_test.go create mode 100644 internal/eval/corpus.go create mode 100644 internal/eval/eval_test.go create mode 100644 internal/eval/fixture_test.go create mode 100644 internal/eval/metrics.go create mode 100644 internal/eval/rank.go create mode 100644 internal/eval/rank_test.go create mode 100644 internal/eval/report.go create mode 100644 internal/eval/report_test.go create mode 100644 internal/eval/testdata/threaded/mailbox.json create mode 100644 internal/eval/testdata/threaded/qrels_conversation.txt create mode 100644 internal/eval/testdata/threaded/qrels_message.txt create mode 100644 internal/eval/testdata/threaded/topics.tsv create mode 100644 internal/store/api_batch_chunk_test.go diff --git a/cmd/msgvault/cmd/eval.go b/cmd/msgvault/cmd/eval.go new file mode 100644 index 000000000..3fd9a7f3e --- /dev/null +++ b/cmd/msgvault/cmd/eval.go @@ -0,0 +1,1499 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/providercredentials" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/hybrid" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +var ( + evalQrels string + evalTopics string + evalModes string + evalDocKey string + evalLimit int + evalJSON bool +) + +var evalCmd = &cobra.Command{ + Use: "eval", + Short: "Evaluate retrieval quality against relevance judgments (qrels)", + Long: `Measure retrieval quality over a set of labeled queries. + +Runs each topic through one or more search modes (fts, vector, hybrid) against +the local archive and scores the ranking against relevance judgments using +standard IR metrics: precision@10, nDCG@10, recall@100, MAP and MRR. This makes +the effect of an indexing, embedding, or fusion change measurable rather than +guessed. MAP and MRR are reported as MAP@n / MRR@n: they take no cutoff of +their own, but the ranking they score stops at -n, so a relevant message below +that rank is as invisible to them as it is to recall. + +Inputs (TREC-style): + --qrels judgments file, one per line: " " + (rel >= 1 means relevant; the iter column is ignored) + --topics queries file, tab-separated: "\t[\t]" + Each qid must appear once — it is the join key to --qrels, so a + repeat would score the same judgments twice and weight that query + twice over in every average; the file is rejected instead. + The optional third column labels the question's shape (e.g. + "pointed" for answerable-from-one-message, "spanning" for + requires-synthesizing-across-messages); when present, results + are also broken down per category. Two-column files work as-is. + +Doc ids in --qrels are matched against each message's source_message_id by +default (--doc-key=message), or its conversation's source_conversation_id +(--doc-key=conversation). Pick the one your judgments actually reference: an +mbox import, for example, keys each imported document by conversation, so +message-keyed qrels would score a flat zero against it. When the judged unit +is the conversation, a thread is counted once, at its best-ranked message — +and retrieval over-fetches messages so that -n really does yield up to n +distinct threads. Reported latency therefore includes that over-fetch. + +Both ids are assigned by the source and are unique only within it, while a +qrels doc id records no source at all. A run therefore stops before scoring +anything if the archive holds an id shared by two connected accounts: merging +two accounts' documents under one key would let an unjudged account's message +inherit a judged one's relevance. Accounts with disjoint id spaces (a mailbox +and a chat archive, say) score normally. + +Metric depths follow -n: the standard P@10 / nDCG@10 / R@100 are reported when +the run retrieves at least that deep, and are clamped to -n below it (a run +that only ever looks 20 deep has no recall@100, and labelling one would invite +a false comparison). MAP and MRR are always at -n, since the truncated ranking +is the whole of what they see. The column headers always name the depth +actually used. + +Each mode runs the same code production search runs. fts is the +relevance-ranked (BM25, subject-weighted) store path behind +/api/v1/search?mode=fts — not a chronological listing — so it honours the same +deletion scope and the same substring address-filter semantics real searches +do; vector and hybrid go through the hybrid engine with the fusion parameters +from your config. Query embedding follows [vector.embeddings] api_format, so an +index built through the Voyage contextual endpoint is queried through it too — +comparing "voyage-context-4" against an OpenAI-compatible model is a matter of +pointing the command at each config in turn. + +Every run reports the embedding model, api format, index settings and index +size that produced it, plus per-query latency, because a quality number is not +comparable — or even interpretable — without them. Those numbers describe what +was searched, not what merely exists: the corpus count covers live messages +only (dedup-hidden duplicates and messages deleted from their source account +are excluded, as they are from every search here), and the vector count covers +the active generation only, not the retired ones vectors.db still holds. That +generation is resolved once at the start of the run; each query still searches +whatever is active at query time, so a rebuild or activation you trigger while +an eval is in flight can leave the reported vector count stale for topics +scored after the swap. +Anything that went wrong without being fatal — unparseable judgment lines, +topics whose query string did not parse or parsed to no search criteria at +all, hits that could not be hydrated from the archive, rankings cut short by +the fusion pool, topics no mode could score — is reported under "Diagnostics" +rather than silently folded into the scores. So is partial qrels coverage: a +topic the judgments never mention cannot be scored, and the diagnostics say +how many of the topics file that leaves the headline numbers standing on. + +Note on topic phrasing: it is an experimental variable, not a constant. FTS5 +matches on AND semantics, so a verbose natural-language topic requires every +one of its words to appear in a message and will usually score near zero, +while its keyword reduction scores well. Dense retrieval can move the other +way. Compare runs only across the same topics file. + +Example: + msgvault eval --qrels qrels.txt --topics topics.tsv --modes fts,vector,hybrid -n 100`, + Args: cobra.NoArgs, + RunE: runEval, +} + +func init() { + rootCmd.AddCommand(evalCmd) + evalCmd.Flags().StringVar(&evalQrels, "qrels", "", "Path to TREC-format relevance judgments (required)") + evalCmd.Flags().StringVar(&evalTopics, "topics", "", "Path to topics TSV: \\t (required)") + evalCmd.Flags().StringVar(&evalModes, "modes", "fts,vector,hybrid", "Comma-separated search modes to evaluate") + // The registry's key set is fixed even though its entries are built per + // run, so rendering the usage string from a throwaway registry is safe. + evalCmd.Flags().StringVar(&evalDocKey, "doc-key", "message", "Which id qrels reference: "+docKeyNames(newDocKeyRegistry())) + evalCmd.Flags().IntVarP(&evalLimit, "limit", "n", 100, "Distinct documents retrieved per query") + evalCmd.Flags().BoolVar(&evalJSON, flagJSON, false, "Output as JSON") + _ = evalCmd.MarkFlagRequired("qrels") + _ = evalCmd.MarkFlagRequired("topics") +} + +// errNoFreeText marks a topic that vector and hybrid modes structurally cannot +// answer: it parsed to filters only, so there is nothing to embed. It is a +// property of one topic, not of the run, so it is reported per cell instead of +// aborting and discarding every score computed so far. +var errNoFreeText = errors.New("topic has no free-text terms to embed") + +// evalHit is the doc-key-relevant projection of one retrieved message. +// +// The retrieval paths disagree about what a hit is: the store's +// relevance-ranked FTS path returns store.APIMessage, while the vector path +// hydrates hybrid hits into query.MessageSummary. A --doc-key has to mean the +// same thing whichever engine produced the hit, so both paths project into +// this one struct and the doc-key registry is defined over it alone. +type evalHit struct { + // MessageID is the archive's own row id. No registered doc-key uses it + // yet; it is carried because the judged-unit extension the registry + // documents (a reconstructed-thread id resolved through an external + // mapping) resolves from an id, not from text. + MessageID int64 + SourceMessageID string + SourceConversationID string +} + +// hitFromAPIMessage projects a store-path (relevance-ranked FTS) result. +func hitFromAPIMessage(m store.APIMessage) evalHit { + return evalHit{ + MessageID: m.ID, + SourceMessageID: m.SourceMessageID, + SourceConversationID: m.SourceConversationID, + } +} + +// hitFromSummary projects a hydrated vector/hybrid result. +func hitFromSummary(m query.MessageSummary) evalHit { + return evalHit{ + MessageID: m.ID, + SourceMessageID: m.SourceMessageID, + SourceConversationID: m.SourceConversationID, + } +} + +// evalIDColumn locates, in the archive, the column a doc-key's ids are read +// from. It exists so the cross-source collision check +// (requireDisjointSourceIDs) can be written once, over any key, without having +// to know what a particular key means. +type evalIDColumn struct { + // table is the alias the collision query gives the id's table: "m" for + // messages, "c" for the conversations reached through join. + table string + // column is the unqualified column name, and doubles as the name the + // error text shows a user. + column string + // join is the extra FROM clause needed to reach table, empty when the id + // lives on messages itself. + join string +} + +// expr renders the qualified column for use in the collision query. +func (c evalIDColumn) expr() string { return c.table + "." + c.column } + +// docKeySpec describes one --doc-key value. +type docKeySpec struct { + // extract pulls, from a retrieved hit, the stable document id that qrels + // judge against. It is the only place a doc-key's meaning lives: the + // scoring core (eval.Evaluate, eval.Aggregate, eval.DedupeKeys) operates + // on the opaque string keys these return and never learns what they + // identify. + extract func(evalHit) string + // collapses is true when the judged unit is coarser than the retrieved + // one, so several hits routinely fold into a single key. Retrieval then + // has to over-fetch messages to fill the requested depth with *distinct* + // keys — see eval.OverFetchPlan and evaluator.rankedKeys. + collapses bool + // idColumn says where in the archive extract's id comes from, so the run + // can check up front that no id in it names documents in two different + // connected sources — see requireDisjointSourceIDs, which refuses to run a + // key that leaves this unset rather than skipping the check. + idColumn evalIDColumn +} + +// newDocKeyRegistry builds the --doc-key registry for one run. +// +// It is a constructor rather than a package-level map so the registry is built +// after flags are parsed. That is what makes the extension story real: a +// future --doc-key=thread, resolving a reconstructed-thread id through an +// externally supplied message-id -> thread-id mapping file, is one more entry +// here, closing over the mapping this function loaded. A map initialised at +// program start could not hold that entry — the mapping file is named by a +// flag, and its contents are unknown until runEval runs. The CLI validation, +// the scoring core and the output paths all pick a new entry up unchanged. +func newDocKeyRegistry() map[string]docKeySpec { + return map[string]docKeySpec{ + // One message, one source_message_id. Duplicates are still collapsed + // (the same message synced from two accounts), but they are rare + // enough that the depth does not need padding for them. + "message": { + extract: func(h evalHit) string { return h.SourceMessageID }, + idColumn: evalIDColumn{table: "m", column: "source_message_id"}, + }, + // Many messages share one conversation, so filling n distinct threads + // takes more than n messages. + "conversation": { + extract: func(h evalHit) string { return h.SourceConversationID }, + collapses: true, + idColumn: evalIDColumn{ + table: "c", + column: "source_conversation_id", + join: "JOIN conversations c ON c.id = m.conversation_id", + }, + }, + } +} + +// docKeyNames renders the valid --doc-key values for usage and error text. +func docKeyNames(registry map[string]docKeySpec) string { + names := make([]string, 0, len(registry)) + for n := range registry { + names = append(names, n) + } + sort.Strings(names) + return strings.Join(names, "|") +} + +// runDiagnostics collects the non-fatal anomalies of a run. Each of these was +// previously either silent or fatal, and both are wrong for an instrument: a +// stale index that quietly drops hits is precisely the failure this command +// exists to expose, and a single unanswerable topic should not throw away +// every other topic's score. +type runDiagnostics struct { + QrelsLoad eval.LoadStats `json:"qrels_load"` + TopicsLoad eval.LoadStats `json:"topics_load"` + UnhydratedHits int `json:"unhydrated_hits,omitempty"` + // DepthShortfalls counts runs that ran out of over-fetch budget with the + // engine still willing to give more. PoolShortfalls counts runs that hit + // the hybrid engine's candidate-pool ceiling instead — the engine came + // back short, but more matching messages exist beyond the pool. The two + // are kept apart because only one of them is fixable by configuration, + // and neither may be confused with "the corpus genuinely ran out", which + // is not a shortfall at all. + DepthShortfalls int `json:"depth_shortfalls,omitempty"` + PoolShortfalls int `json:"pool_shortfalls,omitempty"` + SkippedCells []string `json:"skipped_cells,omitempty"` + // UnjudgedTopics names the topics the qrels file says nothing about. + // Leaving them out of the scoring is correct — an unjudged topic has + // nothing to be scored against — but leaving it *unsaid* is not: as soon + // as one topic is judged the run reports a headline number, and a qrels + // file that matches only a handful of a large topics file therefore + // reports it over a small, self-selected subset while looking like a + // complete run. Naming them makes the coverage of a run readable from its + // own output. + UnjudgedTopics []string `json:"unjudged_topics,omitempty"` + + // kPerSignal is the fusion pool size in force for this run, used only to + // make the PoolShortfalls note actionable. Unexported so it stays out of + // the JSON diagnostics block, where it would duplicate run_config. + kPerSignal int + + // scored is how many topics actually contributed a score, set once after + // the scoring loop finishes. It is not len(TopicsLoad)-len(UnjudgedTopics): + // a judged topic can still fail to score (an empty parsed query, or every + // mode skipping it), so the unjudged-topics note must read this field + // rather than recompute a count the loop already produced correctly — + // the same number the report's own topics_evaluated is built from. + // Unexported for the same reason as kPerSignal: it would duplicate + // topics_evaluated in the JSON diagnostics block. + scored int +} + +// skip records that one topic/mode combination could not be scored. +func (d *runDiagnostics) skip(topicID, mode, reason string) { + d.SkippedCells = append(d.SkippedCells, fmt.Sprintf("topic %s / %s: %s", topicID, mode, reason)) +} + +// skipTopic records that a topic could not be scored by any mode. Unlike skip, +// this is a property of the topic itself (a query string no mode can run), so +// it is reported once rather than once per mode. +func (d *runDiagnostics) skipTopic(topicID, reason string) { + d.SkippedCells = append(d.SkippedCells, fmt.Sprintf("topic %s: %s", topicID, reason)) +} + +// unjudged records a topic this qrels file never mentions. It is not a skip: +// nothing went wrong with the topic, there is simply nothing to score it +// against. It is tracked so the run can report how much of the topics file its +// headline numbers actually cover. +func (d *runDiagnostics) unjudged(topicID string) { + d.UnjudgedTopics = append(d.UnjudgedTopics, topicID) +} + +// notes renders the diagnostics as human-readable lines, empty when the run +// was clean. +func (d *runDiagnostics) notes() []string { + var out []string + for _, l := range []struct { + kind string + stats eval.LoadStats + }{{"qrels", d.QrelsLoad}, {"topics", d.TopicsLoad}} { + if l.stats.Skipped > 0 { + out = append(out, fmt.Sprintf("%s %s: %s — skipped lines did not match the expected format", + l.kind, l.stats.Path, l.stats)) + } + } + // Partial coverage is not an error — a topics file is often larger than + // the judgments gathered for it so far — but it changes what the headline + // numbers mean, so it is stated rather than inferred from the topic count. + if n := len(d.UnjudgedTopics); n > 0 { + // d.scored, not d.TopicsLoad.Parsed-n: a judged topic can still fail + // to score (an empty parsed query, every mode skipping it), so the + // remainder after subtracting only the unjudged ones overstates what + // the run actually covers — see the scored field's own doc comment. + out = append(out, fmt.Sprintf( + "%d of %d topics had no matching qrels entry and were not scored (%s); "+ + "the reported metrics cover %d of the topics file, so they describe a subset of %s — "+ + "check that the qids in both files refer to the same queries", + n, d.TopicsLoad.Parsed, eval.FormatIDList(d.UnjudgedTopics, 10), + d.scored, d.TopicsLoad.Path)) + } + if d.UnhydratedHits > 0 { + out = append(out, fmt.Sprintf( + "%d retrieved hits could not be hydrated back to a message row and were dropped from the ranking; "+ + "this usually means the vector index references deleted or unmigrated messages (re-run `msgvault embed`)", + d.UnhydratedHits)) + } + if d.DepthShortfalls > 0 { + out = append(out, fmt.Sprintf( + "%d topic/mode runs could not fill %d distinct %s keys within the over-fetch budget (%dx -n); "+ + "their metrics are computed over a shallower list than requested", + d.DepthShortfalls, evalLimit, evalDocKey, eval.MaxOverFetchFactor)) + } + if d.PoolShortfalls > 0 { + out = append(out, fmt.Sprintf( + "%d topic/mode runs stopped short of %d distinct %s keys because the fusion candidate pool "+ + "saturated%s: the engine returned fewer hits than asked for, but more matching messages "+ + "exist beyond the pool — this is a reachability limit, not an exhausted corpus. "+ + "Raise [vector.search].k_per_signal to rank deeper, and note that doing so changes the "+ + "fusion, so only compare runs at the same setting", + d.PoolShortfalls, evalLimit, evalDocKey, kPerSignalSuffix(d.kPerSignal))) + } + out = append(out, d.SkippedCells...) + return out +} + +// kPerSignalSuffix renders the fusion pool size for a diagnostic line, or +// nothing when the run never opened the vector path (so the number is unknown +// rather than zero). +func kPerSignalSuffix(k int) string { + if k <= 0 { + return "" + } + return fmt.Sprintf(" at k_per_signal=%d", k) +} + +// ftsSearcher is the production relevance-ranked full-text path. +// +// It is deliberately the Store's search, not query.Engine.Search: those are +// two different searches. query.Engine.Search returns matches in reverse +// chronological order with no relevance component at all, so scoring it as a +// *ranking* measures the archive's date distribution rather than its retrieval +// quality. Store.SearchMessagesQueryContext is the path /api/v1/search?mode=fts +// serves, ordering by the dialect's BM25 expression (subject-weighted) before +// falling back to recency — the same messages_fts index and the same weights +// the hybrid engine's BM25 leg fuses. It also matches production on the two +// semantics that silently move scores: it honours search.DeletionScope +// (active-only by default, so source-deleted messages are excluded), and its +// from:/to:/cc: filters are substring matches rather than exact-address +// equality. +// +// *store.Store satisfies this; the interface exists so tests can drive the +// over-fetch loop without a database. +type ftsSearcher interface { + SearchMessagesQueryContext( + ctx context.Context, q *search.Query, offset, limit int, + ) ([]store.APIMessage, int64, error) +} + +// evaluator bundles the engines and config needed to turn a query string into +// a ranked list of document ids for a given search mode. +type evaluator struct { + ctx context.Context + fts ftsSearcher + qeng query.Engine + heng *hybrid.Engine + key docKeySpec + limit int + prov eval.RunConfig + diag *runDiagnostics +} + +// fetchResult is one attempt at pulling raw hits out of a search engine. +type fetchResult struct { + // keys are the doc keys in the engine's rank order, before collapsing; + // duplicates and empty strings are expected and handled by the caller. + keys []string + // raw is how many hits the engine returned. It is the count *before* key + // extraction, so "the engine gave back fewer than we asked for" — the + // signal that a deeper fetch cannot help — stays accurate even when some + // hits fail to hydrate. + raw int + // dropped is how many hits could not be hydrated back to a message row. + dropped int + // saturated reports that the engine filled its own candidate pool: it + // had at least one more candidate than it was willing to consider. It is + // what separates "the corpus ran out" from "the engine stopped looking", + // which look identical from the hit count alone. The hybrid engine's + // fused query caps each signal at k_per_signal, so it can hand back fewer + // hits than requested while the corpus still holds plenty more — see + // hybrid.ResultMeta.PoolSaturated. + saturated bool +} + +// rankedKeys turns a search engine into up to limit *distinct* doc keys, in +// ranked order. +// +// The collapse must happen before the truncation, not after: retrieving n +// messages and then collapsing them yields however many distinct threads +// happen to sit inside those n, which is not what "-n 100" claims. So for a +// collapsing doc-key this over-fetches raw hits (eval.OverFetchPlan), collapses, +// and only then cuts to the requested depth, growing the pool while the engine +// still has more to give and the depth is still unfilled. +func (e *evaluator) rankedKeys(fetch func(n int) (fetchResult, error)) ([]string, error) { + plan := eval.OverFetchPlan(e.limit, e.key.collapses) + for i, n := range plan { + res, err := fetch(n) + if err != nil { + return nil, err + } + deduped := eval.DedupeKeys(res.keys) + filled := len(deduped) >= e.limit + // The engine came back short. Why it came back short decides both + // whether to retry and what to report, and the hit count alone cannot + // tell the two apart: + // + // - not saturated: it gave everything it had. The corpus is + // exhausted, a deeper fetch returns the same list, and a short + // ranking is the honest answer — not a shortfall. + // - saturated: it filled its own candidate pool and stopped. More + // matching messages exist, but no value of n reaches them, + // because the ceiling is k_per_signal, not the page size. Retrying + // deeper would only burn queries, so stop — and say so, because + // scoring this as an exhausted corpus reports a shallow ranking + // as if it were the whole of what retrieval could find. + short := res.raw < n + exhausted := short && !res.saturated + poolCapped := short && res.saturated + if filled || exhausted || poolCapped || i == len(plan)-1 { + e.diag.UnhydratedHits += res.dropped + switch { + case filled || exhausted: + // Nothing to report: the depth was met, or there was + // genuinely nothing more to retrieve. + case poolCapped: + e.diag.PoolShortfalls++ + default: + e.diag.DepthShortfalls++ + } + return eval.TruncateKeys(deduped, e.limit), nil + } + } + // Unreachable: OverFetchPlan is never empty and the loop always returns on + // its final step. + return nil, nil +} + +// rankedFTS scores the production relevance-ranked FTS path. See ftsSearcher +// for why that is the Store's search and not query.Engine.Search. +func (e *evaluator) rankedFTS(q *search.Query) ([]string, error) { + return e.rankedKeys(func(n int) (fetchResult, error) { + res, _, err := e.fts.SearchMessagesQueryContext(e.ctx, q, 0, n) + if err != nil { + return fetchResult{}, err + } + keys := make([]string, 0, len(res)) + for _, m := range res { + keys = append(keys, e.key.extract(hitFromAPIMessage(m))) + } + // The store path pages a single ranked list, so a short page means + // the corpus ran out — there is no candidate pool to saturate. + return fetchResult{keys: keys, raw: len(res)}, nil + }) +} + +func (e *evaluator) rankedVector(mode, qstr string, q *search.Query) ([]string, error) { + // Both modes embed the free text, so a filter-only topic (`from:alice`) + // has nothing to embed and hybrid.Engine.Search would return a bare + // "empty query". Detect it here and hand runEval a recognisable error so + // it can skip this one cell instead of aborting the run. + if len(q.TextTerms) == 0 { + return nil, fmt.Errorf("%w: %q parsed to filters only", errNoFreeText, qstr) + } + // Use the engine method rather than the package function: it supplies the + // dialect's placeholder rebind, which the package function now requires. + filter, err := e.heng.BuildFilter(e.ctx, q) + if err != nil { + return nil, fmt.Errorf("build filter: %w", err) + } + subjectTerms := make([]string, 0, len(q.TextTerms)) + for _, t := range q.TextTerms { + subjectTerms = append(subjectTerms, strings.ToLower(t)) + } + freeText := strings.Join(q.TextTerms, " ") + + return e.rankedKeys(func(n int) (fetchResult, error) { + hits, meta, err := e.heng.Search(e.ctx, hybrid.SearchRequest{ + Mode: hybrid.Mode(mode), + FreeText: freeText, + Filter: filter, + Limit: n, + SubjectTerms: subjectTerms, + }) + if err != nil { + return fetchResult{}, err + } + if len(hits) == 0 { + return fetchResult{saturated: meta.PoolSaturated}, nil + } + ids := make([]int64, len(hits)) + for i, h := range hits { + ids[i] = h.MessageID + } + summaries, err := e.qeng.GetMessageSummariesByIDs(e.ctx, ids) + if err != nil { + return fetchResult{}, fmt.Errorf("map message ids: %w", err) + } + byID := make(map[int64]query.MessageSummary, len(summaries)) + for _, m := range summaries { + byID[m.ID] = m + } + // Preserve the engine's ranking order. A hit that cannot be hydrated + // is dropped — but counted, because a vector index pointing at rows + // the archive no longer has is exactly the staleness this command + // exists to surface. + out := fetchResult{ + keys: make([]string, 0, len(hits)), + raw: len(hits), + // Carry the engine's own account of why it stopped. Without + // it, a fused query that ran out of candidate pool is + // indistinguishable from one that ran out of corpus, and + // rankedKeys would report a pool-capped ranking as complete. + saturated: meta.PoolSaturated, + } + for _, h := range hits { + m, ok := byID[h.MessageID] + if !ok { + out.dropped++ + continue + } + out.keys = append(out.keys, e.key.extract(hitFromSummary(m))) + } + return out, nil + }) +} + +// ranked runs one topic through one mode. The topic is parsed once by the +// caller and handed in already validated: re-parsing per mode would let the +// same malformed filter be dropped silently three times over. +func (e *evaluator) ranked(mode, qstr string, q *search.Query) ([]string, error) { + switch mode { + case "fts": + return e.rankedFTS(q) + case "vector", "hybrid": + return e.rankedVector(mode, qstr, q) + default: + return nil, fmt.Errorf("unknown mode %q (want fts|vector|hybrid)", mode) + } +} + +// parseTopic turns one topic's query string into a validated search.Query. +// +// search.Parse never fails outright: an operator it recognises but cannot +// read — `before:invalid`, `larger:5X` — is recorded on the query and the +// filter is simply dropped, leaving a *wider* query behind. That is a +// reasonable default for an interactive search box, where the user sees the +// results and can correct the typo, but it is silent corruption for a +// benchmark: the topic still scores, against a question nobody asked. The +// production front doors (the CLI search command, /api/v1/search, +// /cli/search) all reject such a query via Query.Err(); this one skips the +// topic and says why, so one malformed line cannot quietly move a run's +// headline numbers. +// +// Parsing cleanly is not enough, though: a topic can be non-empty text and +// still parse to no search criteria at all. `subject:""` is the plain case — +// the parser drops an empty operator value rather than building a `LIKE '%%'` +// that matches everything — and the widest possible query is what is left +// behind. That is the same corruption one step further along, and it is worse, +// because the fts path answers an empty query by listing the whole live corpus +// in its default order: the topic scores whatever the archive's date +// distribution happens to give it. Production rejects the identical query +// (cmd/search.go and the /cli/search handler both test Query.IsEmpty), so it +// is skipped and reported here too. +func parseTopic(t eval.Topic, diag *runDiagnostics) (*search.Query, bool) { + q := search.Parse(t.Query) + if err := q.Err(); err != nil { + diag.skipTopic(t.ID, fmt.Sprintf("query %q did not parse: %v — scoring it would have "+ + "silently evaluated the broader query left after the bad filter was dropped", t.Query, err)) + return nil, false + } + if q.IsEmpty() { + diag.skipTopic(t.ID, fmt.Sprintf("query %q parsed to no search criteria at all — scoring it "+ + "would have ranked the whole live corpus in its default order rather than a retrieval "+ + "of this topic; production search rejects the same query as empty", t.Query)) + return nil, false + } + return q, true +} + +func runEval(cmd *cobra.Command, _ []string) error { + registry := newDocKeyRegistry() + keySpec, ok := registry[evalDocKey] + if !ok { + return usageErr(cmd, fmt.Errorf("invalid --doc-key %q (want %s)", evalDocKey, docKeyNames(registry))) + } + // A non-positive depth is not a "use the default" signal: fts would fall + // back to an internal 100 while the vector backend would return nothing + // for k=0, so the same flag would mean two different things. Reject it. + if evalLimit <= 0 { + return usageErr(cmd, fmt.Errorf("--limit must be a positive integer, got %d", evalLimit)) + } + modes, needVec, err := parseEvalModes(evalModes) + if err != nil { + return usageErr(cmd, err) + } + cutoffs := eval.CutoffsForDepth(evalLimit) + + diag := &runDiagnostics{} + qrels, qrelsStats, err := eval.LoadQrels(evalQrels) + if err != nil { + return err + } + diag.QrelsLoad = qrelsStats + topics, topicsStats, err := eval.LoadTopics(evalTopics) + if err != nil { + return err + } + diag.TopicsLoad = topicsStats + + // A file in a near-miss format parses to an empty-but-valid result. Say so + // in terms of the format, because the downstream symptom ("no topics had + // judgments") reads like an id mismatch and sends people looking in the + // wrong place. + if qrelsStats.Parsed == 0 { + return fmt.Errorf("no judgments parsed from %s (%s); expected whitespace-separated "+ + "\" \" — a three-column file without the iteration column is the usual cause", + evalQrels, qrelsStats) + } + if len(topics) == 0 { + return fmt.Errorf("no topics loaded from %s (%s); expected tab-separated "+ + "\"\\t\" — spaces where tabs are expected is the usual cause", + evalTopics, topicsStats) + } + // Warn on stderr so --json output stays machine-readable. + for _, l := range []struct { + kind string + stats eval.LoadStats + }{{"qrels", qrelsStats}, {"topics", topicsStats}} { + if l.stats.Suspect() { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s %s parsed oddly (%s); check the file format\n", + l.kind, l.stats.Path, l.stats) + } + } + + ctx := cmd.Context() + + // Store + query engine: serves FTS search and the rowid -> source-id + // mapping for vector/hybrid hits. Opening it also runs the schema + // migrations the vector backend relies on, and its handle is the one every + // DB read in this command goes through. + s, err := store.Open(cfg.DatabaseDSN()) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer func() { _ = s.Close() }() + // Context-aware forms: this command's Cobra context is already + // cancellable on Ctrl-C, and a long migration or schema init on a large + // archive must actually stop when the user asks it to, not run to + // completion on a background context that can't hear the cancellation. + if err := s.InitSchemaContext(ctx); err != nil { + return fmt.Errorf("init schema: %w", err) + } + if err := runStartupMigrationsContext(ctx, s); err != nil { + return fmt.Errorf("startup migrations: %w", err) + } + if err := requireFTS5ForModes(modes, s.FTS5Available()); err != nil { + return err + } + // Every score this run produces rests on one archive-wide fact about the + // chosen key: that its ids name one document each. Establish it before the + // vector path is opened and before the first topic is scored, so a run that + // cannot be trusted stops instead of printing a number. + if err := requireDisjointSourceIDs(ctx, s.DB(), evalDocKey, keySpec); err != nil { + return err + } + + ev := &evaluator{ + ctx: ctx, + // The store serves --modes fts through the same relevance-ranked + // path /api/v1/search?mode=fts uses; the query engine serves the + // rowid -> source-id hydration the vector/hybrid path needs. + fts: s, + qeng: query.NewEngine(s.DB(), s.IsPostgreSQL()), + key: keySpec, + limit: evalLimit, + diag: diag, + } + + if needVec { + cleanup, err := ev.attachVector(ctx, s) + if err != nil { + return err + } + defer cleanup() + } + + // Record corpus size regardless of mode: recall numbers are unreadable + // without knowing how big the haystack was. + ev.prov.QrelsPath = evalQrels + ev.prov.TopicsPath = evalTopics + ev.collectCorpusStats(s.DB()) + + aggs := make(map[string]*eval.Aggregate, len(modes)) + lats := make(map[string]*eval.LatencyTracker, len(modes)) + for _, m := range modes { + aggs[m] = &eval.Aggregate{} + lats[m] = &eval.LatencyTracker{} + } + // Per-category aggregates (mode -> category), populated only for topics + // that carry a category label. Whether a question is answerable from one + // message or needs a whole thread decides which retrieval levers a run + // can even see, so when the topics file says which is which, report the + // split rather than averaging it away. + catAggs := make(map[string]map[string]*eval.Aggregate, len(modes)) + catCounts := map[string]int{} + scored := 0 + for _, t := range topics { + if !qrels.HasJudgments(t.ID) { + // This qrels file says nothing at all about the topic, so there is + // nothing to score it against. Record it: one judged topic is + // enough to produce a headline number, and a reader has to be able + // to see how much of the topics file that number covers. + diag.unjudged(t.ID) + continue + } + // May legitimately be empty: a topic judged but with every document + // graded non-relevant scores a real zero and belongs in the macro + // average. Dropping it would quietly raise every reported mean — see + // eval.Qrels.HasJudgments. + rel := qrels.RelevantSet(t.ID) + // Parse once, before any mode runs it: a malformed filter is a + // property of the topic, not of the mode, and must not be silently + // widened into a different question three times over. + q, ok := parseTopic(t, diag) + if !ok { + continue + } + anyMode := false + for _, m := range modes { + start := time.Now() + ranked, err := ev.ranked(m, t.Query, q) + elapsed := time.Since(start) + if err != nil { + if errors.Is(err, errNoFreeText) { + // One mode cannot answer this topic. Record it and carry + // on: aborting here would throw away every score already + // computed, for every mode and every earlier topic. + diag.skip(t.ID, m, "no free-text terms to embed (filter-only topic)") + continue + } + return fmt.Errorf("topic %s, mode %s: %w", t.ID, m, err) + } + lats[m].Add(elapsed) + s := eval.Evaluate(ranked, rel, cutoffs) + aggs[m].Add(s) + if t.Category != "" { + if catAggs[m] == nil { + catAggs[m] = map[string]*eval.Aggregate{} + } + if catAggs[m][t.Category] == nil { + catAggs[m][t.Category] = &eval.Aggregate{} + } + catAggs[m][t.Category].Add(s) + } + anyMode = true + } + if !anyMode { + continue // no mode could score this topic + } + if t.Category != "" { + catCounts[t.Category]++ + } + scored++ + } + diag.scored = scored + if scored == 0 { + // Distinguish the two ways a run can end up with nothing: no topic + // matched a judgment, or every topic was skipped by every mode. + if len(diag.SkippedCells) > 0 { + return fmt.Errorf("no topic could be scored by any of the requested modes: %s", + strings.Join(diag.SkippedCells, "; ")) + } + return fmt.Errorf("none of the %d topics had relevance judgments in %s "+ + "(qrels: %s; topics: %s); check that the qids in both files refer to the same queries", + len(topics), evalQrels, qrelsStats, topicsStats) + } + + report := evalReport{ + modes: modes, aggs: aggs, lats: lats, catAggs: catAggs, catCounts: catCounts, + prov: ev.prov, topics: scored, cutoffs: cutoffs, diag: diag, + } + if evalJSON { + return report.json() + } + report.table() + return nil +} + +// parseEvalModes splits and validates the --modes flag, keeping each mode once +// in the order it was first named. +// +// A repeated mode is deduplicated rather than rejected. The scoring loop +// evaluates and aggregates the list entry by entry into a per-mode Aggregate, +// so `--modes fts,fts` would run every topic through fts twice, add each score +// to the same aggregate twice — doubling that mode's topic count while leaving +// its means unchanged — and double the latency work and its sample. Rejecting +// it, as a repeated qid in the topics file is rejected, would be the wrong +// shape here: two topic rows sharing a qid carry different query text, so the +// file is genuinely ambiguous and picking one silently answers a question +// nobody asked, whereas `fts,fts` has exactly one possible reading. There is +// nothing to disambiguate, so it is simply honoured once. +// +// Validation still runs per entry, before the duplicate is dropped, so a +// repeated invalid mode is still an error. Order is preserved because it is the +// order the report's rows come out in, and that belongs to the user. +func parseEvalModes(spec string) (modes []string, needVec bool, err error) { + seen := make(map[string]bool, 3) + for m := range strings.SplitSeq(spec, ",") { + m = strings.TrimSpace(m) + if m == "" { + continue + } + switch m { + case "fts": + case "vector", "hybrid": + needVec = true + default: + return nil, false, fmt.Errorf("invalid mode %q in --modes (want fts|vector|hybrid)", m) + } + if seen[m] { + continue + } + seen[m] = true + modes = append(modes, m) + } + if len(modes) == 0 { + return nil, false, errors.New("--modes is empty") + } + return modes, needVec, nil +} + +// requireFTS5ForModes stops the run before its first topic when --modes fts +// is requested against an archive without FTS5, the same "a run that cannot +// be trusted stops instead of printing a number" precondition +// requireDisjointSourceIDs establishes for --doc-key. +// +// --modes fts scores through Store.SearchMessagesQueryContext (see +// rankedFTS) expecting its production BM25 relevance ranking. A binary +// built with the sqlite_vec tag but not fts5 — or one whose FTS5 shadow +// tables failed to initialize — has that same call silently fall back to a +// LIKE scan ordered by recency instead of erroring, so the run would score a +// date sort while the report still labels the mode "fts" and claims BM25 +// ranking. --modes hybrid needs no equivalent guard: its BM25 leg runs a +// direct messages_fts MATCH inside the vector backend's fused query, which +// errors outright ("no such table: messages_fts") rather than degrading +// silently when FTS5 is unavailable. +// +// This only covers the startup state: searchMessagesQueryNoFTS documents a +// second, narrower fallback inside Store itself, forcing the same LIKE +// branch mid-run if an FTS query that started fine ever errors later +// (shadow-table corruption, an extension fault). Catching that would mean +// either threading a "did this call degrade" signal back through +// Store.SearchMessagesQueryContext — a production API several daemon +// handlers share, not an eval-only seam — or re-probing FTS5Available() +// once per topic, which cannot detect a failure the query itself hasn't hit +// yet either. Both are more machinery than a rare, already-defensive +// mid-run fallback warrants; this guard catches the case that is actually +// common (a build or archive that never had FTS5 to begin with). +func requireFTS5ForModes(modes []string, fts5Available bool) error { + if fts5Available || !slices.Contains(modes, string(hybrid.ModeFTS)) { + return nil + } + return errors.New("--modes fts needs FTS5, but this archive's FTS5 index is unavailable " + + "(binary built without the fts5 tag, or the shadow tables failed to initialize); " + + "rebuild with -tags \"fts5 sqlite_vec\" or drop fts from --modes") +} + +// attachVector wires the sqlite-vec backend and hybrid engine onto the +// evaluator (mirroring the search command's vector path) and returns a +// cleanup closure that closes the resources it opened. +// +// It reuses the caller's store handle rather than opening its own: that handle +// already carries the DSN parameters store.Open applies (busy_timeout, WAL, +// the registered driver's unicode_lower hook), and routing every DB operation +// through the Store is this repo's rule. +func (e *evaluator) attachVector(ctx context.Context, mainStore *store.Store) (func(), error) { + if !cfg.Vector.Enabled { + return nil, errors.New("vector/hybrid modes need [vector].enabled = true in config") + } + mainPath := cfg.DatabaseDSN() + if store.IsPostgresURL(mainPath) { + // This command's vector path is the sqlite-vec one; a PG archive + // stores its embeddings in pgvector, alongside the messages. Fail + // clearly rather than pointing a sqlite-vec backend at a PG handle. + return nil, errors.New("vector/hybrid eval currently supports SQLite archives only; " + + "the configured database is PostgreSQL — run with --modes fts") + } + + // Resolve [vector.embed.scope] accounts to source IDs before deriving the + // build scope or the generation fingerprint, exactly as the serve/embed + // paths do. The fingerprint folds in the scope, so an unresolved config + // would compute a different one and every query would fail as "index + // stale" on any archive that scopes embedding by account. + vecCfg, err := resolvedVectorConfig(mainStore, cfg.Vector) + if err != nil { + return nil, fmt.Errorf("vector embed scope: %w", err) + } + // Validate the resolved config with the same check serve runs before it + // opens anything. It names the offending key and value — including an + // api_format this binary has no client for, which must fail here rather + // than fall back to a client that talks a different protocol to the + // endpoint that built the index. + if err := vecCfg.Validate(); err != nil { + return nil, fmt.Errorf("vector/hybrid modes need a valid [vector] config: %w", err) + } + + // Select the query client by api_format, exactly as the serve path does, + // and before anything is opened. A run scored with the OpenAI-compatible + // client against a voyage-contextual index would measure a protocol + // mismatch, not retrieval quality. Every eval call is query-time, and each + // client's EmbedQuery carries its own query role (Voyage sends + // input_type=query to /contextualizedembeddings), so no document-side + // wiring is needed here. + credentialSnapshot, err := providercredentials.Read(cfg.TokensDir()) + if err != nil { + return nil, fmt.Errorf("load provider credentials: %w", err) + } + embeddingAPIKey, err := resolveProviderCredentialFromSnapshot( + credentialSnapshot, providercredentials.VectorEmbeddingsID, + vecCfg.Embeddings.Endpoint, vecCfg.Embeddings.APIKeyEnv, + ) + if err != nil { + return nil, fmt.Errorf("resolve text embedding credential: %w", err) + } + embedClient, err := newQueryEmbeddingClient(vecCfg, embeddingAPIKey) + if err != nil { + return nil, err + } + mainDB := mainStore.DB() + + vecDBPath := vecCfg.DBPath + if vecDBPath == "" { + vecDBPath = filepath.Join(cfg.Data.DataDir, "vectors.db") + } + if err := sqlitevec.RegisterExtension(); err != nil { + return nil, fmt.Errorf("register sqlite-vec: %w", err) + } + backend, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: vecDBPath, + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + BuildScope: vecCfg.Embed.Scope.BuildScope(), + }) + if err != nil { + return nil, fmt.Errorf("open vectors.db: %w", err) + } + // Keep the resolved generation: it is the one the hybrid engine will + // search, and therefore the only one whose vector count describes this + // run's index. + active, err := vector.ResolveActiveForFingerprint(ctx, backend, vecCfg.GenerationFingerprint()) + if err != nil { + _ = backend.Close() + return nil, fmt.Errorf("resolve active generation: %w", err) + } + + e.heng = hybrid.NewEngine(backend, mainDB, embedClient, hybrid.Config{ + ExpectedFingerprint: vecCfg.GenerationFingerprint(), + RRFK: vecCfg.Search.RRFK, + KPerSignal: vecCfg.Search.KPerSignal, + SubjectBoost: vecCfg.Search.SubjectBoost, + // Without this the engine's index-scope check short-circuits to nil, + // so an out-of-scope filter would run against an index holding no + // vectors for that scope and its near-zero hit count would be scored + // as genuinely poor retrieval instead of erroring. + BuildScope: vecCfg.Embed.Scope.BuildScope(), + }) + e.collectVectorStats(mainDB, vecCfg, backend, vecDBPath, active.ID) + + return func() { _ = backend.Close() }, nil +} + +// collectCorpusStats records how big the searched archive is. It reads the +// store's own handle so it also works for --modes fts, which never opens the +// vector path. Failures are non-fatal: missing provenance should degrade the +// report, never abort a run. +// +// "How big" means how big the haystack retrieval actually searched, not how +// many rows the tables hold. A long-lived archive accumulates dedup-hidden +// duplicates and messages deleted from their source account, and no search +// this command runs returns either: the fts path resolves the default active +// deletion scope to store.LiveMessagesWhere, and the vector path drops +// source-deleted hits after the fact. Counting them would overstate the +// haystack and make recall look harder-won than it was. The predicate is +// borrowed from the store rather than restated here so the two cannot drift. +// +// Conversations are derived from those same live messages for the same +// reason: an emptied conversation row is not a thread retrieval can return, +// and with --doc-key=conversation the thread count is the denominator a +// reader will reach for. +func (e *evaluator) collectCorpusStats(db *sql.DB) { + if db == nil { + return + } + live := store.LiveMessagesWhere("", true) + _ = db.QueryRowContext(e.ctx, + "SELECT COUNT(*) FROM messages WHERE "+live).Scan(&e.prov.Messages) + _ = db.QueryRowContext(e.ctx, + "SELECT COUNT(DISTINCT conversation_id) FROM messages WHERE "+live).Scan(&e.prov.Conversations) +} + +// collidingDocKeyIDs returns doc ids that occur under more than one source in +// the live population, at most limit of them, in id order for stable output. +// +// It counts distinct source_id over the same live messages every search in this +// command draws from, so an id whose only other holder is dedup-hidden or +// deleted from its source is correctly not a collision: neither copy can be +// retrieved, so neither can be scored. +func collidingDocKeyIDs(ctx context.Context, db *sql.DB, col evalIDColumn, limit int) ([]string, error) { + // The id may be NULL (no id assigned) or empty; eval.DedupeKeys drops both + // from a ranking, so neither can collide with anything and both are + // excluded here for the same reason. + expr := col.expr() + q := fmt.Sprintf(` + SELECT %s + FROM messages m + %s + WHERE %s AND %s IS NOT NULL AND %s <> '' + GROUP BY %s + HAVING COUNT(DISTINCT m.source_id) > 1 + ORDER BY %s + LIMIT %d`, + expr, col.join, store.LiveMessagesWhere("m", true), expr, expr, expr, expr, limit) + + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// requireDisjointSourceIDs refuses a run whose archive cannot give the chosen +// --doc-key an unambiguous doc-id space. +// +// A qrels file is flat. " " has nowhere to record +// which connected account belongs to, and neither a TREC-derived +// collection nor judgments written by hand against a personal archive carry +// one. But both ids this command can key on are assigned by the *source* — +// source_message_id by the provider or the sending mail system, +// source_conversation_id by the provider's threading — and are unique only +// within it. msgvault is a multi-source archiver, so one archive routinely +// holds several accounts, and two of them can issue the same id for unrelated +// documents (two chat accounts each numbering their first conversation "1") or +// for related ones (the same mail delivered to two mailboxes). Either way the +// eval folds two documents into one key: a hit from an unjudged account +// inherits a judged account's relevance, or two genuinely distinct documents +// collapse and the ranking quietly loses a rank. Both move the score, both move +// it upward, and neither appears anywhere in the output — which is exactly the +// class of silent corruption this command exists to expose in other people's +// indexes. +// +// The fix is a precondition rather than a new key shape. Composing the source +// id into the key — as query.EntryKeyFacts.EntryKey does for explore entries, +// production's own answer to the same uniqueness problem — would make the key +// sound, but it would also change the shape of every doc id this command +// matches on, so every qrels file already written would stop matching. And it +// would stop matching by scoring a flat zero rather than by failing, which is +// the same silent corruption one level up. +// +// The precondition is disjointness, not single-source. An archive holding a +// Gmail account and a WhatsApp account has two sources and no overlapping ids +// at all; refusing to score it would be a wall built for a hazard that is not +// there. What has to hold is that the id space the qrels address is +// unambiguous, and "no id in it names documents in two sources" is exactly +// that. It is a property of the archive rather than of what a particular topic +// happened to retrieve, so it is established once, up front, instead of +// inferred from hits that may simply have got lucky — and it is established +// before the vector path is opened, so a run that cannot be scored does not +// first pay for an index and an embedding client. +func requireDisjointSourceIDs(ctx context.Context, db *sql.DB, docKey string, spec docKeySpec) error { + if spec.idColumn.column == "" { + // A registered key whose ids do not come from an archive column cannot + // be checked here, and passing it silently would put the collision + // straight back. Fail naming the key, so adding a doc-key forces an + // answer to the question rather than allowing it to be skipped. + return fmt.Errorf("--doc-key %q has no archive column to check for cross-source id collisions; "+ + "a doc-key whose ids come from elsewhere has to establish its own single-id-space guarantee", docKey) + } + // A single connected source cannot collide with itself, and that is the + // common archive shape, so a cheap distinct-source count (backed by + // idx_messages_source) skips the GROUP BY/HAVING scan — and its join, for + // --doc-key=conversation — entirely for the run that does not need it. + var sources int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(DISTINCT source_id) FROM messages WHERE "+store.LiveMessagesWhere("", true), + ).Scan(&sources); err != nil { + return fmt.Errorf("count connected sources: %w", err) + } + if sources <= 1 { + return nil + } + + // Enough ids to make the error concrete without pasting an entire + // re-imported mailbox into a terminal. + const show = 10 + ids, err := collidingDocKeyIDs(ctx, db, spec.idColumn, show+1) + if err != nil { + return fmt.Errorf("check %s for cross-source id collisions: %w", spec.idColumn.column, err) + } + if len(ids) == 0 { + return nil + } + count := strconv.Itoa(len(ids)) + if len(ids) > show { + count = fmt.Sprintf("more than %d", show) + } + return fmt.Errorf("%s document ids in this archive (%s) belong to more than one connected source, "+ + "so --doc-key=%s cannot name a single document: %s is unique only within the source that "+ + "assigned it, while a qrels doc id records no source at all. Scoring this archive would fold "+ + "those sources' hits into one key and let an unjudged account's message inherit a judged one's "+ + "relevance. Evaluate an archive whose accounts do not share ids, or key the run on the other "+ + "--doc-key if its id space is disjoint", + count, eval.FormatIDList(ids, show), docKey, spec.idColumn.column) +} + +// collectVectorStats records the embedding model, fusion parameters and index +// size in force for this run, so a score can never be read without knowing +// what produced it. +func (e *evaluator) collectVectorStats( + mainDB *sql.DB, vecCfg vector.Config, backend *sqlitevec.Backend, vecDBPath string, activeGen vector.GenerationID, +) { + e.prov.VectorEnabled = true + e.prov.EmbeddingModel = vecCfg.Embeddings.Model + e.prov.APIFormat = string(vecCfg.Embeddings.EffectiveAPIFormat()) + e.prov.Dimension = vecCfg.Embeddings.Dimension + e.prov.Endpoint = vecCfg.Embeddings.Endpoint + e.prov.Backend = vecCfg.Backend + e.prov.Fingerprint = vecCfg.GenerationFingerprint() + e.prov.RRFK = vecCfg.Search.RRFK + e.prov.KPerSignal = vecCfg.Search.KPerSignal + e.prov.SubjectBoost = vecCfg.Search.SubjectBoost + e.prov.IndexPath = vecDBPath + // The pool ceiling is what a saturation diagnostic has to name to be + // actionable, so the diagnostics carry it too. + e.diag.kPerSignal = vecCfg.Search.KPerSignal + + if fi, err := os.Stat(vecDBPath); err == nil { + e.prov.IndexSizeBytes = fi.Size() + } + // Backend.DB() is the backend's own accessor for exactly this kind of + // read-only query, so the row count goes through it rather than opening a + // second connection to the same file. + // + // Scope the count to the generation search reads. vectors.db keeps a + // retired generation's rows — vec0 partition-key isolation means retiring + // does not delete them — and a half-finished rebuild sits in the same + // table, so COUNT(*) over the whole table describes the file on disk, not + // the index this run queried. IndexSizeBytes already reports the file; + // this number has to report the index. + if vdb := backend.DB(); vdb != nil { + _ = vdb.QueryRowContext(e.ctx, + "SELECT COUNT(*) FROM embeddings WHERE generation_id = ?", int64(activeGen)). + Scan(&e.prov.IndexedVectors) + e.collectVectorCorpusStats(mainDB, vdb, vecCfg, activeGen) + } +} + +// collectVectorCorpusStats records the live population an account-scoped +// vector generation actually searches, when [vector.embed.scope] narrows it +// below the archive-wide Messages/Conversations collectCorpusStats already +// recorded. +// +// This deliberately does NOT require messages.embed_gen = gen, unlike +// Backend.EmbeddedMessageCount (production's own coverage accessor, built for +// a different question: "is this message's CURRENT content embedded"). A +// content change resets embed_gen to mark a message as needing re-embedding, +// but Backend.Search reads vectors.db purely by generation_id — it returns a +// message's stale vector until the re-embed actually runs, embed_gen or not. +// Requiring the stamp here would undercount relative to what a run can +// actually retrieve, the same "corpus" mismatch this field exists to fix in +// the other direction. So membership is: present in vectors.db for this +// generation, live, and in scope — exactly what Search can return, nothing +// narrower. +// +// Both counts are read from one query — the embedded message ids come from +// vectors.db, same as IndexedVectors reads, intersected once against +// main.db's live, scoped population — rather than a separate call per count, +// so a transient failure can only leave both at zero together, never one +// populated and the other not. +// +// Failures degrade the report rather than the run, same policy as +// collectCorpusStats: provenance is a courtesy to the reader, not a +// precondition for scoring. +func (e *evaluator) collectVectorCorpusStats( + mainDB *sql.DB, vdb *sql.DB, vecCfg vector.Config, gen vector.GenerationID, +) { + if mainDB == nil || vdb == nil { + return + } + + rows, err := vdb.QueryContext(e.ctx, + `SELECT DISTINCT message_id FROM embeddings WHERE generation_id = ?`, int64(gen)) + if err != nil { + return + } + defer func() { _ = rows.Close() }() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil || len(ids) == 0 { + return + } + + blob, err := json.Marshal(ids) + if err != nil { + return + } + // BuildScope, not the raw config fields: it lowercases/trims message + // types and drops non-positive source ids the same way the embed and + // search paths already do, so a config value like "EMAIL" matches the + // lowercase message_type rows the archive actually stores instead of + // silently matching nothing. + scope := vecCfg.Embed.Scope.BuildScope() + where := `id IN (SELECT value FROM json_each(?)) + AND ` + store.LiveMessagesWhere("", true) + args := []any{string(blob)} + if len(scope.MessageTypes) > 0 { + placeholders := make([]string, len(scope.MessageTypes)) + for i, typ := range scope.MessageTypes { + placeholders[i] = "?" + args = append(args, typ) + } + where += fmt.Sprintf(" AND message_type IN (%s)", strings.Join(placeholders, ",")) + } + if len(scope.SourceIDs) > 0 { + placeholders := make([]string, len(scope.SourceIDs)) + for i, id := range scope.SourceIDs { + placeholders[i] = "?" + args = append(args, id) + } + where += fmt.Sprintf(" AND source_id IN (%s)", strings.Join(placeholders, ",")) + } + // One row, one query: reading both counts off the same scan means a + // transient failure here can only leave both fields at their zero value + // together, never one populated and the other not — a partial success + // would print a self-contradictory line no error report explains. + _ = mainDB.QueryRowContext(e.ctx, + "SELECT COUNT(DISTINCT id), COUNT(DISTINCT conversation_id) FROM messages WHERE "+where, args...). + Scan(&e.prov.VectorMessages, &e.prov.VectorConversations) +} + +// sortedCategories returns the category labels seen in a run, sorted for +// stable output. +func sortedCategories(catCounts map[string]int) []string { + cats := make([]string, 0, len(catCounts)) + for c := range catCounts { + cats = append(cats, c) + } + sort.Strings(cats) + return cats +} + +// evalReport is everything one run produced, ready to render. +type evalReport struct { + modes []string + aggs map[string]*eval.Aggregate + lats map[string]*eval.LatencyTracker + catAggs map[string]map[string]*eval.Aggregate + catCounts map[string]int + prov eval.RunConfig + topics int + cutoffs eval.Cutoffs + diag *runDiagnostics +} + +// metricHeaders names the metric columns at the depths this run actually used, +// so a clamped cutoff can never be read as the standard one. +// +// MAP and MRR are qualified too. They take no cutoff, but the ranking handed to +// them is truncated to -n, so a relevant document below that rank is invisible +// to them exactly as it is to R@n: what the run measured is MAP@n and MRR@n. +// Printing them bare would offer them for comparison against a run that +// retrieved deeper, which is the same mislabeling the clamped headers exist to +// prevent. If the depth is somehow unknown there is nothing to qualify them +// with, so they stay bare rather than claiming a depth of zero. +func (r evalReport) metricHeaders() (p, ndcg, recall, mapAt, mrr string) { + mapAt, mrr = "MAP", "MRR" + if r.cutoffs.Depth > 0 { + mapAt = fmt.Sprintf("MAP@%d", r.cutoffs.Depth) + mrr = fmt.Sprintf("MRR@%d", r.cutoffs.Depth) + } + return fmt.Sprintf("P@%d", r.cutoffs.P), + fmt.Sprintf("nDCG@%d", r.cutoffs.NDCG), + fmt.Sprintf("R@%d", r.cutoffs.Recall), + mapAt, mrr +} + +func (r evalReport) table() { + fmt.Printf("Evaluated %d topics (doc-key=%s, n=%d)\n", r.topics, evalDocKey, evalLimit) + if !r.cutoffs.IsStandard() { + fmt.Printf("Metric depths are clamped to -n: the standard P@%d/nDCG@%d/R@%d need -n %d or more.\n", + eval.StandardCutoffs.P, eval.StandardCutoffs.NDCG, eval.StandardCutoffs.Recall, + eval.StandardCutoffs.Recall) + } + + // Provenance first: a score is not interpretable without it. + fmt.Printf("\nRun configuration\n") + pw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintf(pw, " topics\t%s\n", r.prov.TopicsPath) + _, _ = fmt.Fprintf(pw, " qrels\t%s\n", r.prov.QrelsPath) + _, _ = fmt.Fprintf(pw, " corpus\t%d live messages, %d conversations\n", + r.prov.Messages, r.prov.Conversations) + if r.prov.VectorEnabled { + if r.prov.VectorMessages != r.prov.Messages || r.prov.VectorConversations != r.prov.Conversations { + _, _ = fmt.Fprintf(pw, " vector corpus\t%d live messages, %d conversations "+ + "(embed.scope narrows this generation below the archive)\n", + r.prov.VectorMessages, r.prov.VectorConversations) + } + _, _ = fmt.Fprintf(pw, " embedding model\t%s (dim %d)\n", r.prov.EmbeddingModel, r.prov.Dimension) + _, _ = fmt.Fprintf(pw, " embedding api format\t%s\n", r.prov.APIFormat) + _, _ = fmt.Fprintf(pw, " embedding endpoint\t%s\n", r.prov.Endpoint) + _, _ = fmt.Fprintf(pw, " vector backend\t%s\n", r.prov.Backend) + _, _ = fmt.Fprintf(pw, " generation fingerprint\t%s\n", r.prov.Fingerprint) + _, _ = fmt.Fprintf(pw, " fusion\trrf_k=%d k_per_signal=%d subject_boost=%.2f\n", + r.prov.RRFK, r.prov.KPerSignal, r.prov.SubjectBoost) + _, _ = fmt.Fprintf(pw, " vector index\t%d vectors in the active generation, %s on disk (%s)\n", + r.prov.IndexedVectors, formatSize(r.prov.IndexSizeBytes), r.prov.IndexPath) + } else { + _, _ = fmt.Fprintf(pw, " vector index\t(not used; --modes fts only)\n") + } + _ = pw.Flush() + + pCol, ndcgCol, rCol, mapCol, mrrCol := r.metricHeaders() + fmt.Printf("\n") + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + // "topics" is per mode, not per run: a mode that cannot answer some topic + // (a filter-only query has nothing to embed) scores fewer of them, and the + // means are only comparable if the denominators are visible. + header := []string{"MODE", "topics", pCol, ndcgCol, rCol, mapCol, mrrCol, "med ms", "p95 ms"} + _, _ = fmt.Fprintln(w, strings.Join(header, "\t")) + rule := make([]string, len(header)) + for i, h := range header { + rule[i] = strings.Repeat("─", len([]rune(h))) + } + _, _ = fmt.Fprintln(w, strings.Join(rule, "\t")) + for _, m := range r.modes { + s := r.aggs[m].Mean() + l := r.lats[m].Summary() + _, _ = fmt.Fprintf(w, "%s\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.1f\t%.1f\n", + m, r.aggs[m].N, s.P, s.NDCG, s.Recall, s.MAP, s.MRR, l.MedianMS, l.P95MS) + } + _ = w.Flush() + + // Per-category breakdown, only when the topics file carries labels. + // Latency is tracked per mode, not per category, so those columns are + // omitted here. + if len(r.catCounts) > 0 { + fmt.Printf("\nBy query category\n") + cw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintf(cw, "MODE\tCATEGORY\ttopics\t%s\t%s\t%s\t%s\t%s\n", + pCol, ndcgCol, rCol, mapCol, mrrCol) + for _, m := range r.modes { + for _, c := range sortedCategories(r.catCounts) { + agg := r.catAggs[m][c] + if agg == nil { + continue + } + s := agg.Mean() + _, _ = fmt.Fprintf(cw, "%s\t%s\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n", + m, c, agg.N, s.P, s.NDCG, s.Recall, s.MAP, s.MRR) + } + } + _ = cw.Flush() + } + + if notes := r.diag.notes(); len(notes) > 0 { + fmt.Printf("\nDiagnostics\n") + for _, n := range notes { + fmt.Printf(" - %s\n", n) + } + } +} + +func (r evalReport) json() error { + pCol, ndcgCol, rCol, mapCol, mrrCol := r.metricHeaders() + metricsOf := func(a *eval.Aggregate) map[string]any { + s := a.Mean() + return map[string]any{ + pCol: s.P, ndcgCol: s.NDCG, rCol: s.Recall, mapCol: s.MAP, mrrCol: s.MRR, + } + } + results := make(map[string]any, len(r.modes)) + for _, m := range r.modes { + entry := metricsOf(r.aggs[m]) + entry["topics"] = r.aggs[m].N + entry["latency"] = r.lats[m].Summary() + if len(r.catAggs[m]) > 0 { + byCat := make(map[string]any, len(r.catAggs[m])) + for c, a := range r.catAggs[m] { + cm := metricsOf(a) + cm["topics"] = a.N + byCat[c] = cm + } + entry["by_category"] = byCat + } + results[m] = entry + } + out := map[string]any{ + "topics_evaluated": r.topics, + "doc_key": evalDocKey, + "limit": evalLimit, + // One entry per metric, including the two whose depth is the retrieval + // depth rather than a cutoff of their own, so a consumer can read every + // metric's depth the same way instead of knowing which are special. + "cutoffs": map[string]int{ + "precision": r.cutoffs.P, "ndcg": r.cutoffs.NDCG, "recall": r.cutoffs.Recall, + "map": r.cutoffs.Depth, "mrr": r.cutoffs.Depth, + }, + "modes": r.modes, + "run_config": r.prov, + "results": results, + "diagnostics": r.diag, + } + if len(r.catCounts) > 0 { + out["topic_categories"] = r.catCounts + } + return printJSON(out) +} diff --git a/cmd/msgvault/cmd/eval_fts_production_test.go b/cmd/msgvault/cmd/eval_fts_production_test.go new file mode 100644 index 000000000..6bce71ace --- /dev/null +++ b/cmd/msgvault/cmd/eval_fts_production_test.go @@ -0,0 +1,259 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" +) + +// seedRankingDivergenceArchive builds a small real archive in which BM25 +// relevance and reverse-chronological order disagree, and in which one +// matching message has been deleted from its source account. +// +// m1 2020-01-01 subject hit, short body — most relevant, oldest +// m2 2024-01-01 body hit, long body — least relevant, newest +// m3 2022-01-01 subject hit — deleted from source +// +// BM25 weights subject ten times body (see store.SQLiteDialect.FTSSearchClause) +// and normalizes by document length, so the ranking is m1, m3, m2 while the +// date ordering is exactly the reverse. Production FTS search returns the +// former, minus m3. +func seedRankingDivergenceArchive(t *testing.T) *store.Store { + t.Helper() + return seedRankingDivergenceArchiveIn(t, t.TempDir()) +} + +func seedRankingDivergenceArchiveIn(t *testing.T, dataDir string) *store.Store { + t.Helper() + dbPath := filepath.Join(dataDir, "msgvault.db") + s, err := store.Open(dbPath) + require.NoError(t, err, "open store") + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.InitSchema(), "init schema") + + // A body long enough that BM25's length normalization can tell the two + // documents apart, with the query term buried once inside it. + filler := strings.Repeat("quarterly figures and other unrelated correspondence text ", 60) + + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'me@example.com'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 1, 'thread-2', 'email_thread'), + (3, 1, 'thread-3', 'email_thread'); +INSERT INTO messages (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, size_estimate, deleted_from_source_at) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100, NULL), + (2, 2, 1, '', 'email', 'Weekly digest', '2024-01-01T00:00:00Z', 100, NULL), + (3, 3, 1, '', 'email', 'Lease renewal notice','2022-01-01T00:00:00Z', 100, '2023-06-01T00:00:00Z'); +`) + require.NoError(t, err, "seed messages") + + for _, b := range []struct { + id int64 + body string + }{ + {1, "Signed and returned."}, + {2, filler + " renewal " + filler}, + {3, "Notice served."}, + } { + _, err = s.DB().Exec( + `INSERT INTO message_bodies (message_id, body_text) VALUES (?, ?)`, b.id, b.body) + require.NoError(t, err, "seed body %d", b.id) + } + + // Index through the production backfill rather than writing messages_fts + // by hand, so the test scores the documents production would have built. + indexed, err := s.BackfillFTS(nil) + require.NoError(t, err, "backfill FTS") + require.EqualValues(t, 3, indexed, "every message must be indexed") + return s +} + +func keysOfAPIMessages(msgs []store.APIMessage) []string { + out := make([]string, 0, len(msgs)) + for _, m := range msgs { + out = append(out, m.SourceMessageID) + } + return out +} + +func keysOfSummaries(msgs []query.MessageSummary) []string { + out := make([]string, 0, len(msgs)) + for _, m := range msgs { + out = append(out, m.SourceMessageID) + } + return out +} + +// TestRankedFTS_MatchesProductionRelevanceRanking is the regression for the +// mode=fts path having been scored against the wrong search. +// +// query.Engine.Search — what this command used to call — has no relevance +// component at all: it filters, then orders by sent_at DESC. Scoring that as a +// *ranking* measures the archive's date distribution, and reports it as +// retrieval quality. It also diverges from production on which messages are +// eligible at all: it leaves source-deleted messages in the result set, which +// no production search returns by default. +// +// The eval path must instead be the search production runs — the +// BM25-ranked store path behind /api/v1/search?mode=fts, whose messages_fts +// index and subject weighting are the same ones the hybrid engine's BM25 leg +// fuses, so fts and hybrid scores are comparable to each other. +func TestRankedFTS_MatchesProductionRelevanceRanking(t *testing.T) { + s := seedRankingDivergenceArchive(t) + ctx := t.Context() + const topic = "renewal" + + // The two searches genuinely disagree here — without that this test + // would pass no matter which one the eval called. + production, _, err := s.SearchMessagesQueryContext(ctx, search.Parse(topic), 0, 10) + require.NoError(t, err, "production store search") + productionKeys := keysOfAPIMessages(production) + require.Equal(t, []string{"", ""}, productionKeys, + "production ranks the subject hit first and never returns the source-deleted message") + + qeng := query.NewEngine(s.DB(), s.IsPostgreSQL()) + legacy, err := qeng.Search(ctx, search.Parse(topic), 10, 0) + require.NoError(t, err, "query engine search") + legacyKeys := keysOfSummaries(legacy) + require.Equal(t, + []string{"", "", ""}, legacyKeys, + "the chronological path inverts the ranking and includes the source-deleted message") + + ev, diag := newTestEvaluator(t, s, "message") + ev.limit = 10 + + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(t, err) + + assert.Equal(t, productionKeys, ranked, + "the eval's fts mode must rank exactly as production search does") + assert.NotEqual(t, legacyKeys, ranked, + "and must no longer reproduce the chronological ordering") + assert.NotContains(t, ranked, "", + "a message deleted from its source is not something production retrieval returns") + assert.Empty(t, diag.notes(), "a clean archive produces a clean run") +} + +// TestRankedFTS_HonoursProductionAddressFilterSemantics pins the second +// divergence roborev flagged. The store path resolves from: as a substring +// match against the participant address (so `from:landlord` finds +// landlord@example.com); query.Engine.Search requires an exact address or an +// @domain pattern, and would have scored a flat zero for the same topic. +func TestRankedFTS_HonoursProductionAddressFilterSemantics(t *testing.T) { + s := seedRankingDivergenceArchive(t) + ctx := t.Context() + _, err := s.DB().Exec(` +INSERT INTO participants (id, email_address) VALUES (1, 'landlord@example.com'); +INSERT INTO message_recipients (message_id, participant_id, recipient_type) VALUES (1, 1, 'from'); +`) + require.NoError(t, err, "seed sender") + + const topic = "from:landlord renewal" + + production, _, err := s.SearchMessagesQueryContext(ctx, search.Parse(topic), 0, 10) + require.NoError(t, err, "production store search") + require.Equal(t, []string{""}, keysOfAPIMessages(production), + "production matches the address by substring") + + qeng := query.NewEngine(s.DB(), s.IsPostgreSQL()) + legacy, err := qeng.Search(ctx, search.Parse(topic), 10, 0) + require.NoError(t, err, "query engine search") + require.Empty(t, legacy, "the exact-match path finds nothing for the same topic") + + ev, _ := newTestEvaluator(t, s, "message") + ev.limit = 10 + + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(t, err) + assert.Equal(t, []string{""}, ranked, + "the eval must score the hits production returns, not zero") +} + +// writeEvalFile writes one of the run's input files and returns its path. +func writeEvalFile(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600), "write %s", name) + return path +} + +// TestRunEval_EndToEnd_RanksByRelevanceAndSkipsAMalformedTopic drives the whole +// command against a real archive, so the two fixes are pinned where they +// actually have to hold: at the call sites, not just in the helpers. +// +// - q1 is scored through the production relevance-ranked FTS path. Its one +// judged message is the subject hit, which BM25 puts at rank 1 (MRR 1.0) +// and the old chronological path put at rank 3 (MRR 0.333). +// - q2 carries a malformed date filter. search.Parse drops it and leaves the +// broader query `renewal` behind, which would have scored exactly like q1 +// under a question nobody asked. It must be skipped and reported instead. +func TestRunEval_EndToEnd_RanksByRelevanceAndSkipsAMalformedTopic(t *testing.T) { + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + + savedCfg := cfg + t.Cleanup(func() { cfg = savedCfg }) + cfg = config.NewDefaultConfig() + cfg.Data.DataDir = dir + + savedFlags := [...]any{evalQrels, evalTopics, evalModes, evalDocKey, evalLimit, evalJSON} + t.Cleanup(func() { + evalQrels, _ = savedFlags[0].(string) + evalTopics, _ = savedFlags[1].(string) + evalModes, _ = savedFlags[2].(string) + evalDocKey, _ = savedFlags[3].(string) + evalLimit, _ = savedFlags[4].(int) + evalJSON, _ = savedFlags[5].(bool) + }) + evalQrels = writeEvalFile(t, dir, "qrels.txt", + "q1 0 1\nq2 0 1\n") + evalTopics = writeEvalFile(t, dir, "topics.tsv", + "q1\trenewal\nq2\tbefore:invalid renewal\n") + evalModes = "fts" + evalDocKey = "message" + evalLimit = 10 + evalJSON = true + + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(t, err, "eval run") + + var report struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + Topics int `json:"topics"` + } `json:"results"` + Diagnostics struct { + SkippedCells []string `json:"skipped_cells"` + } `json:"diagnostics"` + } + require.NoError(t, json.Unmarshal([]byte(out), &report), "parse report: %s", out) + + assert.Equal(t, 1, report.TopicsEvaluated, "only the well-formed topic is scored") + assert.Equal(t, 1, report.Results["fts"].Topics) + assert.InDelta(t, 1.0, report.Results["fts"].MRR, 1e-9, + "the relevance-ranked path puts the subject hit first; the chronological one scored 0.333 here") + + require.Len(t, report.Diagnostics.SkippedCells, 1, "the malformed topic must be reported, not dropped") + assert.Contains(t, report.Diagnostics.SkippedCells[0], "topic q2") + assert.Contains(t, report.Diagnostics.SkippedCells[0], "before:invalid renewal") +} diff --git a/cmd/msgvault/cmd/eval_key_test.go b/cmd/msgvault/cmd/eval_key_test.go new file mode 100644 index 000000000..cc8d35f6a --- /dev/null +++ b/cmd/msgvault/cmd/eval_key_test.go @@ -0,0 +1,91 @@ +//go:build sqlite_vec + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +// TestDocKeyFuncs pins the doc-key registry: the scoring core only ever sees +// the opaque ids these extractors return, so each --doc-key value must map to +// the field its qrels actually reference. A future judged unit (for example a +// reconstructed-thread id resolved through an external mapping) is added as +// one more entry here and must not require touching this contract. +func TestDocKeyFuncs(t *testing.T) { + m := evalHit{ + MessageID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + } + registry := newDocKeyRegistry() + + msgKey, ok := registry["message"] + require.True(t, ok) + assert.Equal(t, "", msgKey.extract(m)) + + convKey, ok := registry["conversation"] + require.True(t, ok) + assert.Equal(t, "thread-42", convKey.extract(m)) + + _, ok = registry["thread"] + assert.False(t, ok, "thread scoring is a future extension, not yet registered") +} + +// TestDocKeySpec_Collapses pins which keys need the over-fetch: a conversation +// id is shared by every message in a thread, a source message id is not. +func TestDocKeySpec_Collapses(t *testing.T) { + registry := newDocKeyRegistry() + assert.False(t, registry["message"].collapses) + assert.True(t, registry["conversation"].collapses) +} + +// TestNewDocKeyRegistry_IsPerRun backs the extensibility claim in the +// registry's doc comment: it is built by a call, not fixed at program init, so +// an entry closing over state that only exists after flags are parsed (a +// loaded message-id -> thread-id mapping, say) is possible. Two calls must +// therefore produce independent maps. +func TestNewDocKeyRegistry_IsPerRun(t *testing.T) { + first := newDocKeyRegistry() + second := newDocKeyRegistry() + require.Len(t, second, len(first)) + + first["thread"] = docKeySpec{extract: func(evalHit) string { return "x" }} + _, leaked := newDocKeyRegistry()["thread"] + assert.False(t, leaked, "a run's registry must not mutate the next run's") +} + +// TestDocKeyNames keeps usage/error text in step with the registry, sorted so +// the rendering is stable. +func TestDocKeyNames(t *testing.T) { + assert.Equal(t, "conversation|message", docKeyNames(newDocKeyRegistry())) +} + +// TestEvalHitProjections_AgreeAcrossRetrievalPaths backs the claim evalHit +// exists to make: a --doc-key means the same thing whichever engine produced +// the hit. fts hits arrive as store.APIMessage and vector/hybrid hits as +// query.MessageSummary, and if those two projected differently the same +// message would score under two different ids. +func TestEvalHitProjections_AgreeAcrossRetrievalPaths(t *testing.T) { + fromFTS := hitFromAPIMessage(store.APIMessage{ + ID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + }) + fromVector := hitFromSummary(query.MessageSummary{ + ID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + }) + assert.Equal(t, fromFTS, fromVector, "the same message must project identically") + + for name, spec := range newDocKeyRegistry() { + assert.Equal(t, spec.extract(fromFTS), spec.extract(fromVector), + "--doc-key=%s must not depend on which engine returned the hit", name) + } +} diff --git a/cmd/msgvault/cmd/eval_labels_test.go b/cmd/msgvault/cmd/eval_labels_test.go new file mode 100644 index 000000000..840860d89 --- /dev/null +++ b/cmd/msgvault/cmd/eval_labels_test.go @@ -0,0 +1,86 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunEval_LabelsMAPAndMRRAtTheRetrievalDepth pins the labels against the +// depth the run actually looked to. +// +// MAP and MRR take no cutoff, so they were reported bare — but the ranking they +// score is truncated to -n before they ever see it, which makes a relevant +// message below that rank invisible to them exactly as it is to recall. A run +// at -n 5 reports MAP@5 and MRR@5; calling them "MAP" and "MRR" offers them for +// comparison against a run that retrieved a hundred deep, which is the same +// mislabeling the clamped P/nDCG/R headers already exist to prevent. +func TestRunEval_LabelsMAPAndMRRAtTheRetrievalDepth(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, "q1 0 1\n", "q1\trenewal\n") + evalLimit = 5 + + var report struct { + Cutoffs map[string]int `json:"cutoffs"` + Results map[string]map[string]any `json:"results"` + } + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + require.NoError(json.Unmarshal([]byte(out), &report), "parse report: %s", out) + + scored := report.Results[evalTestMode] + assert.Contains(scored, "MAP@5", "the JSON metric key names the depth; %v", scored) + assert.Contains(scored, "MRR@5") + assert.NotContains(scored, "MAP", "an unqualified key would claim an untruncated ranking") + assert.NotContains(scored, "MRR") + // Every metric's depth is readable the same way, including the two whose + // depth is the retrieval depth rather than a cutoff of their own. + assert.Equal(map[string]int{"precision": 5, "ndcg": 5, "recall": 5, "map": 5, "mrr": 5}, + report.Cutoffs) +} + +// TestEvalReport_TableLabelsMAPAndMRRAtTheRetrievalDepth is the same rule for +// the human-readable output, which builds its header row separately from the +// JSON keys. +func TestEvalReport_TableLabelsMAPAndMRRAtTheRetrievalDepth(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, "q1 0 1\n", "q1\trenewal\n") + evalLimit = 5 + evalJSON = false + + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + + header := "" + for line := range strings.SplitSeq(out, "\n") { + if strings.HasPrefix(line, "MODE") { + header = line + break + } + } + require.NotEmpty(header, "no results table in:\n%s", out) + assert.Contains(header, "MAP@5") + assert.Contains(header, "MRR@5") +} diff --git a/cmd/msgvault/cmd/eval_modes_test.go b/cmd/msgvault/cmd/eval_modes_test.go new file mode 100644 index 000000000..62dc9c31f --- /dev/null +++ b/cmd/msgvault/cmd/eval_modes_test.go @@ -0,0 +1,112 @@ +//go:build sqlite_vec + +package cmd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/vector/hybrid" +) + +// parsedEvalModes runs the flag parser and returns the accepted modes as the +// comma-separated list they came in as, so an expectation can be written as one +// string rather than as a slice of bare mode names. +func parsedEvalModes(t *testing.T, spec string) (string, bool) { + t.Helper() + modes, needVec, err := parseEvalModes(spec) + require.NoError(t, err, "parse --modes %q", spec) + return strings.Join(modes, ","), needVec +} + +// TestParseEvalModes_DeduplicatesPreservingOrder is the regression for a +// repeated mode being evaluated twice. +// +// Each entry in the list gets its own pass through the scoring loop but shares +// one Aggregate and one LatencyTracker per mode name, so a duplicate added +// every topic's score to that mode a second time — doubling the topic count the +// report prints beside the means, and doubling the queries actually run. Order +// is the order the report's rows appear in, so the first mention keeps its +// position. +func TestParseEvalModes_DeduplicatesPreservingOrder(t *testing.T) { + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, "hybrid,fts,hybrid,fts") + assert.Equal("hybrid,fts", modes, + "each mode once, at the position it was first named") + assert.True(needVec, "the surviving hybrid entry still opens the vector path") +} + +// TestParseEvalModes_DedupesASingleRepeatedMode covers the flag as a user is +// most likely to mistype it, and pins that dropping the duplicate does not also +// drop the vector requirement it carried. +func TestParseEvalModes_DedupesASingleRepeatedMode(t *testing.T) { + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, " fts , fts ") + assert.Equal("fts", modes, "surrounding whitespace does not make a second mode") + assert.False(needVec, "fts alone never opens the vector path") + + modes, needVec = parsedEvalModes(t, "vector,vector") + assert.Equal("vector", modes) + assert.True(needVec, "the surviving entry still needs the index") +} + +// TestParseEvalModes_StillRejectsARepeatedInvalidMode keeps the dedupe from +// swallowing the validation: the switch has to run on every entry, before the +// duplicate check, or `--modes fts,bogus,bogus` would quietly report on fts +// alone. +func TestParseEvalModes_StillRejectsARepeatedInvalidMode(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + modes, _, err := parseEvalModes("fts,bogus,bogus") + require.Error(err, "an unknown mode is an error however often it appears") + assert.Empty(modes) + assert.Contains(err.Error(), `"bogus"`) +} + +// TestParseEvalModes_AcceptsTheFullSetAndRejectsAnEmptyOne pins the two ends of +// the flag the dedupe must leave alone: distinct modes all survive, and a list +// naming none of them is still rejected rather than silently scoring nothing. +func TestParseEvalModes_AcceptsTheFullSetAndRejectsAnEmptyOne(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, "fts,vector,hybrid") + assert.Equal("fts,vector,hybrid", modes, "distinct modes all survive, in order") + assert.True(needVec) + + _, _, err := parseEvalModes(" , ,") + require.Error(err, "a list of nothing but separators names no mode") + assert.Contains(err.Error(), "--modes is empty") +} + +// TestRequireFTS5ForModes_RejectsFTSModeWithoutFTS5 is the regression for a +// run that would otherwise silently score a LIKE-and-recency fallback while +// its own report still calls the mode "fts" and implies BM25 ranking. +func TestRequireFTS5ForModes_RejectsFTSModeWithoutFTS5(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + err := requireFTS5ForModes([]string{evalTestMode}, false) + require.Error(err, "fts without FTS5 must stop the run, not silently degrade it") + assert.Contains(err.Error(), "FTS5") + assert.Contains(err.Error(), "--modes fts") +} + +// TestRequireFTS5ForModes_AllowsWhatDoesNotNeedFTS5 pins the two ways this +// guard must stay out of the way: fts5 actually being available, and a +// --modes list that never asked for fts in the first place (vector/hybrid +// alone must not be blocked by an FTS5 outage they don't depend on). +func TestRequireFTS5ForModes_AllowsWhatDoesNotNeedFTS5(t *testing.T) { + require := require.New(t) + + require.NoError(requireFTS5ForModes([]string{evalTestMode, string(hybrid.ModeVector)}, true), + "fts is fine once FTS5 is actually available") + require.NoError(requireFTS5ForModes([]string{string(hybrid.ModeVector), string(hybrid.ModeHybrid)}, false), + "neither mode here reads through Store's FTS path") +} diff --git a/cmd/msgvault/cmd/eval_provenance_test.go b/cmd/msgvault/cmd/eval_provenance_test.go new file mode 100644 index 000000000..db33972a9 --- /dev/null +++ b/cmd/msgvault/cmd/eval_provenance_test.go @@ -0,0 +1,200 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// seedLifecycleShapedArchive builds an archive with the row shapes a long-lived +// install accumulates and a fresh benchmark corpus never has: +// +// thread-1 m1 live, m2 live +// thread-2 m3 live, m4 deleted from its source account +// thread-3 m5 dedup-hidden (deleted_at set — a losing duplicate) +// thread-4 no messages left at all +// +// Three messages are retrievable, across two threads. The tables hold five and +// four. +func seedLifecycleShapedArchive(t *testing.T) *store.Store { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "msgvault.db")) + require.NoError(t, err, "open store") + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.InitSchema(), "init schema") + + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'me@example.com'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 1, 'thread-2', 'email_thread'), + (3, 1, 'thread-3', 'email_thread'), + (4, 1, 'thread-4', 'email_thread'); +INSERT INTO messages + (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, + size_estimate, deleted_at, deleted_from_source_at) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100, NULL, NULL), + (2, 1, 1, '', 'email', 'Re: Lease renewal', '2020-01-02T00:00:00Z', 100, NULL, NULL), + (3, 2, 1, '', 'email', 'Insurance certificate','2020-02-01T00:00:00Z', 100, NULL, NULL), + (4, 2, 1, '', 'email', 'Deleted upstream', '2020-02-02T00:00:00Z', 100, NULL, '2023-06-01T00:00:00Z'), + (5, 3, 1, '', 'email', 'Duplicate copy', '2020-03-01T00:00:00Z', 100, '2023-01-01T00:00:00Z', NULL); +`) + require.NoError(t, err, "seed archive") + return s +} + +// TestCollectCorpusStats_CountsOnlyTheSearchablePopulation is the regression for +// provenance that described the tables instead of the haystack. +// +// COUNT(*) over messages and conversations includes dedup-hidden duplicates, +// messages deleted from their source, and conversations with nothing left in +// them — none of which any search this command runs can return. Reporting them +// as the corpus size inflates the denominator a reader mentally divides recall +// by, and does it invisibly: on the flat TREC corpus this branch is normally +// exercised against, every count coincides. +func TestCollectCorpusStats_CountsOnlyTheSearchablePopulation(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + s := seedLifecycleShapedArchive(t) + + // What the raw tables hold, so a change to the fixture cannot quietly + // make the assertions below trivially true. + var rawMessages, rawConversations int64 + require.NoError(s.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&rawMessages)) + require.NoError(s.DB().QueryRow(`SELECT COUNT(*) FROM conversations`).Scan(&rawConversations)) + require.EqualValues(5, rawMessages) + require.EqualValues(4, rawConversations) + + ev, _ := newTestEvaluator(t, s, "message") + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(3, ev.prov.Messages, + "the dedup-hidden and source-deleted rows are not part of the searched corpus") + assert.EqualValues(2, ev.prov.Conversations, + "and neither is a thread with no live message left in it") +} + +// TestCollectCorpusStats_MatchesWhatSearchCanReturn ties the count to the +// retrieval path rather than to a hand-written expectation: whatever the +// production FTS search is willing to return over the whole archive is the +// population the provenance block must be describing. +func TestCollectCorpusStats_MatchesWhatSearchCanReturn(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + s := seedLifecycleShapedArchive(t) + + // An empty query with no text terms filters on nothing but the deletion + // scope, so the result set is exactly the live population. + found, _, err := s.SearchMessagesQueryContext(t.Context(), &search.Query{}, 0, 100) + require.NoError(err, "production store search") + + threads := map[int64]struct{}{} + for _, m := range found { + threads[m.ConversationID] = struct{}{} + } + + ev, _ := newTestEvaluator(t, s, "message") + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(len(found), ev.prov.Messages, + "the reported corpus must be the one search draws from") + assert.EqualValues(len(threads), ev.prov.Conversations) +} + +// seedTwoGenerations writes an index that has been rebuilt once: an older +// generation carrying three vectors, retired but retained (sqlitevec keeps a +// retired generation's rows — vec0 partition-key isolation means retiring does +// not delete them), and the active generation carrying one. Returns the total +// row count and the active generation's. +func seedTwoGenerations( + t *testing.T, dataDir, mainPath string, mainDB *sql.DB, vecCfg vector.Config, +) (total, active int64) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + dim := vecCfg.Embeddings.Dimension + chunk := func(msgID int64) vector.Chunk { + v := make([]float32, dim) + for i := range v { + v[i] = float32(msgID) / 10 + } + return vector.Chunk{MessageID: msgID, Vector: v, SourceCharLen: 32} + } + newGen := func(msgIDs ...int64) { + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, dim, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + chunks := make([]vector.Chunk, 0, len(msgIDs)) + for _, id := range msgIDs { + chunks = append(chunks, chunk(id)) + } + require.NoError(t, b.Upsert(ctx, gen, chunks), "Upsert") + // force: the coverage gate is about messages.embed_gen, which this + // test never stamps; the generation lifecycle is what is under test. + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") + } + newGen(1, 2, 3) // superseded by the next activation, rows retained + newGen(1) // active + + require.NoError(t, b.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM embeddings`).Scan(&total), "count all embeddings") + require.NoError(t, b.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM embeddings WHERE generation_id = + (SELECT id FROM index_generations WHERE state = 'active')`).Scan(&active), + "count active embeddings") + return total, active +} + +// TestAttachVector_CountsOnlyTheActiveGenerationsVectors is the regression for +// the index-size half of the same problem. Search reads exactly one generation +// — the active one attachVector resolves — but the reported vector count was a +// COUNT(*) over the whole embeddings table, which also holds every retired +// generation's rows and any rebuild in progress. On a rebuilt archive that +// reports an index several times the size of the one the scores came from. +func TestAttachVector_CountsOnlyTheActiveGenerationsVectors(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := t.Context() + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + withTestConfig(t, c) + + total, active := seedTwoGenerations(t, dataDir, c.DatabaseDSN(), s.DB(), c.Vector) + require.EqualValues(4, total, "the retired generation's rows are retained") + require.EqualValues(1, active, "and the active generation is the smaller one") + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.Equal(active, ev.prov.IndexedVectors, + "the run must report the index it searched, not every generation on disk") + assert.Positive(ev.prov.IndexSizeBytes, + "the whole-file measure stays available separately") +} diff --git a/cmd/msgvault/cmd/eval_qrels_test.go b/cmd/msgvault/cmd/eval_qrels_test.go new file mode 100644 index 000000000..8129934ba --- /dev/null +++ b/cmd/msgvault/cmd/eval_qrels_test.go @@ -0,0 +1,125 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" +) + +// evalTestMode is the single search mode these end-to-end runs score. It is +// named once so the flag the run is configured with and the report key its +// assertions read can never drift apart. +const evalTestMode = "fts" + +// configureEvalRun points the eval command at one scratch directory: its +// archive as the configured data dir, and the given qrels and topics content +// written into it, for a message-keyed JSON run over the fts mode alone. The +// command's package-level config and flag variables are snapshotted and put +// back when the test ends, so a test that drives runEval directly cannot leak +// its settings into whatever runs next. +func configureEvalRun(t *testing.T, dir, qrels, topics string) { + t.Helper() + + savedCfg := cfg + t.Cleanup(func() { cfg = savedCfg }) + cfg = config.NewDefaultConfig() + cfg.Data.DataDir = dir + + savedQrels, savedTopics, savedModes := evalQrels, evalTopics, evalModes + savedDocKey, savedLimit, savedJSON := evalDocKey, evalLimit, evalJSON + t.Cleanup(func() { + evalQrels, evalTopics, evalModes = savedQrels, savedTopics, savedModes + evalDocKey, evalLimit, evalJSON = savedDocKey, savedLimit, savedJSON + }) + + evalQrels = writeEvalFile(t, dir, "qrels.txt", qrels) + evalTopics = writeEvalFile(t, dir, "topics.tsv", topics) + evalModes = evalTestMode + evalDocKey = "message" + evalLimit = 10 + evalJSON = true +} + +// TestRunEval_ScoresATopicJudgedEntirelyNonRelevant is the regression for +// conflating "this topic was never judged" with "this topic was judged and +// nothing was relevant". +// +// Both produce an empty relevant set, and the command used to skip on exactly +// that. But an all-non-relevant topic is a real measurement — the run looked, +// and found nothing it should have found — and it can only ever score zero. +// Dropping it therefore removes a zero from every macro average and reports a +// better run than happened. TREC semantics are to score it; only a qid the +// qrels file never mentions has nothing to score against. +// +// - q1 is judged relevant on the BM25 top hit: MRR 1.0, P@10 0.1. +// - q2 is judged, every grade 0: MRR 0, P@10 0 — and must be counted. +// - q3 appears in the topics file only, and must still be skipped. +// +// Before the fix this reported one topic at MRR 1.0; the honest answer is two +// topics at 0.5. +func TestRunEval_ScoresATopicJudgedEntirelyNonRelevant(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 0\n"+ + "q2 0 0\n", + "q1\trenewal\nq2\trenewal\nq3\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + + var report struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + P float64 `json:"P@10"` + Topics int `json:"topics"` + } `json:"results"` + } + require.NoError(json.Unmarshal([]byte(out), &report), "parse report: %s", out) + scored := report.Results[evalTestMode] + + assert.Equal(2, report.TopicsEvaluated, + "the all-non-relevant topic is scored; only the unjudged q3 is skipped") + assert.Equal(2, scored.Topics) + assert.InDelta(0.5, scored.MRR, 1e-9, + "q2 contributes a real zero to the macro average; skipping it reported 1.0") + assert.InDelta(0.05, scored.P, 1e-9, + "same for precision: (0.1 + 0) / 2, not 0.1") +} + +// TestRunEval_FailsWhenNoTopicIsJudged pins the other side of the split: when +// the qrels file mentions none of the topics there is genuinely nothing to +// score, and the run must still fail with the id-mismatch guidance rather than +// report an empty run full of zeroes. +func TestRunEval_FailsWhenNoTopicIsJudged(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, "other-1 0 0\n", "q1\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + + err := runEval(cmd, nil) + require.Error(err, "no topic was judged, so there is nothing to report") + assert.Contains(err.Error(), "relevance judgments") +} diff --git a/cmd/msgvault/cmd/eval_rank_test.go b/cmd/msgvault/cmd/eval_rank_test.go new file mode 100644 index 000000000..5eadf7caf --- /dev/null +++ b/cmd/msgvault/cmd/eval_rank_test.go @@ -0,0 +1,411 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/query/querytest" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/hybrid" +) + +// vectorTestGeneration is the active generation the fake backend serves; the +// hybrid engine refuses to search unless the fingerprint matches its config. +var vectorTestGeneration = vector.Generation{ + ID: 1, Model: "test-model", Dimension: 4, + Fingerprint: "test-model:4", State: vector.GenerationActive, +} + +// saturatingFusingBackend is a vector.FusingBackend that returns a fixed hit +// list together with a caller-chosen saturation flag, so a test can pin what +// the eval path does with the flag the real engine computes. Only the two +// methods the hybrid search path calls are implemented; the embedded interface +// panics on anything else, which is the point — a widened call is a test bug, +// not something to silently stub. +type saturatingFusingBackend struct { + vector.Backend + + generation vector.Generation + hits []vector.FusedHit + saturated bool + fusedCalls int +} + +func (b *saturatingFusingBackend) ActiveGeneration(context.Context) (vector.Generation, error) { + return b.generation, nil +} + +func (b *saturatingFusingBackend) FusedSearch( + context.Context, vector.FusedRequest, +) ([]vector.FusedHit, bool, error) { + b.fusedCalls++ + return b.hits, b.saturated, nil +} + +// stubEmbedder returns a fixed query vector; the fake backend never looks at +// it, but the engine insists on embedding before it will search. +type stubEmbedder struct{} + +func (stubEmbedder) EmbedQuery(context.Context, string) ([]float32, error) { + return []float32{1, 0, 0, 0}, nil +} + +// threadedCorpus builds a ranked corpus of threads*perThread messages, ordered +// thread by thread: the first perThread hits all belong to one conversation. +// That is the shape that breaks a truncate-then-collapse ranking. +func threadedCorpus(threads, perThread int) []store.APIMessage { + out := make([]store.APIMessage, 0, threads*perThread) + var id int64 + for t := range threads { + for m := range perThread { + id++ + out = append(out, store.APIMessage{ + ID: id, + SourceMessageID: fmt.Sprintf("", t, m), + SourceConversationID: fmt.Sprintf("thread-%03d", t), + }) + } + } + return out +} + +// pagingFTS serves the first n results of corpus and records the depths it was +// asked for, so a test can assert both the answer and the work done to get it. +// It stands in for *store.Store on the production relevance-ranked FTS path. +type pagingFTS struct { + corpus []store.APIMessage + depths []int +} + +func (p *pagingFTS) SearchMessagesQueryContext( + _ context.Context, _ *search.Query, _, limit int, +) ([]store.APIMessage, int64, error) { + p.depths = append(p.depths, limit) + if limit > len(p.corpus) { + limit = len(p.corpus) + } + return p.corpus[:limit], int64(len(p.corpus)), nil +} + +func newPagingFTS(corpus []store.APIMessage) *pagingFTS { + return &pagingFTS{corpus: corpus} +} + +// evalTestLimit is the retrieval depth every ranking test uses; it matches the +// command's own -n default, so the over-fetch arithmetic in the assertions is +// the arithmetic a real run does. +const evalTestLimit = 100 + +func newTestEvaluator(t *testing.T, fts ftsSearcher, docKey string) (*evaluator, *runDiagnostics) { + t.Helper() + spec, ok := newDocKeyRegistry()[docKey] + require.True(t, ok) + diag := &runDiagnostics{} + return &evaluator{ctx: t.Context(), fts: fts, key: spec, limit: evalTestLimit, diag: diag}, diag +} + +// evalTestQuery parses a topic the way runEval does, so the ranking tests +// exercise the same already-validated query object the command builds. +func evalTestQuery(t *testing.T, qstr string) *search.Query { + t.Helper() + q := search.Parse(qstr) + require.NoError(t, q.Err()) + return q +} + +// TestRankedFTS_ConversationKeyFillsTheRequestedDepth is the regression for the +// ordering bug. The corpus has 4 messages per thread, so a raw fetch of 100 +// covers only 25 threads. Collapsing after truncating returned those 25 and +// reported them as "R@100"; the ranking must instead over-fetch, collapse, and +// hand back 100 distinct threads. +func TestRankedFTS_ConversationKeyFillsTheRequestedDepth(t *testing.T) { + corpus := threadedCorpus(200, 4) + eng := newPagingFTS(corpus) + ev, diag := newTestEvaluator(t, eng, "conversation") + + // For contrast, what the old ordering produced: collapsing the first + // --limit raw hits only ever reached a quarter of the requested depth. + firstPage := make([]string, 0, 100) + for _, m := range corpus[:100] { + firstPage = append(firstPage, m.SourceConversationID) + } + require.Len(t, eval.DedupeKeys(firstPage), 25, "truncate-then-collapse caps out at 25 threads here") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + + require.Len(t, ranked, 100, "-n 100 with --doc-key=conversation means 100 distinct threads") + assert.Equal(t, "thread-000", ranked[0], "best rank wins") + assert.Equal(t, "thread-099", ranked[99]) + seen := map[string]struct{}{} + for _, k := range ranked { + _, dup := seen[k] + require.False(t, dup, "collapsed ranking must not repeat a thread: %s", k) + seen[k] = struct{}{} + } + assert.Equal(t, []int{400}, eng.depths, "one over-fetch was enough here") + assert.Zero(t, diag.DepthShortfalls) +} + +// TestRankedFTS_MessageKeyDoesNotOverFetch: the message doc-key is 1:1 with +// hits, so padding the query would only inflate the latency this command +// reports. +func TestRankedFTS_MessageKeyDoesNotOverFetch(t *testing.T) { + eng := newPagingFTS(threadedCorpus(200, 5)) + ev, _ := newTestEvaluator(t, eng, "message") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 100) + assert.Equal(t, []int{100}, eng.depths, "no over-fetch for a 1:1 doc-key") +} + +// TestRankedFTS_GrowsThePoolUntilTheDepthIsFilled: one over-fetch is not always +// enough. With 50 messages per thread, limit*4 still only covers 8 threads, so +// the pool has to grow. +func TestRankedFTS_GrowsThePoolUntilTheDepthIsFilled(t *testing.T) { + eng := newPagingFTS(threadedCorpus(200, 50)) + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 100) + assert.Equal(t, []int{400, 1600, 6400}, eng.depths, "the pool grows geometrically") + assert.Zero(t, diag.DepthShortfalls, "the depth was filled on the last attempt") +} + +// TestRankedFTS_StopsWhenTheEngineIsExhausted: a corpus smaller than the pool +// must not trigger pointless retries, and the short answer is not a shortfall +// worth warning about — there is simply nothing more to retrieve. +func TestRankedFTS_StopsWhenTheEngineIsExhausted(t *testing.T) { + eng := newPagingFTS(threadedCorpus(6, 5)) // 30 messages, 6 threads + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 6, "six threads exist; six threads come back") + assert.Equal(t, []int{400}, eng.depths, "the engine came back short, so stop") + assert.Zero(t, diag.DepthShortfalls) + assert.Zero(t, diag.PoolShortfalls, "an unsaturated short page is an exhausted corpus, not a shortfall") +} + +// TestRankedFTS_ReportsADepthShortfall: when even the largest pool cannot fill +// the depth, the metrics are computed over a shallower list than requested and +// the run has to say so rather than passing the number off as full depth. +func TestRankedFTS_ReportsADepthShortfall(t *testing.T) { + // One single thread, deeper than the biggest pool: every fetch is + // saturated, and every fetch collapses to one key. + eng := newPagingFTS(threadedCorpus(1, 20_000)) + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Equal(t, []string{"thread-000"}, ranked) + assert.Equal(t, []int{400, 1600, 6400}, eng.depths, "the pool grows to the documented ceiling and stops") + assert.Equal(t, 1, diag.DepthShortfalls) + assert.Contains(t, diag.notes()[0], "could not fill") +} + +// TestRankedKeys_SaturatedShortFetchIsNotCorpusExhaustion is the regression for +// the discarded PoolSaturated flag. A fused query caps each signal at +// k_per_signal, so it can return fewer hits than asked for while the corpus +// still holds plenty more. Reading that short page as "the corpus ran out" +// reported a pool-capped ranking as if it were everything retrieval could +// find — the one reading that makes the resulting metric silently wrong. +func TestRankedKeys_SaturatedShortFetchIsNotCorpusExhaustion(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + diag.kPerSignal = 250 + + var asked []int + ranked, err := ev.rankedKeys(func(n int) (fetchResult, error) { + asked = append(asked, n) + // 40 hits back for a request of 400, spread over 10 threads, with + // the engine reporting that its candidate pool was full. + keys := make([]string, 0, 40) + for i := range 40 { + keys = append(keys, fmt.Sprintf("thread-%03d", i%10)) + } + return fetchResult{keys: keys, raw: len(keys), saturated: true}, nil + }) + require.NoError(t, err) + + assert.Len(t, ranked, 10, "the ten reachable threads are still scored") + assert.Equal(t, []int{400}, asked, + "a bigger page cannot get past k_per_signal, so do not burn another query on it") + assert.Equal(t, 1, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls, "this is a pool ceiling, not an exhausted over-fetch budget") + + notes := diag.notes() + require.Len(t, notes, 1) + assert.Contains(t, notes[0], "candidate pool") + assert.Contains(t, notes[0], "k_per_signal=250", "the note names the setting that caused it") + assert.Contains(t, notes[0], "not an exhausted corpus", + "the whole point of the flag is to keep the two apart") +} + +// TestRankedKeys_UnsaturatedShortFetchIsExhaustion pins the other half of the +// pair: the same short page, with the engine reporting it had nothing left, +// is not a shortfall at all and must stay silent. +func TestRankedKeys_UnsaturatedShortFetchIsExhaustion(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + + ranked, err := ev.rankedKeys(func(int) (fetchResult, error) { + return fetchResult{keys: []string{"thread-000", "thread-001"}, raw: 2}, nil + }) + require.NoError(t, err) + + assert.Equal(t, []string{"thread-000", "thread-001"}, ranked) + assert.Zero(t, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls) + assert.Empty(t, diag.notes(), "an exhausted corpus is an answer, not an anomaly") +} + +// TestRankedKeys_SaturatedFullDepthIsNotAShortfall: saturation only matters +// when the depth went unfilled. A run that got everything it asked for has +// nothing to warn about, however full the engine's pool was. +func TestRankedKeys_SaturatedFullDepthIsNotAShortfall(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + + keys := make([]string, 0, evalTestLimit) + for i := range evalTestLimit { + keys = append(keys, fmt.Sprintf("thread-%03d", i)) + } + ranked, err := ev.rankedKeys(func(int) (fetchResult, error) { + return fetchResult{keys: keys, raw: len(keys), saturated: true}, nil + }) + require.NoError(t, err) + + assert.Len(t, ranked, evalTestLimit) + assert.Zero(t, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls) +} + +// TestRankedVector_CarriesPoolSaturationFromTheEngine checks the wiring the +// classification above depends on: hybrid.ResultMeta.PoolSaturated has to +// survive the trip from the engine into fetchResult. It used to be dropped on +// the floor at the call site, which no amount of correct classification +// downstream could recover from. +func TestRankedVector_CarriesPoolSaturationFromTheEngine(t *testing.T) { + // One hit for a request of 400, with the backend reporting a full pool. + backend := &saturatingFusingBackend{ + generation: vectorTestGeneration, + hits: []vector.FusedHit{{MessageID: 1, RRFScore: 0.9}}, + saturated: true, + } + qeng := &querytest.MockEngine{ + GetMessageSummariesByIDsFunc: func(_ context.Context, ids []int64) ([]query.MessageSummary, error) { + out := make([]query.MessageSummary, 0, len(ids)) + for _, id := range ids { + out = append(out, query.MessageSummary{ + ID: id, + SourceMessageID: fmt.Sprintf("", id), + SourceConversationID: "thread-000", + }) + } + return out, nil + }, + } + ev, diag := newTestEvaluator(t, nil, "conversation") + ev.qeng = qeng + ev.heng = hybrid.NewEngine(backend, nil, stubEmbedder{}, hybrid.Config{ + ExpectedFingerprint: vectorTestGeneration.Fingerprint, + }) + + ranked, err := ev.rankedVector("hybrid", "lease renewal", evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + + assert.Equal(t, []string{"thread-000"}, ranked) + assert.Equal(t, 1, backend.fusedCalls, "a saturated pool must not be retried at a deeper page") + assert.Equal(t, 1, diag.PoolShortfalls, "the engine's own saturation flag has to reach the diagnostics") + assert.Zero(t, diag.DepthShortfalls) +} + +// TestRankedVector_FilterOnlyTopicIsRecoverable: a topic that parses to filters +// only has nothing to embed. It must surface as errNoFreeText — which runEval +// turns into a skipped cell — and never as an opaque failure that aborts the +// run and discards every score computed so far. +func TestRankedVector_FilterOnlyTopicIsRecoverable(t *testing.T) { + const topic = "from:alice@example.com" + parsed := evalTestQuery(t, topic) + require.Empty(t, parsed.TextTerms, "fixture assumption: this topic is filter-only") + + // heng is deliberately nil: the check must happen before the engine is + // ever touched. + ev, _ := newTestEvaluator(t, nil, "message") + ev.qeng = &querytest.MockEngine{} + + _, err := ev.rankedVector("vector", topic, parsed) + require.Error(t, err) + require.ErrorIs(t, err, errNoFreeText) + assert.Contains(t, err.Error(), topic, "the error names the offending topic") +} + +// TestParseTopic_RejectsAMalformedFilter is the regression for silently +// widened topics. search.Parse drops an operator value it cannot read and +// carries on, so `before:invalid renewal` becomes the unfiltered query +// `renewal` — a different question, scored under the original topic's id. +func TestParseTopic_RejectsAMalformedFilter(t *testing.T) { + const bad = "before:invalid renewal" + + // What the old code would have run: the date filter is gone, and only + // the bare term survives. This is the query that must NOT be scored. + widened := search.Parse(bad) + require.Error(t, widened.Err(), "fixture assumption: this topic does not parse cleanly") + require.Nil(t, widened.BeforeDate, "the malformed filter is dropped, not honoured") + require.Equal(t, []string{"renewal"}, widened.TextTerms, "leaving a strictly broader query behind") + + diag := &runDiagnostics{} + q, ok := parseTopic(eval.Topic{ID: "q7", Query: bad}, diag) + assert.False(t, ok, "a topic that does not parse must not be scored") + assert.Nil(t, q) + + notes := diag.notes() + require.Len(t, notes, 1) + assert.Contains(t, notes[0], "topic q7", "the report names the offending topic") + assert.Contains(t, notes[0], "before", "and the operator that failed") +} + +// TestParseTopic_AcceptsAWellFormedFilter: the guard rejects malformed values, +// not filters in general. A topic with a valid date filter parses through with +// the filter intact. +func TestParseTopic_AcceptsAWellFormedFilter(t *testing.T) { + diag := &runDiagnostics{} + q, ok := parseTopic(eval.Topic{ID: "q8", Query: "before:2024-01-01 renewal"}, diag) + require.True(t, ok) + require.NotNil(t, q) + assert.NotNil(t, q.BeforeDate, "a filter that parses is kept") + assert.Equal(t, []string{"renewal"}, q.TextTerms) + assert.Empty(t, diag.notes(), "a clean topic says nothing") +} + +// TestRunDiagnostics_Notes checks that every silent failure mode this run can +// hit produces a line a user will actually read. +func TestRunDiagnostics_Notes(t *testing.T) { + assert.Empty(t, (&runDiagnostics{}).notes(), "a clean run says nothing") + + d := &runDiagnostics{UnhydratedHits: 7} + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "7 retrieved hits could not be hydrated") + + d = &runDiagnostics{} + d.skip("q3", "hybrid", "no free-text terms to embed (filter-only topic)") + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "topic q3 / hybrid") + + // The pool note stays readable when the run never opened the vector + // path, so an unknown k_per_signal is never rendered as "k_per_signal=0". + d = &runDiagnostics{PoolShortfalls: 2} + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "candidate pool") + assert.NotContains(t, d.notes()[0], "k_per_signal=0") +} diff --git a/cmd/msgvault/cmd/eval_source_scope_test.go b/cmd/msgvault/cmd/eval_source_scope_test.go new file mode 100644 index 000000000..ef1948c5f --- /dev/null +++ b/cmd/msgvault/cmd/eval_source_scope_test.go @@ -0,0 +1,217 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/store" +) + +// seedTwoSourceArchiveIn builds an archive with two connected accounts whose +// source-assigned ids overlap, which is the shape a multi-source archiver +// routinely ends up in and no single-account benchmark corpus ever has. +// +// source 1 (gmail) thread-1 / , thread-1 / +// source 2 (whatsapp) thread-1 / +// +// Both the message id and the conversation id are shared across the two +// sources, and every message is live, so retrieval can return either copy. +// Passing shareMessageID=false keeps the message ids disjoint while leaving the +// conversation ids shared, so a test can tell the two doc-keys apart. +func seedTwoSourceArchiveIn(t *testing.T, dataDir string, shareMessageID bool) *store.Store { + t.Helper() + require := require.New(t) + + s, err := store.Open(filepath.Join(dataDir, "msgvault.db")) + require.NoError(err, "open store") + t.Cleanup(func() { require.NoError(s.Close()) }) + require.NoError(s.InitSchema(), "init schema") + + secondMessageID := "" + if !shareMessageID { + secondMessageID = "" + } + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES + (1, 'gmail', 'me@example.com'), + (2, 'whatsapp', '+15550100'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 2, 'thread-1', 'whatsapp_chat'); +INSERT INTO messages + (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, size_estimate) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100), + (2, 1, 1, '', 'email', 'Re: Lease renewal', '2020-01-02T00:00:00Z', 100), + (3, 2, 2, ?, 'whatsapp', 'Lease renewal chat', '2020-01-03T00:00:00Z', 100); +`, secondMessageID) + require.NoError(err, "seed two-source archive") + + for id, body := range map[int64]string{ + 1: "Signed and returned.", + 2: "Counter-signed.", + 3: "Unrelated chat that happens to reuse the id.", + } { + _, err = s.DB().Exec( + `INSERT INTO message_bodies (message_id, body_text) VALUES (?, ?)`, id, body) + require.NoError(err, "seed body %d", id) + } + indexed, err := s.BackfillFTS(nil) + require.NoError(err, "backfill FTS") + require.EqualValues(3, indexed, "every message must be indexed") + return s +} + +// TestRankedFTS_CrossSourceIDsCollapseIntoOneKey characterizes the hazard the +// precondition exists to prevent, so the guard below is not asserting the +// absence of a problem nobody has shown to exist. +// +// Message 1 (gmail) and message 3 (whatsapp) are unrelated documents that +// happen to carry the same source-assigned id. Retrieval finds both, and the +// doc-key extraction reduces them to one key: the ranking hands the scoring +// core two hits' worth of evidence under a single id, so a judgment written +// about the gmail message silently grades the whatsapp one as well, and the +// depth quietly loses a rank. +func TestRankedFTS_CrossSourceIDsCollapseIntoOneKey(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + const topic = "renewal" + + found, _, err := s.SearchMessagesQueryContext(t.Context(), evalTestQuery(t, topic), 0, 10) + require.NoError(err, "production store search") + require.Len(found, 3, "all three messages are live and match the topic") + + ev, _ := newTestEvaluator(t, s, "message") + ev.limit = 10 + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(err, "rankedFTS") + + assert.Len(ranked, 2, + "three retrieved messages reduce to two keys: the two sources' ids collided") + assert.Contains(ranked, "", + "and the surviving key names a document in each source at once") +} + +// TestRequireDisjointSourceIDs_RejectsCollidingMessageIDs is the guard for that +// collapse under --doc-key=message. The run must stop with an error that names +// the colliding id, the key, and the column whose uniqueness does not hold. +func TestRequireDisjointSourceIDs_RejectsCollidingMessageIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + registry := newDocKeyRegistry() + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "message", registry["message"]) + require.Error(err, "a colliding message id must stop the run, not be scored") + assert.Contains(err.Error(), "", "the offending id is named") + assert.Contains(err.Error(), "source_message_id", "and so is the column whose uniqueness failed") + assert.NotContains(err.Error(), "", + "an id held by one source only is not a collision") +} + +// TestRequireDisjointSourceIDs_RejectsCollidingConversationIDs pins the same +// guard for the coarser key, whose id lives on conversations rather than +// messages. Here the message ids are disjoint and only the thread ids overlap, +// so the two keys must reach opposite verdicts on the same archive — a check +// that quietly looked at one column for both keys would pass this. +func TestRequireDisjointSourceIDs_RejectsCollidingConversationIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + registry := newDocKeyRegistry() + + require.NoError(requireDisjointSourceIDs(t.Context(), s.DB(), "message", registry["message"]), + "the message ids are disjoint in this archive") + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "conversation", registry["conversation"]) + require.Error(err, "the thread ids are not") + assert.Contains(err.Error(), "thread-1") + assert.Contains(err.Error(), "source_conversation_id") +} + +// TestRequireDisjointSourceIDs_AllowsSeveralSourcesWithDisjointIDs is the other +// half of the decision: the precondition is disjointness, not single-source. +// msgvault archives normally hold several accounts, and refusing to score one +// because it has two sources — when nothing in it can collide — would be a wall +// built for a hazard that is not there. +func TestRequireDisjointSourceIDs_AllowsSeveralSourcesWithDisjointIDs(t *testing.T) { + require := require.New(t) + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + + var sources int + require.NoError(s.DB().QueryRow( + `SELECT COUNT(DISTINCT source_id) FROM messages`).Scan(&sources)) + require.Equal(2, sources, "the archive really does hold two connected accounts") + + require.NoError(requireDisjointSourceIDs( + t.Context(), s.DB(), "message", newDocKeyRegistry()["message"]), + "two accounts that share no message id are scorable") +} + +// TestRequireDisjointSourceIDs_IgnoresUnretrievableCopies pins the population +// the check runs over. A dedup-hidden or source-deleted copy is not something +// any search here returns, so it cannot reach a ranking and cannot collide with +// anything — counting it would refuse to score an archive that is fine. +func TestRequireDisjointSourceIDs_IgnoresUnretrievableCopies(t *testing.T) { + require := require.New(t) + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + + // Hide the second source's copy the way dedup does, leaving the row in + // place. + _, err := s.DB().Exec( + `UPDATE messages SET deleted_at = '2023-01-01T00:00:00Z' WHERE id = 3`) + require.NoError(err, "hide the duplicate") + + require.NoError(requireDisjointSourceIDs( + t.Context(), s.DB(), "message", newDocKeyRegistry()["message"]), + "the surviving copy is the only retrievable one, so the id names one document") +} + +// TestRequireDisjointSourceIDs_RejectsAKeyWithNoArchiveColumn keeps the check +// from being skipped by omission. A doc-key added without saying where its ids +// live would otherwise sail past the guard and reintroduce the collapse; it has +// to fail loudly instead, naming itself. +func TestRequireDisjointSourceIDs_RejectsAKeyWithNoArchiveColumn(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + unbacked := docKeySpec{extract: func(evalHit) string { return "x" }} + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "thread", unbacked) + require.Error(err, "an unbacked key must not silently skip the check") + assert.Contains(err.Error(), `"thread"`) +} + +// TestRunEval_StopsOnCrossSourceIDCollisions drives the whole command, so the +// guard is pinned at the call site and not just in its own helper. Without it +// the run completes and reports a perfect MRR for q1: the whatsapp message +// carries the gmail message's id, so its hit is scored against the gmail +// message's judgment. +func TestRunEval_StopsOnCrossSourceIDCollisions(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedTwoSourceArchiveIn(t, dir, true) + configureEvalRun(t, dir, + "q1 0 1\n", + "q1\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + + err := runEval(cmd, nil) + require.Error(err, "the run must stop rather than print a number it cannot justify") + assert.Contains(err.Error(), "more than one connected source") + assert.Contains(err.Error(), "--doc-key=message") +} diff --git a/cmd/msgvault/cmd/eval_stub.go b/cmd/msgvault/cmd/eval_stub.go new file mode 100644 index 000000000..4f0e35803 --- /dev/null +++ b/cmd/msgvault/cmd/eval_stub.go @@ -0,0 +1,25 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// evalCmd is a stub for builds that lack the sqlite_vec build tag. The eval +// command exercises vector/hybrid retrieval, which needs the sqlite-vec +// extension; binaries from `make build` (which sets `-tags "fts5 sqlite_vec"`) +// use the real implementation in eval.go. +var evalCmd = &cobra.Command{ + Use: "eval", + Short: "Evaluate retrieval quality against relevance judgments (requires sqlite_vec build)", + RunE: func(_ *cobra.Command, _ []string) error { + return fmt.Errorf("eval requires sqlite-vec support; rebuild with `go build -tags \"fts5 sqlite_vec\"`") + }, +} + +func init() { + rootCmd.AddCommand(evalCmd) +} diff --git a/cmd/msgvault/cmd/eval_topics_test.go b/cmd/msgvault/cmd/eval_topics_test.go new file mode 100644 index 000000000..25a3578d4 --- /dev/null +++ b/cmd/msgvault/cmd/eval_topics_test.go @@ -0,0 +1,193 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/eval" +) + +// evalTopicReport is the slice of the JSON report the topic-handling tests +// read: how many topics were scored, what the scored ones came to, and what the +// run said about the ones it did not score. +type evalTopicReport struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + Topics int `json:"topics"` + } `json:"results"` + Diagnostics struct { + SkippedCells []string `json:"skipped_cells"` + UnjudgedTopics []string `json:"unjudged_topics"` + } `json:"diagnostics"` +} + +// runEvalForReport drives the configured eval run and decodes its JSON report. +func runEvalForReport(t *testing.T, out *evalTopicReport) { + t.Helper() + cmd := &cobra.Command{} + cmd.SetContext(t.Context()) + + done := captureStdout(t) + err := runEval(cmd, nil) + text := done() + require.NoError(t, err, "eval run") + require.NoError(t, json.Unmarshal([]byte(text), out), "parse report: %s", text) +} + +// TestRunEval_SkipsATopicThatParsedToAnEmptyQuery is the regression for a topic +// that is non-empty text, parses without error, and still carries no search +// criteria. +// +// `subject:""` is the plain case: the parser drops an empty operator value +// rather than building a `LIKE '%%'` that matches everything, so Query.Err() is +// nil and Query.IsEmpty() is true. Scoring it does not measure retrieval at +// all — the fts path answers an empty query by listing the whole live corpus, +// newest first — so the topic collects whatever the archive's date +// distribution hands it. Here that is a spurious MRR of 0.5 for q2: its judged +// message is the archive's oldest, so the corpus listing puts it second, and +// the run reported a headline 0.75 over "two" topics. +// +// The honest answer is one topic at MRR 1.0, with q2 reported as unscorable. +func TestRunEval_SkipsATopicThatParsedToAnEmptyQuery(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 1\n", + "q1\trenewal\nq2\tsubject:\"\"\n") + + var report evalTopicReport + runEvalForReport(t, &report) + scored := report.Results[evalTestMode] + + assert.Equal(1, report.TopicsEvaluated, "the criteria-less topic is not a measurement") + assert.Equal(1, scored.Topics) + assert.InDelta(1.0, scored.MRR, 1e-9, + "only q1 is scored; folding in the full-corpus scan reported 0.75") + + require.Len(report.Diagnostics.SkippedCells, 1, "the skip must be reported, not silent") + assert.Contains(report.Diagnostics.SkippedCells[0], "topic q2") + assert.Contains(report.Diagnostics.SkippedCells[0], "no search criteria") +} + +// TestRunEval_ReportsTopicsWithNoMatchingJudgments pins the coverage half of +// the qrels join. Not scoring an unjudged topic is right — there is nothing to +// score it against — but one judged topic is enough for the command to print a +// headline number, so a qrels file that matches a fraction of the topics file +// reports a mean over a small, self-selected subset and looks like a complete +// run. Here three of four topics go unjudged and the survivor scores a perfect +// 1.0; the run has to say which three it dropped and out of how many. +func TestRunEval_ReportsTopicsWithNoMatchingJudgments(t *testing.T) { + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, + "q1 0 1\n", + "q1\trenewal\nq2\trenewal\nq3\trenewal\nq4\trenewal\n") + + var report evalTopicReport + runEvalForReport(t, &report) + + assert.Equal(1, report.TopicsEvaluated) + assert.Equal([]string{"q2", "q3", "q4"}, report.Diagnostics.UnjudgedTopics, + "the unscored topics are named, in topics-file order") + assert.Empty(report.Diagnostics.SkippedCells, + "an unjudged topic is not a skip: nothing went wrong with it") + + // The same fact has to reach the table output, which renders notes() and + // never sees the JSON block. + note := findEvalNote(t, report.Diagnostics.UnjudgedTopics, report.TopicsEvaluated, 4) + assert.Contains(note, "3 of 4 topics had no matching qrels entry") + assert.Contains(note, "q2, q3, q4") + assert.Contains(note, "cover 1 of the topics file") +} + +// TestRunEval_UnjudgedCoverageNoteCountsWhatWasActuallyScored is the +// regression for conflating "judged" with "scored". A judged topic that +// parses to no search criteria is skipped, same as an unjudged one, so +// subtracting only the unjudged count from the topics-file total overstates +// what the headline numbers cover. Here q1 scores, q2 is judged but +// criteria-less, q3 is unjudged: TopicsEvaluated is 1, not +// Parsed-len(unjudged) = 3, and the note has to say 1, matching the same +// number the JSON report's own topics_evaluated is built from. +func TestRunEval_UnjudgedCoverageNoteCountsWhatWasActuallyScored(t *testing.T) { + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 1\n", + "q1\trenewal\nq2\tsubject:\"\"\nq3\trenewal\n") + + var report evalTopicReport + runEvalForReport(t, &report) + + assert.Equal(1, report.TopicsEvaluated, "only q1 scores; q2 is judged but criteria-less") + assert.Equal([]string{"q3"}, report.Diagnostics.UnjudgedTopics) + + note := findEvalNote(t, report.Diagnostics.UnjudgedTopics, report.TopicsEvaluated, 3) + assert.Contains(note, "1 of 3 topics had no matching qrels entry") + assert.Contains(note, "cover 1 of the topics file", + "not 2 (Parsed-unjudged): q2 was judged but never scored") +} + +// TestRunEval_CountsEachTopicOnceForARepeatedMode is the end-to-end regression +// for a duplicated --modes value. The aggregates are keyed by mode name, so +// `--modes fts,fts` added every topic's score to the fts aggregate twice: the +// report claimed two topics for a two-topic run over one mode, over a run that +// had also issued every query twice. The count beside the means is the +// denominator a reader compares two runs by, so it has to be the number of +// topics, not the number of times the list mentioned the mode. +func TestRunEval_CountsEachTopicOnceForARepeatedMode(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + configureEvalRun(t, dir, + "q1 0 1\nq2 0 1\n", + "q1\trenewal\nq2\trenewal\n") + // configureEvalRun snapshots and restores every eval flag, so overriding + // one after it is safe. + evalModes = evalTestMode + "," + evalTestMode + + var report evalTopicReport + runEvalForReport(t, &report) + + require.Len(report.Results, 1, "a repeated mode is still one mode") + assert.Equal(2, report.TopicsEvaluated) + assert.Equal(2, report.Results[evalTestMode].Topics, + "two topics scored once each; counting the mode twice reported 4") +} + +// findEvalNote re-renders the diagnostics the table output would print and +// returns the line covering the given unjudged topics. scored is the number +// of topics the run actually scored and parsed is the topics file's total, +// both matching runEval's own diag.scored assignment and TopicsLoad.Parsed — +// a caller passing report.TopicsEvaluated and the real topics count keeps +// the synthetic diagnostics consistent with the run it is standing in for. +func findEvalNote(t *testing.T, unjudged []string, scored, parsed int) string { + t.Helper() + diag := &runDiagnostics{UnjudgedTopics: unjudged, scored: scored} + diag.TopicsLoad = eval.LoadStats{Path: "topics.tsv", Lines: parsed, Parsed: parsed} + for _, n := range diag.notes() { + if strings.Contains(n, "no matching qrels entry") { + return n + } + } + t.Fatalf("no coverage note in %v", diag.notes()) + return "" +} diff --git a/cmd/msgvault/cmd/eval_vector_config_test.go b/cmd/msgvault/cmd/eval_vector_config_test.go new file mode 100644 index 000000000..507337a08 --- /dev/null +++ b/cmd/msgvault/cmd/eval_vector_config_test.go @@ -0,0 +1,133 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// evalVectorConfig builds a config whose vector section is valid apart from +// the embeddings fields the caller overrides, pointed at a scratch data dir. +func evalVectorConfig(t *testing.T, format vector.EmbeddingAPIFormat, model string) *config.Config { + t.Helper() + c := &config.Config{} + c.Data.DataDir = t.TempDir() + c.Vector.Enabled = true + c.Vector.Embeddings.Endpoint = "http://127.0.0.1:1/v1" + c.Vector.Embeddings.Model = model + c.Vector.Embeddings.APIFormat = format + c.Vector.Embeddings.Dimension = 1024 + c.Vector.ApplyDefaults() + return c +} + +// TestAttachVector_RejectsUnusableEmbeddingConfig pins the eval command's +// fail-fast contract on the resolved vector config. An api_format this binary +// cannot build a client for, and a contextual format paired with a model the +// contextual endpoint does not serve, must both stop the run with an error +// naming the offending value — before any index is opened, so a bad config can +// never be scored as poor retrieval. +func TestAttachVector_RejectsUnusableEmbeddingConfig(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f, _, _ := setupScopeFixture(t) + + c := evalVectorConfig(t, "voyage", "voyage-context-4") + withTestConfig(t, c) + + ev := &evaluator{ctx: context.Background()} + cleanup, err := ev.attachVector(context.Background(), f.Store) + require.Error(err, "an unsupported api_format must not fall back to the OpenAI-compatible client") + assert.Nil(cleanup) + assert.Contains(err.Error(), "api_format") + assert.Contains(err.Error(), `"voyage"`) + + _, statErr := os.Stat(filepath.Join(c.Data.DataDir, "vectors.db")) + assert.True(os.IsNotExist(statErr), "the config check must run before the index is opened") +} + +// seedActiveGeneration activates an empty generation carrying the config's +// own fingerprint, so attachVector gets past the active-generation check and +// on to the part under test. +func seedActiveGeneration(t *testing.T, dataDir, mainPath string, mainDB *sql.DB, vecCfg vector.Config) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, vecCfg.Embeddings.Dimension, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") +} + +// TestAttachVector_EmbedsQueriesThroughConfiguredAPIFormat is the end-to-end +// binding: run the eval command's own vector setup against a config that says +// api_format = "voyage-contextual", then embed a query through the engine it +// wired and watch what goes over the wire. Before this fix the request landed +// on /v1/embeddings as a flat OpenAI-compatible body, so a contextual index was +// scored with query vectors from a different endpoint and a different role. +func TestAttachVector_EmbedsQueriesThroughConfiguredAPIFormat(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}]}`) + + c := evalVectorConfig(t, vector.APIFormatVoyageContextual, "voyage-context-4") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Endpoint = endpoint + c.Vector.Embeddings.Dimension = 3 + withTestConfig(t, c) + seedActiveGeneration(t, dataDir, c.DatabaseDSN(), s.DB(), c.Vector) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + vec, err := ev.heng.EmbedQuery(ctx, "lease renewal") + require.NoError(err, "EmbedQuery") + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/contextualizedembeddings", path, "queries must go to the contextual endpoint") + assert.Equal("query", body["input_type"], "queries must carry the query role, not the document role") + assert.Equal("voyage-contextual", ev.prov.APIFormat, "the run reports the format that produced its scores") +} + +// TestAttachVector_RejectsContextualModelMismatch covers the other resolved +// config the eval tool must refuse: api_format = "voyage-contextual" with a +// non-contextual model. +func TestAttachVector_RejectsContextualModelMismatch(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f, _, _ := setupScopeFixture(t) + + withTestConfig(t, evalVectorConfig(t, vector.APIFormatVoyageContextual, "voyage-large-4")) + + ev := &evaluator{ctx: context.Background()} + _, err := ev.attachVector(context.Background(), f.Store) + require.Error(err) + assert.Contains(err.Error(), "voyage-large-4") +} diff --git a/cmd/msgvault/cmd/eval_vector_corpus_test.go b/cmd/msgvault/cmd/eval_vector_corpus_test.go new file mode 100644 index 000000000..7e4a584c0 --- /dev/null +++ b/cmd/msgvault/cmd/eval_vector_corpus_test.go @@ -0,0 +1,237 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// seedEmbeddedGeneration embeds exactly msgIDs, activates the result, and +// stamps embed_gen on those rows in the main DB the way a real embed run +// would — needed because EmbeddedMessageCount requires the stamp, not just a +// vectors.db row, to count a message as embedded. +func seedEmbeddedGeneration( + t *testing.T, dataDir, mainPath string, mainDB *store.Store, vecCfg vector.Config, msgIDs ...int64, +) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB.DB(), + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, vecCfg.Embeddings.Dimension, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + + dim := vecCfg.Embeddings.Dimension + chunks := make([]vector.Chunk, 0, len(msgIDs)) + for _, id := range msgIDs { + v := make([]float32, dim) + for i := range v { + v[i] = float32(id) / 10 + } + chunks = append(chunks, vector.Chunk{MessageID: id, Vector: v, SourceCharLen: 32}) + } + require.NoError(t, b.Upsert(ctx, gen, chunks), "Upsert") + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") + + for _, id := range msgIDs { + _, err = mainDB.DB().Exec(`UPDATE messages SET embed_gen = ? WHERE id = ?`, int64(gen), id) + require.NoError(t, err, "stamp embed_gen on message %d", id) + } +} + +// TestAttachVector_ReportsScopedCorpusSeparatelyFromTheArchive is the +// regression for provenance that described the archive when a run's vector +// generation only ever searches part of it. An account-scoped +// [vector.embed.scope] means retrieval can return source 1's two messages +// and never source 2's, so a report that prints the whole three-message, +// two-conversation archive as "the corpus" overstates what vector mode's +// recall is actually measured against. +func TestAttachVector_ReportsScopedCorpusSeparatelyFromTheArchive(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedTwoSourceArchiveIn(t, dataDir, false) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.SourceIDs = []int64{1} + withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(3, ev.prov.Messages, "the archive holds three live messages across both sources") + assert.EqualValues(2, ev.prov.Conversations, "and two conversations") + assert.EqualValues(2, ev.prov.VectorMessages, + "only source 1's two messages are embedded in the active generation") + assert.EqualValues(1, ev.prov.VectorConversations, + "and only source 1's one conversation") +} + +// TestAttachVector_CorpusScopeFilterExcludesOutOfScopeStampedMessages pins +// the scope-filter clause in collectVectorCorpusStats specifically, as +// opposed to the embed_gen stamp doing all the work by coincidence. Message +// 3 (source 2) is embedded and stamped into the very same generation as +// messages 1 and 2 — as if a stray embed run, or a since-narrowed scope, +// left an out-of-scope message carrying this generation's stamp — while +// [vector.embed.scope] declares only source 1. Without the source_id filter +// this would count all three messages and both conversations; with it, the +// out-of-scope stamp must not move the reported corpus at all. +func TestAttachVector_CorpusScopeFilterExcludesOutOfScopeStampedMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedTwoSourceArchiveIn(t, dataDir, false) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.SourceIDs = []int64{1} + withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2, 3) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "source 2's stamped-but-out-of-scope message must not count") + assert.EqualValues(1, ev.prov.VectorConversations, + "nor must its conversation") +} + +// TestAttachVector_MatchesArchiveCorpusWhenScopeCoversIt pins the other +// half: a generation with no [vector.embed.scope] restriction embeds every +// live message in the archive, so VectorMessages/VectorConversations equal +// Messages/Conversations exactly and the table print (eval.go's table +// method) has nothing narrower to add. +func TestAttachVector_MatchesArchiveCorpusWhenScopeCoversIt(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + withTestConfig(t, c) + + // seedRankingDivergenceArchiveIn's two live messages (1 and 2) sit in + // separate conversations; message 3 is deleted from its source and + // excluded from both the archive count and the embedded set. + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + ev.collectCorpusStats(s.DB()) + + require.EqualValues(2, ev.prov.Messages) + require.EqualValues(2, ev.prov.Conversations) + assert.Equal(ev.prov.Messages, ev.prov.VectorMessages, + "an unscoped generation embeds the whole archive") + assert.Equal(ev.prov.Conversations, ev.prov.VectorConversations) +} + +// TestAttachVector_CorpusScopeNormalizesMessageTypeCase is the regression for +// reading vecCfg.Embed.Scope.MessageTypes directly instead of through +// BuildScope(): the archive stores message_type lowercase ("email"), but +// [vector.embed.scope] is user-typed TOML with no case convention enforced. +// A raw, unnormalized "EMAIL" would match nothing and silently zero the +// vector corpus even though every message is actually in scope. +func TestAttachVector_CorpusScopeNormalizesMessageTypeCase(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.MessageTypes = []string{"EMAIL"} + withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "an uppercase configured message type must still match the archive's lowercase rows") + assert.EqualValues(2, ev.prov.VectorConversations) +} + +// TestAttachVector_CorpusIncludesMessagesWithStaleEmbedGenStamp is the +// regression for requiring messages.embed_gen = gen in the corpus query. +// Content changes reset a message's embed_gen to mark it for re-embedding, +// but Backend.Search reads vectors.db purely by generation_id: the old +// vector stays searchable, embed_gen or not, until a re-embed run actually +// replaces it. A count that required the stamp would report a smaller +// corpus than what the run's own search can retrieve — message 1's stale +// stamp here must not remove it. +func TestAttachVector_CorpusIncludesMessagesWithStaleEmbedGenStamp(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + // Simulate a content change on message 1 after it was embedded: the + // backfill machinery resets embed_gen to mark it for re-embedding, but + // its vector row in vectors.db is untouched until that re-embed runs. + _, err := s.DB().Exec(`UPDATE messages SET embed_gen = NULL WHERE id = 1`) + require.NoError(err, "reset embed_gen to simulate a pending re-embed") + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "message 1's stale embed_gen must not drop it from a corpus its own stale vector is still searchable in") + assert.EqualValues(2, ev.prov.VectorConversations) +} diff --git a/cmd/msgvault/cmd/serve_vector.go b/cmd/msgvault/cmd/serve_vector.go index 69b57aab3..b27d724ce 100644 --- a/cmd/msgvault/cmd/serve_vector.go +++ b/cmd/msgvault/cmd/serve_vector.go @@ -249,14 +249,7 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e apiKey := deps.APIKey switch vectorCfg.Embeddings.EffectiveAPIFormat() { case vector.APIFormatOpenAI: - clientConfig := embed.Config{ - Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, - Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, - Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, - DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, - QueryPrefix: vectorCfg.Embeddings.QueryPrefix, - RejectRedirects: true, - } + clientConfig := openAIEmbedConfig(vectorCfg, apiKey) messageClient := embed.NewClient(clientConfig) documentClientConfig := clientConfig documentClientConfig.BeforeRequest = deps.DocumentGate @@ -290,24 +283,15 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e QuerySemanticClient: queryClient, }, nil case vector.APIFormatVoyageContextual: - if vectorCfg.Embeddings.Model != "voyage-context-4" { - return nil, fmt.Errorf("vector.embeddings.model: api_format=%q requires %q, got %q", - vector.APIFormatVoyageContextual, "voyage-context-4", vectorCfg.Embeddings.Model) + clientConfig, err := voyageContextualEmbedConfig(vectorCfg, apiKey) + if err != nil { + return nil, err } publisher, ok := deps.Backend.(vector.DocumentPublisher) if !ok { return nil, errors.New("voyage contextual embeddings require a document publisher backend") } - clientConfig := embed.VoyageConfig{ - Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, - Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, - Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, - DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, - QueryPrefix: vectorCfg.Embeddings.QueryPrefix, - RejectRedirects: true, - Limits: embed.RequestLimits{MaxDocuments: vectorCfg.Embeddings.BatchSize, - MaxChunks: 16_000, MaxUTF8Bytes: contextualDocumentUTF8Limit}, - } + messageClient := embed.NewVoyageClient(clientConfig) documentClientConfig := clientConfig documentClientConfig.BeforeRequest = deps.DocumentGate @@ -351,6 +335,62 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e } } +// openAIEmbedConfig keeps indexing and query clients on the same settings. +func openAIEmbedConfig(vectorCfg vector.Config, apiKey string) embed.Config { + return embed.Config{ + Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, + Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, + Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, + DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, + QueryPrefix: vectorCfg.Embeddings.QueryPrefix, + RejectRedirects: true, + } +} + +// voyageContextualEmbedConfig keeps indexing and query clients on the same settings. +func voyageContextualEmbedConfig(vectorCfg vector.Config, apiKey string) (embed.VoyageConfig, error) { + if vectorCfg.Embeddings.Model != "voyage-context-4" { + return embed.VoyageConfig{}, fmt.Errorf("vector.embeddings.model: api_format=%q requires %q, got %q", + vector.APIFormatVoyageContextual, "voyage-context-4", vectorCfg.Embeddings.Model) + } + return embed.VoyageConfig{ + Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, + Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, + Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, + DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, + QueryPrefix: vectorCfg.Embeddings.QueryPrefix, + RejectRedirects: true, + Limits: embed.RequestLimits{MaxDocuments: vectorCfg.Embeddings.BatchSize, + MaxChunks: 16_000, MaxUTF8Bytes: contextualDocumentUTF8Limit}, + }, nil +} + +// newQueryEmbeddingClient selects the query-time embedding client for the +// configured vector.embeddings.api_format, using the same constructors +// newEmbeddingRuntime uses for the indexing side. +// +// Query-only callers (search, eval) need this rather than newEmbeddingRuntime: +// they never embed a document, so they must not require the document +// publisher backend or build an embed worker. Constructing embed.NewClient +// unconditionally here would send an OpenAI-compatible request body to +// Voyage's /contextualizedembeddings endpoint under a config that indexed with +// the contextual one — the wrong endpoint, the wrong request shape, and no +// input_type=query role. +func newQueryEmbeddingClient(vectorCfg vector.Config, apiKey string) (hybrid.EmbeddingClient, error) { + switch vectorCfg.Embeddings.EffectiveAPIFormat() { + case vector.APIFormatOpenAI: + return embed.NewClient(openAIEmbedConfig(vectorCfg, apiKey)), nil + case vector.APIFormatVoyageContextual: + clientConfig, err := voyageContextualEmbedConfig(vectorCfg, apiKey) + if err != nil { + return nil, err + } + return embed.NewVoyageClient(clientConfig), nil + default: + return nil, fmt.Errorf("unsupported embedding api format %q", vectorCfg.Embeddings.APIFormat) + } +} + func newConvergenceChecker( vectorCfg vector.Config, mainStore *store.Store, diff --git a/cmd/msgvault/cmd/serve_vector_query_client_test.go b/cmd/msgvault/cmd/serve_vector_query_client_test.go new file mode 100644 index 000000000..ef882ab68 --- /dev/null +++ b/cmd/msgvault/cmd/serve_vector_query_client_test.go @@ -0,0 +1,134 @@ +//go:build sqlite_vec || pgvector + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/vector" +) + +// embedRequestRecorder captures the one request a query-time embedding call +// makes, so a test can assert the protocol on the wire rather than the Go +// type of the client. +type embedRequestRecorder struct { + mu sync.Mutex + path string + body map[string]any +} + +func (r *embedRequestRecorder) record(req *http.Request) { + r.mu.Lock() + defer r.mu.Unlock() + r.path = req.URL.Path + r.body = map[string]any{} + _ = json.NewDecoder(req.Body).Decode(&r.body) +} + +func (r *embedRequestRecorder) seen() (string, map[string]any) { + r.mu.Lock() + defer r.mu.Unlock() + return r.path, r.body +} + +// embedTestServer serves one canned embedding response and records the +// request that asked for it. +func embedTestServer(t *testing.T, response string) (*embedRequestRecorder, string) { + t.Helper() + rec := &embedRequestRecorder{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.record(r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(srv.Close) + return rec, srv.URL + "/v1" +} + +// queryClientTestConfig is a minimal vector config for the query-side client, +// defaults applied the way config load applies them. +func queryClientTestConfig(endpoint string, format vector.EmbeddingAPIFormat, model string) vector.Config { + var c vector.Config + c.Enabled = true + c.Embeddings.Endpoint = endpoint + c.Embeddings.Model = model + c.Embeddings.APIFormat = format + c.Embeddings.Dimension = 3 + c.ApplyDefaults() + return c +} + +// TestNewQueryEmbeddingClient_VoyageContextualUsesContextualQueryRole pins the +// contract a Voyage-contextual eval or search run depends on: the query goes to +// the contextual endpoint, in the nested request shape, tagged with the query +// role. Constructing the OpenAI-compatible client for this config instead — +// which is what the eval command used to do unconditionally — posts a flat +// {"input": [...]} body to /v1/embeddings with no input_type, so the query +// vector would come from a different endpoint and a different role than the +// documents it is compared against, if the request succeeded at all. +func TestNewQueryEmbeddingClient_VoyageContextualUsesContextualQueryRole(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}]}`) + client, err := newQueryEmbeddingClient( + queryClientTestConfig(endpoint, vector.APIFormatVoyageContextual, "voyage-context-4"), "") + require.NoError(err) + + vec, err := client.EmbedQuery(context.Background(), "who signed the lease?") + require.NoError(err) + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/contextualizedembeddings", path, "the contextual endpoint, not /embeddings") + assert.Equal("query", body["input_type"], "a query must be embedded in the query role, not the document role") + assert.Equal([]any{[]any{"who signed the lease?"}}, body["inputs"], "the contextual request nests chunks per document") +} + +// TestNewQueryEmbeddingClient_DefaultFormatStaysOpenAICompatible pins the other +// half: an omitted api_format is still the OpenAI-compatible path, unchanged. +func TestNewQueryEmbeddingClient_DefaultFormatStaysOpenAICompatible(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}`) + client, err := newQueryEmbeddingClient(queryClientTestConfig(endpoint, "", "bge-m3"), "") + require.NoError(err) + + vec, err := client.EmbedQuery(context.Background(), "who signed the lease?") + require.NoError(err) + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/embeddings", path) + assert.Equal([]any{"who signed the lease?"}, body["input"]) + assert.NotContains(body, "input_type", "the OpenAI-compatible body carries no role") +} + +// TestNewQueryEmbeddingClient_RejectsFormatsItCannotBuild proves the selector +// fails loudly on a format it has no client for, naming the offending value, +// rather than silently falling back to the OpenAI-compatible client and +// scoring a protocol mismatch as retrieval quality. +func TestNewQueryEmbeddingClient_RejectsFormatsItCannotBuild(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + _, err := newQueryEmbeddingClient(queryClientTestConfig("http://127.0.0.1:1/v1", "voyage", "voyage-context-4"), "") + require.Error(err, "an api_format with no client must not fall back") + assert.Contains(err.Error(), `"voyage"`, "the error names the unsupported value") + + // A contextual format with a model the contextual endpoint does not serve + // is the same class of silent mismatch, and fails the same way. + _, err = newQueryEmbeddingClient( + queryClientTestConfig("http://127.0.0.1:1/v1", vector.APIFormatVoyageContextual, "voyage-large-4"), "") + require.Error(err) + assert.Contains(err.Error(), "voyage-large-4") +} diff --git a/cmd/msgvault/cmd/store_resolver.go b/cmd/msgvault/cmd/store_resolver.go index 2ee0ae025..61c961bbd 100644 --- a/cmd/msgvault/cmd/store_resolver.go +++ b/cmd/msgvault/cmd/store_resolver.go @@ -41,8 +41,16 @@ const ( // // Always returns nil unless the migration itself errors. func runStartupMigrations(s *store.Store) error { + return runStartupMigrationsContext(context.Background(), s) +} + +// runStartupMigrationsContext is the context-aware form of +// runStartupMigrations, for callers — like eval, whose Cobra context is +// already cancellable on Ctrl-C — that must not let a long-running +// migration ignore a cancellation the user actually asked for. +func runStartupMigrationsContext(ctx context.Context, s *store.Store) error { addrs := cfg.Identity.Addresses - res, err := s.RunStartupMigrations(addrs) + res, err := s.RunStartupMigrationsContext(ctx, addrs) if err != nil { logger.Warn("startup migration failed", "error", err) return err diff --git a/internal/eval/corpus.go b/internal/eval/corpus.go new file mode 100644 index 000000000..7f0eaa228 --- /dev/null +++ b/internal/eval/corpus.go @@ -0,0 +1,227 @@ +package eval + +import ( + "bufio" + "fmt" + "os" + "slices" + "strconv" + "strings" +) + +// Qrels maps a query id to its document-id → relevance-grade judgments. +type Qrels map[string]map[string]int + +// HasJudgments reports whether qid was judged at all in this qrels file. +// +// It exists because "no relevant documents" and "no judgments" are different +// facts that RelevantSet cannot tell apart: a topic whose every line grades +// rel=0 produces an empty relevant set, exactly like a qid the file never +// mentions. Conflating them drops the all-non-relevant topic from the run, +// which silently raises every macro average — the topic can only score zero, +// so excluding it removes a zero from the mean. TREC semantics are to score +// such a topic normally; only an unjudged qid has nothing to score against. +func (q Qrels) HasJudgments(qid string) bool { + return len(q[qid]) > 0 +} + +// RelevantSet returns the set of doc ids judged relevant (grade >= 1) for qid. +// An empty result is ambiguous on its own — see HasJudgments. +func (q Qrels) RelevantSet(qid string) map[string]struct{} { + out := make(map[string]struct{}) + for d, r := range q[qid] { + if r >= 1 { + out[d] = struct{}{} + } + } + return out +} + +// LoadStats records how a corpus file parsed. Both loaders skip lines they +// cannot understand, which is the right behaviour for real-world TREC files +// (comments, trailing junk) but makes a whole-file format mismatch silent: a +// three-column qrels file — no iteration column — parses to an empty-but-valid +// Qrels, and the only downstream symptom is "none of the topics had relevance +// judgments", which reads like an id mismatch rather than a format problem. +// Returning the counts lets a caller tell those two apart and say so. +// +// Blank lines are not counted at all, so Lines == Parsed + Skipped always +// holds. +type LoadStats struct { + Path string `json:"path,omitempty"` + Lines int `json:"lines"` // non-blank lines read + Parsed int `json:"parsed"` // lines that produced a record + Skipped int `json:"skipped"` // lines the format check rejected +} + +// String renders the counts for a warning or error message. +func (s LoadStats) String() string { + return fmt.Sprintf("%d lines, %d parsed, %d skipped", s.Lines, s.Parsed, s.Skipped) +} + +// Suspect reports whether the file parsed badly enough to be worth telling the +// user about: nothing usable came out, or a skipped line outnumbered a parsed +// one. Either shape usually means the file is not in the format the loader +// expects, rather than merely containing a stray line. +func (s LoadStats) Suspect() bool { + return s.Lines > 0 && (s.Parsed == 0 || s.Skipped > s.Parsed) +} + +// LoadQrels reads TREC-format relevance judgments: whitespace-separated +// " " per line (the iter column is ignored). Lines +// with fewer than four fields, or a non-integer relevance, are skipped and +// counted in the returned LoadStats — see that type for why the count matters. +// +// A query/document pair graded differently on two lines is rejected, the same +// way LoadTopics rejects a repeated qid: a plain map assignment would let the +// later line silently overwrite the earlier one, so which grade a run scored +// against would depend on nothing but file order, with no diagnostic that it +// happened. An identical repeat (the same grade twice) is harmless — files +// get merged and re-exported — so only a genuine conflict is an error. +func LoadQrels(path string) (Qrels, LoadStats, error) { + stats := LoadStats{Path: path} + f, err := os.Open(path) + if err != nil { + return nil, stats, fmt.Errorf("open qrels: %w", err) + } + defer func() { _ = f.Close() }() + + q := Qrels{} + var conflicts []string + seenConflict := make(map[string]struct{}) + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for sc.Scan() { + line := sc.Text() + if strings.TrimSpace(line) == "" { + continue + } + stats.Lines++ + fields := strings.Fields(line) + if len(fields) < 4 { + stats.Skipped++ + continue + } + rel, err := strconv.Atoi(fields[3]) + if err != nil { + stats.Skipped++ + continue + } + qid, docid := fields[0], fields[2] + if q[qid] == nil { + q[qid] = make(map[string]int) + } + // Read the whole file before complaining, so one message names every + // conflicting pair instead of stopping at the first — same policy + // LoadTopics uses for a repeated qid. + if prev, exists := q[qid][docid]; exists && prev != rel { + pair := qid + "/" + docid + if _, already := seenConflict[pair]; !already { + seenConflict[pair] = struct{}{} + conflicts = append(conflicts, fmt.Sprintf("%s (%d vs %d)", pair, prev, rel)) + } + } + q[qid][docid] = rel + stats.Parsed++ + } + if err := sc.Err(); err != nil { + return nil, stats, fmt.Errorf("read qrels: %w", err) + } + if len(conflicts) > 0 { + return nil, stats, fmt.Errorf("qrels file %s grades the same query/document pair differently on "+ + "different lines: %s — the run would score against whichever grade happened to load last, "+ + "which depends on nothing but file order; fix the file so each pair is graded once", + path, FormatIDList(conflicts, 10)) + } + return q, stats, nil +} + +// Topic is a query whose ID matches a qrels query id. +// +// Category is an optional free-form label for the question's shape — e.g. +// "pointed" (answerable from one message) versus "spanning" (requires +// synthesizing across several). The distinction matters because it decides +// which retrieval levers a benchmark can even see: a topic set made entirely +// of pointed questions is structurally blind to thread-level improvements. +// Empty for topics files that don't carry the column. +type Topic struct { + ID string + Query string + Category string +} + +// LoadTopics reads a tab-separated topics file: +// "\t[\t]" per line. The third column is an +// optional query-category label (see Topic.Category); two-column files — +// the original format — load exactly as before, with Category empty. +// Blank lines are ignored; lines without a tab (or with an empty id or query) +// are skipped and counted in the returned LoadStats, so a space-separated file +// that would otherwise load as zero topics is diagnosable. +// +// A repeated query id is rejected outright, unlike a malformed line. The two +// are not the same kind of problem: a malformed line contributes nothing, so +// dropping and counting it leaves the run intact, whereas a repeated qid +// contributes twice. Every judgment for that qid would be applied once per +// occurrence, weighting the query two or more times in a macro average that is +// meant to be per-query — and when the repeated lines carry different query +// text, whichever policy picked a winner would silently answer one question +// and report it against the other. The qid is the join key to the qrels file, +// so it has to identify exactly one query; a caller cannot repair that +// ambiguity, only the file can. +func LoadTopics(path string) ([]Topic, LoadStats, error) { + stats := LoadStats{Path: path} + f, err := os.Open(path) + if err != nil { + return nil, stats, fmt.Errorf("open topics: %w", err) + } + defer func() { _ = f.Close() }() + + var topics []Topic + seen := make(map[string]struct{}) + var duplicates []string + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for sc.Scan() { + line := sc.Text() + if strings.TrimSpace(line) == "" { + continue + } + stats.Lines++ + parts := strings.SplitN(line, "\t", 3) + if len(parts) < 2 { + stats.Skipped++ + continue + } + id := strings.TrimSpace(parts[0]) + query := strings.TrimSpace(parts[1]) + if id == "" || query == "" { + stats.Skipped++ + continue + } + category := "" + if len(parts) == 3 { + category = strings.TrimSpace(parts[2]) + } + // Read the whole file before complaining, so one message names every + // offending qid instead of sending the user round the loop once per + // duplicate. Each is named once however many times it repeats. + if _, repeat := seen[id]; repeat { + if !slices.Contains(duplicates, id) { + duplicates = append(duplicates, id) + } + } + seen[id] = struct{}{} + topics = append(topics, Topic{ID: id, Query: query, Category: category}) + stats.Parsed++ + } + if err := sc.Err(); err != nil { + return nil, stats, fmt.Errorf("read topics: %w", err) + } + if len(duplicates) > 0 { + return nil, stats, fmt.Errorf("topics file %s repeats query id %s: a qid is the join key to "+ + "the qrels file, so a repeat scores the same judgments more than once and weights that "+ + "query several times over in every macro average — give each topic a unique id", + path, FormatIDList(duplicates, 10)) + } + return topics, stats, nil +} diff --git a/internal/eval/eval_test.go b/internal/eval/eval_test.go new file mode 100644 index 000000000..b49d2c17c --- /dev/null +++ b/internal/eval/eval_test.go @@ -0,0 +1,389 @@ +package eval + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func set(ids ...string) map[string]struct{} { + m := make(map[string]struct{}, len(ids)) + for _, id := range ids { + m[id] = struct{}{} + } + return m +} + +// ranked = [a b c d], relevant = {a, c, x}; x is never retrieved. +func TestMetrics_KnownValues(t *testing.T) { + assert := assert.New(t) + ranked := []string{"a", "b", "c", "d"} + rel := set("a", "c", "x") + + assert.InDelta(1.0, PrecisionAt(ranked, rel, 1), 1e-9) + assert.InDelta(0.5, PrecisionAt(ranked, rel, 2), 1e-9) + assert.InDelta(0.5, PrecisionAt(ranked, rel, 4), 1e-9) + + assert.InDelta(1.0/3.0, RecallAt(ranked, rel, 1), 1e-9) + assert.InDelta(2.0/3.0, RecallAt(ranked, rel, 4), 1e-9) + + // MRR: first relevant ("a") is at rank 1. + assert.InDelta(1.0, ReciprocalRank(ranked, rel), 1e-9) + + // AP = (1/|rel|) * (P@1 + P@3) = (1/3) * (1/1 + 2/3) = 0.555... + assert.InDelta((1.0+2.0/3.0)/3.0, AveragePrecision(ranked, rel), 1e-9) + + // nDCG@4: DCG = 1/log2(2) + 1/log2(4) = 1 + 0.5 = 1.5 + // IDCG (3 relevant) = 1/log2(2)+1/log2(3)+1/log2(4) = 2.13092975 + assert.InDelta(1.5/2.1309297535714578, NDCGAt(ranked, rel, 4), 1e-9) +} + +func TestMetrics_EdgeCases(t *testing.T) { + assert := assert.New(t) + rel := set("a") + // No relevant docs at all -> every metric is 0. + empty := map[string]struct{}{} + assert.Zero(PrecisionAt([]string{"a"}, empty, 10)) + assert.Zero(RecallAt([]string{"a"}, empty, 10)) + assert.Zero(NDCGAt([]string{"a"}, empty, 10)) + assert.Zero(AveragePrecision([]string{"a"}, empty)) + assert.Zero(ReciprocalRank([]string{"a"}, empty)) + + // Empty ranking -> 0. + assert.Zero(PrecisionAt(nil, rel, 10)) + assert.Zero(RecallAt(nil, rel, 10)) + assert.Zero(ReciprocalRank(nil, rel)) + + // Perfect ranking -> P@1 = 1, MRR = 1, nDCG@1 = 1. + assert.InDelta(1.0, PrecisionAt([]string{"a"}, rel, 1), 1e-9) + assert.InDelta(1.0, NDCGAt([]string{"a"}, rel, 1), 1e-9) +} + +func TestEvaluateAndAggregate(t *testing.T) { + assert := assert.New(t) + a := &Aggregate{} + a.Add(Evaluate([]string{"a", "b"}, set("a"), StandardCutoffs)) // P@10=0.1, MRR=1 + a.Add(Evaluate([]string{"b", "a"}, set("a"), StandardCutoffs)) // MRR=0.5 + assert.Equal(2, a.N) + mean := a.Mean() + assert.InDelta((1.0+0.5)/2.0, mean.MRR, 1e-9) + assert.InDelta(0.1, mean.P, 1e-9) // each had exactly 1 hit in top 10 + // Zero-N aggregate is safe. + assert.Equal(Scores{}, (&Aggregate{}).Mean()) +} + +func TestLoadQrels(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + p := filepath.Join(dir, "qrels.txt") + // qid iter docid rel — iter ignored; a malformed line is skipped. + content := "301 0 docA 1\n301 0 docB 0\n302 0 docC 1\nmalformed line\n301 0 docD 1\n" + require.NoError(os.WriteFile(p, []byte(content), 0o644)) + + q, stats, err := LoadQrels(p) + require.NoError(err) + assert.Equal(1, q["301"]["docA"]) + assert.Equal(0, q["301"]["docB"]) + + rel := q.RelevantSet("301") + assert.Equal(set("docA", "docD"), rel) // docB (grade 0) excluded + assert.Equal(set("docC"), q.RelevantSet("302")) + assert.Empty(q.RelevantSet("999")) // unknown qid -> empty set + + // The skipped line is counted, not silently swallowed. + assert.Equal(LoadStats{Path: p, Lines: 5, Parsed: 4, Skipped: 1}, stats) + assert.False(stats.Suspect(), "one stray line in a good file is not suspicious") +} + +// TestLoadQrels_RejectsConflictingGradesForTheSamePair is the regression for +// silent last-write-wins: a plain map assignment let a later line for the +// same query/document pair overwrite an earlier, differently-graded one with +// no diagnostic, so which grade a run scored against depended on nothing but +// file order. +func TestLoadQrels_RejectsConflictingGradesForTheSamePair(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "qrels.txt") + require.NoError(os.WriteFile(p, []byte("301 0 docA 1\n301 0 docA 0\n"), 0o644)) + + q, _, err := LoadQrels(p) + require.Error(err, "docA must not silently end up graded 0 just because that line came second") + assert.Nil(q) + assert.Contains(err.Error(), "301/docA") + assert.Contains(err.Error(), "1 vs 0") +} + +// TestLoadQrels_AllowsAnIdenticalRepeat pins the other half: the same +// query/document pair graded the same way twice — the shape a merged or +// re-exported judgments file produces — is harmless and must not be rejected +// alongside a genuine conflict. +func TestLoadQrels_AllowsAnIdenticalRepeat(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "qrels.txt") + require.NoError(os.WriteFile(p, []byte("301 0 docA 1\n301 0 docA 1\n"), 0o644)) + + q, stats, err := LoadQrels(p) + require.NoError(err) + assert.Equal(1, q["301"]["docA"]) + assert.Equal(2, stats.Parsed, "both lines still count toward the parsed total") +} + +// TestLoadQrels_WrongColumnCount is the diagnosability regression. A qrels file +// in the common three-column variant (no iteration column) used to load as an +// empty-but-valid Qrels with no signal at all, and the only downstream symptom +// — "no topics had relevance judgments" — reads like an id mismatch. The counts +// have to make the format problem visible. +func TestLoadQrels_WrongColumnCount(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "qrels.txt") + require.NoError(os.WriteFile(p, []byte("301 docA 1\n301 docB 1\n302 docC 1\n"), 0o644)) + + q, stats, err := LoadQrels(p) + require.NoError(err, "a format mismatch is reported through the stats, not as a read error") + assert.Empty(q) + assert.Equal(3, stats.Lines) + assert.Equal(0, stats.Parsed) + assert.Equal(3, stats.Skipped) + assert.True(stats.Suspect(), "nothing parsed: the caller must be able to say so") + assert.Equal("3 lines, 0 parsed, 3 skipped", stats.String()) +} + +// TestQrels_HasJudgmentsSeparatesUnjudgedFromAllNonRelevant is the regression +// for topics that were judged and found to contain nothing relevant. Their +// RelevantSet is empty, which used to be read as "this qrels file has nothing +// to say about the topic" and the topic was dropped from the run — and because +// such a topic can only ever score zero, dropping it lifts every macro average. +// HasJudgments has to tell the two apart on the qrels alone. +func TestQrels_HasJudgmentsSeparatesUnjudgedFromAllNonRelevant(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "qrels.txt") + // 301 has a relevant document; 302 was judged and nothing was relevant. + require.NoError(os.WriteFile(p, + []byte("301 0 docA 1\n302 0 docB 0\n302 0 docC 0\n"), 0o644)) + + q, _, err := LoadQrels(p) + require.NoError(err) + + assert.True(q.HasJudgments("301")) + assert.True(q.HasJudgments("302"), "all-zero grades are judgments, not an absent qid") + assert.False(q.HasJudgments("999"), "a qid the file never mentions is unjudged") + + assert.Empty(q.RelevantSet("302"), "and its relevant set is empty either way") + assert.Empty(q.RelevantSet("999")) + + // What such a topic contributes once it is scored: a real zero across the + // board, which is exactly what excluding it used to hide. + assert.Equal(Scores{}, Evaluate([]string{"docB", "docC"}, q.RelevantSet("302"), StandardCutoffs)) +} + +// TestLoadStats_Suspect pins when a caller should warn: nothing usable came +// out, or the skipped lines outnumber the parsed ones. A clean file, or one +// with a stray line, must not trip it. +func TestLoadStats_Suspect(t *testing.T) { + assert := assert.New(t) + assert.False(LoadStats{}.Suspect(), "an empty file is a different complaint") + assert.False(LoadStats{Lines: 10, Parsed: 10}.Suspect()) + assert.False(LoadStats{Lines: 10, Parsed: 9, Skipped: 1}.Suspect()) + assert.True(LoadStats{Lines: 10, Parsed: 4, Skipped: 6}.Suspect()) + assert.True(LoadStats{Lines: 3, Skipped: 3}.Suspect()) +} + +func TestLoadTopics(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + p := filepath.Join(dir, "topics.tsv") + content := "301\toil and gas drilling\n\n302\tspill response\nno_tab_line\n" + require.NoError(os.WriteFile(p, []byte(content), 0o644)) + + topics, stats, err := LoadTopics(p) + require.NoError(err) + require.Len(topics, 2) + assert.Equal("301", topics[0].ID) + assert.Equal("oil and gas drilling", topics[0].Query) + assert.Empty(topics[0].Category, "two-column format has no category") + assert.Equal("302", topics[1].ID) + // The blank line is not counted at all; the tabless line is a skip. + assert.Equal(LoadStats{Path: p, Lines: 3, Parsed: 2, Skipped: 1}, stats) +} + +// TestLoadTopics_SpaceSeparated covers the topics-side format mismatch: a file +// written with spaces instead of tabs parses to zero topics, and the counts are +// the only way to tell that apart from an empty file. +func TestLoadTopics_SpaceSeparated(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "topics.tsv") + require.NoError(os.WriteFile(p, []byte("301 oil and gas\n302 spill response\n"), 0o644)) + + topics, stats, err := LoadTopics(p) + require.NoError(err) + assert.Empty(topics) + assert.Equal(LoadStats{Path: p, Lines: 2, Parsed: 0, Skipped: 2}, stats) + assert.True(stats.Suspect()) +} + +// TestLoadTopics_CategoryColumn covers the optional third column. A file may +// mix labeled and unlabeled lines; old two-column files must load unchanged. +func TestLoadTopics_CategoryColumn(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + p := filepath.Join(dir, "topics.tsv") + content := "q1\tlease renewal terms\tpointed\n" + // labeled + "q2\thow did the negotiation conclude\tspanning\n" + // labeled + "q3\tforklift servicing\n" + // plain two-column line in the same file + "q4\tinsurance quote\t\n" + // trailing tab, empty category + "q5\t padded query \t padded \n" // whitespace trimmed + require.NoError(os.WriteFile(p, []byte(content), 0o644)) + + topics, stats, err := LoadTopics(p) + require.NoError(err) + require.Len(topics, 5) + assert.Equal(Topic{ID: "q1", Query: "lease renewal terms", Category: "pointed"}, topics[0]) + assert.Equal("spanning", topics[1].Category) + assert.Empty(topics[2].Category, "unlabeled line stays unlabeled") + assert.Empty(topics[3].Category, "a trailing tab is not a category") + assert.Equal(Topic{ID: "q5", Query: "padded query", Category: "padded"}, topics[4]) + assert.Equal(LoadStats{Path: p, Lines: 5, Parsed: 5, Skipped: 0}, stats) +} + +// TestCutoffsForDepth pins the anti-mislabelling rule: a run that only ever +// looks 20 deep has no recall@100, so the depth it reports must be the depth it +// used. Above the standard depths nothing is clamped. +// It also pins Depth, which is not a clamp but the depth itself: MAP and MRR +// take no cutoff, so the retrieval depth is the only thing bounding them and +// the only thing a report can honestly label them with. +func TestCutoffsForDepth(t *testing.T) { + assert := assert.New(t) + assert.Equal(Cutoffs{P: 10, NDCG: 10, Recall: 100, Depth: 100}, CutoffsForDepth(100)) + assert.Equal(Cutoffs{P: 10, NDCG: 10, Recall: 100, Depth: 1000}, CutoffsForDepth(1000)) + assert.Equal(Cutoffs{P: 10, NDCG: 10, Recall: 20, Depth: 20}, CutoffsForDepth(20)) + assert.Equal(Cutoffs{P: 5, NDCG: 5, Recall: 5, Depth: 5}, CutoffsForDepth(5)) + assert.Equal(Cutoffs{P: 1, NDCG: 1, Recall: 1, Depth: 1}, CutoffsForDepth(1)) + // A non-positive depth is not a depth; the CLI rejects it before we get + // here, so fall back to the standard set rather than inventing a zero one. + assert.Equal(StandardCutoffs, CutoffsForDepth(0)) + assert.Equal(StandardCutoffs, CutoffsForDepth(-1)) + assert.Zero(StandardCutoffs.Depth, "the standard set names metrics, not a run") +} + +// TestCutoffs_IsStandard pins what the "depths were clamped" notice keys off: +// the named cutoffs alone. A run at -n 100 and one at -n 500 both report the +// standard P@10/nDCG@10/R@100 and neither is clamped, even though their MAP@n +// and MRR@n differ — so Depth must not enter the comparison. +func TestCutoffs_IsStandard(t *testing.T) { + assert := assert.New(t) + assert.True(StandardCutoffs.IsStandard()) + assert.True(CutoffsForDepth(100).IsStandard()) + assert.True(CutoffsForDepth(500).IsStandard(), "a deeper run is not a clamped one") + assert.False(CutoffsForDepth(20).IsStandard()) + assert.False(CutoffsForDepth(1).IsStandard()) +} + +// TestEvaluate_HonoursCutoffs shows why the clamp matters: with 20 results and +// 40 relevant documents, "recall@100" and recall@20 are different numbers, and +// only one of them is a thing this run measured. +func TestEvaluate_HonoursCutoffs(t *testing.T) { + assert := assert.New(t) + rel := map[string]struct{}{} + var ranked []string + for i := range 40 { + id := fmt.Sprintf("d%02d", i) + rel[id] = struct{}{} + if i < 20 { + ranked = append(ranked, id) + } + } + atStandard := Evaluate(ranked, rel, StandardCutoffs) + atDepth := Evaluate(ranked, rel, CutoffsForDepth(20)) + + assert.InDelta(0.5, atStandard.Recall, 1e-9, "R@100 over a 20-deep list is bounded by the depth") + assert.InDelta(0.5, atDepth.Recall, 1e-9) + // P and nDCG are unaffected here because 10 <= 20; the point is that the + // reported label, not the value, is what changes. + assert.InDelta(atStandard.P, atDepth.P, 1e-9) + assert.InDelta(atStandard.NDCG, atDepth.NDCG, 1e-9) + + // Below the precision depth the clamp does change the value: 3 retrieved, + // all relevant, is precision 1.0 at depth 3 and 0.3 at depth 10. + short := []string{"d00", "d01", "d02"} + assert.InDelta(0.3, Evaluate(short, rel, StandardCutoffs).P, 1e-9) + assert.InDelta(1.0, Evaluate(short, rel, CutoffsForDepth(3)).P, 1e-9) +} + +func TestLoadQrels_MissingFile(t *testing.T) { + _, _, err := LoadQrels(filepath.Join(t.TempDir(), "nope.txt")) + require.Error(t, err) +} + +// TestLoadTopics_RejectsDuplicateIDs pins the duplicate-qid policy. A repeated +// qid is not a malformed line to be dropped and counted: it loads fine and +// contributes twice, so every judgment for that qid is applied once per +// occurrence and the query is weighted several times over in a macro average +// that is meant to be per-query. Worse, when the repeated lines ask different +// questions, any dedupe policy scores one of them and reports it against the +// other. The qid is the join key to the qrels file, so the file is rejected +// with every offending id named. +func TestLoadTopics_RejectsDuplicateIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "topics.tsv") + require.NoError(os.WriteFile(p, []byte( + "301\toil and gas drilling\n"+ + "302\tspill response\n"+ + "301\tsomething else entirely\n"+ + "302\tspill response\n"), 0o644)) + + topics, _, err := LoadTopics(p) + require.Error(err, "a repeated qid must not load as two independent topics") + assert.Nil(topics, "nothing is returned: a partial load would score a biased subset") + assert.Contains(err.Error(), "repeats query id 301, 302", + "every offending id is named once, in file order") + assert.Contains(err.Error(), "unique id") +} + +// TestLoadTopics_ManyDuplicateIDsAreSummarised keeps the rejection readable: a +// file whose ids are wholesale duplicated must not print a wall of them over +// the sentence that explains the problem. +func TestLoadTopics_ManyDuplicateIDsAreSummarised(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var b strings.Builder + for pass := range 2 { + for i := range 12 { + fmt.Fprintf(&b, "q%02d\tquery %d pass %d\n", i, i, pass) + } + } + p := filepath.Join(t.TempDir(), "topics.tsv") + require.NoError(os.WriteFile(p, []byte(b.String()), 0o644)) + + _, _, err := LoadTopics(p) + require.Error(err) + assert.Contains(err.Error(), "q00, q01, q02, q03, q04, q05, q06, q07, q08, q09, and 2 more") +} + +// TestLoadTopics_DistinctIDsStillLoad guards the other side: uniqueness is +// checked on the id alone, so two topics that happen to ask the same question +// under different ids are perfectly valid. +func TestLoadTopics_DistinctIDsStillLoad(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := filepath.Join(t.TempDir(), "topics.tsv") + require.NoError(os.WriteFile(p, []byte("301\tsame question\n302\tsame question\n"), 0o644)) + + topics, stats, err := LoadTopics(p) + require.NoError(err) + require.Len(topics, 2) + assert.Equal("301", topics[0].ID) + assert.Equal(LoadStats{Path: p, Lines: 2, Parsed: 2}, stats) +} diff --git a/internal/eval/fixture_test.go b/internal/eval/fixture_test.go new file mode 100644 index 000000000..95c88848a --- /dev/null +++ b/internal/eval/fixture_test.go @@ -0,0 +1,255 @@ +package eval + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The threaded fixture is a small, checked-in, msgvault-shaped mailbox: real +// In-Reply-To/References chains, quoted replies, varied dates and multiple +// participants. The TREC legal collection msgvault is otherwise evaluated +// against is flat — exactly one message per judged document — so it cannot +// exercise any of the threading behaviour these tests cover. + +const fixtureDir = "testdata/threaded" + +type fixtureMessage struct { + MessageID string `json:"message_id"` + ConversationID string `json:"conversation_id"` + InReplyTo string `json:"in_reply_to"` + References []string `json:"references"` + Date string `json:"date"` + From string `json:"from"` + To []string `json:"to"` + Cc []string `json:"cc"` + Subject string `json:"subject"` + Body string `json:"body"` +} + +type fixtureMailbox struct { + Description string `json:"description"` + Messages []fixtureMessage `json:"messages"` +} + +func loadFixture(t *testing.T) fixtureMailbox { + t.Helper() + raw, err := os.ReadFile(filepath.Join(fixtureDir, "mailbox.json")) + require.NoError(t, err) + var mb fixtureMailbox + require.NoError(t, json.Unmarshal(raw, &mb)) + require.NotEmpty(t, mb.Messages) + return mb +} + +// TestFixture_IsThreadShaped guards the fixture itself. If someone flattens it +// (one message per thread, no quotes, one participant) it silently stops +// testing the thing it exists to test, so assert its shape explicitly. +func TestFixture_IsThreadShaped(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + mb := loadFixture(t) + + byID := make(map[string]fixtureMessage, len(mb.Messages)) + threads := make(map[string]int) + senders := make(map[string]struct{}) + dates := make(map[string]struct{}) + quoted, replies := 0, 0 + + for _, m := range mb.Messages { + require.NotEmpty(m.MessageID, "every message needs a Message-ID") + require.NotEmpty(m.ConversationID, "every message needs a conversation") + require.NotEmpty(m.Date, "every message needs a date") + byID[m.MessageID] = m + threads[m.ConversationID]++ + senders[m.From] = struct{}{} + dates[m.Date] = struct{}{} + if strings.Contains(m.Body, "\n>") { + quoted++ + } + if m.InReplyTo != "" { + replies++ + } + } + + // Threading: replies must point at a real parent in the same thread, and + // References must contain the parent. + for _, m := range mb.Messages { + if m.InReplyTo == "" { + assert.Empty(m.References, "root %s should have no References", m.MessageID) + continue + } + parent, ok := byID[m.InReplyTo] + require.True(ok, "%s replies to unknown parent %s", m.MessageID, m.InReplyTo) + assert.Equal(parent.ConversationID, m.ConversationID, + "%s must share its parent's conversation", m.MessageID) + assert.Contains(m.References, m.InReplyTo, + "%s References must include its parent", m.MessageID) + } + + // Shape: multi-message threads, singletons, quoted replies, many people. + var multi, single int + for _, n := range threads { + if n > 1 { + multi++ + } else { + single++ + } + } + assert.GreaterOrEqual(multi, 2, "need several multi-message threads") + assert.GreaterOrEqual(single, 1, "need at least one singleton thread") + assert.GreaterOrEqual(replies, 4, "need real reply chains") + assert.GreaterOrEqual(quoted, 4, "need quoted-reply bodies") + assert.GreaterOrEqual(len(senders), 4, "need multiple participants") + assert.Len(dates, len(mb.Messages), "dates should be varied, not cloned") +} + +// TestFixture_QrelsMatchMailbox keeps the judgments and the mailbox in step: +// every judged id must exist, so a renamed message or thread fails loudly +// instead of quietly scoring zero. +func TestFixture_QrelsMatchMailbox(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + mb := loadFixture(t) + msgIDs := make(map[string]struct{}, len(mb.Messages)) + convIDs := make(map[string]struct{}) + for _, m := range mb.Messages { + msgIDs[m.MessageID] = struct{}{} + convIDs[m.ConversationID] = struct{}{} + } + + topics, _, err := LoadTopics(filepath.Join(fixtureDir, "topics.tsv")) + require.NoError(err) + require.Len(topics, 3) + + // The fixture deliberately mixes labeled and unlabeled topics: the + // optional category column (pointed vs spanning question shape) must + // coexist with plain two-column lines in one file. + assert.Equal("spanning", topics[0].Category, + "q1's relevant set is a whole thread, not one message") + assert.Equal("pointed", topics[1].Category) + assert.Empty(topics[2].Category, "q3 pins the unlabeled two-column form") + + for _, tc := range []struct { + file string + valid map[string]struct{} + }{ + {"qrels_message.txt", msgIDs}, + {"qrels_conversation.txt", convIDs}, + } { + q, _, err := LoadQrels(filepath.Join(fixtureDir, tc.file)) + require.NoError(err) + for _, top := range topics { + require.NotEmpty(q[top.ID], "%s: topic %s has no judgments", tc.file, top.ID) + for docID := range q[top.ID] { + assert.Contains(tc.valid, docID, "%s: judged id %q is not in the mailbox", tc.file, docID) + } + assert.NotEmpty(q.RelevantSet(top.ID), "%s: topic %s has no relevant docs", tc.file, top.ID) + } + } +} + +// TestThreadCollapsing_ConversationKey is the regression the flat TREC corpus +// cannot catch. msgvault retrieves messages; with --doc-key=conversation the +// judged unit is the thread. A four-message thread filling the top of the +// ranking is ONE retrieved thread, not four. Scoring the un-collapsed list +// counts it four times, which inflates precision and drives recall above 1.0. +func TestThreadCollapsing_ConversationKey(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + qrels, _, err := LoadQrels(filepath.Join(fixtureDir, "qrels_conversation.txt")) + require.NoError(err) + rel := qrels.RelevantSet("q1") + require.Len(rel, 1, "q1 has exactly one relevant thread") + + // A message-level engine answering q1: the whole renewal thread ranks + // first, then unrelated threads. + perMessage := []string{ + "thread-renewal", "thread-renewal", "thread-renewal", "thread-renewal", + "thread-insurance", "thread-forklift", "thread-newsletter", "thread-offsite", + } + + // Un-collapsed: visibly broken. + assert.InDelta(0.4, PrecisionAt(perMessage, rel, 10), 1e-9, + "un-collapsed P@10 counts the same thread four times") + assert.InDelta(4.0, RecallAt(perMessage, rel, 100), 1e-9, + "un-collapsed recall exceeds 1.0, which is impossible") + + // Collapsed: one thread, one slot, at its best rank. + collapsed := DedupeKeys(perMessage) + assert.Equal([]string{ + "thread-renewal", "thread-insurance", "thread-forklift", + "thread-newsletter", "thread-offsite", + }, collapsed) + assert.InDelta(0.1, PrecisionAt(collapsed, rel, 10), 1e-9) + assert.InDelta(1.0, RecallAt(collapsed, rel, 100), 1e-9, "recall must never exceed 1.0") + assert.InDelta(1.0, ReciprocalRank(collapsed, rel), 1e-9) + assert.InDelta(1.0, NDCGAt(collapsed, rel, 10), 1e-9) +} + +// TestDedupeKeys_PreservesBestRank pins the collapse rule for real keys: +// first occurrence wins, since that is the thread's best rank. +func TestDedupeKeys_PreservesBestRank(t *testing.T) { + assert := assert.New(t) + assert.Equal([]string{"b", "a", "c"}, DedupeKeys([]string{"b", "a", "b", "c", "a"})) + assert.Empty(DedupeKeys(nil)) +} + +// TestDedupeKeys_KeepsUnscorableHitsAsUniqueOccupiedSlots is the regression +// for treating an empty key — a hit that carried no id for the chosen +// doc-key — as absent from the ranking entirely. It still occupied a rank a +// real user would have seen, so dropping it would let every relevant +// document below it shift up and inflate MRR/AP/nDCG by a rank position the +// run didn't earn. It must survive as a key unique to its own position: +// present in the output (so later ranks don't shift), but never equal to +// another such placeholder (two hits missing an id are not one document). +func TestDedupeKeys_KeepsUnscorableHitsAsUniqueOccupiedSlots(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + ranked := DedupeKeys([]string{"", "a", "", "a"}) + require.Len(ranked, 3, "both unscorable hits occupy a slot; the second \"a\" is still a real duplicate") + assert.NotEqual(ranked[0], ranked[2], "two hits missing an id must not collapse into one document") + assert.Equal("a", ranked[1]) + + rel := map[string]struct{}{"a": {}} + assert.InDelta(0.5, ReciprocalRank(ranked, rel), 1e-9, + "the relevant doc is genuinely second, not first — dropping the leading empty key would have inflated this to 1.0") + + assert.Len(DedupeKeys([]string{"", ""}), 2, "neither unscorable hit is dropped") +} + +// TestQuotedReplyDistractor_HurtsPrecision covers the other product-specific +// failure the legal corpus misses. Every reply in this fixture quotes its +// parent, so an engine that indexes quoted text without attributing it to the +// original message will surface a reply from an unrelated thread whose only +// match is the quotation. Judged non-relevant, so a quote-stripping regression +// shows up as a precision drop rather than passing silently. +func TestQuotedReplyDistractor_HurtsPrecision(t *testing.T) { + assert := assert.New(t) + qrels, _, err := LoadQrels(filepath.Join(fixtureDir, "qrels_message.txt")) + require.NoError(t, err) + rel := qrels.RelevantSet("q1") + + // t2-b is in the forklift thread and merely mentions the lease renewal + // signing; t5-a is a newsletter whose subject contains the query words. + assert.NotContains(rel, "") + assert.NotContains(rel, "") + + clean := []string{"", "", "", ""} + leaky := []string{"", "", "", ""} + + cleanScore := Evaluate(clean, rel, StandardCutoffs) + leakyScore := Evaluate(leaky, rel, StandardCutoffs) + + assert.InDelta(0.4, cleanScore.P, 1e-9) + assert.InDelta(0.2, leakyScore.P, 1e-9) + assert.Less(leakyScore.NDCG, cleanScore.NDCG, "quote leakage must cost nDCG") + assert.InDelta(1.0, cleanScore.MRR, 1e-9) + assert.InDelta(1.0/3.0, leakyScore.MRR, 1e-9) +} diff --git a/internal/eval/metrics.go b/internal/eval/metrics.go new file mode 100644 index 000000000..0c1ec71ee --- /dev/null +++ b/internal/eval/metrics.go @@ -0,0 +1,224 @@ +// Package eval provides retrieval-quality evaluation for msgvault: standard +// information-retrieval metrics (precision@k, recall@k, nDCG@k, MAP, MRR) +// computed over a ranked result list scored against relevance judgments +// (qrels). Every one of them is bounded by the depth the ranking was +// retrieved to, MAP and MRR included even though they take no k of their own, +// so Cutoffs records that depth for a report to label them with. +// +// The metric functions are pure — no I/O, no engine or database dependencies — +// so they are unit-testable in isolation and reused by the `msgvault eval` +// command, which supplies the rankings by running the search engine. +// +// Judgments are binary here: a document is relevant (in the set) or not. This +// matches TREC-style qrels where a positive grade means "relevant". Graded +// relevance can be layered on later without changing the command surface. +package eval + +import "math" + +// Cutoffs are the rank depths one run scores at. They travel with the run +// rather than being baked into the metric names because a cutoff deeper than +// the retrieval depth is a mislabeled number, not a hard number: a run that +// only ever retrieves 20 results per query cannot have a recall@100, and +// reporting one as "R@100" invites a reader to compare it against a run that +// really did look 100 deep. +type Cutoffs struct { + P int // precision depth + NDCG int // nDCG depth + Recall int // recall depth + // Depth is the retrieval depth itself: the rank past which the run never + // looked, so the length of the longest ranking Evaluate can be handed. + // + // It is here because MAP and MRR have no cutoff of their own — they run + // over the whole list they are given — which makes the list's length their + // cutoff whether or not anyone names it. Reporting them unqualified claims + // they saw every relevant document, when a run at -n 20 cannot see one at + // rank 21 and its "MRR" is really MRR@20. Zero when the depth is not known + // (the standard set), in which case a caller has nothing to qualify them + // with and should print them bare. + Depth int +} + +// StandardCutoffs is the conventional depth set: P@10, nDCG@10, R@100. It +// carries no Depth: it names the metrics wanted, not a run that produced them. +var StandardCutoffs = Cutoffs{P: 10, NDCG: 10, Recall: 100} + +// CutoffsForDepth clamps StandardCutoffs to the number of results a run +// actually retrieves per query, so the reported metric is always one the run +// could in principle have maximised, and records that depth for the metrics +// that are bounded by it without being named after it. A non-positive limit is +// meaningless as a depth and falls back to the standard set. +func CutoffsForDepth(limit int) Cutoffs { + if limit <= 0 { + return StandardCutoffs + } + return Cutoffs{ + P: min(StandardCutoffs.P, limit), + NDCG: min(StandardCutoffs.NDCG, limit), + Recall: min(StandardCutoffs.Recall, limit), + Depth: limit, + } +} + +// IsStandard reports whether the named cutoffs are the conventional ones, so a +// caller can tell a clamped run from a full-depth one. Depth is deliberately +// ignored: runs at -n 100 and -n 500 both report P@10/nDCG@10/R@100, and only +// the clamp is worth warning about. +func (c Cutoffs) IsStandard() bool { + return c.P == StandardCutoffs.P && + c.NDCG == StandardCutoffs.NDCG && + c.Recall == StandardCutoffs.Recall +} + +// Scores holds the standard metric set for a single query's ranking. The +// depths every field was measured at are not stored here — they are a property +// of the run, carried in Cutoffs, and every Scores folded into one Aggregate +// must share them. +type Scores struct { + P float64 // precision@Cutoffs.P + NDCG float64 // normalized DCG@Cutoffs.NDCG (binary gains) + Recall float64 // recall@Cutoffs.Recall + // MAP is average precision (the "AP" that MAP averages) and MRR the + // reciprocal rank of the first relevant hit. Neither takes a cutoff, but + // both are bounded by one all the same: they are computed over the ranking + // as handed in, and a run that retrieves Cutoffs.Depth results cannot see a + // relevant document below that rank. They are therefore MAP@Depth and + // MRR@Depth, and a report that labels them otherwise invites comparison + // against a run that looked deeper. + MAP float64 + MRR float64 +} + +// Evaluate computes the standard metric set for one query. ranked is the +// ordered list of retrieved document ids (best first); rel is the set of +// document ids judged relevant for the query; c gives the rank depths to +// score at (see CutoffsForDepth). +func Evaluate(ranked []string, rel map[string]struct{}, c Cutoffs) Scores { + return Scores{ + P: PrecisionAt(ranked, rel, c.P), + NDCG: NDCGAt(ranked, rel, c.NDCG), + Recall: RecallAt(ranked, rel, c.Recall), + MAP: AveragePrecision(ranked, rel), + MRR: ReciprocalRank(ranked, rel), + } +} + +func isRel(rel map[string]struct{}, d string) bool { + _, ok := rel[d] + return ok +} + +// hitsInTopK counts relevant docs among the first k of ranked. +func hitsInTopK(ranked []string, rel map[string]struct{}, k int) int { + hit := 0 + for i := 0; i < k && i < len(ranked); i++ { + if isRel(rel, ranked[i]) { + hit++ + } + } + return hit +} + +// PrecisionAt returns the fraction of the top-k results that are relevant. +func PrecisionAt(ranked []string, rel map[string]struct{}, k int) float64 { + if k <= 0 { + return 0 + } + return float64(hitsInTopK(ranked, rel, k)) / float64(k) +} + +// RecallAt returns the fraction of all relevant docs found in the top-k. +func RecallAt(ranked []string, rel map[string]struct{}, k int) float64 { + if len(rel) == 0 { + return 0 + } + return float64(hitsInTopK(ranked, rel, k)) / float64(len(rel)) +} + +// NDCGAt returns normalized discounted cumulative gain at k with binary gains. +func NDCGAt(ranked []string, rel map[string]struct{}, k int) float64 { + dcg := 0.0 + for i := 0; i < k && i < len(ranked); i++ { + if isRel(rel, ranked[i]) { + dcg += 1.0 / math.Log2(float64(i)+2.0) + } + } + idcg := 0.0 + for i := 0; i < k && i < len(rel); i++ { + idcg += 1.0 / math.Log2(float64(i)+2.0) + } + if idcg == 0 { + return 0 + } + return dcg / idcg +} + +// AveragePrecision returns the average of the precision values computed at +// each rank where a relevant document is retrieved, divided by the total +// number of relevant documents (the standard AP that MAP averages). +// +// It has no cutoff argument because it scores the whole of ranked. That makes +// the caller's retrieval depth its effective cutoff — see Cutoffs.Depth for +// why the reported label has to say so. +func AveragePrecision(ranked []string, rel map[string]struct{}) float64 { + if len(rel) == 0 { + return 0 + } + hit := 0 + sum := 0.0 + for i, d := range ranked { + if isRel(rel, d) { + hit++ + sum += float64(hit) / float64(i+1) + } + } + return sum / float64(len(rel)) +} + +// ReciprocalRank returns 1/rank of the first relevant hit, or 0 if none. Like +// AveragePrecision it scores the whole of ranked, so the caller's retrieval +// depth is its effective cutoff. +func ReciprocalRank(ranked []string, rel map[string]struct{}) float64 { + for i, d := range ranked { + if isRel(rel, d) { + return 1.0 / float64(i+1) + } + } + return 0 +} + +// Aggregate accumulates per-query Scores and reports their mean (macro-average +// over queries, the standard way MAP/mean-nDCG are reported). +type Aggregate struct { + N int + sumP float64 + sumNDCG float64 + sumRecall float64 + sumMAP float64 + sumMRR float64 +} + +// Add folds one query's scores into the running totals. +func (a *Aggregate) Add(s Scores) { + a.N++ + a.sumP += s.P + a.sumNDCG += s.NDCG + a.sumRecall += s.Recall + a.sumMAP += s.MAP + a.sumMRR += s.MRR +} + +// Mean returns the per-query average of every metric (zero value if N == 0). +func (a *Aggregate) Mean() Scores { + if a.N == 0 { + return Scores{} + } + n := float64(a.N) + return Scores{ + P: a.sumP / n, + NDCG: a.sumNDCG / n, + Recall: a.sumRecall / n, + MAP: a.sumMAP / n, + MRR: a.sumMRR / n, + } +} diff --git a/internal/eval/rank.go b/internal/eval/rank.go new file mode 100644 index 000000000..162a0ee98 --- /dev/null +++ b/internal/eval/rank.go @@ -0,0 +1,124 @@ +package eval + +import "fmt" + +// DedupeKeys collapses a ranked list of document keys so each distinct key +// appears once, at its best (earliest) rank, preserving relative order. +// +// This matters when the unit being scored is coarser than the unit being +// retrieved. msgvault retrieves *messages*, but qrels may judge *threads* +// (--doc-key=conversation). A four-message thread that fills ranks 1-4 is one +// retrieved document, not four: scoring the raw list would count the same +// thread four times, inflating precision@k and — because the duplicates +// displace other threads out of the top-k window — distorting recall@k too. +// +// The bug is invisible on flat, one-message-per-document corpora such as the +// TREC Legal collection, which is exactly why msgvault needs a threaded +// fixture of its own to catch it. +// +// Ordering matters, and it is the caller's job to get it right: collapsing +// must happen BEFORE the ranked list is cut to the requested depth. Retrieving +// N messages and then collapsing them yields however many distinct threads +// happen to sit among those N — not N distinct threads — so "R@100" would +// silently measure a much shallower list than it claims. Callers therefore +// over-fetch raw hits, collapse, and only then TruncateKeys to the depth the +// user asked for. +func DedupeKeys(keys []string) []string { + seen := make(map[string]struct{}, len(keys)) + out := make([]string, 0, len(keys)) + for i, k := range keys { + if k == "" { + // A hit with no id for this doc-key can never be judged + // relevant — no qrels file names an empty docid — but it still + // occupied a rank a real user would have seen. Dropping it + // outright, rather than giving it a key, would let every + // relevant document below it shift up to fill the hole, + // inflating MRR/AP/nDCG by a rank position the run didn't earn. + // A key unique to its position keeps the slot occupied (it can + // never collide with a real key, and giving two such hits the + // same placeholder would wrongly collapse them as one document) + // while still resolving to non-relevant, exactly like any other + // hit no qrels row names. + out = append(out, fmt.Sprintf("\x00unscorable:%d", i)) + continue + } + if _, dup := seen[k]; dup { + continue + } + seen[k] = struct{}{} + out = append(out, k) + } + return out +} + +// TruncateKeys cuts an already-collapsed ranked list to at most n keys. It is +// the second half of the collapse-then-truncate rule described on DedupeKeys: +// applied to the deduped list it yields n distinct documents, applied to the +// raw list it would not. A non-positive n returns nothing, matching the +// "retrieve nothing" reading of a zero depth rather than silently meaning +// "everything". +func TruncateKeys(keys []string, n int) []string { + if n <= 0 { + return nil + } + if len(keys) <= n { + return keys + } + return keys[:n] +} + +const ( + // OverFetchFactor is how many raw hits are requested per requested + // distinct key on the first attempt, and the factor by which the pool + // grows on each retry. + OverFetchFactor = 4 + // MaxOverFetchFactor bounds the pool: a query that cannot fill the + // requested depth with distinct keys costs a bounded amount of work + // instead of walking the whole corpus. + MaxOverFetchFactor = 64 + // maxRawFetch is an absolute ceiling on one raw fetch, so an absurd + // --limit cannot overflow the multiplication or ask an engine for a + // nonsensical page. + maxRawFetch = 1 << 20 +) + +// OverFetchPlan returns the successive raw-hit depths a caller should try in +// order to end up with `limit` DISTINCT doc keys. +// +// When the doc-key is 1:1 with retrieved hits there is nothing to collapse and +// the plan is just [limit] — no wasted work, and no latency inflation in a +// command that reports latency. When it is not (--doc-key=conversation, where +// one thread can occupy many consecutive ranks), the plan over-fetches so the +// collapse still has `limit` distinct keys left afterwards, growing while the +// engine has more to give: +// +// limit*4, limit*16, limit*64 +// +// The caller stops early as soon as the depth is filled or the engine returns +// fewer hits than asked for. A plan is never empty. +func OverFetchPlan(limit int, collapses bool) []int { + if limit <= 0 { + return []int{0} + } + if !collapses { + return []int{limit} + } + var plan []int + for factor := OverFetchFactor; factor <= MaxOverFetchFactor; factor *= OverFetchFactor { + n := maxRawFetch + if limit <= maxRawFetch/factor { + n = limit * factor + } + if n < limit { + n = limit + } + if len(plan) > 0 && n <= plan[len(plan)-1] { + break // the pool ceiling stopped the depth growing + } + plan = append(plan, n) + } + if len(plan) == 0 { + plan = []int{limit} + } + return plan +} diff --git a/internal/eval/rank_test.go b/internal/eval/rank_test.go new file mode 100644 index 000000000..8893617e2 --- /dev/null +++ b/internal/eval/rank_test.go @@ -0,0 +1,87 @@ +package eval + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestTruncateKeys pins the second half of the collapse-then-truncate rule. +func TestTruncateKeys(t *testing.T) { + assert := assert.New(t) + keys := []string{"a", "b", "c"} + assert.Equal([]string{"a", "b"}, TruncateKeys(keys, 2)) + assert.Equal(keys, TruncateKeys(keys, 3)) + assert.Equal(keys, TruncateKeys(keys, 10), "a short list is not padded") + assert.Empty(TruncateKeys(keys, 0)) + assert.Empty(TruncateKeys(keys, -1)) + assert.Empty(TruncateKeys(nil, 5)) +} + +// TestDedupeThenTruncate_Order is the ordering regression. Retrieving n +// messages and *then* collapsing them yields however many distinct threads +// happen to sit inside those n — here 2 — which is not what "-n 4" claims. +// Collapsing first and truncating after is what makes the depth mean what the +// metric labels say it means. +func TestDedupeThenTruncate_Order(t *testing.T) { + // Four threads, three messages each, interleaved the way a real ranking + // interleaves them: the first four raw hits cover only two threads. + raw := []string{ + "t1", "t1", "t1", + "t2", "t2", "t2", + "t3", "t3", "t3", + "t4", "t4", "t4", + } + const want = 4 + + truncateFirst := DedupeKeys(TruncateKeys(raw, want)) + assert.Equal(t, []string{"t1", "t2"}, truncateFirst, + "truncating the raw list first caps the result at the threads inside it") + + dedupeFirst := TruncateKeys(DedupeKeys(raw), want) + assert.Equal(t, []string{"t1", "t2", "t3", "t4"}, dedupeFirst, + "collapsing first yields the requested number of DISTINCT threads") +} + +// TestOverFetchPlan_NonCollapsingKey: with a doc-key that is 1:1 with hits +// there is nothing to collapse, so the plan must not over-fetch. This command +// reports latency, and padding every query would inflate it for no gain. +func TestOverFetchPlan_NonCollapsingKey(t *testing.T) { + assert.Equal(t, []int{100}, OverFetchPlan(100, false)) + assert.Equal(t, []int{1}, OverFetchPlan(1, false)) +} + +// TestOverFetchPlan_CollapsingKey: the plan grows geometrically and stops at +// the documented ceiling, so a query that cannot fill the depth costs a +// bounded amount of work. +func TestOverFetchPlan_CollapsingKey(t *testing.T) { + assert := assert.New(t) + assert.Equal([]int{400, 1600, 6400}, OverFetchPlan(100, true)) + assert.Equal([]int{4, 16, 64}, OverFetchPlan(1, true)) + + plan := OverFetchPlan(10, true) + require.NotEmpty(t, plan) + assert.Equal(10*OverFetchFactor, plan[0], "the first attempt already over-fetches") + assert.Equal(10*MaxOverFetchFactor, plan[len(plan)-1], "the last attempt is the documented ceiling") + for i := 1; i < len(plan); i++ { + assert.Greater(plan[i], plan[i-1], "each attempt must ask for strictly more") + } +} + +// TestOverFetchPlan_Bounded keeps the plan sane for degenerate depths: never +// empty, never smaller than the requested depth, and never overflowing the +// multiplication on an absurd --limit. +func TestOverFetchPlan_Bounded(t *testing.T) { + for _, limit := range []int{0, 1, 7, 100, 1 << 19, 1 << 21, 1 << 40} { + for _, collapses := range []bool{false, true} { + plan := OverFetchPlan(limit, collapses) + name := fmt.Sprintf("limit=%d collapses=%v", limit, collapses) + require.NotEmpty(t, plan, name) + for _, n := range plan { + assert.GreaterOrEqual(t, n, limit, name+": a fetch is never shallower than the depth asked for") + } + } + } +} diff --git a/internal/eval/report.go b/internal/eval/report.go new file mode 100644 index 000000000..dc47cd56d --- /dev/null +++ b/internal/eval/report.go @@ -0,0 +1,145 @@ +package eval + +import ( + "fmt" + "math" + "slices" + "strings" + "time" +) + +// FormatIDList renders a list of ids for an error or diagnostic message, +// naming at most limit of them and summarising the rest. +// +// The ids these messages carry come from user-supplied files, so their number +// is unbounded: a topics file whose qids drifted from the qrels can leave +// hundreds unmatched, and printing all of them buries the sentence that +// explains what is wrong. A handful is enough to recognise the pattern and go +// look. +func FormatIDList(ids []string, limit int) string { + if limit < 1 || len(ids) <= limit { + return strings.Join(ids, ", ") + } + return fmt.Sprintf("%s, and %d more", strings.Join(ids[:limit], ", "), len(ids)-limit) +} + +// RunConfig records exactly what produced a set of numbers: which embedding +// model and index settings were in force, and how big the index was. +// +// A retrieval score is meaningless on its own — "nDCG@10 = 0.22" says nothing +// unless you also know the embedding model, its dimension, the fusion +// parameters and the size of the haystack. Emitting this alongside every run +// makes results comparable across machines and across time, and makes it +// impossible to read a number without knowing its provenance. +type RunConfig struct { + // Inputs. How a topic is phrased is an experimental variable, not a + // constant: FTS5 uses AND semantics, so a verbose natural-language topic + // matches almost nothing while its keyword reduction scores well, and the + // dense side can move the other way. A run is only comparable to another + // run over the same topic file, so record which one was used. + QrelsPath string `json:"qrels_path,omitempty"` + TopicsPath string `json:"topics_path,omitempty"` + + // Corpus. These are the live population — dedup-hidden duplicates and + // messages deleted from their source account are excluded, because no + // search the run performed could return them, and Conversations counts + // only threads that still hold such a message. A recall figure is read + // against the haystack that was actually searched, so counting rows the + // query path filters out would make the run look harder than it was. + Messages int64 `json:"messages"` + Conversations int64 `json:"conversations"` + + // VectorMessages and VectorConversations report the population an + // account-scoped vector generation actually searches, when + // [vector.embed.scope] narrows it below the whole archive Messages/ + // Conversations above describe. A pure-vector run over a scoped + // generation can only ever return documents from the accounts that + // generation embedded, so the archive-wide count overstates its + // searchable corpus the same way an unfiltered table COUNT(*) once + // overstated Messages/Conversations themselves. Left at zero when vector + // mode did not run, and equal to Messages/Conversations when the + // generation's scope covers the whole archive — the report then prints + // only the shared corpus line rather than repeating an identical one. + VectorMessages int64 `json:"vector_messages,omitempty"` + VectorConversations int64 `json:"vector_conversations,omitempty"` + + // Embeddings / vector index (zero-valued when only --modes fts is run) + VectorEnabled bool `json:"vector_enabled"` + EmbeddingModel string `json:"embedding_model,omitempty"` + // APIFormat is the embeddings protocol the run's queries were embedded + // with ("openai" or "voyage-contextual"). Two models are not the only + // variable an operator changes: the same corpus indexed and queried + // through the contextual endpoint is a different retrieval system from + // the same model over the OpenAI-compatible one, so the format belongs + // beside the model name. + APIFormat string `json:"embedding_api_format,omitempty"` + Dimension int `json:"embedding_dimension,omitempty"` + Endpoint string `json:"embedding_endpoint,omitempty"` + Backend string `json:"vector_backend,omitempty"` + Fingerprint string `json:"generation_fingerprint,omitempty"` + RRFK int `json:"rrf_k,omitempty"` + KPerSignal int `json:"k_per_signal,omitempty"` + SubjectBoost float64 `json:"subject_boost,omitempty"` + // IndexedVectors counts the rows in the active generation alone — the one + // search resolved and queried. A vectors.db retains retired generations + // and may hold a rebuild in progress, so a whole-table count describes + // the file on disk, which is what IndexSizeBytes is for. The generation + // is resolved once when the run starts; each query still searches + // whatever is active at query time, so an activation or rebuild + // triggered mid-run can leave this count stale for later topics. + IndexedVectors int64 `json:"indexed_vectors,omitempty"` + IndexSizeBytes int64 `json:"index_size_bytes,omitempty"` + IndexPath string `json:"index_path,omitempty"` +} + +// Latency summarises per-query wall-clock cost for one search mode. Quality +// improvements routinely cost latency or index size; reporting them beside +// the quality metrics stops an "improvement" from hiding an operational +// regression. +type Latency struct { + Queries int `json:"queries"` + MedianMS float64 `json:"median_ms"` + P95MS float64 `json:"p95_ms"` + TotalMS float64 `json:"total_ms"` +} + +// LatencyTracker accumulates per-query durations for a single mode. +type LatencyTracker struct { + samples []time.Duration +} + +// Add records one query's wall-clock duration. +func (l *LatencyTracker) Add(d time.Duration) { l.samples = append(l.samples, d) } + +// Summary reports median, p95 and total over the recorded samples. The p95 +// uses nearest-rank on the sorted samples, which is well-defined for the +// small sample counts an eval run produces (it degenerates to "the slowest +// query" below 20 samples — honest, rather than interpolating a percentile +// the data cannot support). +func (l *LatencyTracker) Summary() Latency { + n := len(l.samples) + if n == 0 { + return Latency{} + } + sorted := make([]time.Duration, n) + copy(sorted, l.samples) + slices.Sort(sorted) + + ms := func(d time.Duration) float64 { return float64(d.Microseconds()) / 1000.0 } + + var median float64 + if n%2 == 1 { + median = ms(sorted[n/2]) + } else { + median = (ms(sorted[n/2-1]) + ms(sorted[n/2])) / 2 + } + + // Nearest-rank p95: ceil(0.95*n), clamped into range. + rank := min(max(int(math.Ceil(0.95*float64(n)))-1, 0), n-1) + + var total time.Duration + for _, d := range l.samples { + total += d + } + return Latency{Queries: n, MedianMS: median, P95MS: ms(sorted[rank]), TotalMS: ms(total)} +} diff --git a/internal/eval/report_test.go b/internal/eval/report_test.go new file mode 100644 index 000000000..08f2f17a2 --- /dev/null +++ b/internal/eval/report_test.go @@ -0,0 +1,108 @@ +package eval + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLatencyTracker_Summary(t *testing.T) { + assert := assert.New(t) + var l LatencyTracker + // Odd count -> median is the middle sample. + for _, ms := range []int{30, 10, 20} { + l.Add(time.Duration(ms) * time.Millisecond) + } + s := l.Summary() + assert.Equal(3, s.Queries) + assert.InDelta(20.0, s.MedianMS, 1e-6) + assert.InDelta(60.0, s.TotalMS, 1e-6) + // Nearest-rank p95 on 3 samples is the slowest one. + assert.InDelta(30.0, s.P95MS, 1e-6) +} + +func TestLatencyTracker_EvenCountMedian(t *testing.T) { + assert := assert.New(t) + var l LatencyTracker + for _, ms := range []int{40, 10, 30, 20} { + l.Add(time.Duration(ms) * time.Millisecond) + } + s := l.Summary() + assert.Equal(4, s.Queries) + assert.InDelta(25.0, s.MedianMS, 1e-6) // (20+30)/2 + assert.InDelta(100.0, s.TotalMS, 1e-6) + assert.InDelta(40.0, s.P95MS, 1e-6) +} + +func TestLatencyTracker_Empty(t *testing.T) { + var l LatencyTracker + assert.Equal(t, Latency{}, l.Summary()) +} + +func TestLatencyTracker_P95NearestRank(t *testing.T) { + var l LatencyTracker + // 100 samples, 1..100 ms: nearest-rank p95 is the 95th slowest. + for i := 1; i <= 100; i++ { + l.Add(time.Duration(i) * time.Millisecond) + } + s := l.Summary() + assert.InDelta(t, 95.0, s.P95MS, 1e-6) + assert.InDelta(t, 50.5, s.MedianMS, 1e-6) +} + +// A result must never be readable without knowing what produced it, so the +// provenance block has to survive serialisation. +func TestRunConfig_JSONCarriesProvenance(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + p := RunConfig{ + Messages: 18401, Conversations: 18401, + VectorEnabled: true, EmbeddingModel: "bge-m3", Dimension: 1024, + Endpoint: "http://127.0.0.1:8123/v1", Backend: "sqlite-vec", + Fingerprint: "abc123", RRFK: 60, KPerSignal: 100, SubjectBoost: 1.25, + IndexedVectors: 18401, IndexSizeBytes: 81_500_000, IndexPath: "/tmp/vectors.db", + } + raw, err := json.Marshal(p) + require.NoError(err) + + var back map[string]any + require.NoError(json.Unmarshal(raw, &back)) + for _, k := range []string{ + "messages", "conversations", "embedding_model", "embedding_dimension", + "vector_backend", "generation_fingerprint", "rrf_k", "k_per_signal", + "subject_boost", "indexed_vectors", "index_size_bytes", + } { + assert.Contains(back, k, "provenance field %q must be serialised", k) + } + assert.Equal("bge-m3", back["embedding_model"]) +} + +// An fts-only run has no vector config; the block must degrade cleanly rather +// than reporting a zero-dimension model that never ran. +func TestRunConfig_FTSOnlyOmitsVectorFields(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + raw, err := json.Marshal(RunConfig{Messages: 10, Conversations: 4}) + require.NoError(err) + var back map[string]any + require.NoError(json.Unmarshal(raw, &back)) + assert.Equal(false, back["vector_enabled"]) + assert.NotContains(back, "embedding_model") + assert.NotContains(back, "index_size_bytes") + assert.Contains(back, "messages") +} + +// TestFormatIDList pins the cap: enough ids to recognise the pattern, then a +// count, so a diagnostic naming hundreds of unmatched qids stays readable. +func TestFormatIDList(t *testing.T) { + assert := assert.New(t) + assert.Empty(FormatIDList(nil, 3)) + assert.Equal("q1", FormatIDList([]string{"q1"}, 3)) + assert.Equal("q1, q2, q3", FormatIDList([]string{"q1", "q2", "q3"}, 3)) + assert.Equal("q1, q2, q3, and 2 more", FormatIDList([]string{"q1", "q2", "q3", "q4", "q5"}, 3)) + // A non-positive cap is not a cap; list everything rather than nothing. + assert.Equal("q1, q2", FormatIDList([]string{"q1", "q2"}, 0)) +} diff --git a/internal/eval/testdata/threaded/mailbox.json b/internal/eval/testdata/threaded/mailbox.json new file mode 100644 index 000000000..074351dea --- /dev/null +++ b/internal/eval/testdata/threaded/mailbox.json @@ -0,0 +1,129 @@ +{ + "description": "A tiny msgvault-shaped mailbox: real In-Reply-To/References chains, quoted replies, varied dates and multiple participants. Deliberately unlike the flat, one-message-per-document TREC legal collection, so it exercises the threading behaviour that corpus cannot reach.", + "messages": [ + { + "message_id": "", + "conversation_id": "thread-renewal", + "in_reply_to": "", + "references": [], + "date": "2026-01-12T09:14:00Z", + "from": "dana@northwind.example", + "to": ["priya@contoso.example"], + "cc": [], + "subject": "Warehouse lease renewal - terms for 2027", + "body": "Priya,\n\nThe Dock Street warehouse lease expires in March 2027. The landlord has proposed a 9% uplift and a five-year term with no break clause.\n\nCan we discuss before the board pack goes out?\n\nDana" + }, + { + "message_id": "", + "conversation_id": "thread-renewal", + "in_reply_to": "", + "references": [""], + "date": "2026-01-12T11:02:00Z", + "from": "priya@contoso.example", + "to": ["dana@northwind.example"], + "cc": ["marcus@contoso.example"], + "subject": "Re: Warehouse lease renewal - terms for 2027", + "body": "A 9% uplift is above the index. I would counter at 4% and insist on a break at year three.\n\nAdding Marcus for the finance view.\n\nPriya\n\nOn 12 Jan 2026, Dana wrote:\n> The Dock Street warehouse lease expires in March 2027. The landlord has\n> proposed a 9% uplift and a five-year term with no break clause.\n> Can we discuss before the board pack goes out?" + }, + { + "message_id": "", + "conversation_id": "thread-renewal", + "in_reply_to": "", + "references": ["", ""], + "date": "2026-01-13T08:40:00Z", + "from": "marcus@contoso.example", + "to": ["priya@contoso.example", "dana@northwind.example"], + "cc": [], + "subject": "Re: Warehouse lease renewal - terms for 2027", + "body": "Finance can support 4%. Anything above 6% breaches the facilities budget envelope.\n\nMarcus\n\n> A 9% uplift is above the index. I would counter at 4% and insist on a\n> break at year three.\n>> The Dock Street warehouse lease expires in March 2027. The landlord has\n>> proposed a 9% uplift and a five-year term with no break clause." + }, + { + "message_id": "", + "conversation_id": "thread-renewal", + "in_reply_to": "", + "references": ["", "", ""], + "date": "2026-01-19T16:25:00Z", + "from": "dana@northwind.example", + "to": ["priya@contoso.example", "marcus@contoso.example"], + "cc": [], + "subject": "Re: Warehouse lease renewal - terms for 2027", + "body": "Landlord accepted 4.5% with a break at year three. Signing Friday.\n\nDana\n\n> Finance can support 4%. Anything above 6% breaches the facilities budget\n> envelope.\n>> A 9% uplift is above the index. I would counter at 4% and insist on a\n>> break at year three." + }, + + { + "message_id": "", + "conversation_id": "thread-forklift", + "in_reply_to": "", + "references": [], + "date": "2026-02-03T07:55:00Z", + "from": "ops@northwind.example", + "to": ["dana@northwind.example"], + "cc": [], + "subject": "Forklift servicing schedule", + "body": "Annual servicing for the three counterbalance forklifts is booked for 18 February. The Dock Street warehouse floor will be partly closed that morning." + }, + { + "message_id": "", + "conversation_id": "thread-forklift", + "in_reply_to": "", + "references": [""], + "date": "2026-02-03T09:31:00Z", + "from": "dana@northwind.example", + "to": ["ops@northwind.example"], + "cc": [], + "subject": "Re: Forklift servicing schedule", + "body": "Fine. Please avoid the week the lease renewal signing happens.\n\n> Annual servicing for the three counterbalance forklifts is booked for\n> 18 February. The Dock Street warehouse floor will be partly closed." + }, + + { + "message_id": "", + "conversation_id": "thread-insurance", + "in_reply_to": "", + "references": [], + "date": "2026-03-06T13:10:00Z", + "from": "broker@sureline.example", + "to": ["marcus@contoso.example"], + "cc": [], + "subject": "Property insurance renewal quotation", + "body": "Attached is the property insurance quotation covering the Dock Street warehouse for the 2027 policy year. Premium is up 3% year on year." + }, + { + "message_id": "", + "conversation_id": "thread-insurance", + "in_reply_to": "", + "references": [""], + "date": "2026-03-09T10:05:00Z", + "from": "marcus@contoso.example", + "to": ["broker@sureline.example"], + "cc": ["priya@contoso.example"], + "subject": "Re: Property insurance renewal quotation", + "body": "Noted, 3% is acceptable. Please confirm the sum insured reflects the new racking.\n\n> Attached is the property insurance quotation covering the Dock Street\n> warehouse for the 2027 policy year. Premium is up 3% year on year." + }, + + { + "message_id": "", + "conversation_id": "thread-offsite", + "in_reply_to": "", + "references": [], + "date": "2026-04-21T18:47:00Z", + "from": "priya@contoso.example", + "to": ["marcus@contoso.example", "dana@northwind.example"], + "cc": [], + "subject": "Team offsite - venue options", + "body": "Three venue options for the June offsite. None of them are warehouses. Please rank by Friday." + }, + + { + "message_id": "", + "conversation_id": "thread-newsletter", + "in_reply_to": "", + "references": [], + "date": "2026-05-02T06:00:00Z", + "from": "news@logisticsweekly.example", + "to": ["dana@northwind.example"], + "cc": [], + "subject": "Logistics Weekly: warehouse lease rates hit five-year high", + "body": "This week: warehouse lease rates hit a five-year high, forklift safety recalls, and an interview on break clauses. Unsubscribe at any time." + } + ] +} diff --git a/internal/eval/testdata/threaded/qrels_conversation.txt b/internal/eval/testdata/threaded/qrels_conversation.txt new file mode 100644 index 000000000..fad9575f9 --- /dev/null +++ b/internal/eval/testdata/threaded/qrels_conversation.txt @@ -0,0 +1,8 @@ +q1 0 thread-renewal 1 +q1 0 thread-forklift 0 +q1 0 thread-newsletter 0 +q1 0 thread-offsite 0 +q2 0 thread-forklift 1 +q2 0 thread-newsletter 0 +q3 0 thread-insurance 1 +q3 0 thread-renewal 0 diff --git a/internal/eval/testdata/threaded/qrels_message.txt b/internal/eval/testdata/threaded/qrels_message.txt new file mode 100644 index 000000000..5f03967b0 --- /dev/null +++ b/internal/eval/testdata/threaded/qrels_message.txt @@ -0,0 +1,13 @@ +q1 0 1 +q1 0 1 +q1 0 1 +q1 0 1 +q1 0 0 +q1 0 0 +q1 0 0 +q2 0 1 +q2 0 1 +q2 0 0 +q3 0 1 +q3 0 1 +q3 0 0 diff --git a/internal/eval/testdata/threaded/topics.tsv b/internal/eval/testdata/threaded/topics.tsv new file mode 100644 index 000000000..0c9bf1d1a --- /dev/null +++ b/internal/eval/testdata/threaded/topics.tsv @@ -0,0 +1,3 @@ +q1 warehouse lease renewal uplift and break clause spanning +q2 forklift servicing schedule pointed +q3 property insurance renewal for the warehouse diff --git a/internal/query/sqlite.go b/internal/query/sqlite.go index 4f3f5c7b6..10c7e04fb 100644 --- a/internal/query/sqlite.go +++ b/internal/query/sqlite.go @@ -1064,6 +1064,17 @@ func (e *SQLiteEngine) ListMessages(ctx context.Context, filter MessageFilter) ( return results, nil } +// messageSummaryIDChunk caps how many ids GetMessageSummariesByIDs binds into +// a single IN-list statement — for the base summary query and for the +// per-message label/participant hydration that follows it. SQLite refuses a +// statement carrying more than 32766 bound parameters by default, and one id +// is one parameter here; the eval command's dense vector/hybrid modes can +// over-fetch a ranked result set well past that at a large -n. This engine +// is dialect-agnostic (SQLite and PostgreSQL share it), and PostgreSQL's own +// parameter ceiling is far higher, so chunking — rather than a SQLite-only +// rewrite of the IN clause — is the one code path that stays correct on both. +const messageSummaryIDChunk = 500 + // GetMessageSummariesByIDs returns summary rows (no body, no raw // MIME) for the supplied IDs in the same order as ids. Missing IDs // are silently dropped. Designed for vector/hybrid search hit @@ -1074,6 +1085,38 @@ func (e *SQLiteEngine) GetMessageSummariesByIDs(ctx context.Context, ids []int64 if len(ids) == 0 { return nil, nil } + byID := make(map[int64]MessageSummary, len(ids)) + for start := 0; start < len(ids); start += messageSummaryIDChunk { + end := min(start+messageSummaryIDChunk, len(ids)) + if err := e.fetchMessageSummariesByIDsInto(ctx, ids[start:end], byID); err != nil { + return nil, err + } + } + + // Reassemble in caller-order so search rank is preserved. + results := make([]MessageSummary, 0, len(byID)) + for _, id := range ids { + if m, ok := byID[id]; ok { + results = append(results, m) + } + } + // Chunked for the same reason the base query above is: fetchLabelsForMessages + // binds one parameter per message id into its own IN-list, and results here + // can carry as many ids as the caller originally asked to hydrate. + for start := 0; start < len(results); start += messageSummaryIDChunk { + end := min(start+messageSummaryIDChunk, len(results)) + if err := e.fetchLabelsForMessages(ctx, results[start:end]); err != nil { + return nil, fmt.Errorf("fetch labels: %w", err) + } + } + return results, nil +} + +// fetchMessageSummariesByIDsInto runs one chunk's IN-list query and merges +// its rows into byID, keyed by message id. +func (e *SQLiteEngine) fetchMessageSummariesByIDsInto( + ctx context.Context, ids []int64, byID map[int64]MessageSummary, +) error { placeholders := make([]string, len(ids)) args := make([]any, len(ids)) for i, id := range ids { @@ -1107,11 +1150,10 @@ func (e *SQLiteEngine) GetMessageSummariesByIDs(ctx context.Context, ids []int64 rows, err := e.queryContext(ctx, q, args...) if err != nil { - return nil, fmt.Errorf("get message summaries by ids: %w", err) + return fmt.Errorf("get message summaries by ids: %w", err) } defer func() { _ = rows.Close() }() - byID := make(map[int64]MessageSummary, len(ids)) for rows.Next() { var msg MessageSummary var sentAt sql.NullTime @@ -1135,7 +1177,7 @@ func (e *SQLiteEngine) GetMessageSummariesByIDs(ctx context.Context, ids []int64 &msg.MessageType, &msg.ConversationTitle, ); err != nil { - return nil, fmt.Errorf("scan message: %w", err) + return fmt.Errorf("scan message: %w", err) } if sentAt.Valid { msg.SentAt = sentAt.Time @@ -1146,22 +1188,9 @@ func (e *SQLiteEngine) GetMessageSummariesByIDs(ctx context.Context, ids []int64 byID[msg.ID] = msg } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate messages: %w", err) - } - - // Reassemble in caller-order so search rank is preserved. - results := make([]MessageSummary, 0, len(byID)) - for _, id := range ids { - if m, ok := byID[id]; ok { - results = append(results, m) - } + return fmt.Errorf("iterate messages: %w", err) } - if len(results) > 0 { - if err := e.fetchLabelsForMessages(ctx, results); err != nil { - return nil, fmt.Errorf("fetch labels: %w", err) - } - } - return results, nil + return nil } func (e *SQLiteEngine) fetchLabelsForMessages(ctx context.Context, messages []MessageSummary) error { diff --git a/internal/query/sqlite_crud_test.go b/internal/query/sqlite_crud_test.go index 6aa26e5e7..00805a2d1 100644 --- a/internal/query/sqlite_crud_test.go +++ b/internal/query/sqlite_crud_test.go @@ -1840,6 +1840,63 @@ func TestSQLiteMessageSummariesIncludeSourceID(t *testing.T) { assert.Equal(sourceID, detail.SourceID) } +// TestGetMessageSummariesByIDs_ChunksLargeIDSetsAndPreservesOrder is the +// regression for the SQLite bound-parameter ceiling: one id is one bound +// parameter in this query's IN-list, and a caller hydrating a large ranked +// result set (eval's dense vector/hybrid modes at a large -n) can ask for far +// more ids than SQLite's default 32766-parameter-per-statement limit allows +// in a single call. Requesting more ids than messageSummaryIDChunk must still +// return every one of them, in the caller's own order — not chunk order — +// since that order is the search rank the caller reassembles by. The label +// hydration that follows the base query binds ids into its own IN-list too, +// so a message on each side of the chunk boundary carries a label to prove +// that pass is chunked as well, not just the base fetch. +func TestGetMessageSummariesByIDs_ChunksLargeIDSetsAndPreservesOrder(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + env := newTestEnv(t) + sourceID := env.AddSource(dbtest.SourceOpts{Identifier: "chunk-source"}) + conversationID := env.AddConversation(dbtest.ConversationOpts{SourceID: sourceID}) + + const total = messageSummaryIDChunk + 3 + ids := make([]int64, total) + for i := range total { + ids[i] = env.AddMessage(dbtest.MessageOpts{ + SourceID: sourceID, + ConversationID: conversationID, + Subject: fmt.Sprintf("chunk-regression-%d", i), + }) + } + + firstChunkLabel := env.AddLabel(dbtest.LabelOpts{SourceID: sourceID, Name: "first-chunk"}) + env.AddMessageLabel(ids[0], firstChunkLabel) + lastChunkLabel := env.AddLabel(dbtest.LabelOpts{SourceID: sourceID, Name: "second-chunk"}) + env.AddMessageLabel(ids[total-1], lastChunkLabel) + + // Reverse the request order so a bug that silently reassembled results in + // chunk (insertion) order rather than caller order would fail loudly. + requested := make([]int64, total) + for i, id := range ids { + requested[total-1-i] = id + } + + hydrated, err := env.Engine.GetMessageSummariesByIDs(env.Ctx, requested) + require.NoError(err, "GetMessageSummariesByIDs") + require.Len(hydrated, total, "every id across the chunk boundary must come back") + for i, m := range hydrated { + assert.Equal(requested[i], m.ID, "result order must match the caller's request order, not chunk order") + } + + byID := make(map[int64]MessageSummary, len(hydrated)) + for _, m := range hydrated { + byID[m.ID] = m + } + assert.Equal([]string{"first-chunk"}, byID[ids[0]].Labels, + "a message hydrated in the first label chunk must carry its label") + assert.Equal([]string{"second-chunk"}, byID[ids[total-1]].Labels, + "a message hydrated in the second label chunk must carry its label too") +} + func TestGetTotalStats_SearchScopeCountsMatchingLabelsAndSources(t *testing.T) { env := newTestEnv(t) source2 := env.AddSource(dbtest.SourceOpts{ diff --git a/internal/store/api.go b/internal/store/api.go index fcaec71b8..924f1a264 100644 --- a/internal/store/api.go +++ b/internal/store/api.go @@ -1094,12 +1094,37 @@ func (s *Store) batchPopulateContext(ctx context.Context, messages []APIMessage, return nil } -// batchGetRecipients loads recipients for multiple messages in a single query. +// batchQueryIDChunk caps how many message ids batchGetRecipients and +// batchGetLabels bind into a single IN-list statement. SQLite refuses a +// statement carrying more than 32766 bound parameters by default, and one id +// is one parameter here; eval's FTS mode over-fetches a ranked page well past +// that at a large -n (the same over-fetch plan documented on rankedKeys). +// Mirrors messageSummaryIDChunk in internal/query — same limit, same cause, +// a different package's copy of the batch-hydration pattern. +const batchQueryIDChunk = 500 + +// batchGetRecipients loads recipients for multiple messages, chunked to stay +// under the SQLite bound-parameter ceiling. func (s *Store) batchGetRecipients(ctx context.Context, messageIDs []int64, recipientType string) (map[int64][]string, error) { if len(messageIDs) == 0 { return map[int64][]string{}, nil } + result := make(map[int64][]string, len(messageIDs)) + for start := 0; start < len(messageIDs); start += batchQueryIDChunk { + end := min(start+batchQueryIDChunk, len(messageIDs)) + if err := s.fetchRecipientsInto(ctx, messageIDs[start:end], recipientType, result); err != nil { + return nil, err + } + } + return result, nil +} +// fetchRecipientsInto runs one chunk's IN-list query and merges its rows +// into result, keyed by message id. Every id belongs to exactly one chunk, +// so no key is ever written by more than one call. +func (s *Store) fetchRecipientsInto( + ctx context.Context, messageIDs []int64, recipientType string, result map[int64][]string, +) error { placeholders := make([]string, len(messageIDs)) args := make([]any, 0, len(messageIDs)+1) for i, id := range messageIDs { @@ -1117,33 +1142,45 @@ func (s *Store) batchGetRecipients(ctx context.Context, messageIDs []int64, reci rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { - return nil, fmt.Errorf("batch get recipients: %w", err) + return fmt.Errorf("batch get recipients: %w", err) } defer func() { _ = rows.Close() }() - result := make(map[int64][]string, len(messageIDs)) for rows.Next() { var msgID int64 var display string if err := rows.Scan(&msgID, &display); err != nil { - return nil, fmt.Errorf("scan recipient: %w", err) + return fmt.Errorf("scan recipient: %w", err) } if display != "" { result[msgID] = append(result[msgID], display) } } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate recipients: %w", err) + return fmt.Errorf("iterate recipients: %w", err) } - return result, nil + return nil } -// batchGetLabels loads labels for multiple messages in a single query. +// batchGetLabels loads labels for multiple messages, chunked to stay under +// the SQLite bound-parameter ceiling. func (s *Store) batchGetLabels(ctx context.Context, messageIDs []int64) (map[int64][]string, error) { if len(messageIDs) == 0 { return map[int64][]string{}, nil } + result := make(map[int64][]string, len(messageIDs)) + for start := 0; start < len(messageIDs); start += batchQueryIDChunk { + end := min(start+batchQueryIDChunk, len(messageIDs)) + if err := s.fetchLabelsInto(ctx, messageIDs[start:end], result); err != nil { + return nil, err + } + } + return result, nil +} +// fetchLabelsInto runs one chunk's IN-list query and merges its rows into +// result, keyed by message id. +func (s *Store) fetchLabelsInto(ctx context.Context, messageIDs []int64, result map[int64][]string) error { placeholders := make([]string, len(messageIDs)) args := make([]any, 0, len(messageIDs)) for i, id := range messageIDs { @@ -1160,23 +1197,22 @@ func (s *Store) batchGetLabels(ctx context.Context, messageIDs []int64) (map[int rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { - return nil, fmt.Errorf("batch get labels: %w", err) + return fmt.Errorf("batch get labels: %w", err) } defer func() { _ = rows.Close() }() - result := make(map[int64][]string, len(messageIDs)) for rows.Next() { var msgID int64 var name string if err := rows.Scan(&msgID, &name); err != nil { - return nil, fmt.Errorf("scan label: %w", err) + return fmt.Errorf("scan label: %w", err) } result[msgID] = append(result[msgID], name) } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate labels: %w", err) + return fmt.Errorf("iterate labels: %w", err) } - return result, nil + return nil } // Single-message helpers (still used by GetMessage for single PK lookups) diff --git a/internal/store/api_batch_chunk_test.go b/internal/store/api_batch_chunk_test.go new file mode 100644 index 000000000..0c2841cfe --- /dev/null +++ b/internal/store/api_batch_chunk_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBatchGetRecipientsAndLabels_ChunkLargeIDSets is the regression for the +// SQLite bound-parameter ceiling in batchGetRecipients/batchGetLabels: one +// message id is one bound parameter in each function's IN-list, and eval's +// FTS mode over-fetches a ranked page well past batchQueryIDChunk at a large +// -n (see rankedFTS / eval.OverFetchPlan). A message on each side of the +// chunk boundary carries a recipient and a label, so a bug that silently +// dropped the second chunk, or mis-merged results across chunks, would leave +// one of the two empty instead of populated. +func TestBatchGetRecipientsAndLabels_ChunkLargeIDSets(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := openTestStore(t) + ctx := context.Background() + + src, err := st.GetOrCreateSource("gmail", "batchchunk@example.com") + require.NoError(err, "GetOrCreateSource") + convID, err := st.EnsureConversation(src.ID, "thread-batchchunk", "Thread BatchChunk") + require.NoError(err, "EnsureConversation") + aliceID, err := st.EnsureParticipant("alice@example.com", "Alice", "example.com") + require.NoError(err, "EnsureParticipant alice") + bobID, err := st.EnsureParticipant("bob@example.com", "Bob", "example.com") + require.NoError(err, "EnsureParticipant bob") + + const total = batchQueryIDChunk + 3 + sentAt := time.Date(2024, 9, 2, 12, 0, 0, 0, time.UTC) + ids := make([]int64, total) + for i := range total { + id, err := st.UpsertMessage(&Message{ + ConversationID: convID, + SourceID: src.ID, + SourceMessageID: fmt.Sprintf("batchchunk-msg-%d", i), + MessageType: "email", + SentAt: sql.NullTime{Time: sentAt, Valid: true}, + Subject: sql.NullString{String: "batchchunk", Valid: true}, + SizeEstimate: 100, + }) + require.NoError(err, "UpsertMessage %d", i) + ids[i] = id + } + + require.NoError(st.ReplaceMessageRecipients(ids[0], "to", []int64{aliceID}, []string{"Alice"}), + "recipient on the first chunk's message") + require.NoError(st.ReplaceMessageRecipients(ids[total-1], "to", []int64{bobID}, []string{"Bob"}), + "recipient on the second chunk's message") + + firstLabelID, err := st.EnsureLabel(src.ID, "first-chunk", "first-chunk", "user") + require.NoError(err, "EnsureLabel first-chunk") + require.NoError(st.LinkMessageLabel(ids[0], firstLabelID)) + secondLabelID, err := st.EnsureLabel(src.ID, "second-chunk", "second-chunk", "user") + require.NoError(err, "EnsureLabel second-chunk") + require.NoError(st.LinkMessageLabel(ids[total-1], secondLabelID)) + + recipients, err := st.batchGetRecipients(ctx, ids, "to") + require.NoError(err, "batchGetRecipients") + assert.Equal([]string{"Alice "}, recipients[ids[0]], + "a message hydrated in the first recipient chunk must carry its recipient") + assert.Equal([]string{"Bob "}, recipients[ids[total-1]], + "a message hydrated in the second recipient chunk must carry its recipient too") + + labels, err := st.batchGetLabels(ctx, ids) + require.NoError(err, "batchGetLabels") + assert.Equal([]string{"first-chunk"}, labels[ids[0]], + "a message hydrated in the first label chunk must carry its label") + assert.Equal([]string{"second-chunk"}, labels[ids[total-1]], + "a message hydrated in the second label chunk must carry its label too") +} From 61df40990d902ac13c3e566ee144b96ad844540a Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 21 Sep 2026 23:08:31 -0400 Subject: [PATCH 009/125] fix(pst): treat empty attachment tables as no attachments (#888) PST import now treats a message flagged with attachments but carrying an empty attachment table as having no attachments. It no longer increments the import error count for that case. go-pst wraps `ErrTableContextNoRows` inside the attachment iterator error, while `ReadAttachments` recognized only `ErrAttachmentsNotFound`. It now matches either sentinel by identity. Other attachment table, read, and iterator failures remain counted, and source identity and message content inputs stay unchanged. On the EDRM Enron PST from #886 the import error count drops from 3 to 1; the remaining error comes from a search-folder entry that a separate change handles. Refs #886 Co-authored-by: Rod Boev --- internal/importer/pst_integration_test.go | 64 +++++++++++++++++++++ internal/pst/mime_test.go | 70 +++++++++++++++++++++++ internal/pst/reader.go | 13 ++++- 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/internal/importer/pst_integration_test.go b/internal/importer/pst_integration_test.go index 2a6305366..96676222f 100644 --- a/internal/importer/pst_integration_test.go +++ b/internal/importer/pst_integration_test.go @@ -1,12 +1,18 @@ package importer import ( + "bytes" "context" + "errors" + "log/slog" + "os" "path/filepath" "testing" + pstlib "github.com/mooijtech/go-pst/v6/pkg" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + pstreader "go.kenn.io/msgvault/internal/pst" "go.kenn.io/msgvault/internal/store" ) @@ -39,10 +45,68 @@ func TestImportPst_SupportPST(t *testing.T) { assert.Equal(int64(17), summary.MessagesProcessed, "MessagesProcessed") assert.Equal(int64(17), summary.MessagesAdded, "MessagesAdded") assert.Equal(int64(0), summary.MessagesSkipped, "MessagesSkipped on first import") + assert.Equal(int64(0), summary.Errors, "Errors") assert.False(summary.HardErrors, "HardErrors") assert.Positive(summary.FoldersImported, "FoldersImported") } +func countEmptyAttachmentTables(t *testing.T, path string) int { + t.Helper() + require := require.New(t) + + pst, err := pstreader.Open(path) + require.NoError(err, "open PST") + t.Cleanup(func() { + require.NoError(pst.Close(), "close PST") + }) + + count := 0 + require.NoError(pst.WalkFolders(func(entry pstreader.FolderEntry, folder *pstlib.Folder) error { + msgIter, err := folder.GetMessageIterator() + if err != nil { + return nil //nolint:nilerr // skip folders whose message iterator cannot be read + } + for msgIter.Next() { + msg := msgIter.Value() + if pstreader.ExtractMessage(msg, entry.Path) == nil { + continue + } + _, err := msg.GetAttachmentIterator() + if errors.Is(err, pstlib.ErrTableContextNoRows) { + count++ + } + } + return nil + }), "walk folders") + + return count +} + +// TestImportPst_EmptyAttachmentTableArchive imports a real PST containing a +// message flagged with attachments whose attachment table has no rows. Set +// MSGVAULT_TEST_PST_EMPTY_ATTACHMENT_TABLE to its path to run it. +func TestImportPst_EmptyAttachmentTableArchive(t *testing.T) { + pstPath := os.Getenv("MSGVAULT_TEST_PST_EMPTY_ATTACHMENT_TABLE") + if pstPath == "" { + t.Skip("set MSGVAULT_TEST_PST_EMPTY_ATTACHMENT_TABLE to a PST containing an empty attachment table") + } + require := require.New(t) + assert := assert.New(t) + require.Positive(countEmptyAttachmentTables(t, pstPath), "archive has no empty attachment table") + + var logs bytes.Buffer + st := openIntegrationStore(t) + summary, err := ImportPst(context.Background(), st, pstPath, PstImportOptions{ + Identifier: "user@example.com", + NoResume: true, + Logger: slog.New(slog.NewTextHandler(&logs, nil)), + }) + require.NoError(err, "ImportPst") + assert.NotContains(logs.String(), "read attachments failed") + assert.Positive(summary.MessagesProcessed, "MessagesProcessed") + assert.False(summary.HardErrors, "HardErrors") +} + // TestImportPst_SupportPST_Idempotent verifies that re-importing the same PST // skips all messages (content-hash deduplication). func TestImportPst_SupportPST_Idempotent(t *testing.T) { diff --git a/internal/pst/mime_test.go b/internal/pst/mime_test.go index 3fb30d42a..b70e0b637 100644 --- a/internal/pst/mime_test.go +++ b/internal/pst/mime_test.go @@ -2,14 +2,84 @@ package pst import ( "bytes" + "errors" + "fmt" "strings" "testing" "time" + pstlib "github.com/mooijtech/go-pst/v6/pkg" + "github.com/rotisserie/eris" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func goPSTIteratorErr(cause error) error { + err := eris.Wrap(cause, "failed to get attachment table context") + err = eris.Wrap(err, "failed to get attachment table context") + return eris.Wrap(err, "failed to get attachment count") +} + +func TestNoAttachments(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "empty attachment table", + err: goPSTIteratorErr(pstlib.ErrTableContextNoRows), + want: true, + }, + { + name: "empty attachment table rewrapped", + err: fmt.Errorf("get attachment iterator: %w", goPSTIteratorErr(pstlib.ErrTableContextNoRows)), + want: true, + }, + { + name: "attachment flag unset", + err: pstlib.ErrAttachmentsNotFound, + want: true, + }, + { + name: "same text without sentinel", + err: goPSTIteratorErr(errors.New("go-pst: there are no rows in this table context")), + want: false, + }, + { + name: "table without columns", + err: goPSTIteratorErr(pstlib.ErrTableContextNoColumns), + want: false, + }, + { + name: "missing local descriptor", + err: goPSTIteratorErr(eris.Wrap(pstlib.ErrLocalDescriptorNotFound, "failed to find attachment local descriptor")), + want: false, + }, + { + name: "invalid attachment index", + err: goPSTIteratorErr(pstlib.ErrAttachmentIndexInvalid), + want: false, + }, + { + name: "size cap", + err: errAttachmentTooLarge, + want: false, + }, + { + name: "nil", + err: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, noAttachments(tt.err)) + }) + } +} + func TestWindowsFiletimeToTime(t *testing.T) { tests := []struct { name string diff --git a/internal/pst/reader.go b/internal/pst/reader.go index 43d2593a0..a4bf079ef 100644 --- a/internal/pst/reader.go +++ b/internal/pst/reader.go @@ -236,12 +236,21 @@ func (lw *limitWriter) Write(p []byte) (int, error) { return n, err } +// noAttachments reports whether err from GetAttachmentIterator means the +// message has no attachments to read: either its has-attachments flag is +// unset, or the flag is set but its attachment table has no rows. +func noAttachments(err error) bool { + return eris.Is(err, pstlib.ErrAttachmentsNotFound) || + eris.Is(err, pstlib.ErrTableContextNoRows) +} + // ReadAttachments reads all attachments from a pstlib.Message into memory. -// Returns an empty slice (not an error) when there are no attachments. +// Returns an empty slice (not an error) when there are no attachments, +// including a message flagged with attachments whose table has no rows. // Individual attachment read errors are returned as a non-nil error. func ReadAttachments(msg *pstlib.Message, maxBytes int64) ([]AttachmentEntry, error) { iter, err := msg.GetAttachmentIterator() - if eris.Is(err, pstlib.ErrAttachmentsNotFound) { + if noAttachments(err) { return nil, nil } if err != nil { From 9e8427573cf4ce9e7befd57b50fb8c2e63b6e0a0 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 21 Sep 2026 23:08:48 -0400 Subject: [PATCH 010/125] fix(pst): skip search folders when walking PST archives (#887) `msgvault import-pst` now skips PST search folders. On the EDRM Enron file from #886, it still imports all 2,178 messages, reports 2 errors instead of 3, and avoids creating an empty `All Messages` label. The PST folder walker documented this behavior but never checked the folder type. It passed `Search Root/All Messages` with a stored count of 2,178 to go-pst, which refuses to read search folders. The walker now filters `IdentifierTypeSearchFolder`, matching go-pst's check. The two remaining errors on that file come from unreadable attachment tables and are handled in a separate change. Refs #886 Co-authored-by: Rod Boev --- internal/importer/pst_integration_test.go | 88 +++++++++++++++++++++++ internal/pst/reader.go | 6 ++ internal/pst/reader_internal_test.go | 80 +++++++++++++++++++++ internal/pst/reader_test.go | 25 +++++++ 4 files changed, 199 insertions(+) create mode 100644 internal/pst/reader_internal_test.go diff --git a/internal/importer/pst_integration_test.go b/internal/importer/pst_integration_test.go index 96676222f..88e59a21f 100644 --- a/internal/importer/pst_integration_test.go +++ b/internal/importer/pst_integration_test.go @@ -48,6 +48,94 @@ func TestImportPst_SupportPST(t *testing.T) { assert.Equal(int64(0), summary.Errors, "Errors") assert.False(summary.HardErrors, "HardErrors") assert.Positive(summary.FoldersImported, "FoldersImported") + assert.Equal(int64(0), summary.Errors, "Errors") + assert.Equal(2, summary.FoldersTotal, "FoldersTotal") +} + +func TestImportPst_SearchFolderResume(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + pstPath := filepath.Join(pstTestdataDir, "support.pst") + absPath, err := filepath.Abs(pstPath) + require.NoError(err, "abs PST path") + archiveID, err := pstArchiveFingerprint(absPath) + require.NoError(err, "PST fingerprint") + + tests := []struct { + name string + folderIndex int + folderPath string + messageIndex int64 + wantCalls int + wantFolders int + wantProcessed int64 + }{ + { + name: "matching checkpoint resumes in the saved folder", + folderIndex: 1, + folderPath: "ROOT_FOLDER/Top of Personal Folders/Sent Messages", + messageIndex: 3, + wantCalls: 8, + wantFolders: 1, + wantProcessed: 8, + }, + { + name: "shifted checkpoint restarts after path mismatch", + folderIndex: 0, + folderPath: "ROOT_FOLDER/Top of Personal Folders/Sent Messages", + messageIndex: 3, + wantCalls: 17, + wantFolders: 2, + wantProcessed: 17, + }, + { + name: "out of range checkpoint restarts", + folderIndex: 99, + folderPath: "ROOT_FOLDER/Top of Personal Folders/Sent Messages", + messageIndex: 3, + wantCalls: 17, + wantFolders: 2, + wantProcessed: 17, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + st := openIntegrationStore(t) + src, err := st.GetOrCreateSource("pst", "resume@example.com") + require.NoError(err, "get/create source") + syncID, err := st.StartSync(src.ID, "import-pst") + require.NoError(err, "start sync") + cp := store.Checkpoint{ + MessagesProcessed: 9, + MessagesAdded: 9, + ErrorsCount: 1, + } + require.NoError(savePstCheckpoint( + st, syncID, absPath, archiveID, + tc.folderIndex, tc.folderPath, tc.messageIndex, &cp, + ), "save checkpoint") + require.NoError(st.FailSync(syncID, "worker stopped"), "fail prior sync") + + mock := &mockIngestFunc{} + summary, err := ImportPst(context.Background(), st, pstPath, PstImportOptions{ + Identifier: "resume@example.com", + CheckpointInterval: 1, + IngestFunc: mock.fn, + }) + require.NoError(err, "ImportPst") + require.True(summary.WasResumed, "expected checkpoint resume") + assert.Len(mock.calls, tc.wantCalls, "ingest calls") + assert.Equal(tc.wantFolders, summary.FoldersImported, "FoldersImported") + assert.Equal(tc.wantProcessed, summary.MessagesProcessed, "MessagesProcessed") + + var errorsCount int64 + require.NoError(st.DB().QueryRow( + `SELECT errors_count FROM sync_runs ORDER BY id DESC LIMIT 1`, + ).Scan(&errorsCount), "read resumed checkpoint") + assert.Equal(int64(1), errorsCount, "prior errors retained") + }) + } } func countEmptyAttachmentTables(t *testing.T, path string) int { diff --git a/internal/pst/reader.go b/internal/pst/reader.go index a4bf079ef..995bafbdc 100644 --- a/internal/pst/reader.go +++ b/internal/pst/reader.go @@ -137,6 +137,12 @@ func (f *File) WalkFolders(fn WalkFolderFunc) error { } func walkFoldersRecursive(folder *pstlib.Folder, parentPath string, fn WalkFolderFunc) error { + // Search folders are saved queries over messages stored in real folders. + // go-pst refuses to iterate them even when their stored count is nonzero. + if folder.Identifier.GetType() == pstlib.IdentifierTypeSearchFolder { + return nil + } + path := folder.Name if parentPath != "" { path = parentPath + "/" + folder.Name diff --git a/internal/pst/reader_internal_test.go b/internal/pst/reader_internal_test.go new file mode 100644 index 000000000..02921a1d2 --- /dev/null +++ b/internal/pst/reader_internal_test.go @@ -0,0 +1,80 @@ +package pst + +import ( + "path/filepath" + "testing" + + pstlib "github.com/mooijtech/go-pst/v6/pkg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalkFoldersRecursive_SearchFolderRecord(t *testing.T) { + t.Run("search folder is skipped", func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + folder := pstlib.Folder{ + Identifier: 1827, + Name: "All Messages", + MessageCount: 2178, + } + var got []FolderEntry + + err := walkFoldersRecursive(&folder, "ROOT_FOLDER/Search Root", func(entry FolderEntry, _ *pstlib.Folder) error { + got = append(got, entry) + return nil + }) + require.NoError(err) + assert.Empty(got) + + _, err = folder.GetMessageIterator() + require.ErrorIs(err, pstlib.ErrMessagesNotFound) + }) + + t.Run("normal folder with same name and count is visited", func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + folder := pstlib.Folder{ + Identifier: 1826, + Name: "All Messages", + MessageCount: 2178, + } + var got []FolderEntry + + err := walkFoldersRecursive(&folder, "ROOT_FOLDER/Search Root", func(entry FolderEntry, _ *pstlib.Folder) error { + got = append(got, entry) + return nil + }) + require.NoError(err) + require.Len(got, 1) + assert.Equal("ROOT_FOLDER/Search Root/All Messages", got[0].Path) + assert.Equal(int32(2178), got[0].MsgCount) + }) +} + +func TestWalkFolders_MatchesLibraryTraversalWithoutSearchFolders(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f, err := Open(filepath.Join("testdata", "support.pst")) + require.NoError(err, "Open") + defer func() { _ = f.Close() }() + + var got []pstlib.Identifier + err = f.WalkFolders(func(_ FolderEntry, folder *pstlib.Folder) error { + got = append(got, folder.Identifier) + return nil + }) + require.NoError(err, "WalkFolders") + + var all, want []pstlib.Identifier + err = f.pstFile.WalkFolders(func(folder *pstlib.Folder) error { + all = append(all, folder.Identifier) + if folder.Identifier.GetType() != pstlib.IdentifierTypeSearchFolder { + want = append(want, folder.Identifier) + } + return nil + }) + require.NoError(err, "go-pst WalkFolders") + require.Less(len(want), len(all)) + assert.Equal(want, got) +} diff --git a/internal/pst/reader_test.go b/internal/pst/reader_test.go index bf73d9a83..7a73ad82f 100644 --- a/internal/pst/reader_test.go +++ b/internal/pst/reader_test.go @@ -92,6 +92,31 @@ func TestWalkFolders_PathsAreSlashSeparated(t *testing.T) { assert.True(t, depth2, "expected at least one folder path with depth >= 2 (slash-separated)") } +// TestWalkFolders_SkipsSearchFolders_SupportPST verifies that real search +// folders are skipped while ordinary folders remain visible. +func TestWalkFolders_SkipsSearchFolders_SupportPST(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f, err := pstreader.Open(supportPST(t)) + require.NoError(err, "Open") + defer func() { _ = f.Close() }() + + seen := make(map[string]pstlib.IdentifierType) + err = f.WalkFolders(func(entry pstreader.FolderEntry, folder *pstlib.Folder) error { + seen[entry.Path] = folder.Identifier.GetType() + return nil + }) + require.NoError(err, "WalkFolders") + + assert.NotContains(seen, "ROOT_FOLDER/SPAM Search Folder 2") + assert.Contains(seen, "ROOT_FOLDER/Search Root") + assert.Contains(seen, "ROOT_FOLDER/Top of Personal Folders/Drafts") + assert.Contains(seen, "ROOT_FOLDER/Top of Personal Folders/Sent Messages") + for _, typ := range seen { + assert.NotEqual(pstlib.IdentifierTypeSearchFolder, typ) + } +} + // TestExtractMessages_SupportPST verifies that email messages are extracted // with the expected properties from support.pst. func TestExtractMessages_SupportPST(t *testing.T) { From 56d3f0b8b4f82b63d316daafdbef86bf5d619644 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Tue, 22 Sep 2026 08:19:29 -0400 Subject: [PATCH 011/125] feat(imap): recover identified draft operations (#880) `draft-recover` resumes an interrupted IMAP draft edit or delete from its saved receipt. It can publish a known replacement or finish cleanup after the original copy is gone, without appending another copy. The daemon checks the caller's source-scoped `draft.edit` or `draft.delete` grant before returning delegated revision or policy results. It then verifies the revision, source identity, mailbox generation, exact UID, and draft flags. Delegated responses include lifecycle metadata while omitting draft content, raw MIME, and candidate bytes. Recovery uses only saved receipts and bytes. Unknown UIDs, moved copies, mailbox searches, and sync-archived replacements remain unresolved, so recovery cannot adopt an unrelated message. Refs https://github.com/kenn-io/msgvault/issues/666#issuecomment-5585767456, slice 3c Closes #862 Co-authored-by: Rod Boev --- cmd/msgvault/cmd/draft_lifecycle.go | 14 + cmd/msgvault/cmd/root.go | 2 +- cmd/msgvault/cmd/root_test.go | 7 + cmd/msgvault/cmd/serve_draft_lifecycle.go | 462 ++++++- .../cmd/serve_draft_lifecycle_cache_test.go | 171 +++ .../cmd/serve_draft_lifecycle_review_test.go | 79 +- cmd/msgvault/cmd/serve_draft_recover_test.go | 1147 +++++++++++++++++ docs/changelog.md | 15 +- docs/cli-reference.md | 24 +- docs/usage/imap.md | 62 +- internal/agentgrant/agentgrant.go | 4 + internal/agentgrant/agentgrant_test.go | 24 + internal/api/cli_allowlist_drafts_test.go | 18 + internal/api/cli_run_draft.go | 21 +- internal/api/middleware.go | 5 +- internal/store/imap_draft_review_test.go | 44 + 16 files changed, 2027 insertions(+), 72 deletions(-) create mode 100644 cmd/msgvault/cmd/serve_draft_recover_test.go diff --git a/cmd/msgvault/cmd/draft_lifecycle.go b/cmd/msgvault/cmd/draft_lifecycle.go index 9fa293bde..c6fc0ec61 100644 --- a/cmd/msgvault/cmd/draft_lifecycle.go +++ b/cmd/msgvault/cmd/draft_lifecycle.go @@ -6,6 +6,7 @@ func init() { rootCmd.AddCommand(newDraftGetCommand()) rootCmd.AddCommand(newDraftEditCommand()) rootCmd.AddCommand(newDraftDeleteCommand()) + rootCmd.AddCommand(newDraftRecoverCommand()) } func newDraftGetCommand() *cobra.Command { @@ -46,3 +47,16 @@ func newDraftDeleteCommand() *cobra.Command { command.Flags().Bool("json", false, "emit one JSON result") return command } + +func newDraftRecoverCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-recover ", + Short: "Recover an interrupted managed IMAP draft edit or delete", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + _ = command.MarkFlagRequired("revision") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} diff --git a/cmd/msgvault/cmd/root.go b/cmd/msgvault/cmd/root.go index 7f5b77c19..d5a00d70c 100644 --- a/cmd/msgvault/cmd/root.go +++ b/cmd/msgvault/cmd/root.go @@ -243,7 +243,7 @@ func skipsConfigLoad(cmd *cobra.Command) bool { // must be run by the owner. func agentDelegatedCapable(cmd *cobra.Command) bool { switch cmd.Name() { - case "draft-reply": + case "draft-reply", "draft-recover": return true } return false diff --git a/cmd/msgvault/cmd/root_test.go b/cmd/msgvault/cmd/root_test.go index f405c1526..20fadd7d2 100644 --- a/cmd/msgvault/cmd/root_test.go +++ b/cmd/msgvault/cmd/root_test.go @@ -669,6 +669,13 @@ func TestAgentDelegatedCapableCommandSucceeds(t *testing.T) { require.NoError(t, err, "draft-reply with agent flags should succeed in PersistentPreRunE") } +func TestAgentDelegatedRecoveryCommandSucceeds(t *testing.T) { + withAgentFlags(t, "http://daemon.example:8080", "/tmp/token") + + err := rootCmd.PersistentPreRunE(&cobra.Command{Use: "draft-recover"}, nil) + require.NoError(t, err) +} + // TestAgentDelegatedNonCapableCommandReturnsError verifies that a command // not in the delegated-capable set (serve) returns "not available in // agent-delegated mode" when agent flags are present. diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle.go b/cmd/msgvault/cmd/serve_draft_lifecycle.go index ca4696ec7..396d7fe75 100644 --- a/cmd/msgvault/cmd/serve_draft_lifecycle.go +++ b/cmd/msgvault/cmd/serve_draft_lifecycle.go @@ -11,6 +11,7 @@ import ( "time" "unicode/utf8" + "go.kenn.io/msgvault/internal/agentgrant" "go.kenn.io/msgvault/internal/api" imaplib "go.kenn.io/msgvault/internal/imap" msgmime "go.kenn.io/msgvault/internal/mime" @@ -61,6 +62,7 @@ type draftLifecycleOutput struct { CandidateContent string `json:"candidate_content,omitempty"` PendingOperation string `json:"pending_operation,omitempty"` PendingCode string `json:"pending_code,omitempty"` + RefusalCode string `json:"refusal_code,omitempty"` PendingReceipt *draftLifecycleReceipt `json:"pending_receipt,omitempty"` ProviderObservation *draftLifecycleObservation `json:"provider_observation,omitempty"` Observation *draftLifecycleObservation `json:"observation,omitempty"` @@ -144,6 +146,10 @@ func parseDraftLifecycleArgs(args []string) (draftLifecycleIntent, error) { if !revisionSet || bodySet { return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-delete requires --revision and no body")) } + case api.CLIRunDraftRecoverCommand: + if !revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-recover requires --revision and no body")) + } } return intent, nil } @@ -173,21 +179,12 @@ func draftLifecycleObservationCode(observation imaplib.DraftObservation, fallbac return fallback } -func (a *storeAPIAdapter) draftLifecycleOutput( - ctx context.Context, +func draftLifecycleMetadata( draft store.IMAPDraft, status string, providerObservation *draftLifecycleObservation, observation *draftLifecycleObservation, -) (draftLifecycleOutput, error) { - message, err := a.store.GetMessageContext(ctx, draft.CurrentMessageID) - if err != nil { - return draftLifecycleOutput{}, fmt.Errorf("load managed draft message: %w", err) - } - raw, err := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) - if err != nil { - return draftLifecycleOutput{}, fmt.Errorf("load managed draft MIME: %w", err) - } +) draftLifecycleOutput { lifecycle := draftLifecycleActive if draft.DiscardedAt != nil { lifecycle = "discarded" @@ -196,23 +193,57 @@ func (a *storeAPIAdapter) draftLifecycleOutput( Status: status, DraftID: draft.DraftID, Revision: draft.Revision, Lifecycle: lifecycle, MessageID: draft.CurrentMessageID, SourceID: draft.SourceID, Receipt: draftLifecycleReceiptOutput(draft.CurrentReceipt), - Content: message.BodyText, RawMIME: string(raw), ProviderObservation: providerObservation, Observation: observation, } if draft.Pending != nil { output.PendingOperation = draft.Pending.Operation output.PendingCode = draft.Pending.Code - if len(draft.Pending.Raw) > 0 { - output.CandidateContent = string(draft.Pending.Raw) - } if draft.Pending.ReplacementReceipt != nil { receipt := draftLifecycleReceiptOutput(*draft.Pending.ReplacementReceipt) output.PendingReceipt = &receipt } } + return output +} + +func (a *storeAPIAdapter) draftLifecycleOutput( + ctx context.Context, + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, +) (draftLifecycleOutput, error) { + message, err := a.store.GetMessageContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftLifecycleOutput{}, fmt.Errorf("load managed draft message: %w", err) + } + raw, err := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftLifecycleOutput{}, fmt.Errorf("load managed draft MIME: %w", err) + } + output := draftLifecycleMetadata(draft, status, providerObservation, observation) + output.Content = message.BodyText + output.RawMIME = string(raw) + if draft.Pending != nil { + output.CandidateContent = string(draft.Pending.Raw) + } return output, nil } +func (a *storeAPIAdapter) draftRecoveryOutput( + ctx context.Context, + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, + grant *agentgrant.Grant, +) (draftLifecycleOutput, error) { + if grant != nil { + return draftLifecycleMetadata(draft, status, providerObservation, observation), nil + } + return a.draftLifecycleOutput(ctx, draft, status, providerObservation, observation) +} + func emitDraftLifecycleOutput( emit func(api.CLIRunEvent) error, stream string, @@ -259,6 +290,9 @@ func emitDraftLifecycleOutput( fmt.Fprintf(&data, "old provider receipt: %s\n", textutil.SanitizeTerminal(formatDraftLifecycleObservationReceipt(*output.Observation))) } + if output.RefusalCode != "" { + fmt.Fprintf(&data, "recovery refusal: %s\n", textutil.SanitizeTerminal(output.RefusalCode)) + } providerOutcome := output.PendingCode if providerOutcome == "" { observations := []*draftLifecycleObservation{output.ProviderObservation, output.Observation} @@ -302,21 +336,28 @@ func (a *storeAPIAdapter) loadManagedDraftSource(ctx context.Context, draft stor if err != nil { return nil, draftReplyError("invalid_source", err) } + if err := a.validateManagedDraftSource(draft, source); err != nil { + return nil, err + } + return source, nil +} + +func (a *storeAPIAdapter) validateManagedDraftSource(draft store.IMAPDraft, source *store.Source) error { mailbox, err := authorizeIMAPDraft(a.draftPolicy, source.ID, source.SourceType) if err != nil { - return nil, err + return err } if mailbox != draft.CurrentReceipt.Mailbox { - return nil, draftReplyError("invalid_mailbox", errors.New("draft receipt mailbox is outside the current owner grant")) + return draftReplyError("invalid_mailbox", errors.New("draft receipt mailbox is outside the current owner grant")) } if !source.SyncConfig.Valid { - return nil, draftReplyError("invalid_source", errors.New("source has no sync config")) + return draftReplyError("invalid_source", errors.New("source has no sync config")) } config, err := imaplib.ConfigFromJSON(source.SyncConfig.String) if err != nil || config.Identifier() != source.Identifier { - return nil, draftReplyError("invalid_source", errors.New("source sync config identity does not match the source")) + return draftReplyError("invalid_source", errors.New("source sync config identity does not match the source")) } - return source, nil + return nil } func localDraftEvidenceContext(ctx context.Context) (context.Context, context.CancelFunc) { @@ -347,6 +388,9 @@ func (a *storeAPIAdapter) runCLIDraftLifecycle( } draft, err := a.store.GetIMAPDraftContext(ctx, intent.DraftID) if err != nil { + if req.Grant != nil { + return draftReplyNotPermitted(err) + } return draftReplyError("draft_not_found", err) } if intent.Operation == api.CLIRunDraftGetCommand { @@ -357,6 +401,9 @@ func (a *storeAPIAdapter) runCLIDraftLifecycle( } return emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output) } + if intent.Operation == api.CLIRunDraftRecoverCommand { + return a.runDraftRecover(ctx, intent, draft, req.Grant, emit) + } if draft.Revision != intent.Revision { return draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) } @@ -604,7 +651,7 @@ func (a *storeAPIAdapter) runDraftEdit( if err != nil || !removed.Complete { code := draftLifecycleObservationCode(removed, "cleanup_incomplete") recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, published.Revision, code, nil) - a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) if err == nil { err = errors.New("provider cleanup is incomplete") } @@ -616,12 +663,12 @@ func (a *storeAPIAdapter) runDraftEdit( if err := a.store.RecordIMAPDraftOutcomeContext( evidenceCtx, intent.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil, ); err != nil { - a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) return draftReplyError("local_persistence_failed", err) } finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, published.Revision) if err != nil { - a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) return draftReplyError("cleanup_local_failed", err) } output, err := a.draftLifecycleOutput(evidenceCtx, finished, "edited", nil, draftLifecycleObservationOutput(removed)) @@ -638,6 +685,7 @@ func (a *storeAPIAdapter) emitDraftLifecyclePending( ctx context.Context, intent draftLifecycleIntent, draft store.IMAPDraft, + grant *agentgrant.Grant, providerObservation *draftLifecycleObservation, observation imaplib.DraftObservation, emit func(api.CLIRunEvent) error, @@ -645,27 +693,13 @@ func (a *storeAPIAdapter) emitDraftLifecyclePending( if latest, err := a.store.GetIMAPDraftContext(ctx, draft.DraftID); err == nil { draft = latest } - output, err := a.draftLifecycleOutput(ctx, draft, "pending", providerObservation, draftLifecycleObservationOutput(observation)) + output, err := a.draftRecoveryOutput(ctx, draft, "pending", providerObservation, draftLifecycleObservationOutput(observation), grant) if err != nil { - output = draftLifecycleOutput{ - Status: "pending", DraftID: draft.DraftID, Revision: draft.Revision, Lifecycle: draftLifecycleActive, - MessageID: draft.CurrentMessageID, SourceID: draft.SourceID, Receipt: draftLifecycleReceiptOutput(draft.CurrentReceipt), - ProviderObservation: providerObservation, Observation: draftLifecycleObservationOutput(observation), - } - if draft.Pending != nil { - output.PendingOperation = draft.Pending.Operation + output = draftLifecycleMetadata(draft, "pending", providerObservation, draftLifecycleObservationOutput(observation)) + if grant == nil && draft.Pending != nil { output.CandidateContent = string(draft.Pending.Raw) - if draft.Pending.ReplacementReceipt != nil { - receipt := draftLifecycleReceiptOutput(*draft.Pending.ReplacementReceipt) - output.PendingReceipt = &receipt - } } } - if observation.Code != "" { - output.PendingCode = observation.Code - } else if output.PendingCode == "" { - output.PendingCode = "cleanup_incomplete" - } output.ManualReconciliation = true _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) } @@ -692,7 +726,7 @@ func (a *storeAPIAdapter) runDraftDelete( if !removed.WriteAttempted { active, abortErr := a.store.AbortIMAPDraftContext(evidenceCtx, intent.DraftID, intent.Revision, "not_attempted") if abortErr != nil { - a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) return draftReplyError("local_persistence_failed", errors.Join(err, abortErr)) } output, outputErr := a.draftLifecycleOutput(evidenceCtx, active, draftLifecycleActive, draftLifecycleObservationOutput(inspection), draftLifecycleObservationOutput(removed)) @@ -703,7 +737,7 @@ func (a *storeAPIAdapter) runDraftDelete( return draftReplyError(code, err) } recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, nil) - a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) if err == nil { err = errors.New("provider cleanup is incomplete") } @@ -717,12 +751,12 @@ func (a *storeAPIAdapter) runDraftDelete( if err := a.store.RecordIMAPDraftOutcomeContext( evidenceCtx, intent.DraftID, intent.Revision, store.IMAPDraftCodeRemoved, nil, ); err != nil { - a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) return draftReplyError("local_persistence_failed", err) } finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, intent.Revision) if err != nil { - a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, draftLifecycleObservationOutput(inspection), removed, emit) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) return draftReplyError("cleanup_local_failed", err) } defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) @@ -736,6 +770,346 @@ func (a *storeAPIAdapter) runDraftDelete( return nil } +func draftRecoveryPermission(draft store.IMAPDraft) (agentgrant.Permission, error) { + if draft.DiscardedAt != nil { + return agentgrant.PermissionDraftDelete, nil + } + if draft.Pending == nil { + return agentgrant.PermissionDraftEdit, nil + } + switch draft.Pending.Operation { + case store.IMAPDraftOperationEdit: + return agentgrant.PermissionDraftEdit, nil + case store.IMAPDraftOperationDelete: + return agentgrant.PermissionDraftDelete, nil + default: + return "", draftReplyError("invalid_state", fmt.Errorf("unknown pending draft operation %q", draft.Pending.Operation)) + } +} + +func draftProviderReceipt(receipt store.IMAPDraftReceipt) imaplib.DraftReceipt { + return imaplib.DraftReceipt{ + Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID, + } +} + +func (a *storeAPIAdapter) authorizeDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, +) (*store.Source, error) { + permission, err := draftRecoveryPermission(draft) + if err != nil { + return nil, err + } + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + if grant != nil { + return nil, draftReplyNotPermitted(fmt.Errorf("load source %d: %w", draft.SourceID, err)) + } + return nil, draftReplyError("invalid_source", err) + } + if grant != nil { + ref := agentgrant.SourceRef{ID: source.ID, Type: source.SourceType, Identifier: source.Identifier} + if !grant.Allows(permission, ref) { + return nil, draftReplyNotPermitted(fmt.Errorf("source %d is not in grant %s", source.ID, grant.ID)) + } + } + if draft.Revision != intent.Revision { + return nil, draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if err := a.validateManagedDraftSource(draft, source); err != nil { + return nil, err + } + return source, nil +} + +func (a *storeAPIAdapter) settleDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + emit func(api.CLIRunEvent) error, +) (bool, error) { + var status string + switch { + case draft.DiscardedAt != nil: + status = "already_discarded" + case draft.Pending == nil: + status = draftLifecycleActive + case draft.Pending.Operation == store.IMAPDraftOperationEdit && draft.Pending.ReplacementReceipt == nil: + output, err := a.draftRecoveryOutput(ctx, draft, "unknown_replacement", nil, nil, grant) + if err != nil { + return true, draftReplyError("draft_read_failed", err) + } + output.RefusalCode = "unknown_replacement" + output.ManualReconciliation = true + if err := emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output); err != nil { + return true, draftReplyError("output_failed", err) + } + return true, draftReplyError("unknown_replacement", errors.New("pending edit has no recorded replacement receipt")) + default: + return false, nil + } + output, err := a.draftRecoveryOutput(ctx, draft, status, nil, nil, grant) + if err != nil { + return true, draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); err != nil { + return true, draftReplyError("output_failed", err) + } + return true, nil +} + +func (a *storeAPIAdapter) refuseDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + code string, + cause error, + observation *imaplib.DraftObservation, + emit func(api.CLIRunEvent) error, +) error { + var providerObservation *draftLifecycleObservation + if observation != nil { + providerObservation = draftLifecycleObservationOutput(*observation) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + output, outputErr := a.draftRecoveryOutput(evidenceCtx, draft, "refused", providerObservation, nil, grant) + if outputErr != nil { + return draftReplyError("draft_read_failed", errors.Join(cause, outputErr)) + } + output.RefusalCode = code + output.ManualReconciliation = true + if emitErr := emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output); emitErr != nil { + return draftReplyError("output_failed", errors.Join(cause, emitErr)) + } + if cause == nil { + cause = errors.New("provider refused draft recovery") + } + return draftReplyError(code, cause) +} + +func (a *storeAPIAdapter) publishRecoveredDraftReplacement( + ctx context.Context, + draft store.IMAPDraft, +) (store.IMAPDraft, error) { + if draft.Pending == nil || draft.Pending.Operation != store.IMAPDraftOperationEdit || draft.Pending.ReplacementReceipt == nil { + return store.IMAPDraft{}, errors.New("known replacement is required for draft publication") + } + parsed, err := msgmime.Parse(draft.Pending.Raw) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("parse recorded draft replacement: %w", err) + } + replacement := imaplib.ReplyDraft{ + Raw: append([]byte(nil), draft.Pending.Raw...), + Parsed: parsed, + } + message, err := a.store.GetMessageContext(ctx, draft.CurrentMessageID) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("load draft message for replacement: %w", err) + } + replyTo, err := a.store.GetMessageReplyToMessageIDContext(ctx, draft.CurrentMessageID) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("load draft reply link for replacement: %w", err) + } + receipt := *draft.Pending.ReplacementReceipt + participants, build := draftLifecyclePersistData(message.ConversationID, replyTo, replacement, receipt) + return a.store.PublishIMAPDraftReplacementContext(ctx, draft.DraftID, draft.Revision, participants, build) +} + +func (a *storeAPIAdapter) runDraftRecover( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + emit func(api.CLIRunEvent) error, +) error { + source, err := a.authorizeDraftRecovery(ctx, intent, draft, grant) + if err != nil { + return err + } + settled, err := a.settleDraftRecovery(ctx, intent, draft, grant, emit) + if settled || err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + refreshScheduled := false + defer func() { + if refreshScheduled { + a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + return + } + _ = execution.Release() + }() + + draft, err = a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + if grant != nil { + return draftReplyNotPermitted(err) + } + return draftReplyError("draft_not_found", err) + } + source, err = a.authorizeDraftRecovery(ctx, intent, draft, grant) + if err != nil { + return err + } + settled, err = a.settleDraftRecovery(ctx, intent, draft, grant, emit) + if settled || err != nil { + return err + } + + if draft.Pending.Code == store.IMAPDraftCodeRemoved { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + finished, finishErr := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, draft.DraftID, draft.Revision) + if finishErr != nil { + return draftReplyError("cleanup_local_failed", finishErr) + } + refreshScheduled = true + status := "edited" + if finished.DiscardedAt != nil { + status = "deleted" + } + output, outputErr := a.draftRecoveryOutput(evidenceCtx, finished, status, nil, nil, grant) + if outputErr != nil { + return draftReplyError("draft_read_failed", outputErr) + } + if outputErr := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); outputErr != nil { + return draftReplyError("output_failed", outputErr) + } + return nil + } + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + + originalObservation, err := client.InspectDraft(ctx, draftProviderReceipt(draft.Pending.OriginalReceipt)) + if err != nil { + code := draftLifecycleObservationCode(originalObservation, "provider_refused") + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, err, &originalObservation, emit) + } + published := false + if draft.Pending.Operation == store.IMAPDraftOperationEdit { + replacementReceipt := *draft.Pending.ReplacementReceipt + published = draft.CurrentReceipt == replacementReceipt + if !published { + if draft.Pending.OriginalReceipt.UIDValidity != replacementReceipt.UIDValidity { + return a.refuseDraftRecovery( + ctx, intent, draft, grant, "uidvalidity_mismatch", + errors.New("recorded original and replacement generations differ"), + &originalObservation, emit, + ) + } + replacementObservation, replacementErr := client.InspectDraft(ctx, draftProviderReceipt(replacementReceipt)) + if replacementErr != nil || !replacementObservation.Present || !replacementObservation.Draft || replacementObservation.Deleted { + code := draftLifecycleObservationCode(replacementObservation, "provider_refused") + if replacementErr == nil { + switch { + case !replacementObservation.Present: + code = "absent" + case replacementObservation.Deleted: + code = "already_deleted" + case !replacementObservation.Draft: + code = "not_draft" + } + replacementErr = errors.New("recorded replacement is unavailable for publication") + } + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, replacementErr, &replacementObservation, emit) + } + publicationCtx, cancel := localDraftEvidenceContext(ctx) + publishedDraft, publishErr := a.publishRecoveredDraftReplacement(publicationCtx, draft) + if publishErr != nil { + output, outputErr := a.draftRecoveryOutput(publicationCtx, draft, "accepted_local_failed", draftLifecycleObservationOutput(replacementObservation), nil, grant) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent.JSON, output) + } + cancel() + return draftReplyError("accepted_local_failed", publishErr) + } + cancel() + draft = publishedDraft + published = true + refreshScheduled = true + } + } + + cleanupObservation := originalObservation + if originalObservation.Present { + cleanupObservation, err = client.RemoveDraft(ctx, draftProviderReceipt(draft.Pending.OriginalReceipt)) + if err != nil || !cleanupObservation.Complete { + code := draftLifecycleObservationCode(cleanupObservation, "cleanup_incomplete") + if !cleanupObservation.WriteAttempted { + if published { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + if err == nil { + err = errors.New("provider cleanup was not attempted") + } + return draftReplyError(code, err) + } + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, err, &cleanupObservation, emit) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, draft.DraftID, draft.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + if recordErr != nil { + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + return draftReplyError(code, err) + } + } + + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, draft.DraftID, draft.Revision, store.IMAPDraftCodeRemoved, nil); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, draft.DraftID, draft.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + return draftReplyError("cleanup_local_failed", err) + } + refreshScheduled = true + status := "edited" + if finished.DiscardedAt != nil { + status = "deleted" + } + output, err := a.draftRecoveryOutput(evidenceCtx, finished, status, nil, draftLifecycleObservationOutput(cleanupObservation), grant) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent.JSON, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + func draftLifecyclePersistData( conversationID int64, replyTo sql.NullInt64, diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go index 92a9598e8..316aac33d 100644 --- a/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/search" "go.kenn.io/msgvault/internal/store" @@ -197,3 +198,173 @@ func TestDraftDeleteOutputFailureRefreshesCache(t *testing.T) { }) } } + +func TestDraftRecoverRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "recovercache") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + removed, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(draft.CurrentReceipt)) + requirements.NoError(err) + requirements.True(removed.Complete) + requirements.NoError(client.Close()) + + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + baselineEngine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + baselineResults, err := baselineEngine.SearchFast(t.Context(), search.Parse("recovercache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + requirements.Len(baselineResults, 1) + baselineRows, err := baselineEngine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var baselineDraftCount int64 + for _, row := range baselineRows { + if row.Key == "Drafts" { + baselineDraftCount = row.Count + } + } + assertions.Equal(int64(1), baselineDraftCount) + requirements.NoError(baselineEngine.Close()) + + events, err := runReviewLifecycle(t, adapter, "draft-recover", created.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("recovercache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Empty(results) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(0), draftCount) +} + +func TestDraftRecoverPublishedEditCleanupRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + adapter.draftCacheRefresh = func(context.Context, string) error { return nil } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + if loadErr == nil && latest.Revision == 2 { + cancel() + } + }, + })) + var editEvents []api.CLIRunEvent + err = adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "recoverpublishedcache", "--json"}, + }, func(event api.CLIRunEvent) error { + editEvents = append(editEvents, event) + return nil + }) + slog.SetDefault(previous) + requirements.ErrorContains(err, "cancelled") + requirements.NotEmpty(editEvents) + published, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.Equal(int64(2), published.Revision) + requirements.NotNil(published.Pending) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + removed, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(draft.CurrentReceipt)) + requirements.NoError(err) + requirements.True(removed.Complete) + requirements.NoError(client.Close()) + + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + baselineEngine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + baselineResults, err := baselineEngine.SearchFast(t.Context(), search.Parse("recoverpublishedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Len(baselineResults, 1) + baselineRows, err := baselineEngine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var baselineDraftCount int64 + for _, row := range baselineRows { + if row.Key == "Drafts" { + baselineDraftCount = row.Count + } + } + assertions.Equal(int64(2), baselineDraftCount) + requirements.NoError(baselineEngine.Close()) + + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftRecoverCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("recoverpublishedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + assertions.Len(results, 1) + assertions.Equal(latest.CurrentMessageID, results[0].ID) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(1), draftCount) +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go index 0515025ec..7ef2a93e1 100644 --- a/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go @@ -807,7 +807,6 @@ func TestDraftLifecycleCleanupOutcomeEvidence(t *testing.T) { requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) assertions.Equal("pending", output.Status) assertions.True(output.ManualReconciliation) - assertions.Equal("store_conflict", output.PendingCode) requirements.NotNil(output.Observation) assertions.Equal("store_conflict", output.Observation.Code) assertions.Equal(uint32(1), output.Observation.UID) @@ -819,6 +818,7 @@ func TestDraftLifecycleCleanupOutcomeEvidence(t *testing.T) { latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) requirements.NoError(loadErr) requirements.NotNil(latest.Pending) + assertions.Equal(latest.Pending.Code, output.PendingCode) assertions.Equal(latest.Revision, output.Revision) if fault != "none" { assertions.Equal("local_persistence_failed", err.Error()) @@ -1391,6 +1391,83 @@ type reviewDropAppendSession struct { conn *imapserver.Conn } +type reviewUIDValidityChangeControl struct { + selects atomic.Int32 +} + +type reviewUIDValidityChangeSession struct { + imapserver.Session + + control *reviewUIDValidityChangeControl +} + +func (s *reviewUIDValidityChangeSession) Select( + mailbox string, + options *emersionimap.SelectOptions, +) (*emersionimap.SelectData, error) { + data, err := s.Session.Select(mailbox, options) + if err != nil { + return nil, fmt.Errorf("select mailbox: %w", err) + } + if s.control.selects.Add(1) == 2 { + changed := *data + changed.UIDValidity++ + return &changed, nil + } + return data, nil +} + +type reviewExpungeFailureSession struct { + imapserver.Session +} + +func (s *reviewExpungeFailureSession) Expunge( + *imapserver.ExpungeWriter, + *emersionimap.UIDSet, +) error { + return errors.New("synthetic UID EXPUNGE failure") +} + +func startReviewCustomIMAPServer( + t *testing.T, + wrap func(imapserver.Session) imapserver.Session, +) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return wrap(memServer.NewSession()), nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +func startReviewUIDValidityChangeServer( + t *testing.T, + control *reviewUIDValidityChangeControl, +) (string, *imapmemserver.User) { + t.Helper() + return startReviewCustomIMAPServer(t, func(session imapserver.Session) imapserver.Session { + return &reviewUIDValidityChangeSession{Session: session, control: control} + }) +} + +func startReviewExpungeFailureServer(t *testing.T) (string, *imapmemserver.User) { + t.Helper() + return startReviewCustomIMAPServer(t, func(session imapserver.Session) imapserver.Session { + return &reviewExpungeFailureSession{Session: session} + }) +} + func (s *reviewDropAppendSession) Append( mailbox string, r emersionimap.LiteralReader, diff --git a/cmd/msgvault/cmd/serve_draft_recover_test.go b/cmd/msgvault/cmd/serve_draft_recover_test.go new file mode 100644 index 000000000..2b81fb5c9 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_recover_test.go @@ -0,0 +1,1147 @@ +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +const draftRecoverCommand = "draft-recover" + +func newDraftRecoveryFixture(t *testing.T) (reviewManagedLifecycleFixture, *imapmemserver.User) { + t.Helper() + addr, user := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + return fixture, user +} + +func removeRecoveryOriginal(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(fixture.draft.CurrentReceipt)) + require.NoError(t, err) + require.True(t, observation.Complete) + require.NoError(t, client.Close()) +} + +func recoveryTestReceipt(receipt store.IMAPDraftReceipt) imaplib.DraftReceipt { + return imaplib.DraftReceipt{Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID} +} + +func recoveryUIDNext(t *testing.T, fixture reviewManagedLifecycleFixture) uint32 { + t.Helper() + client, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + status, err := client.Status("Drafts", &emersionimap.StatusOptions{UIDNext: true}).Wait() + require.NoError(t, err) + return uint32(status.UIDNext) +} + +func newDraftRecoveryHTTPServer(t *testing.T, fixture reviewManagedLifecycleFixture) *httptest.Server { + t.Helper() + return httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) +} + +func issueDraftRecoveryToken(t *testing.T, server *httptest.Server, sourceID int64, permission string) string { + t.Helper() + body, err := json.Marshal(map[string]any{ + "label": "recovery-agent", + "permissions": []string{permission}, + "source_ids": []int64{sourceID}, + }) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-key") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusCreated, resp.StatusCode) + var issued struct { + Secret string `json:"secret"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret +} + +func runDraftRecoveryHTTP(t *testing.T, server *httptest.Server, secret, draftID string, revision int64) []api.CLIRunEvent { + t.Helper() + body, err := json.Marshal(api.CLIRunRequest{Args: []string{ + draftRecoverCommand, draftID, "--revision", strconv.FormatInt(revision, 10), "--json", + }}) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + require.NoError(t, json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + require.NoError(t, scanner.Err()) + return events +} + +func TestDraftRecoverReconcilesManualCleanup(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal("discarded", output.Lifecycle) + assertions.Equal(uint32(0), reviewDraftMailboxCount(t, fixture.config.Addr())) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.Nil(latest.Pending) + requirements.NotNil(latest.DiscardedAt) +} + +func TestDraftRecoverThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(permission string) string { + body, err := json.Marshal(map[string]any{ + "label": "recovery-agent", + "permissions": []string{permission}, + "source_ids": []int64{fixture.source.ID}, + }) + requirements.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + requirements.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-key") + resp, err := http.DefaultClient.Do(req) + requirements.NoError(err) + defer func() { _ = resp.Body.Close() }() + requirements.Equal(http.StatusCreated, resp.StatusCode) + var issued struct { + Secret string `json:"secret"` + } + requirements.NoError(json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret + } + + run := func(secret, draftID string, revision int64) []api.CLIRunEvent { + body, err := json.Marshal(api.CLIRunRequest{Args: []string{draftRecoverCommand, draftID, "--revision", strconv.FormatInt(revision, 10), "--json"}}) + requirements.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + requirements.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + requirements.NoError(err) + defer func() { _ = resp.Body.Close() }() + requirements.Equal(http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + requirements.NoError(scanner.Err()) + return events + } + + allowed := run(issue("draft.edit"), fixture.draft.DraftID, 1) + requirements.Len(allowed, 2) + assertions.Equal(cliStreamStdout, allowed[0].Type) + assertions.Contains(allowed[0].Data, `"status":"active"`) + var delegatedOutput draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(allowed[0].Data), &delegatedOutput)) + assertions.Empty(delegatedOutput.Content) + assertions.Empty(delegatedOutput.RawMIME) + assertions.Empty(delegatedOutput.CandidateContent) + assertions.Equal("complete", allowed[1].Type) + + denied := run(issue("draft.delete"), fixture.draft.DraftID, 1) + requirements.Len(denied, 1) + assertions.Empty(denied[0].Data) + assertions.Equal("not_permitted", denied[0].Error) + + otherSource, err := fixture.store.GetOrCreateSource("imap", "imap://other@example.com:143") + requirements.NoError(err) + conversationID, err := fixture.store.EnsureConversation(otherSource.ID, "other-draft", "Other draft") + requirements.NoError(err) + otherReceipt := store.IMAPDraftReceipt{SourceID: otherSource.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + otherDraft, err := fixture.store.PersistIMAPDraftContext(t.Context(), otherReceipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: otherSource.ID, SourceMessageID: store.IMAPDraftSourceMessageID(otherReceipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + RawMIME: []byte("From: alice@example.com\r\n\r\nother\r\n"), + } + }) + requirements.NoError(err) + for _, tc := range []struct { + name string + draftID string + revision int64 + }{ + {name: "missing", draftID: "missing-draft", revision: 1}, + {name: "out of scope current", draftID: otherDraft.DraftID, revision: 1}, + {name: "out of scope stale", draftID: otherDraft.DraftID, revision: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Helper() + events := run(issue("draft.edit"), tc.draftID, tc.revision) + requirements.Len(events, 1) + assertions.Empty(events[0].Data) + assertions.Equal("not_permitted", events[0].Error) + }) + } +} + +func TestDraftRecoverDelegatedUnknownReplacementRedactsCandidate(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + server := newDraftRecoveryHTTPServer(t, fixture) + defer server.Close() + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, string(agentgrant.PermissionDraftEdit)) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + + events := runDraftRecoveryHTTP(t, server, secret, fixture.draft.DraftID, 1) + requirements.Len(events, 2) + assertions.Equal(cliStreamStderr, events[0].Type) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Empty(output.PendingCode) + assertions.Equal("unknown_replacement", output.Status) + assertions.Equal("unknown_replacement", output.RefusalCode) + assertions.Empty(output.Content) + assertions.Empty(output.RawMIME) + assertions.Empty(output.CandidateContent) + assertions.Equal("unknown_replacement", events[1].Error) +} + +func TestDraftRecoverPolicyPrecedesSettledOutputThroughHTTP(t *testing.T) { + for _, tc := range []struct { + name string + permission string + prepare func(*testing.T, reviewManagedLifecycleFixture) (string, int64) + }{ + { + name: "active", + permission: string(agentgrant.PermissionDraftEdit), + prepare: func(_ *testing.T, fixture reviewManagedLifecycleFixture) (string, int64) { + return fixture.draft.DraftID, fixture.draft.Revision + }, + }, + { + name: "discarded", + permission: string(agentgrant.PermissionDraftDelete), + prepare: func(t *testing.T, fixture reviewManagedLifecycleFixture) (string, int64) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + _, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + return latest.DraftID, latest.Revision + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + draftID, revision := tc.prepare(t, fixture) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("policy refusal must not connect") + } + fixture.adapter.draftPolicy = nil + execution, err := fixture.store.AcquireSyncExecutionContext(t.Context(), fixture.source.ID) + requirements.NoError(err) + t.Cleanup(func() { _ = execution.Release() }) + server := newDraftRecoveryHTTPServer(t, fixture) + t.Cleanup(server.Close) + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, tc.permission) + + events := runDraftRecoveryHTTP(t, server, secret, draftID, revision) + requirements.Len(events, 1) + assertions.Empty(events[0].Data) + assertions.Equal("draft_disabled", events[0].Error) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverPublishesKnownReplacement(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + uidNextBefore := recoveryUIDNext(t, fixture) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("edited", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal(uint32(2), output.Receipt.UID) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.Nil(latest.Pending) + requirements.Equal(replacement, latest.CurrentReceipt) +} + +func TestDraftRecoverCleansUpPublishedEdit(t *testing.T) { + for _, replacementState := range []string{"live", "absent", "deleted", "not draft"} { + t.Run(replacementState, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.Equal(int64(2), published.Revision) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + defer func() { _ = client.Close() }() + switch replacementState { + case "absent": + _, err = client.RemoveDraft(t.Context(), recoveryTestReceipt(replacement)) + requirements.NoError(err) + case "deleted": + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + case "not draft": + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsDel, emersionimap.FlagDraft) + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("edited", output.Status) + assertions.Equal(int64(2), output.Revision) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.Equal(replacement, latest.CurrentReceipt) + original, err := client.InspectDraft(t.Context(), recoveryTestReceipt(fixture.draft.CurrentReceipt)) + requirements.NoError(err) + assertions.False(original.Present) + }) + } +} + +func TestDraftRecoverRefusesUnavailableReplacement(t *testing.T) { + for _, tc := range []struct { + name string + code string + }{ + {name: "absent", code: "absent"}, + {name: "deleted", code: "already_deleted"}, + {name: "not draft", code: "not_draft"}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + if tc.name == "deleted" || tc.name == "not draft" { + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + if tc.name == "deleted" { + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + } + } + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + uidNextBefore := recoveryUIDNext(t, fixture) + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal(tc.code, err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"refused"`) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), latest.Revision) + assertions.Equal(fixture.draft.CurrentReceipt, latest.CurrentReceipt) + requirements.NotNil(latest.Pending) + assertions.Equal(store.IMAPDraftOperationEdit, latest.Pending.Operation) + assertions.Equal("append_uidplus", latest.Pending.Code) + assertions.Equal(replacement, *latest.Pending.ReplacementReceipt) + }) + } +} + +func TestDraftRecoverRemovesPresentOriginal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + uidNextBefore := recoveryUIDNext(t, fixture) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + assertions.Equal(uint32(0), reviewDraftMailboxCount(t, fixture.config.Addr())) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.NotNil(latest.DiscardedAt) +} + +func TestDraftRecoverArchivedReplacementReportsAcceptedLocalFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + message, err := fixture.store.GetMessageContext(t.Context(), fixture.draft.CurrentMessageID) + requirements.NoError(err) + archivedReceipt := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + _, err = fixture.store.PersistIMAPDraftContext(t.Context(), archivedReceipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(archivedReceipt), + MessageType: store.MessageTypeEmail, ConversationID: message.ConversationID, + }, + RawMIME: candidate, + } + }) + requirements.NoError(err) + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &archivedReceipt)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"accepted_local_failed"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), latest.Revision) + assertions.Equal(fixture.draft.CurrentReceipt, latest.CurrentReceipt) + requirements.NotNil(latest.Pending) + assertions.Equal("append_uidplus", latest.Pending.Code) + assertions.Equal(archivedReceipt, *latest.Pending.ReplacementReceipt) +} + +func TestDraftRecoverRefusesDifferingRecordedGenerations(t *testing.T) { + requirements := require.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 2, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assert.Equal(t, "uidvalidity_mismatch", err.Error()) + requirements.Len(events, 1) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assert.Equal(t, int64(1), latest.Revision) +} + +func TestDraftRecoverKeepsRowUnchangedOnRemovalGenerationRefusal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + control := &reviewUIDValidityChangeControl{} + addr, user := startReviewUIDValidityChangeServer(t, control) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + control.selects.Store(0) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + before, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + var beforeUpdatedAt string + requirements.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind(` + SELECT CAST(updated_at AS TEXT) FROM imap_drafts WHERE draft_id = ? + `), fixture.draft.DraftID).Scan(&beforeUpdatedAt)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("uidvalidity_mismatch", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"code":"uidvalidity_mismatch"`) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(before.Pending.Code, output.PendingCode) + assertions.Equal("uidvalidity_mismatch", output.RefusalCode) + assertions.Equal(int32(2), control.selects.Load()) + after, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + var afterUpdatedAt string + requirements.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind(` + SELECT CAST(updated_at AS TEXT) FROM imap_drafts WHERE draft_id = ? + `), fixture.draft.DraftID).Scan(&afterUpdatedAt)) + assertions.Equal(before.Revision, after.Revision) + assertions.Equal(before.CurrentReceipt, after.CurrentReceipt) + requirements.NotNil(after.Pending) + assertions.Equal(before.Pending.Code, after.Pending.Code) + assertions.Equal(before.Pending.OriginalReceipt, after.Pending.OriginalReceipt) + assertions.Equal(before.Pending.ReplacementReceipt, after.Pending.ReplacementReceipt) + assertions.Equal(beforeUpdatedAt, afterUpdatedAt) +} + +func TestDraftRecoverRecordsWriteAttemptedRemovalFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewExpungeFailureServer(t) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("expunge_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"pending_code":"expunge_failed"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Equal("expunge_failed", latest.Pending.Code) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverReportsProviderRefusal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(&imaplib.Config{ + Host: "127.0.0.1", Port: 1, Username: testutil.IMAPTestUsername, + }, testutil.IMAPTestPassword), nil + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("provider_refused", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"refused"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Empty(latest.Pending.Code) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverRefusesUnknownReplacement(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, []byte("candidate")) + requirements.NoError(err) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("unknown replacement must not connect") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("unknown_replacement", err.Error()) + assertions.Equal(0, providerCalls) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverPolicy(t *testing.T) { + for _, tc := range []struct { + name string + setup func(*testing.T, reviewManagedLifecycleFixture) + code string + }{ + {name: "draft disabled", setup: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftPolicy = nil + }, code: "draft_disabled"}, + {name: "invalid mailbox", setup: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftPolicy[0].Mailbox = "\n" + }, code: "invalid_mailbox"}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + tc.setup(t, fixture) + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal(tc.code, err.Error()) + assertions.Empty(events) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + }) + } +} + +func TestDraftRecoverParser(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + for _, args := range [][]string{ + {draftRecoverCommand, "draft-1"}, + {draftRecoverCommand, "draft-1", "--revision", "1", "--body", "blocked"}, + {draftRecoverCommand, "draft-1", "--revision", "0"}, + } { + _, err := parseDraftLifecycleArgs(args) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } + intent, err := parseDraftLifecycleArgs([]string{draftRecoverCommand, "draft-1", "--revision", "1", "--json"}) + requirements.NoError(err) + assertions.Equal(draftRecoverCommand, intent.Operation) +} + +func TestDraftRecoverSavedRemovedDeleteFinishesWithoutProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("saved removal must finish locally") + } + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"deleted"`) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) +} + +func TestDraftRecoverSavedRemovedEditFinishesWithoutProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.Equal(int64(2), published.Revision) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil)) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("saved removal must finish locally") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"edited"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.Equal(int64(2), latest.Revision) + assertions.Equal(replacement, latest.CurrentReceipt) +} + +func TestDraftRecoverReloadsSettledStateAfterSourceLock(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil)) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("settled recovery must not connect") + } + fired := false + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if fired { + return + } + fired = true + _, finishErr := fixture.store.FinishIMAPDraftRemovalContext(t.Context(), fixture.draft.DraftID, published.Revision) + requirements.NoError(finishErr) + }, + })) + defer slog.SetDefault(previous) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.True(fired) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"active"`) + assertions.Zero(providerCalls) +} + +func TestDraftRecoverReloadsActionAndSourceAfterSourceLock(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*testing.T, reviewManagedLifecycleFixture) + }{ + { + name: "action", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts + SET pending_operation = 'delete', pending_raw = NULL, + pending_replacement_mailbox = NULL, + pending_replacement_uidvalidity = NULL, + pending_replacement_uid = NULL + WHERE draft_id = ? + `), fixture.draft.DraftID) + require.NoError(t, err) + }, + }, + { + name: "draft removed", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(`DELETE FROM imap_drafts WHERE draft_id = ?`), fixture.draft.DraftID) + require.NoError(t, err) + }, + }, + { + name: "source type", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE sources SET source_type = 'gmail' WHERE id = ? + `), fixture.source.ID) + require.NoError(t, err) + }, + }, + { + name: "source identifier", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE sources SET identifier = 'imap://changed@example.test:143' WHERE id = ? + `), fixture.source.ID) + require.NoError(t, err) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + grant := &agentgrant.Grant{ + ID: "reload-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit}, + Sources: []agentgrant.SourceRef{{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier}}, + } + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("reload denial must not connect") + } + fired := false + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if fired { + return + } + fired = true + tc.mutate(t, fixture) + }, + })) + defer slog.SetDefault(previous) + + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + Grant: grant, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + assertions.True(fired) + assertions.Empty(events) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverRejectsInvalidPendingOperation(t *testing.T) { + testutil.SkipIfPostgres(t, "invalid pending operation injection uses SQLite check-constraint bypass") + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(func() error { + _, err := fixture.store.DB().Exec("PRAGMA ignore_check_constraints = ON") + return err + }()) + _, err = fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts SET pending_operation = 'invalid' WHERE draft_id = ? + `), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NoError(func() error { + _, err := fixture.store.DB().Exec("PRAGMA ignore_check_constraints = OFF") + return err + }()) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("invalid operation must not connect") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("invalid_state", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls) +} + +func TestDraftRecoverRejectsWrongSourceAndAction(t *testing.T) { + for _, tc := range []struct { + name string + permission agentgrant.Permission + source func(reviewManagedLifecycleFixture) agentgrant.SourceRef + }{ + { + name: "wrong action", + permission: agentgrant.PermissionDraftDelete, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier} + }, + }, + { + name: "wrong source type", + permission: agentgrant.PermissionDraftEdit, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: "gmail", Identifier: fixture.source.Identifier} + }, + }, + { + name: "wrong source identifier", + permission: agentgrant.PermissionDraftEdit, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: "imap://changed@example.test:143"} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + grant := &agentgrant.Grant{ + ID: "denial-grant", + Permissions: []agentgrant.Permission{tc.permission}, + Sources: []agentgrant.SourceRef{tc.source(fixture)}, + } + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("denial must not connect") + } + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + Grant: grant, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverSettledRepeatNoProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + normalFactory := fixture.adapter.draftClientFactory + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("settled recovery must not connect") + } + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"active"`) + assertions.Equal(0, providerCalls) + + fixture.adapter.draftClientFactory = normalFactory + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + _, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + + providerCalls = 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("discarded recovery must not connect") + } + events, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"already_discarded"`) + assertions.Equal(0, providerCalls) +} + +func TestDraftRecoverRejectsEnvAndCwd(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + adapter := &storeAPIAdapter{} + for _, request := range []api.CLIRunRequest{ + {Args: []string{draftRecoverCommand, "draft-1", "--revision", "1"}, Env: map[string]string{"HOME": "blocked"}}, + {Args: []string{draftRecoverCommand, "draft-1", "--revision", "1"}, Cwd: `C:\blocked`}, + } { + err := adapter.runCLIDraftLifecycle(t.Context(), request, nil) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } +} + +func publishRecoveryReplacement(t *testing.T, fixture reviewManagedLifecycleFixture, candidate []byte) store.IMAPDraft { + t.Helper() + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + require.NoError(t, err) + parsed, err := msgmime.Parse(candidate) + require.NoError(t, err) + message, err := fixture.store.GetMessageContext(t.Context(), draft.CurrentMessageID) + require.NoError(t, err) + replyTo, err := fixture.store.GetMessageReplyToMessageIDContext(t.Context(), draft.CurrentMessageID) + require.NoError(t, err) + replacement := *draft.Pending.ReplacementReceipt + participants, build := draftLifecyclePersistData(message.ConversationID, replyTo, imaplib.ReplyDraft{Raw: candidate, Parsed: parsed}, replacement) + published, err := fixture.store.PublishIMAPDraftReplacementContext(t.Context(), draft.DraftID, draft.Revision, participants, build) + require.NoError(t, err) + return published +} + +func TestDraftRecoverRefusalAfterCancellation(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + cancel() + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + } + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.EqualError(err, "cancelled") + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("refused", output.Status) + assertions.Equal("cancelled", output.RefusalCode) + assertions.Empty(output.PendingCode) + requirements.NotNil(output.ProviderObservation) + assertions.Equal("cancelled", output.ProviderObservation.Code) +} + +func TestDraftRecoverDelegatedCompletionWithoutContentReads(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + _, err = fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE message_raw SET raw_data = ?, compression = 'zlib' WHERE message_id = ? + `), []byte("invalid compressed content"), fixture.draft.CurrentMessageID) + requirements.NoError(err) + server := newDraftRecoveryHTTPServer(t, fixture) + defer server.Close() + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, string(agentgrant.PermissionDraftDelete)) + + events := runDraftRecoveryHTTP(t, server, secret, fixture.draft.DraftID, 1) + requirements.Len(events, 2) + assertions.Equal(cliStreamStdout, events[0].Type) + assertions.Equal("complete", events[1].Type) + assertions.Empty(events[1].Error) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Empty(output.Content) + assertions.Empty(output.RawMIME) + assertions.Empty(output.CandidateContent) +} + +func TestDraftRecoverAlreadyDeletedNeedsManualExpunge(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "expunge_failed", nil)) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.EqualError(err, "already_deleted") + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("expunge_failed", output.PendingCode) + assertions.Equal("already_deleted", output.RefusalCode) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + + client, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + requirements.NoError(err) + defer func() { _ = client.Close() }() + requirements.NoError(client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select("Drafts", nil).Wait() + requirements.NoError(err) + requirements.NoError(client.UIDExpunge(emersionimap.UIDSetNum(1)).Close()) + events, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.NotNil(latest.DiscardedAt) +} diff --git a/docs/changelog.md b/docs/changelog.md index 94bac4654..c84ac1798 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -212,16 +212,19 @@ See [Web UI](web-ui.md), [TUI](usage/tui.md), to a specific source and Drafts folder. The server must support UIDPLUS. Msgvault stores an archived copy; it never sends the email. - Add restricted agent grants. Owner-only `agent-token issue/list/revoke` - commands manage in-memory tokens with `draft.create` permission for named - sources. Agents use `--agent-url` and `--agent-token-file` to run only - `draft-reply`; grants expire when revoked or when the daemon restarts. + commands manage in-memory tokens with `draft.create`, `draft.edit`, and + `draft.delete` permissions for named sources. Agents use `--agent-url` and + `--agent-token-file` to run `draft-reply` or `draft-recover`; grants expire + when revoked or when the daemon restarts. Enable `[server] agent_access = true` with a non-empty `api_key`; see [agent-token](cli-reference.md#agent-token) and [configuration](configuration.md#server). - Read, edit, and delete managed IMAP drafts with `draft-get`, `draft-edit`, and - `draft-delete`. Reads use retained archive content; edits and deletes require - the reported revision and exact provider receipt. Uncertain provider results - keep candidate evidence and block the draft until recovery support is added. + `draft-delete`, plus `draft-recover` for interrupted operations. Reads use + retained archive content; edits, deletes, and recovery require the reported + revision and exact provider receipt. Recovery uses saved replacement bytes and + never replays APPEND. Uncertain provider results retain their evidence for a + later recovery attempt. - Refresh the archived body, recipients, and attachments when a trusted outgoing IMAP copy is edited or moves from Drafts to Sent. Ordinary received-mail and All Mail copies cannot replace that content. Historical rows that already lost diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 96539e68a..b772d5175 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -48,7 +48,7 @@ Commands that access archive state keep their usual stdout/stderr output while u 1. If `[remote].url` is configured and `--local` is not passed, the CLI talks to that remote server. 2. Otherwise, archive-access commands discover or start the local background daemon and talk to it over HTTP. 3. `--local` selects the local daemon even when `[remote].url` is configured; it is not a request to open SQLite in the CLI process. -4. When `--agent-url` and `--agent-token-file` are both supplied, the CLI connects to that remote daemon as a restricted delegated caller using the token from the file. Only `draft-reply` is available in this mode. Owner configuration (`--config`, `--home`, `--local`) is rejected, and the token is never written to logs or argv. The token is transmitted in the `X-Msgvault-Agent-Token` request header; this header is not modeled in the generated OpenAPI clients — it is a transport detail that the CLI handles internally. +4. When `--agent-url` and `--agent-token-file` are both supplied, the CLI connects to that remote daemon as a restricted delegated caller using the token from the file. Only `draft-reply` and `draft-recover` are available in this mode. Owner configuration (`--config`, `--home`, `--local`) is rejected, and the token is never written to logs or argv. The token is transmitted in the `X-Msgvault-Agent-Token` request header; this header is not modeled in the generated OpenAPI clients — it is a transport detail that the CLI handles internally. This makes local and remote msgvault behavior the same from the CLI's point of view and avoids opening a large SQLite database from foreground CLI processes. @@ -165,7 +165,7 @@ changes. Draft creation never moves an IMAP cursor. --- -## draft-get, draft-edit, and draft-delete +## draft-get, draft-edit, draft-delete, and draft-recover Read, edit, or delete an IMAP draft created by `draft-reply`: @@ -173,11 +173,13 @@ Read, edit, or delete an IMAP draft created by `draft-reply`: msgvault draft-get [--json] msgvault draft-edit --revision --body [--json] msgvault draft-delete --revision [--json] +msgvault draft-recover --revision [--json] ``` The creation result supplies the opaque `draft_id` and initial revision. -- `--revision` is required for edit and delete; use the current positive revision. +- `--revision` is required for edit, delete, and recover; use the current + positive revision. - `--body` is required for edit; `--body=` sets an empty plain-text body. - `--json` emits one JSON result. @@ -185,8 +187,18 @@ The creation result supplies the opaque `draft_id` and initial revision. connecting to IMAP or requiring the source's draft mutation grant. Edit and delete require the same source policy as `draft-reply`. Delete removes the provider draft and retains its archived content. These commands never send mail. -See [Manage a created draft](usage/imap.md#manage-a-created-draft) for revision, -provider checks, retention, and retry behavior. +Recovery resumes a pending operation from recorded receipts. It can publish a +known replacement or finish confirmed removal without APPEND. Delegated +recovery requires `draft.edit` for an edit or active repeat and `draft.delete` +for a delete or discarded repeat, scoped to the source in the grant. An active +draft with no pending operation requires `draft.edit`, including after a delete +was aborted before writing. `draft-get`, `draft-edit`, and `draft-delete` remain +owner-only; delegated edit and delete commands are outside this release's scope. +Recovery output keeps the saved `pending_code`. Refused results use +`refusal_code`; pending cleanup results describe the current provider result in +`observation.code`. See +[Manage a created draft](usage/imap.md#manage-a-created-draft) for revision, +provider checks, retention, and recovery limits. --- @@ -2985,7 +2997,7 @@ msgvault agent-token issue --label \ | Flag | Description | |---|---| | `--label ` | (required) Human-readable name for the grant | -| `--permissions ` | Comma-separated permission names to grant; accepted values: `draft.create` | +| `--permissions ` | Comma-separated permissions: `draft.create` for `draft-reply`; `draft.edit` and `draft.delete` for `draft-recover` only (see [draft recovery](#draft-get-draft-edit-draft-delete-and-draft-recover)) | | `--source-ids ` | Comma-separated source IDs that the permissions apply to | The grant is valid until revoked or until the daemon restarts. diff --git a/docs/usage/imap.md b/docs/usage/imap.md index de87972b5..c57b655da 100644 --- a/docs/usage/imap.md +++ b/docs/usage/imap.md @@ -265,11 +265,13 @@ Use the current revision from `draft-get` or the last successful operation: ```bash msgvault draft-edit --revision 1 --body 'Updated text' --json msgvault draft-delete --revision 2 --json +msgvault draft-recover --revision 1 --json ``` Edit and delete require the same daemon-host `[[imap.drafts]]` grant and provider configuration as draft creation. Policy changes take effect after a daemon -restart. Neither command sends mail. +restart. Recovery uses the same policy after it checks the delegated action +grant. None of these commands sends mail. Editing supports plain-text drafts without attachments. `--body=` sets an empty body. The edit preserves the From, To, Cc, Bcc, Reply-To, Subject, In-Reply-To, and @@ -303,6 +305,52 @@ Every removal path checks the mailbox generation and requires both `\Draft` and `UID EXPUNGE` itself is not conditional, so another client can still change flags after the last check. +### Recover an interrupted edit or delete + +Recovery uses only the original receipt and a replacement receipt already saved +on the managed draft row. It checks the requested revision, source type and +identifier, mailbox UIDVALIDITY, exact UID, and current draft flags before it +publishes or removes anything. A pending edit without a known replacement +returns `unknown_replacement` without opening an IMAP connection. + +```bash +msgvault draft-recover --revision 1 --json +``` + +If another client already removed the exact original UID, recovery records that +absence and finishes the Store transition locally. A known unpublished +replacement is published from its saved bytes, then the original is removed +with the existing conditional `UID STORE` and `UID EXPUNGE` sequence. Recovery +never searches a mailbox, adopts a moved copy, calls APPEND, or rolls back a +publication. Once a replacement is published locally, cleanup of the original +can finish even if another client has since sent or deleted the replacement. +Only an unpublished replacement must still be a live draft. + +`draft.edit` authorizes active and pending edits. `draft.delete` authorizes +discarded and pending deletes. A delegated token must also name the exact source +type and identifier. Wrong actions, sources, or revisions return before draft +content, policy errors, locks, or provider work are disclosed. + +If the original already has `\Deleted`, recovery returns `already_deleted` +without a provider write. This includes removal interrupted after `UID STORE` +succeeded. Repeating recovery cannot clear that state: use an IMAP client to +expunge the exact original UID in the recorded mailbox and UIDVALIDITY, then run +`draft-recover` with the revision from the recovery output or `draft-get`. +Recovery may have published a replacement locally and advanced the revision +before cleanup was refused. For a published edit, remove the original +copy, not the replacement. Check the receipt before expunging; do not purge all +deleted messages in the mailbox. Saved cancellation evidence does not establish +who set the flag, so automatic removal of an already-deleted UID is outside this +command's scope. + +If cleanup fails after publication, the new revision and pending evidence stay +saved. Recovery can retry when the original is still an undeleted draft or is +confirmed absent; an already-deleted original needs the manual step above. +A generation or SELECT refusal before `UID STORE` leaves the pending code, +receipts, timestamp, and revision unchanged. Output keeps the saved +`pending_code`. Refused results use `refusal_code`; pending cleanup results use +`observation.code` for the current provider result. + ### If an edit or delete does not finish - A delete failure before any remote write clears the pending claim. After @@ -312,11 +360,13 @@ after the last check. blocks further changes. Inspect the provider state before any manual reconciliation; general recovery for uncertain writes is not available. - If `draft-get --json` reports `pending_code: "removed"`, removal was - confirmed and saved, but local completion is still pending. Repeat the matching - `draft-edit` or `draft-delete` command with the revision from that read to finish - locally without another remote write. The source policy still applies. For an - edit, `--body` must match the already published replacement after MIME - normalization. A `removed` observation in an error response alone is not enough; + confirmed and saved, but local completion is still pending. `draft-recover` is + the preferred local completion route. Run it with the revision from that + read; it finishes locally without another remote write. You can also repeat + the matching `draft-edit` or `draft-delete` command with that revision. The + source policy still applies. For an edit, `--body` must match the already + published replacement after MIME normalization. A `removed` observation in + an error response alone is not enough; `draft-get` must report the saved pending code. ## Keep edited outgoing mail current diff --git a/internal/agentgrant/agentgrant.go b/internal/agentgrant/agentgrant.go index 2d7904e0f..47691b26e 100644 --- a/internal/agentgrant/agentgrant.go +++ b/internal/agentgrant/agentgrant.go @@ -16,10 +16,14 @@ type Permission string const ( PermissionDraftCreate Permission = "draft.create" + PermissionDraftEdit Permission = "draft.edit" + PermissionDraftDelete Permission = "draft.delete" ) var knownPermissions = map[string]Permission{ string(PermissionDraftCreate): PermissionDraftCreate, + string(PermissionDraftEdit): PermissionDraftEdit, + string(PermissionDraftDelete): PermissionDraftDelete, } func KnownPermission(s string) (Permission, bool) { diff --git a/internal/agentgrant/agentgrant_test.go b/internal/agentgrant/agentgrant_test.go index 848653220..b8ad5f1a2 100644 --- a/internal/agentgrant/agentgrant_test.go +++ b/internal/agentgrant/agentgrant_test.go @@ -194,3 +194,27 @@ func TestGrantAllowsExactOriginalTriple(t *testing.T) { assert.True(t, g.Allows(PermissionDraftCreate, original), "exact original (ID, Type, Identifier) triple must be allowed") } + +func TestDraftPermissionsRemainIndependent(t *testing.T) { + assertions := assert.New(t) + src := SourceRef{ID: 1, Type: "imap", Identifier: "alice@example.com"} + for _, permission := range []Permission{PermissionDraftCreate, PermissionDraftEdit, PermissionDraftDelete} { + grant := Grant{Permissions: []Permission{permission}, Sources: []SourceRef{src}} + assertions.True(grant.Allows(permission, src)) + for _, other := range []Permission{PermissionDraftCreate, PermissionDraftEdit, PermissionDraftDelete} { + if other == permission { + continue + } + assertions.False(grant.Allows(other, src)) + } + } + assertions.Equal(PermissionDraftEdit, mustKnownPermission(t, "draft.edit")) + assertions.Equal(PermissionDraftDelete, mustKnownPermission(t, "draft.delete")) +} + +func mustKnownPermission(t *testing.T, name string) Permission { + t.Helper() + permission, ok := KnownPermission(name) + require.True(t, ok) + return permission +} diff --git a/internal/api/cli_allowlist_drafts_test.go b/internal/api/cli_allowlist_drafts_test.go index 42a6ffcaa..57124b248 100644 --- a/internal/api/cli_allowlist_drafts_test.go +++ b/internal/api/cli_allowlist_drafts_test.go @@ -24,11 +24,29 @@ func TestCLIRunDraftAllowlist(t *testing.T) { assertions.True(IsCLIRunDraftLifecycle([]string{"draft-get", "draft-abc"})) assertions.True(cliRunCommandAllowed([]string{"draft-edit", "draft-abc", "--revision=1", "--body=body"})) assertions.True(cliRunCommandAllowed([]string{"draft-delete", "draft-abc", "--revision=1"})) + assertions.True(IsCLIRunDraftLifecycle([]string{"draft-recover", "draft-abc", "--revision=1"})) + assertions.True(cliRunCommandAllowed([]string{"draft-recover", "draft-abc", "--revision=1"})) assertions.False(cliRunCommandAllowed([]string{"configure-imap-drafts"})) assertions.False(cliRunCommandAllowed([]string{"draft-reply"})) assertions.False(cliRunCommandAllowed([]string{"draft-get"})) } +func TestDelegatedDraftRecoverRequiresActionPermission(t *testing.T) { + source := agentgrant.SourceRef{ID: 1, Type: "imap", Identifier: "alice@example.com"} + assert.True(t, delegatedCLIRunAdmitted( + []string{"draft-recover", "draft-abc", "--revision=1"}, + &agentgrant.Grant{Permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit}}, + )) + assert.True(t, delegatedCLIRunAdmitted( + []string{"draft-recover", "draft-abc", "--revision=1"}, + &agentgrant.Grant{Permissions: []agentgrant.Permission{agentgrant.PermissionDraftDelete}}, + )) + assert.False(t, delegatedCLIRunAdmitted( + []string{"draft-recover", "draft-abc", "--revision=1"}, + &agentgrant.Grant{Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, Sources: []agentgrant.SourceRef{source}}, + )) +} + // newDelegatedTestServer creates a server with agentGrants enabled and issues a grant. func newDelegatedTestServer(t *testing.T) (*Server, string) { t.Helper() diff --git a/internal/api/cli_run_draft.go b/internal/api/cli_run_draft.go index 1ac6d20b9..17828076a 100644 --- a/internal/api/cli_run_draft.go +++ b/internal/api/cli_run_draft.go @@ -7,9 +7,10 @@ import "go.kenn.io/msgvault/internal/agentgrant" const CLIRunDraftReplyCommand = "draft-reply" const ( - CLIRunDraftGetCommand = "draft-get" - CLIRunDraftEditCommand = "draft-edit" - CLIRunDraftDeleteCommand = "draft-delete" + CLIRunDraftGetCommand = "draft-get" + CLIRunDraftEditCommand = "draft-edit" + CLIRunDraftDeleteCommand = "draft-delete" + CLIRunDraftRecoverCommand = "draft-recover" ) // IsCLIRunDraftReply reports whether args invoke the in-process draft-reply @@ -19,7 +20,17 @@ func IsCLIRunDraftReply(args []string) bool { } func delegatedCLIRunAdmitted(args []string, grant *agentgrant.Grant) bool { - return grant != nil && IsCLIRunDraftReply(args) && grant.HasPermission(agentgrant.PermissionDraftCreate) + if grant == nil || len(args) == 0 { + return false + } + switch args[0] { + case CLIRunDraftReplyCommand: + return grant.HasPermission(agentgrant.PermissionDraftCreate) + case CLIRunDraftRecoverCommand: + return grant.HasPermission(agentgrant.PermissionDraftEdit) || grant.HasPermission(agentgrant.PermissionDraftDelete) + default: + return false + } } // IsCLIRunDraftLifecycle reports whether args invoke one of the managed draft @@ -29,7 +40,7 @@ func IsCLIRunDraftLifecycle(args []string) bool { return false } switch args[0] { - case CLIRunDraftGetCommand, CLIRunDraftEditCommand, CLIRunDraftDeleteCommand: + case CLIRunDraftGetCommand, CLIRunDraftEditCommand, CLIRunDraftDeleteCommand, CLIRunDraftRecoverCommand: return true default: return false diff --git a/internal/api/middleware.go b/internal/api/middleware.go index bc75e6bd0..9b7185b3b 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -334,9 +334,8 @@ func (s *Server) apiRequestAuthorized(r *http.Request) bool { // requestGateEligible reports whether the request should participate in the // operation gate. Owner, session, and loopback requests register as waiters or // holders on any gated route. Delegated callers reach this predicate only on -// /api/v1/cli/run; cliRunGateDecision further restricts gate entry to the one -// command the caller may reach (draft-reply), so any other body skips the gate -// and the handler issues the rejection. All other gated routes reject delegated +// /api/v1/cli/run; cliRunGateDecision further restricts gate entry to draft-reply +// and draft-recover. Other bodies skip the gate and the handler rejects them. All other gated routes reject delegated // callers at the auth layer without touching gate state. // Unauthenticated requests (AuthModeRequired) pass straight through so they // reach the API auth layer without touching gate state. diff --git a/internal/store/imap_draft_review_test.go b/internal/store/imap_draft_review_test.go index a1b79fe44..1d26f8ff7 100644 --- a/internal/store/imap_draft_review_test.go +++ b/internal/store/imap_draft_review_test.go @@ -267,6 +267,50 @@ func TestManagedIMAPDraftSourceCascade(t *testing.T) { requirements.ErrorIs(err, store.ErrIMAPDraftNotFound) } +func TestManagedIMAPDraftRecoveryCompletionIsIdempotent(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st := testutil.NewTestStore(t) + + edit := newReviewManagedDraftOnStore(t, st, "recovery-edit", 81, "original") + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := st.ClaimIMAPDraftContext(t.Context(), edit.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: edit.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 82} + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), edit.draft.DraftID, 1, "append_uidplus", &replacement)) + published, err := st.PublishIMAPDraftReplacementContext(t.Context(), edit.draft.DraftID, 1, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: edit.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(replacement), + MessageType: store.MessageTypeEmail, ConversationID: edit.conversationID, + }, + BodyText: sql.NullString{String: "candidate", Valid: true}, RawMIME: candidate, + } + }) + requirements.NoError(err) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), edit.draft.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil)) + finished, err := st.FinishIMAPDraftRemovalContext(t.Context(), edit.draft.DraftID, published.Revision) + requirements.NoError(err) + assertions.Equal(int64(2), finished.Revision) + assertions.Nil(finished.Pending) + assertions.Equal(replacement, finished.CurrentReceipt) + oldRaw, err := st.GetMessageRaw(edit.draft.CurrentMessageID) + requirements.NoError(err) + assertions.Contains(string(oldRaw), "original") + + deleted := newReviewManagedDraftOnStore(t, st, "recovery-delete", 83, "to delete") + claimed, err := st.ClaimIMAPDraftContext(t.Context(), deleted.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(st.RecordIMAPDraftOutcomeContext(t.Context(), claimed.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + finished, err = st.FinishIMAPDraftRemovalContext(t.Context(), claimed.DraftID, 1) + requirements.NoError(err) + assertions.Equal(int64(2), finished.Revision) + assertions.NotNil(finished.DiscardedAt) + deleteRaw, err := st.GetMessageRaw(deleted.draft.CurrentMessageID) + requirements.NoError(err) + assertions.Contains(string(deleteRaw), "to delete") +} + func newReviewManagedDraft( t *testing.T, name string, From a36a394122b6880e41ec042870537573f64e2afa Mon Sep 17 00:00:00 2001 From: Thomas Heinrichsdobler Date: Tue, 22 Sep 2026 15:25:40 +0200 Subject: [PATCH 012/125] fix(emlx): restore partial-message attachments from Apple Mail's Attachments/ directory (#883) Fixes #878. Apple Mail keeps some attachments outside `.partial.emlx` files, in a sibling `Attachments/` directory. This change restores cached top-level attachments when importing those messages and reports how many it restored. Re-importing adds newly cached attachments to the existing message. Its ID and fallback conversation key come from the original MIME bytes, so restoration does not create duplicates. Updates retain existing labels and fill placeholders in the stored MIME, preserving attachments even if Apple Mail later removes their cached files. Restored parts use base64 with a matching Content-Transfer-Encoding header. Missing files keep their placeholders; unreadable files or directories produce warnings. Attachment reads are bounded by the remaining message budget, and their actual base64 size is checked before restoring a part. Filename validation and the single-file fallback remain in place. Restoration supports direct children of the outer multipart only. Nested parts are not restored, and the documentation states that limit. A sanitized Apple Mail directory listing is needed to verify the numbering of nested parts before adding support. LF and CRLF messages retain their line endings; mixed line endings can leave placeholders unchanged. Co-authored-by: Thomas Heinrichsdobler --- cmd/msgvault/cmd/import_emlx.go | 4 +- docs/changelog.md | 5 + docs/cli-reference.md | 14 +- docs/usage/importing.md | 2 +- internal/emlx/attachments.go | 286 +++++++++++++++++ internal/emlx/attachments_test.go | 431 ++++++++++++++++++++++++++ internal/emlx/reader.go | 35 ++- internal/emlx/reader_test.go | 4 +- internal/importer/emlx_import.go | 85 +++-- internal/importer/emlx_import_test.go | 144 +++++++++ internal/store/messages.go | 18 ++ 11 files changed, 1001 insertions(+), 27 deletions(-) create mode 100644 internal/emlx/attachments.go create mode 100644 internal/emlx/attachments_test.go diff --git a/cmd/msgvault/cmd/import_emlx.go b/cmd/msgvault/cmd/import_emlx.go index 1df0b82f1..af3f1c44c 100644 --- a/cmd/msgvault/cmd/import_emlx.go +++ b/cmd/msgvault/cmd/import_emlx.go @@ -373,6 +373,7 @@ func importAutoAccounts( grandTotal.MessagesUpdated += summary.MessagesUpdated grandTotal.MessagesSkipped += summary.MessagesSkipped grandTotal.PartialFiles += summary.PartialFiles + grandTotal.AttachmentsRestored += summary.AttachmentsRestored grandTotal.Errors += summary.Errors if summary.HardErrors { grandTotal.HardErrors = true @@ -449,8 +450,9 @@ func printImportStats(out io.Writer, summary importer.EmlxImportSummary) { ) if summary.PartialFiles > 0 { _, _ = fmt.Fprintf(out, - " Partial files: %d (body imported; attachments not cached by Apple Mail)\n", + " Partial files: %d (%d attachments restored from Apple Mail's Attachments/ directory)\n", summary.PartialFiles, + summary.AttachmentsRestored, ) } _, _ = fmt.Fprintf(out, diff --git a/docs/changelog.md b/docs/changelog.md index c84ac1798..bfc935ca2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -195,6 +195,11 @@ See [Web UI](web-ui.md), [TUI](usage/tui.md), RFC Message-ID and link unambiguous replies within a source. Re-import and rebuild the cache to fill missing IDs in existing Apple Mail archives; see [message identifier recovery](usage/importing.md#message-identifiers-and-replies). +- Apple Mail imports restore cached top-level attachments of `.partial.emlx` + messages from the sibling `Attachments/` directory. Re-importing adds restored + attachments to existing messages without duplicates. Nested attachments are + not restored. The import summary reports how many attachments were restored, + and unreadable cached files produce warnings. - Import Apple WhatsApp ChatStorage text, including URL messages, with available participant and push names. Contact-number matching requires country codes; missing group-participant tables no longer block otherwise usable exports. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index b772d5175..aa480dfbf 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -960,10 +960,18 @@ msgvault import-emlx The mail directory should be an Apple Mail mailbox tree containing `.mbox` or `.imapmbox` directories, each with a `Messages/` subdirectory of `.emlx` files. You can also point directly at a single `.mbox` directory. Labels are derived from directory names. -Apple Mail's `N.partial.emlx` files are also imported: their message body is -complete even when uncached attachment parts are absent. If both `N.emlx` and +Apple Mail's `N.partial.emlx` files are also imported. Apple Mail keeps the +attachments of these messages outside the MIME payload, in a sibling +`Attachments/N/` directory. The importer restores cached attachments directly +inside the message's outer multipart, within the message size limit. +Attachments in nested parts, such as inside some forwarded messages, are not +restored. An attachment without a cached file stays absent; unreadable files +or directories produce a warning. Re-importing a partial message adds restored +attachments to the existing message without creating another copy. +If both `N.emlx` and `N.partial.emlx` exist, the complete `N.emlx` copy wins. The command summary -reports the number of partial files imported. +reports the number of partial files imported and how many attachments were +restored. | Flag | Default | Description | |---|---|---| diff --git a/docs/usage/importing.md b/docs/usage/importing.md index 584dc22e9..99fa8e76e 100644 --- a/docs/usage/importing.md +++ b/docs/usage/importing.md @@ -306,7 +306,7 @@ Both layouts are supported. The importer discovers all `.mbox` and `.imapmbox` d Apple Mail stores its data at `~/Library/Mail/` on macOS. The auto-discover mode reads `~/Library/Accounts/Accounts4.sqlite` (the macOS accounts database) to map V10 directory GUIDs to email addresses. You can also use a Time Machine backup or a copy of the Mail directory from another machine. !!! note - Apple Mail stores IMAP and Gmail messages whose attachments have not been downloaded as `.partial.emlx` files. The message body in these files is complete, so they are imported normally — only the uncached attachment parts are absent. When both `N.emlx` and `N.partial.emlx` exist for the same message, the fully-downloaded copy is used. The import summary reports how many partial files were imported. + Apple Mail stores IMAP and Gmail messages with larger attachments as `.partial.emlx` files: the message body is complete, and the attachments live in a sibling `Attachments/` directory. The importer restores cached top-level attachments; nested attachments are not restored. Re-import to add newly downloaded attachments to existing messages. See [import-emlx](../cli-reference.md#import-emlx) for limits and warnings. When both `N.emlx` and `N.partial.emlx` exist for the same message, the fully-downloaded copy is used. The import summary reports how many partial files were imported and how many attachments were restored. ## Deduplication diff --git a/internal/emlx/attachments.go b/internal/emlx/attachments.go new file mode 100644 index 000000000..385edea56 --- /dev/null +++ b/internal/emlx/attachments.go @@ -0,0 +1,286 @@ +package emlx + +import ( + "bytes" + "encoding/base64" + "errors" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// applePlaceholderHeader marks an attachment part whose body Apple Mail did +// not write into the .partial.emlx. The attachment bytes live next to the +// Messages/ directory instead, under Attachments///. +const applePlaceholderHeader = "X-Apple-Content-Length:" + +var ( + boundaryRe = regexp.MustCompile(`(?i)boundary\s*=\s*"?([^";\s]+)"?`) + filenameRe = regexp.MustCompile(`(?i)filename="?([^";]+)"?`) +) + +// attachmentsDir returns Apple Mail's Attachments/ directory for the +// message at path, without checking whether the directory exists. It returns +// "" when path is not a Messages/.partial.emlx file. +func attachmentsDir(path string) string { + base := filepath.Base(path) + if !IsPartial(base) { + return "" + } + num := strings.TrimSuffix(base, ".partial.emlx") + if _, err := strconv.Atoi(num); err != nil { + return "" + } + msgDir := filepath.Dir(path) + if filepath.Base(msgDir) != "Messages" { + return "" + } + return filepath.Join(filepath.Dir(msgDir), "Attachments", num) +} + +// RestoreAttachments fills top-level attachment parts carrying an +// X-Apple-Content-Length placeholder with cached bodies beside messagePath. +// Parts whose file cannot be found are left as they are, and so are parts +// whose base64-encoded size would push the message past maxBytes. All other +// bytes, including the message's line-ending style, are preserved. +func RestoreAttachments(raw []byte, messagePath string, maxBytes int64) ([]byte, int, error) { + if !bytes.Contains(raw, []byte(applePlaceholderHeader)) { + return raw, 0, nil + } + attDir := attachmentsDir(messagePath) + if attDir == "" { + return raw, 0, nil + } + nl := "\n" + if bytes.Contains(raw, []byte("\r\n")) { + nl = "\r\n" + } + remaining := maxBytes - int64(len(raw)) + if remaining <= 0 { + return raw, 0, nil + } + lines := strings.Split(string(raw), nl) + + // Locate the top-level boundary in the message header. + hdrEnd := indexBlank(lines, 0) + if hdrEnd < 0 { + return raw, 0, nil + } + boundary := findBoundary(lines[:hdrEnd]) + if boundary == "" { + return raw, 0, nil + } + open, closeB := "--"+boundary, "--"+boundary+"--" + + var out []string + restored := 0 + var restoreErr error + partIndex := 0 + i := 0 + for i < len(lines) { + line := lines[i] + if line != open { + out = append(out, line) + i++ + continue + } + // Start of a part: copy the boundary line, then read its header. + partIndex++ + out = append(out, line) + i++ + phEnd := indexBlank(lines, i) + if phEnd < 0 { + out = append(out, lines[i:]...) + break + } + header := lines[i:phEnd] + if !hasPlaceholder(header) { + out = append(out, header...) + i = phEnd + continue + } + // Replace the encoding header, including folded continuations, to + // match the base64 body written below. + var restoredHeader []string + drop := false + for _, h := range header { + if !strings.HasPrefix(h, " ") && !strings.HasPrefix(h, "\t") { + name, _, _ := strings.Cut(h, ":") + drop = strings.EqualFold(name, "X-Apple-Content-Length") || + strings.EqualFold(name, "Content-Transfer-Encoding") + } + if !drop { + restoredHeader = append(restoredHeader, h) + } + } + restoredHeader = append(restoredHeader, "Content-Transfer-Encoding: base64") + // Skip files already known to exceed the budget, then bound the read + // and charge its actual size in case the cache changed after Stat. + file, size, err := resolveAttachment(attDir, strconv.Itoa(partIndex), findFilename(header)) + headerGrowth := len(strings.Join(restoredHeader, nl)) - len(strings.Join(header, nl)) + cost := encodedSize(size, len(nl)) + int64(headerGrowth) + var content []byte + if err == nil && file != "" && cost <= remaining { + content, err = readAttachment(file, remaining-int64(headerGrowth)) + cost = encodedSize(int64(len(content)), len(nl)) + int64(headerGrowth) + } + if err != nil || file == "" || cost > remaining { + restoreErr = errors.Join(restoreErr, err) + out = append(out, header...) + i = phEnd + continue + } + remaining -= cost + // Emit the header without the placeholder, then the base64 body, + // and skip the original (empty) body up to the next boundary line. + out = append(out, restoredHeader...) + out = append(out, "") + out = append(out, base64Lines(content)...) + i = phEnd + 1 + for i < len(lines) && lines[i] != open && lines[i] != closeB { + i++ + } + restored++ + } + return []byte(strings.Join(out, nl)), restored, restoreErr +} + +// readAttachment reads at most maxBytes+1 bytes. The extra byte ensures that +// truncating an oversized file cannot make its encoded content fit the budget. +func readAttachment(path string, maxBytes int64) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + return io.ReadAll(io.LimitReader(f, maxBytes+1)) +} + +// indexBlank returns the index of the first empty line at or after from. +func indexBlank(lines []string, from int) int { + for i := from; i < len(lines); i++ { + if lines[i] == "" { + return i + } + } + return -1 +} + +func hasPlaceholder(header []string) bool { + for _, h := range header { + if strings.HasPrefix(h, applePlaceholderHeader) { + return true + } + } + return false +} + +// unfold joins folded header lines so regexps can match across continuations. +func unfold(header []string) string { + var b strings.Builder + for _, h := range header { + if strings.HasPrefix(h, " ") || strings.HasPrefix(h, "\t") { + b.WriteString(strings.TrimLeft(h, " \t")) + continue + } + b.WriteString("\n") + b.WriteString(h) + } + return b.String() +} + +func findBoundary(header []string) string { + for l := range strings.SplitSeq(unfold(header), "\n") { + if !strings.HasPrefix(strings.ToLower(l), "content-type:") { + continue + } + if m := boundaryRe.FindStringSubmatch(l); m != nil { + return m[1] + } + } + return "" +} + +// findFilename returns the attachment's filename from the part header, or "" +// when there is none or when the value is not a plain file name. The header +// is sender-controlled, so anything with a path separator or a parent +// reference is rejected here rather than being joined onto a path later. +func findFilename(header []string) string { + m := filenameRe.FindStringSubmatch(unfold(header)) + if m == nil { + return "" + } + name := strings.TrimSpace(m[1]) + if name == "" || name == "." || name == ".." || + strings.ContainsAny(name, `/\`) || name != filepath.Base(name) { + return "" + } + return name +} + +// resolveAttachment returns the path and size of attDir// +// without reading it. When the exact name cannot be resolved but the part directory +// holds exactly one file, that file is used, since Apple Mail stores one file +// per part and may have decoded the name differently than the raw header +// spells it. name must already have passed findFilename's validation. +func resolveAttachment(attDir, partID, name string) (string, int64, error) { + dir := filepath.Join(attDir, partID) + var nameErr error + if name != "" { + full := filepath.Join(dir, name) + if fi, err := os.Stat(full); err == nil && fi.Mode().IsRegular() { + return full, fi.Size(), nil + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + // Encoded header names can be invalid on the local filesystem. + // Try the cached filename before reporting this lookup failure. + nameErr = err + } + } + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", 0, nil + } + return "", 0, err + } + var files []os.DirEntry + for _, e := range entries { + if !e.IsDir() { + files = append(files, e) + } + } + if len(files) != 1 { + return "", 0, nameErr + } + full := filepath.Join(dir, files[0].Name()) + fi, err := os.Stat(full) + if err != nil || !fi.Mode().IsRegular() { + return "", 0, err + } + return full, fi.Size(), nil +} + +// encodedSize is the number of bytes size raw bytes occupy once base64-encoded +// in 76-character lines, each terminated by a newline of nlLen bytes. +func encodedSize(size int64, nlLen int) int64 { + enc := int64(base64.StdEncoding.EncodedLen(int(size))) + lines := enc / 76 + if enc%76 != 0 { + lines++ + } + return enc + lines*int64(nlLen) +} + +// base64Lines encodes b as RFC 2045 base64 with 76-character lines. +func base64Lines(b []byte) []string { + enc := base64.StdEncoding.EncodeToString(b) + var lines []string + for len(enc) > 76 { + lines = append(lines, enc[:76]) + enc = enc[76:] + } + return append(lines, enc) +} diff --git a/internal/emlx/attachments_test.go b/internal/emlx/attachments_test.go new file mode 100644 index 000000000..f9854cfff --- /dev/null +++ b/internal/emlx/attachments_test.go @@ -0,0 +1,431 @@ +package emlx + +import ( + "bytes" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/mime" +) + +// writePartial writes Messages/.partial.emlx with the given MIME body +// and, for each entry in attachments, Attachments///. +// It returns the path of the .partial.emlx file. +func writePartial(t *testing.T, root string, num int, mime string, attachments map[string][]byte) string { + t.Helper() + msgDir := filepath.Join(root, "Messages") + require.NoError(t, os.MkdirAll(msgDir, 0o755)) + path := filepath.Join(msgDir, fmt.Sprintf("%d.partial.emlx", num)) + data := fmt.Sprintf("%d\n%s", len(mime), mime) + require.NoError(t, os.WriteFile(path, []byte(data), 0o600)) + + for rel, content := range attachments { + p := filepath.Join(root, "Attachments", strconv.Itoa(num), filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, content, 0o600)) + } + return path +} + +// unwrapped returns raw with all line breaks removed, so a base64 payload +// that Raw wraps at 76 characters can be compared against EncodeToString. +func unwrapped(raw []byte) string { + return strings.NewReplacer("\r\n", "", "\n", "").Replace(string(raw)) +} + +// placeholderMIME builds a two-part multipart/mixed message the way Apple Mail +// writes a .partial.emlx: a text body followed by an attachment part whose +// content is replaced by an X-Apple-Content-Length header and an empty body. +func placeholderMIME(nl, boundary, filename string, contentLength int) string { + lines := []string{ + "From: alice@example.com", + "Subject: Invoice", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="` + boundary + `"`, + "", + "--" + boundary, + "Content-Type: text/plain; charset=utf-8", + "", + "Please find the invoice attached.", + "", + "--" + boundary, + "Content-Transfer-Encoding: base64", + "Content-Disposition: attachment;", + "\tfilename=\"" + filename + "\"", + "Content-Type: application/pdf;", + "\tname=\"" + filename + "\"", + fmt.Sprintf("X-Apple-Content-Length: %d", contentLength), + "", + "", + "--" + boundary + "--", + "", + } + return strings.Join(lines, nl) +} + +func TestParseFile_PartialRestoresAttachmentFromSiblingDir(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + pdf := []byte("%PDF-1.6\n%fake pdf content for testing\n") + mime := placeholderMIME("\n", "=-boundary42", "report.pdf", 60) + path := writePartial(t, t.TempDir(), 42, mime, map[string][]byte{ + "2/report.pdf": pdf, + }) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + + raw := string(msg.Raw) + assert.NotContains(raw, "X-Apple-Content-Length", "placeholder header must be removed") + assert.Contains(raw, base64.StdEncoding.EncodeToString(pdf), "attachment bytes must be inlined as base64") + assert.Contains(raw, "Content-Transfer-Encoding: base64") + assert.Contains(raw, "Please find the invoice attached.", "text part must be untouched") + assert.Equal(1, msg.RestoredAttachments) +} + +func TestParseFile_RestoredAttachmentEncoding(t *testing.T) { + for _, encoding := range []string{"base64", "quoted-printable", "7bit", ""} { + t.Run(encoding, func(t *testing.T) { + require := require.New(t) + content := []byte("Meeting notes: bring a pen.\n") + raw := placeholderMIME("\n", "boundary", "notes.txt", len(content)) + header := "" + if encoding != "" { + header = "content-transfer-encoding:\n\t" + encoding + "\n" + } + raw = strings.Replace(raw, "Content-Transfer-Encoding: base64\n", header, 1) + raw = strings.Replace(raw, "application/pdf", "text/plain", 1) + path := writePartial(t, t.TempDir(), 3, raw, map[string][]byte{"2/notes.txt": content}) + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + parsed, err := mime.Parse(msg.Raw) + require.NoError(err) + require.Len(parsed.Attachments, 1) + assert.Equal(t, content, parsed.Attachments[0].Content) + }) + } +} + +func TestParseFile_PartialWithoutAttachmentsDirIsUnchanged(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + mime := placeholderMIME("\n", "=-b", "report.pdf", 60) + path := writePartial(t, t.TempDir(), 7, mime, nil) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + assert.Equal(mime, string(msg.Raw), "no Attachments/ dir: bytes must be untouched") + assert.Equal(0, msg.RestoredAttachments) +} + +func TestParseFile_PartialMissingFileKeepsPlaceholder(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + mime := placeholderMIME("\n", "=-b", "report.pdf", 60) + // Attachments/7 exists but holds a file for a different part. + path := writePartial(t, t.TempDir(), 7, mime, map[string][]byte{ + "3/other.bin": []byte("x"), + }) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + assert.Equal(mime, string(msg.Raw), "missing file: part must stay a placeholder") + assert.Equal(0, msg.RestoredAttachments) +} + +func TestParseFile_FullEmlxNextToAttachmentsDirIsNotTouched(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + root := t.TempDir() + mime := placeholderMIME("\n", "=-b", "report.pdf", 60) + writePartial(t, root, 7, mime, map[string][]byte{"2/report.pdf": []byte("pdf")}) + // A full .emlx with the same number must never be rewritten. + full := filepath.Join(root, "Messages", "7.emlx") + require.NoError(os.WriteFile(full, []byte(fmt.Sprintf("%d\n%s", len(mime), mime)), 0o600)) + + msg, err := ParseFile(full, 1<<20) + require.NoError(err) + assert.Equal(mime, string(msg.Raw)) + assert.Equal(0, msg.RestoredAttachments) +} + +func TestParseFile_PartIndexCountsTopLevelChildrenNotLeaves(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + // mixed -> [ alternative(text/plain, text/html), application/pdf ] + // Apple names the pdf's directory "2" (second child of the top-level + // multipart), not "3" (third leaf). + nl := "\n" + lines := []string{ + "From: alice@example.com", + "Subject: Invoice", + `Content-Type: multipart/mixed; boundary="outer"`, + "", + "--outer", + `Content-Type: multipart/alternative; boundary="inner"`, + "", + "--inner", + "Content-Type: text/plain", + "", + "plain body", + "--inner", + "Content-Type: text/html", + "", + "

html body

", + "--inner--", + "--outer", + "Content-Transfer-Encoding: base64", + "Content-Disposition: attachment;", + "\tfilename=\"invoice.pdf\"", + "Content-Type: application/pdf", + "X-Apple-Content-Length: 12", + "", + "", + "--outer--", + "", + } + mime := strings.Join(lines, nl) + pdf := []byte("%PDF-nested") + path := writePartial(t, t.TempDir(), 9, mime, map[string][]byte{ + "2/invoice.pdf": pdf, + }) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + raw := string(msg.Raw) + assert.Equal(1, msg.RestoredAttachments) + assert.Contains(raw, base64.StdEncoding.EncodeToString(pdf)) + assert.Contains(raw, "

html body

", "nested parts must be untouched") + assert.Contains(raw, "--inner--", "inner boundary must survive") +} + +func TestParseFile_RestoresTwoAttachments(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + nl := "\n" + part := func(name string, n int) []string { + return []string{ + "--b", + "Content-Transfer-Encoding: base64", + "Content-Disposition: attachment;", + "\tfilename=\"" + name + "\"", + "Content-Type: application/pdf", + fmt.Sprintf("X-Apple-Content-Length: %d", n), + "", + "", + } + } + lines := []string{ + "From: a@example.com", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/html", + "", + "

two invoices

", + } + lines = append(lines, part("one.pdf", 10)...) + lines = append(lines, part("two.pdf", 10)...) + lines = append(lines, "--b--", "") + mime := strings.Join(lines, nl) + one, two := []byte("%PDF-one"), []byte("%PDF-two") + path := writePartial(t, t.TempDir(), 11, mime, map[string][]byte{ + "2/one.pdf": one, + "3/two.pdf": two, + }) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + raw := string(msg.Raw) + assert.Equal(2, msg.RestoredAttachments) + assert.Contains(raw, base64.StdEncoding.EncodeToString(one)) + assert.Contains(raw, base64.StdEncoding.EncodeToString(two)) + assert.NotContains(raw, "X-Apple-Content-Length") +} + +func TestParseFile_PreservesCRLF(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + pdf := []byte("%PDF-crlf") + mime := placeholderMIME("\r\n", "=-b", "report.pdf", 12) + path := writePartial(t, t.TempDir(), 13, mime, map[string][]byte{ + "2/report.pdf": pdf, + }) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + raw := string(msg.Raw) + assert.Equal(1, msg.RestoredAttachments) + assert.NotContains(strings.ReplaceAll(raw, "\r\n", ""), "\n", "every line ending must stay CRLF") + assert.Contains(raw, base64.StdEncoding.EncodeToString(pdf)+"\r\n") +} + +func TestParseFile_PicksSingleFileWhenNameDiffers(t *testing.T) { + // Apple may decode the filename differently than the raw header spells + // it. The encoded spelling can be invalid on Windows or exceed the + // filesystem's filename limit even when the decoded name is valid. + for _, tt := range []struct{ name, encoded, cached string }{ + {"encoded", "=?utf-8?Q?Rechnung=5F1.pdf?=", "Rechnung_1.pdf"}, + {"long_encoded", "=?utf-8?Q?" + strings.Repeat("=61", 100) + ".pdf?=", strings.Repeat("a", 100) + ".pdf"}, + } { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + pdf := []byte("%PDF-renamed") + raw := placeholderMIME("\n", "=-b", tt.encoded, 12) + path := writePartial(t, t.TempDir(), 15, raw, map[string][]byte{ + "2/" + tt.cached: pdf, + }) + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + require.NoError(msg.RestorationError) + assert.Equal(1, msg.RestoredAttachments) + parsed, err := mime.Parse(msg.Raw) + require.NoError(err) + require.Len(parsed.Attachments, 1) + assert.Equal(pdf, parsed.Attachments[0].Content) + }) + } +} + +func TestParseFile_RejectsPathTraversalInFilename(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + root := t.TempDir() + // A file outside the Attachments/ tree that a hostile sender must never + // be able to pull into the archive via the filename header. + secret := []byte("-----BEGIN PRIVATE KEY----- not for you") + require.NoError(os.WriteFile(filepath.Join(root, "secret.txt"), secret, 0o600)) + + mime := placeholderMIME("\n", "=-b", "../../../secret.txt", 12) + // Attachments/17 exists so the lookup runs, but the part directory holds + // nothing, so the single-file fallback cannot kick in either. + path := writePartial(t, root, 17, mime, nil) + require.NoError(os.MkdirAll(filepath.Join(root, "Attachments", "17", "2"), 0o755)) + + msg, err := ParseFile(path, 1<<20) + require.NoError(err) + assert.Equal(0, msg.RestoredAttachments) + assert.NotContains(string(msg.Raw), base64.StdEncoding.EncodeToString(secret), "traversal filename must not read outside the part directory") + assert.Contains(string(msg.Raw), "X-Apple-Content-Length", "placeholder must survive when nothing is restored") +} + +func TestParseFile_SkipsAttachmentThatExceedsBudget(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + big := bytes.Repeat([]byte("x"), 4000) + mime := placeholderMIME("\n", "=-b", "big.bin", 5334) + path := writePartial(t, t.TempDir(), 19, mime, map[string][]byte{ + "2/big.bin": big, + }) + + // Budget covers the emlx itself plus a little, but not the 4000-byte file + // once base64-encoded. + msg, err := ParseFile(path, int64(len(mime))+1000) + require.NoError(err) + assert.Equal(0, msg.RestoredAttachments) + assert.NotContains(unwrapped(msg.Raw), base64.StdEncoding.EncodeToString(big), "over-budget bytes must not be read") + assert.Contains(string(msg.Raw), "X-Apple-Content-Length", "over-budget part must stay a placeholder") + assert.Equal(mime, string(msg.Raw)) +} + +func TestReadAttachment_FileChangesAfterSizeCheck(t *testing.T) { + for _, replace := range []bool{false, true} { + t.Run(fmt.Sprintf("replace=%t", replace), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + partDir := filepath.Join(dir, "2") + require.NoError(os.Mkdir(partDir, 0700)) + path := filepath.Join(partDir, "notes.txt") + require.NoError(os.WriteFile(path, []byte("note"), 0600)) + resolved, size, err := resolveAttachment(dir, "2", "notes.txt") + require.NoError(err) + require.Equal(int64(4), size) + + // A completed size check cannot prevent a later cache update. + larger := bytes.Repeat([]byte("x"), 4096) + if replace { + replacement := filepath.Join(dir, "replacement") + require.NoError(os.WriteFile(replacement, larger, 0600)) + require.NoError(os.Rename(replacement, path)) + } else { + require.NoError(os.WriteFile(path, larger, 0600)) + } + content, err := readAttachment(resolved, 8) + require.NoError(err) + require.Len(content, 9, "read only the budget plus one overflow byte") + assert.Equal(bytes.Repeat([]byte("x"), 9), content, "read only the budget plus one overflow byte") + + require.NoError(os.WriteFile(path, []byte("12345678"), 0600)) + content, err = readAttachment(resolved, 8) + require.NoError(err) + assert.Equal([]byte("12345678"), content, "an attachment that fits is read in full") + }) + } +} + +func TestParseFile_RestoresFirstAttachmentAndSkipsSecondWhenBudgetRunsOut(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + nl := "\n" + part := func(name string, n int) []string { + return []string{ + "--b", + "Content-Transfer-Encoding: base64", + "Content-Disposition: attachment;", + "\tfilename=\"" + name + "\"", + "Content-Type: application/octet-stream", + fmt.Sprintf("X-Apple-Content-Length: %d", n), + "", + "", + } + } + lines := []string{ + "From: a@example.com", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/plain", + "", + "two files", + } + lines = append(lines, part("one.bin", 1334)...) + lines = append(lines, part("two.bin", 1334)...) + lines = append(lines, "--b--", "") + mime := strings.Join(lines, nl) + one := bytes.Repeat([]byte("1"), 1000) + two := bytes.Repeat([]byte("2"), 1000) + path := writePartial(t, t.TempDir(), 21, mime, map[string][]byte{ + "2/one.bin": one, + "3/two.bin": two, + }) + + // Room for one encoded file (~1370 bytes) but not two. + msg, err := ParseFile(path, int64(len(mime))+2000) + require.NoError(err) + assert.Equal(1, msg.RestoredAttachments) + assert.Contains(unwrapped(msg.Raw), base64.StdEncoding.EncodeToString(one)) + assert.NotContains(unwrapped(msg.Raw), base64.StdEncoding.EncodeToString(two)) + assert.Equal(1, strings.Count(string(msg.Raw), "X-Apple-Content-Length"), "second part keeps its placeholder") +} + +func TestFindBoundary_ToleratesWhitespaceAroundEquals(t *testing.T) { + assert := assert.New(t) + for _, tc := range []struct{ header, want string }{ + {`Content-Type: multipart/mixed; boundary="=-b"`, "=-b"}, + {`Content-Type: multipart/mixed; boundary=plain`, "plain"}, + {`Content-Type: multipart/mixed; boundary = "spaced"`, "spaced"}, + {`Content-Type: multipart/mixed; boundary= tab`, "tab"}, + {`Content-Type: multipart/mixed; charset=utf-8; BOUNDARY="upper"`, "upper"}, + } { + assert.Equal(tc.want, findBoundary([]string{tc.header}), tc.header) + } +} diff --git a/internal/emlx/reader.go b/internal/emlx/reader.go index e335c1368..9e146e367 100644 --- a/internal/emlx/reader.go +++ b/internal/emlx/reader.go @@ -8,6 +8,8 @@ package emlx import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/xml" "errors" "fmt" @@ -22,6 +24,10 @@ type Message struct { // Raw is the RFC 5322 MIME content. Raw []byte + // SourceHash is the SHA-256 of the original MIME bytes, before restoring + // attachments. It remains stable when Apple Mail downloads attachments. + SourceHash string + // PlistDate is the date-sent value from the plist metadata. // Zero if the plist is missing or the field is absent. PlistDate time.Time @@ -31,6 +37,15 @@ type Message struct { // OrigMailbox is the original-mailbox value from the plist. OrigMailbox string + + // RestoredAttachments is the number of attachment parts whose + // placeholder body was replaced with content from Apple Mail's + // sibling Attachments/ directory (see ParseFile). + RestoredAttachments int + + // RestorationError reports unreadable cached attachments. Raw still + // contains the message, with placeholders for parts that could not be read. + RestorationError error } // Parse parses an .emlx file from its raw bytes. @@ -69,6 +84,8 @@ func Parse(data []byte) (*Message, error) { msg := &Message{ Raw: data[mimeStart:mimeEnd], } + sum := sha256.Sum256(msg.Raw) + msg.SourceHash = hex.EncodeToString(sum[:]) // Parse optional plist metadata (best-effort). if mimeEnd < len(data) { @@ -80,12 +97,26 @@ func Parse(data []byte) (*Message, error) { } // ParseFile reads and parses an .emlx file from disk. -func ParseFile(path string) (*Message, error) { +// +// For a Messages/.partial.emlx file, Apple Mail keeps attachment bytes +// out of the MIME payload and stores them in a sibling Attachments// +// directory instead, leaving an X-Apple-Content-Length placeholder in the +// part header. ParseFile restores top-level attachment parts into Raw as +// base64. Nested attachments keep their placeholders. +// Attachments are only restored while the message, with the +// restored parts base64-encoded, stays within maxBytes; a part that would +// exceed the remaining budget keeps its placeholder. +func ParseFile(path string, maxBytes int64) (*Message, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("emlx: read %q: %w", path, err) } - return Parse(data) + msg, err := Parse(data) + if err != nil { + return nil, err + } + msg.Raw, msg.RestoredAttachments, msg.RestorationError = RestoreAttachments(msg.Raw, path, maxBytes) + return msg, nil } // parsePlist extracts metadata from the Apple Mail XML plist. diff --git a/internal/emlx/reader_test.go b/internal/emlx/reader_test.go index ff67ec81b..75df70026 100644 --- a/internal/emlx/reader_test.go +++ b/internal/emlx/reader_test.go @@ -126,13 +126,13 @@ func TestParseFile(t *testing.T) { path := filepath.Join(dir, "1234.emlx") require.NoError(t, os.WriteFile(path, []byte(data), 0600), "write") - msg, err := ParseFile(path) + msg, err := ParseFile(path, 1<<20) require.NoError(t, err, "ParseFile") require.Equal(t, mime, string(msg.Raw)) } func TestParseFile_NotFound(t *testing.T) { - _, err := ParseFile("/nonexistent/12345.emlx") + _, err := ParseFile("/nonexistent/12345.emlx", 1<<20) require.Error(t, err, "expected error for missing file") } diff --git a/internal/importer/emlx_import.go b/internal/importer/emlx_import.go index 0f7af86fb..51b34b801 100644 --- a/internal/importer/emlx_import.go +++ b/internal/importer/emlx_import.go @@ -2,8 +2,6 @@ package importer import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json/v2" "errors" "fmt" @@ -72,9 +70,14 @@ type EmlxImportSummary struct { MessagesSkipped int64 // PartialFiles counts *.partial.emlx files parsed. Their bodies are - // complete; only attachment parts are uncached by Apple Mail. + // complete; attachment parts are either restored from Apple Mail's + // sibling Attachments/ directory or left uncached. PartialFiles int64 + // AttachmentsRestored counts attachment parts of partial files whose + // bytes were inlined from the Attachments/ directory. + AttachmentsRestored int64 + Errors int64 HardErrors bool } @@ -92,7 +95,8 @@ const defaultMaxEmlxBytes int64 = 128 << 20 // 128 MiB // ImportEmlxDir imports .emlx files from an Apple Mail directory tree. // -// Messages are deduplicated by content hash (sha256 of raw MIME). +// Messages are deduplicated by the hash of their original on-disk MIME. +// Restoring attachments preserves that identity and updates existing messages. // When the same message appears in multiple mailboxes, the first // occurrence is fully ingested; subsequent occurrences add their // mailbox label to the existing message. @@ -258,14 +262,15 @@ func ImportEmlxDir( hardErrors := false type pendingEmlxMsg struct { - Raw []byte - RawHash string - SourceMsg string - LabelIDs []int64 - Fallback time.Time - MboxIdx int - MboxPath string - FileName string + Raw []byte + RestorePaths []string + RawHash string + SourceMsg string + LabelIDs []int64 + Fallback time.Time + MboxIdx int + MboxPath string + FileName string } const ( @@ -381,7 +386,7 @@ func ImportEmlxDir( } } - if exists { + if exists && len(p.RestorePaths) == 0 { rfcID, inReplyTo := mime.ParseMessageIDs(p.Raw) if err := st.RecordEmailHeadersContext(ctx, src.ID, existingID, rfcID, inReplyTo); err != nil { cp.ErrorsCount++ @@ -404,9 +409,45 @@ func ImportEmlxDir( continue } - alreadyExists := false + alreadyExists := exists if anyOK { - _, alreadyExists = existingAny[p.SourceMsg] + _, found := existingAny[p.SourceMsg] + alreadyExists = alreadyExists || found + } + + if exists { + // Ingestion replaces labels, so retain labels from earlier + // imports as well as those just added for this mailbox. + labelIDs, err := st.MessageLabelIDsContext(ctx, existingID) + if err != nil { + cp.ErrorsCount++ + summary.Errors++ + hardErrors, checkpointBlocked = true, true + log.Warn("failed to read labels for attachment restoration", "message_id", existingID, "error", err) + continue + } + for _, id := range labelIDs { + if !slices.Contains(p.LabelIDs, id) { + p.LabelIDs = append(p.LabelIDs, id) + } + } + // Fill the stored placeholders so a smaller Apple Mail cache + // cannot remove attachment content archived on an earlier run. + p.Raw, err = st.GetMessageRawContext(ctx, existingID) + if err != nil { + cp.ErrorsCount++ + summary.Errors++ + hardErrors, checkpointBlocked = true, true + log.Warn("failed to read message for attachment restoration", "message_id", existingID, "error", err) + continue + } + } + for _, path := range p.RestorePaths { + var err error + p.Raw, _, err = emlx.RestoreAttachments(p.Raw, path, opts.MaxMessageBytes) + if err != nil { + log.Warn("could not restore cached attachments", "file", path, "error", err) + } } if err := ingestFn( @@ -509,7 +550,7 @@ func ImportEmlxDir( continue } - msg, err := emlx.ParseFile(filePath) + msg, err := emlx.ParseFile(filePath, opts.MaxMessageBytes) if err != nil { cp.ErrorsCount++ summary.Errors++ @@ -519,12 +560,16 @@ func ImportEmlxDir( continue } + if msg.RestorationError != nil { + log.Warn("could not restore cached attachments", "file", filePath, "error", msg.RestorationError) + } + if emlx.IsPartial(filepath.Base(filePath)) { summary.PartialFiles++ + summary.AttachmentsRestored += int64(msg.RestoredAttachments) } - sum := sha256.Sum256(msg.Raw) - rawHash := hex.EncodeToString(sum[:]) + rawHash := msg.SourceHash sourceMsgID := "emlx-" + rawHash var fallbackDate time.Time @@ -558,6 +603,10 @@ func ImportEmlxDir( }) pendingBytes += int64(len(msg.Raw)) } + if msg.RestoredAttachments > 0 { + p := &pending[pendingIdx[sourceMsgID]] + p.RestorePaths = append(p.RestorePaths, filePath) + } if len(pending) >= batchSize || pendingBytes >= batchBytes { if flushPending() { diff --git a/internal/importer/emlx_import_test.go b/internal/importer/emlx_import_test.go index 13c19d714..37949b730 100644 --- a/internal/importer/emlx_import_test.go +++ b/internal/importer/emlx_import_test.go @@ -1,18 +1,22 @@ package importer import ( + "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" "log/slog" "os" "path/filepath" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/mime" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil/email" ) @@ -693,3 +697,143 @@ func TestImportEmlxDir_CheckpointBlockedOnIngestFailure(t *testing.T) { require.NoError(err, "count messages") require.Equal(3, total, "total messages") } + +// Apple Mail keeps the attachment bytes of a .partial.emlx in a sibling +// Attachments/// directory. The importer must inline them so the +// message is stored with its attachment, and report how many it restored. +func TestImportEmlxDir_PartialAttachmentRestoredFromSiblingDir(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, tmp := openTestStore(t) + + root := filepath.Join(tmp, "Mail") + mboxDir := filepath.Join(root, "Mailboxes", "Test.mbox") + msgDir := filepath.Join(mboxDir, "Messages") + require.NoError(os.MkdirAll(msgDir, 0700), "mkdir") + + pdf := []byte("%PDF-1.6\n%synthetic invoice\n") + raw := strings.Join([]string{ + "From: Bob ", + "Subject: Invoice with cached attachment", + "MIME-Version: 1.0", + `Content-Type: multipart/mixed; boundary="=-b"`, + "", + "--=-b", + "Content-Type: text/plain; charset=utf-8", + "", + "see attached", + "", + "--=-b", + "Content-Transfer-Encoding: base64", + "Content-Disposition: attachment;", + "\tfilename=\"invoice.pdf\"", + "Content-Type: application/pdf;", + "\tname=\"invoice.pdf\"", + fmt.Sprintf("X-Apple-Content-Length: %d", len(pdf)*4/3), + "", + "", + "--=-b--", + "", + }, "\n") + raw = strings.Replace(raw, "--=-b--", "--=-b\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename=notes.txt\nContent-Type: text/plain\nX-Apple-Content-Length: 12\n\n\n--=-b--", 1) + mkEmlx(t, msgDir, "3.partial.emlx", []byte(raw)) + // Import the placeholder first, as archives created before restoration did. + opts := EmlxImportOptions{ + Identifier: "alice@example.com", AttachmentsDir: filepath.Join(tmp, "attachments"), + NoResume: true, + } + summary, err := ImportEmlxDir(context.Background(), st, root, opts) + require.NoError(err) + require.Zero(summary.Errors) + require.Equal(int64(1), summary.MessagesAdded) + var originalID, conversationID int64 + require.NoError(st.DB().QueryRow(`SELECT id, conversation_id FROM messages`).Scan(&originalID, &conversationID)) + labelID, err := st.EnsureLabel(summary.SourceID, "Saved", "Saved", "user") + require.NoError(err) + require.NoError(st.AddMessageLabels(originalID, []int64{labelID})) + + attDir := filepath.Join(mboxDir, "Attachments", "3", "2") + require.NoError(os.MkdirAll(attDir, 0700), "mkdir attachments") + require.NoError(os.WriteFile(filepath.Join(attDir, "invoice.pdf"), pdf, 0600), "write pdf") + + // A duplicate without a cached attachment must not hide the restored copy + // later in the same batch. + mkEmlx(t, msgDir, "2.partial.emlx", []byte(raw)) + notes := []byte("More notes.\n") + for run, restored := range []int64{1, 2, 1, 1} { + if run == 1 { + secondDir := filepath.Join(mboxDir, "Attachments", "2", "3") + require.NoError(os.MkdirAll(secondDir, 0700)) + require.NoError(os.WriteFile(filepath.Join(secondDir, "notes.txt"), notes, 0600)) + } + if run == 2 { + // Losing one cache file must not remove already archived bytes. + require.NoError(os.Remove(filepath.Join(attDir, "invoice.pdf"))) + } + summary, err = ImportEmlxDir(context.Background(), st, root, opts) + require.NoError(err) + require.Zero(summary.Errors) + assert.Zero(summary.MessagesAdded) + assert.Equal(int64(1), summary.MessagesUpdated) + assert.Equal(int64(2), summary.PartialFiles) + assert.Equal(restored, summary.AttachmentsRestored) + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&count)) + assert.Equal(1, count) + var sourceMsgID string + require.NoError(st.DB().QueryRow(`SELECT source_message_id FROM messages WHERE id = ?`, originalID).Scan(&sourceMsgID)) + assert.Equal(fmt.Sprintf("emlx-%x", sha256.Sum256([]byte(raw))), sourceMsgID) + var currentConversationID int64 + require.NoError(st.DB().QueryRow(`SELECT conversation_id FROM messages WHERE id = ?`, originalID).Scan(¤tConversationID)) + assert.Equal(conversationID, currentConversationID) + storedRaw, err := st.GetMessageRaw(originalID) + require.NoError(err) + parsed, err := mime.Parse(storedRaw) + require.NoError(err) + require.Len(parsed.Attachments, 2) + assert.Equal(pdf, parsed.Attachments[0].Content) + if run > 0 { + assert.Equal(notes, parsed.Attachments[1].Content) + } + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM message_labels WHERE message_id = ?`, originalID).Scan(&count)) + assert.Equal(2, count, "restoration preserves existing labels") + } + + var filename string + var size int64 + err = st.DB().QueryRow(`SELECT filename, size FROM attachments WHERE filename = 'invoice.pdf'`).Scan(&filename, &size) + require.NoError(err, "query attachment") + assert.Equal("invoice.pdf", filename) + assert.Equal(int64(len(pdf)), size) +} + +func TestImportEmlxDir_WarnsOnAttachmentReadFailure(t *testing.T) { + for _, deniedRel := range []string{"Attachments/3/2/invoice.pdf", "Attachments/3/2", "Attachments"} { + t.Run(deniedRel, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, tmp := openTestStore(t) + root := filepath.Join(tmp, "Test.mbox") + raw := []byte("From: a@example.com\nContent-Type: multipart/mixed; boundary=b\n\n--b\nContent-Type: text/plain\n\nbody\n--b\nContent-Disposition: attachment; filename=invoice.pdf\nX-Apple-Content-Length: 12\n\n\n--b--\n") + mkMailboxDir(t, root, map[string][]byte{"3.partial.emlx": raw}) + file := filepath.Join(root, "Attachments", "3", "2", "invoice.pdf") + require.NoError(os.MkdirAll(filepath.Dir(file), 0700)) + require.NoError(os.WriteFile(file, []byte("%PDF-test"), 0600)) + denied := filepath.Join(root, filepath.FromSlash(deniedRel)) + require.NoError(os.Chmod(denied, 0)) + t.Cleanup(func() { require.NoError(os.Chmod(denied, 0700)) }) + if _, err := os.ReadFile(file); err == nil { + t.Skip("requires a user subject to filesystem permissions") + } + var logs bytes.Buffer + summary, err := ImportEmlxDir(context.Background(), st, root, EmlxImportOptions{ + Identifier: "a@example.com", Logger: slog.New(slog.NewTextHandler(&logs, nil)), + }) + require.NoError(err) + assert.Equal(int64(1), summary.MessagesAdded) + assert.Zero(summary.AttachmentsRestored) + assert.Contains(logs.String(), "level=WARN") + assert.Contains(logs.String(), "permission denied") + }) + } +} diff --git a/internal/store/messages.go b/internal/store/messages.go index 0254745ac..06e06fc27 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -2635,6 +2635,24 @@ func ensureMessageLabelRefsWith( return ids, nil } +// MessageLabelIDsContext returns the labels currently assigned to a message. +func (s *Store) MessageLabelIDsContext(ctx context.Context, messageID int64) ([]int64, error) { + rows, err := s.db.QueryContext(ctx, s.Rebind(`SELECT label_id FROM message_labels WHERE message_id = ?`), messageID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + // ReplaceMessageLabels replaces all labels for a message atomically. func (s *Store) ReplaceMessageLabels(messageID int64, labelIDs []int64) error { return s.withTx(func(tx *loggedTx) error { From 6f1ca1335a636d4603e9a84e588fb6f293b3bc02 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 22 Sep 2026 09:39:56 -0500 Subject: [PATCH 013/125] docs: update documentation for 0.20.0 (#891) Update the documentation for 0.20.0 with the final changelog, upgrade guidance, and acknowledgements for all 16 contributors identified between the release tags. - Fill gaps in the CLI reference, including search evaluation, visual search, contact activity, subset export, and browser message links. - Correct verification limits, IMAP behavior, provider consent, SQL access, recovery advice, Windows builds, and container version selection. - Rewrite the product and lifecycle pages around user outcomes, keep their Markdown companions aligned, and place browser screenshots beside the tasks they illustrate. - Replace prerelease notices with links to the dated 0.20.0 changelog while preserving the detailed upgrade notes and older release history. The ten refreshed browser screenshots are published directly on `docs-assets` at commit cba6c00ad5b919c86835a32509c039ceb3c994d6. The captures use the existing reviewed Enron fixture. Existing terminal illustrations retain their visible v0.19.0 label; the two obsolete IMAP deletion diagrams remain excluded from public pages. --------- Co-authored-by: Codex --- README.md | 12 +- docs/api-server.md | 9 +- docs/architecture/search-ranking.md | 10 +- docs/changelog.md | 337 +++++++-------------- docs/cli-reference.md | 178 ++++++++++- docs/development.md | 21 +- docs/faq.md | 26 +- docs/guides/remote-deployment.md | 40 +-- docs/guides/sources.md | 6 +- docs/guides/verification.md | 35 ++- docs/index.md | 12 +- docs/setup.md | 40 ++- docs/troubleshooting.md | 26 +- docs/usage/analytics.md | 19 +- docs/usage/querying.md | 15 +- docs/usage/vector-search.md | 10 +- docs/web-ui.md | 66 ++-- web/tests/docs-fixture-screenshots.spec.ts | 12 +- website/guide.md | 80 ++--- website/guide/index.html | 33 +- website/index.html | 62 ++-- website/index.md | 119 ++++---- website/llms.txt | 1 + 23 files changed, 661 insertions(+), 508 deletions(-) diff --git a/README.md b/README.md index cd9c53108..b09ffee0c 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@

Documentation · - Setup Guide · + Setup Guide · Interactive TUI

-**The system of record for your communications and relationships.** +**Keep your communications and relationships in an archive you own.** msgvault is a local-first, open-source archive for email, chat, meetings, calendars, and contacts. Keep your history on your own hardware, find messages @@ -25,9 +25,8 @@ and files, and connect the addresses and handles that belong to the same person. Use the browser, terminal, CLI, HTTP API, or an AI assistant through MCP. > **Alpha software.** APIs, storage format, and CLI flags may change. Back up -> your data. This README describes current `main`; see -> [the changelog](docs/changelog.md#unreleased) for unreleased features -> and upgrade steps. +> your data. This README follows current `main`; see the +> [0.20.0 changelog](docs/changelog.md#0200) for features and upgrade steps. ## What you can do @@ -56,7 +55,6 @@ for the choices and consent steps. ## Installation - **macOS / Linux:** ```bash curl -fsSL https://msgvault.io/install.sh | bash @@ -121,7 +119,7 @@ first sync, and running on your own server. | I want to… | Read | |---|---| | Understand the product | [Product overview](https://msgvault.io/) and [archive lifecycle](https://msgvault.io/guide/) | -| Catch up after 0.19 | [Changelog and upgrade notes](docs/changelog.md#unreleased) | +| Catch up after 0.19 | [Changelog and upgrade notes](docs/changelog.md#0200) | | Search messages and attachments | [Searching](docs/usage/searching.md) and [document indexing](docs/usage/document-indexing.md) | | Maintain contacts and relationships | [People and profiles](docs/usage/people.md) | | Configure optional AI features | [Recommended configuration](docs/usage/recommended-configuration.md) | diff --git a/docs/api-server.md b/docs/api-server.md index a93f4fdf5..2cb1e9af8 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-09" +last_edited: "2026-09-22" title: Web UI & API Server description: Daemon-served analytical Web UI and REST API for your msgvault archive, with optional background sync scheduling. --- @@ -29,10 +29,13 @@ browser login, secure remote deployment, search states, and keyboard controls. The API publishes its generated OpenAPI contract at `/openapi.json`. `msgvault openapi` prints the checked-in contract without starting a daemon or opening an archive. OpenAPI `info.version` is the **API schema version**; -it is separate from the binary release version. The current schema is **2.25.0**. +it is separate from the binary release version. The current schema is **2.26.0**. Upgrade clients and daemon together across incompatible schema versions, including remote deployments. +Schema 2.26.0 adds optional `web_url` metadata to message result schemas. The +URL opens that message in the selected daemon's browser interface. + Schema 2.25.0 adds the CardDAV publication review flow: `GET /api/v1/carddav/publications/{person_id}/preview` returns the exact vCard the next write would send plus an approval token, and @@ -2222,10 +2225,12 @@ All server settings go in the `[server]` section of `config.toml`. Account sched | `api_port` | `0` (auto-select) | Port the server listens on; `0` picks an open port at startup and clients discover it automatically. Set a fixed port for remote/NAS deployments. | | `bind_addr` | `127.0.0.1` | Bind address | | `api_key` | — | API key for authentication | +| `agent_access` | `false` | Enable restricted agent grants; requires a non-empty `api_key` and a daemon restart after changes | | `allow_insecure` | `false` | Allow non-loopback binding without `api_key` | | `cors_origins` | `[]` | Allowed CORS origins | | `cors_credentials` | `false` | Allow credentials in CORS requests | | `cors_max_age` | `0` | CORS preflight cache duration in seconds (defaults to `86400` when `cors_origins` is set) | +| `trusted_proxies` | `[]` | IP addresses or CIDRs allowed to supply forwarded HTTPS and host headers | | `daemon_idle_timeout` | `20m` | Idle timeout for lifecycle-managed background daemons; set to `"0s"` to disable | | `daemon_auto_restart` | `newer` | Local daemon restart policy when the CLI finds a different daemon binary version: `newer`, `never`, or `always` | diff --git a/docs/architecture/search-ranking.md b/docs/architecture/search-ranking.md index e492afb20..5e0fde654 100644 --- a/docs/architecture/search-ranking.md +++ b/docs/architecture/search-ranking.md @@ -43,11 +43,13 @@ weights with PostgreSQL `setweight` labels: |---|---| | Subject | `A` | | From address | `B` | -| Body, To, Cc | `D` | +| To, Cc | `C` | +| Body | `D` | -PostgreSQL's default weights are roughly `A=1.0`, `B=0.4`, and `D=0.1`, which -matches SQLite's 10:4:1 field priority. Unlike BM25, default `ts_rank()` does -not penalize long documents. +msgvault passes rank weights of `A=1.0`, `B=0.4`, `C=0.1`, and `D=0.1`, which +matches SQLite's 10:4:1 field priority while retaining distinct PostgreSQL +weight classes for recipients and body text. Unlike BM25, `ts_rank()` without +a normalization flag does not penalize long documents. ## Where Ordering Can Diverge diff --git a/docs/changelog.md b/docs/changelog.md index bfc935ca2..b1a2ad7be 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,21 +1,20 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-09-22" title: Changelog description: Release history for msgvault --- All notable changes to msgvault, grouped by release. -## Unreleased +## 0.20.0 +2026-09-22 -Since **0.19.3**, msgvault has added richer people profiles, document and image -search, background operations, and more ways to import and maintain an archive. -This section covers work on `main` through **September 15, 2026**. These changes -are not included in 0.19.3; this is not a new release announcement. - -macOS analytics cache builds use the existing CSV export without the unusable -SQLite extension download and warning. +Manage people and relationships, search inside attachments, and bring more of +your communications into the archive. Review the upgrade notes before updating +an existing installation. +[GitHub release](https://github.com/kenn-io/msgvault/releases/tag/v0.20.0) · +[All changes since 0.19.3](https://github.com/kenn-io/msgvault/compare/v0.19.3...v0.20.0) ### Upgrade and compatibility @@ -25,7 +24,7 @@ SQLite extension download and warning. users need their own database backup; see [backend limits](architecture/postgresql.md). - **Upgrade clients and daemon together.** The API crossed the 1.x/2.x - compatibility boundary; the current schema is **2.25.0**. Analytical + compatibility boundary; the current schema is **2.26.0**. Analytical `/api/v1/people/*` routes moved to `/api/v1/participants/*`. Durable profiles moved from `/api/v1/persons/*` to `/api/v1/people/*`. The old paths were removed. Clients reject incompatible daemons; authenticated `/api/v1/health` @@ -67,221 +66,109 @@ SQLite extension download and warning. Keyword search and basic browsing do not require a model provider. Optional features can be enabled after the archive is usable. -### People and relationships - -- Maintain structured names, contact details, addresses, dates, categories, - private notes, organizations, employment, and relationships with start and end - dates. Merge or split duplicate profiles, inspect the history, and reverse - supported merges. -- Browse saved people by last contact in Web Directory, `person directory`, and - MCP's `list_directory_people`. Filter by dates and contact details, choose an - order, and page through results. `person list` retains its existing use. -- Find a person by remembered profile facts with semantic search. Find their - files across attachment metadata, document text, and the visual index. -- Keep the evidence behind profile facts, pin corrections, and repair derived - values when their inputs change. Curated names now appear in analytics, search - results, and exported authors. `export-messages --person-id` selects messages - through the person's bound participants. -- Maintain tracked profiles with consented provider sweeps. Profiles support - OpenAI Chat, OpenAI Responses, Anthropic Messages, and Gemini. Codex - app-server configuration exists, but no executable is approved by its release - gate. The CLI can add, update, check, select, and reverify profiles. -- Configure separate Exa or SixtyFour enrichment with exact consent, request - limits, and suppression controls. -- Generate a **“Last time we talked”** brief for an enrolled person and inspect - its cited sources and earlier versions. Briefs use supported chat and text - messages; email, meetings, documents, and your replies are excluded. CLI, TUI, - Web Directory, and API manage briefs; MCP reads saved versions. -- Review Beeper identity candidates across sources. Strong provider or Beeper - identifiers can link automatically; same-service usernames require review, and - conflicting bindings remain separate. -- Import CardDAV contacts and publish selected profiles with conflict review and - lossless vCard handling. Connect **Google Contacts** through OAuth from Web - Settings or the CLI. Contact sync now works with iCloud servers that reject - full URLs in requests for individual cards. -- Record `how_we_met` and up to 280 characters in seeded text attributes. - Multiline Notes retain their separate behavior. - -See [people](usage/people.md), [profile automation](usage/people-automation.md), -[enrichment](usage/people-enrichment.md), [briefs](usage/people-briefs.md), -[Beeper identities](usage/beeper.md#review-identities-across-sources), and -[CardDAV](usage/people-carddav.md). - -### Search and attachments - -- Search Beeper chats and meeting transcripts with embeddings that include - nearby messages. Build embeddings for selected accounts or collections and - configure model-specific task prefixes. Changing scope requires a rebuild; - hybrid search can still find out-of-scope messages through keyword matching. -- Extract attachment text with the shared Docbank engine and search it by - keyword, meaning, or both. CSV files can opt into local PDF conversion while - retaining their original file identity and extraction history. - PowerPoint PPTX attachments can be indexed from their original bytes after a - fresh capability probe proves Docbank's local slide bound. -- Search image and video content through a separately configured visual - provider, after its capabilities have been checked. -- Find mailing-list traffic with `list:` or `list-id:` and browse Lists - grouping. `repair-list-ids` previews an offline backfill; `--apply` writes it. - Use `conversation_id:` to select one local conversation. -- Choose whether search includes, excludes, or shows only messages deleted from - their source. Their local content remains available until purged. -- Preserve remote email images for offline reading after explicit opt-in, with a - separate backfill for existing mail. Stored MIME and HTML remain unchanged. - Fetching these images can activate tracking. - -See [searching](usage/searching.md), [vector search](usage/vector-search.md), -[document indexing](usage/document-indexing.md), and -[remote images](usage/remote-images.md). - -### Browser, terminal, and integrations - -- Share shorter Web UI links. Ordinary tabs use readable workspace and search - mode parameters. Links omit defaults, keyboard focus, scroll position, and - choices from other tabs; browser Back and Forward retain that session context. -- Understand failed archive queries: memory and temporary-disk failures now - explain the relevant settings and recovery steps. Files offers a retry and - shows a file count only after the listing loads successfully. -- Use Web Directory to edit profiles, relationships, and employment, review - identity matches and facts, merge or split profiles, publish CardDAV contacts, - and inspect curated networks and person attachment galleries. -- Monitor sync, extraction, embeddings, enrichment, and CardDAV in Web - Operations. Filter run history, inspect failures, and run the actions the - daemon offers. CardDAV history survives daemon restarts. -- Edit Web Settings in sections with visible numeric limits, switches for - settings that can be off, and schedule presets with time zones and validation. - Track unsaved changes, discard them, and see when a restart is needed. - Credentials can be replaced or removed without revealing the complete stored - value. Host-managed settings stay read-only in the browser. -- Read messages beside the results or below them, resize the preview, and keep - the layout between visits. In dark mode, HTML email uses app colors; switch to - the sender's original colors for the open message when needed. -- Browse People and attachments in the TUI, use semantic search and Emacs-style - navigation, and scope Email by named collections. Multi-source collections - offer Fast search only; empty collections match nothing. Email, Texts, and - Meetings keep independent source selectors. -- Download, open, or export TUI attachments as a ZIP. Press `s` in email detail - to save the archived original as `.eml` in the client's current directory. - Existing files are preserved, and the original email must be archived. -- Share Saved Views between the browser, API, and MCP assistants. Read tools can - list, inspect, and run views. Creation, editing, and deletion are write tools; - HTTP requires `--http-allow-writes`. Stored definitions are validated against - the version-1 vocabulary, and revision checks prevent stale edits. -- Read person profiles, Notes, relationships, and files through MCP. General - profile reads exclude sensitive attributes and private Notes; Notes have a - separate explicit read tool. Profile writes require additional opt-in. -- Discover running HTTP MCP endpoints with `mcp status --json`, including actual - ports, backend URLs, and private token-file paths. Status does not start a - daemon or print token contents. MCP also supports protocol `2026-07-28`, - publishes object-root output schemas, and accepts parameterless inbound - messages. -- Use `setup providers` for consented provider defaults and `setup status` to - understand readiness. Sensitive profile inference requires a separate - `--allow-sensitive` opt-in. -- Browse the new website's product overview, archive lifecycle guide, and - operating documentation under `/docs/`. - -See [Web UI](web-ui.md), [TUI](usage/tui.md), -[recommended configuration](usage/recommended-configuration.md), and -[MCP](usage/chat.md). - -### Sync, imports, and maintenance - -- Sync Notion AI Meeting Notes with available transcripts, verified attendees, - changed-note refresh, and bounded retries for late transcripts. -- Import Slackdump directories or ZIPs, EML files and `.mailbox` trees, and - Maildir or Maildir++ archives. Maildir imports retain folder and flag labels - and recognize previously archived messages after filename changes. -- Import Google Groups Takeout MBOX or ZIP exports. Apple Mail imports preserve - RFC Message-ID and link unambiguous replies within a source. Re-import and - rebuild the cache to fill missing IDs in existing Apple Mail archives; see - [message identifier recovery](usage/importing.md#message-identifiers-and-replies). -- Apple Mail imports restore cached top-level attachments of `.partial.emlx` - messages from the sibling `Attachments/` directory. Re-importing adds restored - attachments to existing messages without duplicates. Nested attachments are - not restored. The import summary reports how many attachments were restored, - and unreadable cached files produce warnings. -- Import Apple WhatsApp ChatStorage text, including URL messages, with available - participant and push names. Contact-number matching requires country codes; - missing group-participant tables no longer block otherwise usable exports. -- Preserve Google Voice voicemail audio supplied in Takeout exports. Missing or - unreadable recordings remain visible as failed attachment records. Discord - voice messages retain their duration and waveform metadata. -- Run bounded historical Gmail or IMAP imports as background jobs through the - API, with durable status and checkpoint-based resumption. -- Keep IMAP sync incremental with QRESYNC where available and avoid refetching - unchanged folders without it. Retry connection failures within bounds, - preserve valid mailbox state after incomplete responses, and treat messages - removed during a fetch as handled. `repair-labels` rebuilds labels from stored - folder membership without contacting the provider. -- Create an IMAP reply draft with `draft-reply` after an operator grants access - to a specific source and Drafts folder. The server must support UIDPLUS. - Msgvault stores an archived copy; it never sends the email. -- Add restricted agent grants. Owner-only `agent-token issue/list/revoke` - commands manage in-memory tokens with `draft.create`, `draft.edit`, and - `draft.delete` permissions for named sources. Agents use `--agent-url` and - `--agent-token-file` to run `draft-reply` or `draft-recover`; grants expire - when revoked or when the daemon restarts. - Enable `[server] agent_access = true` with a non-empty `api_key`; see - [agent-token](cli-reference.md#agent-token) and - [configuration](configuration.md#server). -- Read, edit, and delete managed IMAP drafts with `draft-get`, `draft-edit`, and - `draft-delete`, plus `draft-recover` for interrupted operations. Reads use - retained archive content; edits, deletes, and recovery require the reported - revision and exact provider receipt. Recovery uses saved replacement bytes and - never replays APPEND. Uncertain provider results retain their evidence for a - later recovery attempt. -- Refresh the archived body, recipients, and attachments when a trusted outgoing - IMAP copy is edited or moves from Drafts to Sent. Ordinary received-mail and - All Mail copies cannot replace that content. Historical rows that already lost - their old location are not repaired automatically. See [IMAP](usage/imap.md). -- Authorize Gmail with read-only access using `add-account --readonly`. Existing - write grants require the documented revoke-and-reauthorize steps. Google - Calendar registration now also works directly with Workspace service accounts; - delegation and disabled-API errors stop without lengthy retries. -- Archive Teams self-chat and avoid replaying the last message at each - incremental boundary. Preserve Beeper transcript metadata, resume backfilled - history, store plain text, distinguish link previews, and repair older - classifications from archived payloads. Calendar snippets retain valid UTF-8. -- Select exact sources with `--source-id` where supported. Deletion manifests - preserve source type and identifier. Stage by query or explicit message IDs, - preview counts, and skip ineligible query matches. Eligible messages must - belong to one exact source; TUI and MCP reject cross-source staging. -- Keep messages and attachments after permanent source deletion. Use the - separate SQLite `gc` command to purge source-deleted rows and unreferenced - blobs. Deduplication requires the reviewed plan to remain valid and prefers - attachment-complete survivors only under its documented equivalence rules. -- Repair source identity, sender attribution, Gmail snapshots, and derived - metadata. Gmail sync reconciles expired history, retries failed fetches from - the previous completed run, verifies cached account identities, and bounds - request and OAuth refresh waits. Scheduler errors show copyable - reauthorization commands. -- Recover more malformed MIME messages and report importer read errors. Bound - large analytical listings, rebuild caches less often, maintain SQLite planner - statistics, resolve cached recipient addresses, and prune orphan embeddings. - Tune DuckDB query memory, threads, and disk spill separately from cache-build - limits. - -See [sources](guides/sources.md), [imports](usage/importing.md), -[text messages](usage/text-messages.md), [Calendar](usage/calendar.md), -[OAuth](guides/oauth-setup.md), [deletion](usage/deletion.md), -[deduplication](usage/deduplication.md), and -[analytics configuration](configuration.md#analytics). - -### Building and maintaining msgvault - -- Build with Go 1.27 and the documented Bun, Node.js, and native SQLite - prerequisites. Nix flake packaging has been removed. -- Use Docker Bake to export Linux AMD64 and ARM64 images as OCI archives. Image - builds check database initialization, DuckDB queries, and the embedded Web UI. - Repository release-publishing workflows and tag/changelog scripts have been - removed; local builds and installers remain available. -- Generate the browser API client with Orval and pinned OpenAPI tools. Local - SQLite test scheduling scales to available CPU and memory; PostgreSQL - configurations remain separately covered. Synthetic media examples and - relevance judgments prepare future retrieval evaluation; they do not add audio - transcription or a runnable evaluation workflow. - -See [Development](development.md) for build and check commands. +### New features + +- Manage people in the new Web Directory and TUI People browser. Profiles bring together contact details, custom fields, private Notes, messages, meetings, files, and change history. +- Record organizations, current and past employment, and dated relationships between people. Review uncertain identity matches, merge duplicate profiles, and split mistaken merges while retaining history. +- Browse people by last contact through `msgvault person directory` and MCP. Relationship calendars show contact activity over time. +- Keep tracked profiles current with optional model-assisted updates from archived conversations. Provider setup requires explicit consent and budgets; saved evidence explains profile changes, and pinned facts preserve manual choices. +- Enrich profiles through Exa or SixtyFour using explicitly permitted identity details. External enrichment stays off by default and does not send message bodies or private attributes. +- Catch up with a "Last time we talked" brief that cites archived messages. Briefs require separate enrollment for each person and currently summarize their supported chat and text messages, excluding email, meetings, and your replies. +- Find people by meaning with `msgvault person search`. This requires separate consent for profile embeddings and search queries. +- Sync contacts bidirectionally with a CardDAV address book, including Google Contacts OAuth setup. Publication is explicit per person, conflicts remain reviewable, and profile edits preserve unrelated vCard properties. +- Find a person's files across linked accounts and conversations through the Web Files view, `msgvault person files`, or MCP. Search attachment details, extracted text, or visual content, with links back to the original messages. +- Search text extracted from PDF, CSV, and PPTX attachments. Extraction requires a successful provider capability check, explicit consent, and a manual build. CSV conversion is opt-in, and PPTX requires a verified slide limit. +- Search indexed attachment text by meaning or combine keyword and semantic results. Document embeddings and query uploads require separate consent; default document searches remain local keyword searches. +- Find images and supported MP4 attachments with text or image queries. Visual search requires provider capability checks and consent before archive uploads. +- Preserve conversation context in chat and meeting search with optional Voyage contextual embeddings. The existing embedding method remains the default. +- Filter messages with `list:`, `list-id:`, and `conversation_id:`. Mailing-list views are available in the Web UI and TUI; run `msgvault repair-list-ids --apply` and rebuild the cache to backfill existing mail. +- Compare keyword, semantic, and hybrid search quality with `msgvault eval`, using your own queries and relevance ratings. Reports include ranking quality and query timings in table or JSON form. +- Import EML mailbox trees and Maildir archives directly with `import-eml` and `import-maildir`. Both preserve folder labels and support repeat imports without duplicating archived messages. +- Import Google Groups Takeout MBOX files and ZIPs with `import-mbox --source-type google-groups`. Imports retain group labels, threads, and attachments without treating the group address as your identity. +- Import Slackdump Standard and Mattermost export directories or ZIPs without a live Slack token. +- Import Apple WhatsApp `ChatStorage.sqlite` text conversations, including supported participant names. Media, reactions, and reply relationships remain outside this importer. +- Sync Notion AI Meeting Notes into meeting views and search. Sync retains summaries, notes, and transcripts, retries pending transcripts, and reports Notion's partial 50-meeting discovery window. +- Keep Google Voice Takeout voicemail recordings with their archived messages. Stored audio remains available after the Takeout files move. +- Create, retrieve, edit, and delete plain-text IMAP reply drafts through the CLI. Enable drafts for each source and configure its mailbox first; sending remains in your mail client. `draft-recover` resumes identified interrupted edits or deletions from saved receipts without appending another copy. +- Give an agent account-scoped permission to create reply drafts with `msgvault agent-token`. Delegation requires explicit server configuration and an owner API key. Tokens remain valid until revoked or the daemon restarts. +- Archive remote email images for offline reading with `[sync] archive_remote_images = true`. The feature is off by default because downloads can activate tracking pixels. Backfill existing email separately with `msgvault archive-remote-images --allow-tracking`. +- Authorize Gmail with read-only access using `msgvault add-account --readonly`. Existing write grants require revocation and reauthorization; adding the flag does not narrow an existing grant. +- Start bounded Gmail and IMAP historical imports through the API and retrieve persistent progress and completion status. +- Stage Gmail deletions from search criteria or explicit IDs with `msgvault stage-delete`, including `--dry-run`. Mixed selections stage eligible Gmail messages and report skipped items; each batch remains tied to one source. +- Inspect background work in the new Operations workspace. Filter run history and open outcomes, recovery guidance, and available actions. +- Manage daemon, source, search, and attachment settings in the Web UI or press `,` in the TUI. Web settings include schedule presets, time zones, numeric limits, and masked API-key hints. +- Configure optional search and people providers with `msgvault setup providers`. Use `msgvault setup status` to see missing credentials, checks, consent, and next steps. +- Let MCP clients discover, run, and manage Saved Views, the saved queries created in the Web UI. New people tools expose profiles, recent contacts, saved briefs, and person-specific file search. +- Discover running HTTP MCP listeners with `msgvault mcp status --json`, including their ports and token-file paths without printing tokens. +- Reclaim SQLite space from messages deleted at their source with `msgvault gc`, which requires confirmation and creates a backup by default. Remove embeddings left by hard-deleted messages with `msgvault embeddings prune` without paying for a full embedding rebuild. + +### Improvements + +- Sync large IMAP accounts incrementally with or without QRESYNC. Unchanged folders avoid full scans, and full enumerations update only changed records. In one 96,000-message Microsoft 365 benchmark, syncing after one message moved drops from 1 hour 46 minutes to 24 seconds. +- Scope embedding builds to selected accounts or named collections. Configure `document_prefix` and `query_prefix` for models that require task instructions; changing either requires `msgvault embeddings build --full-rebuild --yes`. +- Control chat attachment downloads by account, conversation size, and file size. Beeper, Slack, Discord, and Teams now skip downloads in conversations with more than 20 participants by default; set `media_max_participants = 0` to remove that cap. +- Retain larger individual attachments by default in Beeper, Slack, and Teams, with the limit rising to 250 MiB. Discord stays at 50 MiB. Existing files remain stored until you explicitly run `purge-excluded-media`. +- Read HTML email with dark colors and a resizable right-side preview in Everything. A per-message "Use original colors" control restores the sender's presentation. +- Share shorter Web UI links while preserving filters and selected items. Back and Forward retain the full session state, and shared links retain their search mode. +- Open messages in the browser from CLI JSON and MCP results through the new `web_url` field. +- Browse, download, and open individual attachments from the TUI with `e`. Press `s` in email detail to save an `.eml` file on the client machine without overwriting existing files. +- Reuse TUI searches with Up and Down, navigate with `Ctrl+n` and `Ctrl+p`, and select named collections with `A`. Semantic search is available when vector search is enabled. +- Stage the current TUI selection with `d`, or resolve every filter or search match with `D` before confirmation. Set `[data].export_dir` to choose the destination for attachment exports. +- Use curated person names in search, relationship analytics, completion, and message exports. Existing analytics caches require one full rebuild; later name changes refresh derived data without rebuilding message shards. +- Tune daemon query resources with `[analytics] query_memory_limit`, `query_threads`, and `query_temp_limit`. Cache builds now allow up to 32 GB of temporary disk use by default, and `min_rebuild_interval` limits automatic rebuild frequency. +- Select an exact source with `--source-id` for sync, account maintenance, and staged deletion when account names are ambiguous. +- Search retained source-deleted mail with `--deletion-scope deleted` or `any` in keyword searches. Message output includes deletion timestamps, and archived raw email remains exportable. +- Enable remote deletion persistently in the invoking CLI's configuration with `[deletion] remote_enabled = true`. The daemon's own configuration does not grant consent to a remote caller. +- Connect MCP clients using protocol 2026-07-28 with sessionless HTTP discovery. HTTP remains read-only unless explicitly enabled. Clients consuming `aggregate` or `search_by_domains` must read `structuredContent.data`. +- Update API integrations to use `/api/v1/participants` for observed-contact analytics and `/api/v1/people` for saved profiles. The old people-analytics routes and `/api/v1/persons` routes are removed. +- Install through the supported scripts, Homebrew, or release binaries instead of Nix, which is no longer supported. Source builds require Go 1.27.0. + +### Bug fixes + +- Preserve the local message, raw email, metadata, and attachments when permanently deleting mail from Gmail. +- Reconcile Gmail source deletions after history expires without removing archived content. Incremental sync also retries failed fetches from the latest completed run recorded as incremental. +- Reject cached Gmail credentials that belong to a different mailbox during `add-account`. Gmail and Google OAuth requests now have bounded waits, and authentication errors include complete recovery commands. +- Audit and repair mismatched Gmail snapshots from archives written between February and May 2026 with `repair-message --audit` and `repair-message `. +- Pick up new Teams chat messages after the first sync and archive self-chats. Existing accounts need no migration or reauthorization. +- Store readable Beeper message text instead of HTML markup and check daily for newly available older history. Existing archives repair derived text on the next sync or through `repair-derived --source-type beeper`. +- Preserve Beeper transcript metadata and Discord voice-message flags and waveforms. Existing Discord records support repair with `repair-derived --source-type discord`. +- Retry transient IMAP connection failures and reconnect after dropped connections during label collection. Confirmed mid-sync message removals no longer discard completed progress. +- Refresh archived IMAP content when an edited outgoing message moves to a trusted Sent folder. Servers without advertised folder roles need an account-specific `sync.trusted_imap_sent_mailboxes` setting. +- Recover cached top-level attachments from Apple Mail partial messages without duplicating messages on re-import. Nested attachment restoration remains unsupported. +- Preserve Apple Mail Message-ID values and reply links. Re-import with the same source identifier and `--no-resume`, then run `build-cache --full-rebuild` to repair missing IDs and expose them in SQL and JSON. +- Report unreadable Apple Mail directories instead of treating them as empty archives, including Full Disk Access guidance for macOS permission errors. +- Import PST archives without counting search folders or empty attachment tables as errors. +- Recover usable email text and attachments with malformed content types. When broken MIME structure prevents full parsing, intact headers still preserve identification and threading. +- Prefer more complete duplicate copies when earlier source and eligibility rules tie. Deduplication previews no longer write derived metadata before confirmation. +- Find recipient addresses in older archives through `message_recipients.email_address`. Use `envelope_address` when you need the original header address; the next cache rebuild adds both. +- Recover Web views automatically after initial cache preparation and reduce memory use for ordinary Everything and Files listings. Query failures explain memory or disk limits, and Files offers Retry instead of displaying an unavailable count as zero. +- Keep the daemon discoverable after VM clock changes, preventing a second daemon startup from failing on the existing archive lock. +- Build macOS analytics caches without repeated DuckDB extension code-signature warnings. Account removal also completes unique-attachment cleanup before rebuilding the cache. +- Preserve valid UTF-8 in calendar previews and repair affected historical snippets with `repair-encoding`. +- Register calendars with service-account apps using domain-wide delegation. Missing Calendar permissions or a disabled API report errors without waiting through the full retry budget. +- Import Android WhatsApp databases without the optional group-participant table and confirm the source identity after recoverable import errors. Contact matching requires explicit country codes. +- Repair missing sender details, account-owner attribution, and IMAP labels with `repair-senders`, `repair-identity`, and `repair-labels`. Sender and label repairs preview changes by default. + +### Acknowledgements + +Thanks to everyone who contributed to this release: + +- [@cpcloud](https://github.com/cpcloud) for making identity-index tests independent of platform memory limits. +- [@css521](https://github.com/css521) for handling MCP messages without parameters. +- [@danshapiro](https://github.com/danshapiro) for preserving UTF-8 calendar previews and repairing older snippets. +- [@elviskahoro](https://github.com/elviskahoro) for adding account-scoped embedding builds. +- [@exactmike](https://github.com/exactmike) for improving incremental IMAP and Teams sync, reconnecting IMAP sessions, and repairing labels. +- [@fucx](https://github.com/fucx) for recovering cached attachments from Apple Mail partial messages. +- [@hansn74](https://github.com/hansn74) for adding identity repair, configurable analytics resources, and Calendar service-account support. +- [@jesserobbins](https://github.com/jesserobbins) for fixing CardDAV contact sync with iCloud. +- [@mariusvniekerk](https://github.com/mariusvniekerk) for improving Web Settings and email previews, adding Google Contacts OAuth and browser message links, and maintaining build and API tooling. +- [@mikemikimike](https://github.com/mikemikimike) for preserving the local archive during permanent Gmail deletion. +- [@rodboev](https://github.com/rodboev) for adding managed IMAP drafts and delegated access, recent-contact browsing, CSV and PPTX indexing, and import and repair improvements. +- [@salmonumbrella](https://github.com/salmonumbrella) for building people profiles, relationships, briefs, CardDAV sync, contextual search, and archive browsing and maintenance workflows. +- [@shntnu](https://github.com/shntnu) for adding Apple WhatsApp text imports and updating contributor guidance. +- [@sweenzor](https://github.com/sweenzor) for repairing Beeper message text, older-history sync, and link-preview handling. +- [@ValentinViennot](https://github.com/ValentinViennot) for making Saved Views available to MCP clients. +- [@wesm](https://github.com/wesm) for adding document and visual search, person file retrieval, search evaluation, and improvements to analytics, imports, and documentation. --- diff --git a/docs/cli-reference.md b/docs/cli-reference.md index aa480dfbf..10b980a5d 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1,25 +1,25 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-09-22" title: CLI Reference description: Complete command reference for all msgvault commands. --- Find a command by task below, or use `msgvault COMMAND --help` for the flags in your installed binary. This reference follows current `main`; see -[the changelog](changelog.md#unreleased) for the release boundary. +[the 0.20.0 changelog](changelog.md#0200) for features and upgrade notes. | Task | Commands and guides | |---|---| | Add and sync a source | [Choose a source](guides/sources.md), [sync](#sync), [sync-full](#sync-full) | | Import local exports | [import-eml](#import-eml), [import-mbox](#import-mbox), [import-maildir](#import-maildir), [import-emlx](#import-emlx), [import-pst](#import-pst), [import-slackdump](#import-slackdump), [text imports](usage/text-messages.md) | -| Search and browse | [search](#search), [tui](#tui), [show-message](#show-message), [documents](#documents), [embeddings](#embeddings) | +| Search and browse | [search](#search), [tui](#tui), [show-message](#show-message), [documents](#documents), [embeddings](#embeddings), [multimodal](#multimodal), [eval](#eval) | | Maintain people and contacts | [person](#person), [people guide](usage/people.md), [CardDAV](usage/people-carddav.md) | | Organize accounts | [identity](#identity), [collection](#collection), [update-account](#update-account) | -| Export | [export-messages](#export-messages), [export-eml](#export-eml), [export-attachments](#export-attachments) | +| Export | [export-messages](#export-messages), [export-eml](#export-eml), [export-attachments](#export-attachments), [create-subset](#create-subset) | | Review and remove mail | [stage-delete](#stage-delete), [delete-staged](#delete-staged), [deduplicate](#deduplicate), [gc](#gc) | | Back up and manage attachment storage | [backup](#backup), [pack-attachments](#pack-attachments), [purge-excluded-media](#purge-excluded-media) | | Repair older records | [repair-identity](#repair-identity), [repair-senders](#repair-senders), [repair-message](#repair-message), [repair-derived](#repair-derived), [repair-labels](#repair-labels), [repair-list-ids](#repair-list-ids), [repair-dates](#repair-dates) | -| Operate or integrate | [setup](#setup), [daemon](#daemon), [serve](#serve), [mcp](#mcp), [query](#query), [openapi](#openapi), [agent-token](#agent-token) | +| Operate or integrate | [setup](#setup), [daemon](#daemon), [serve](#serve), [activity](#activity), [mcp](#mcp), [query](#query), [openapi](#openapi), [agent-token](#agent-token) | ## Global Flags @@ -1337,6 +1337,9 @@ contain spaces. Repeating either alias requires every value to match. `--mode vector` and `--mode hybrid` require at least one free-text term in the query (filter-only queries use `--mode fts`). They do not support pagination (`--offset` is rejected) or non-active deletion scopes because the vector index covers active messages only. Bump `--limit` to retrieve a larger candidate pool instead. See [Searching](/docs/usage/searching/) for the operator reference and [Vector Search](/docs/usage/vector-search/) for semantic setup. +With `--json`, each result also includes `web_url` when the selected daemon can +provide a browser link for that message. + --- ## repair-list-ids @@ -1472,6 +1475,30 @@ and validation contract. --- +## export-discord + +Export a bounded interval from one Discord guild using the older +`msgvault-discord-export/1` JSON envelope. + +```bash +msgvault export-discord \ + --start 2026-01-01T00:00:00Z \ + --end 2026-02-01T00:00:00Z +``` + +| Flag | Description | +|---|---| +| `--start ` | Inclusive lower bound (required) | +| `--end ` | Exclusive upper bound (required) | +| `--format json` | Output format; `json` is the only supported value | + +This compatibility command reads only the archive and does not contact +Discord. New integrations should use [`export-messages`](#export-messages), +which emits the provider-neutral `msgvault-message-export/1` JSONL schema. See +the [Discord export guide](usage/discord.md#export-a-bounded-history-window). + +--- + ## export-eml Export a message as a `.eml` file. Accepts either a numeric database ID or a Gmail message ID. @@ -1522,6 +1549,32 @@ Accepts internal numeric IDs or Gmail message IDs. See [Exporting Data](/docs/us --- +## create-subset + +Create a new SQLite archive containing the requested number of most recent +messages and the records they reference. The destination receives its own +`msgvault.db` and can be opened as a separate msgvault home. + +```bash +msgvault create-subset --output ./subset-vault --rows 1000 +MSGVAULT_HOME=./subset-vault msgvault tui +``` + +| Flag | Description | +|---|---| +| `-o`, `--output ` | Destination directory (required) | +| `--rows ` | Number of most recent messages to copy; must be positive (required) | +| `--include-identity` | Copy complete identity clusters for included participants | +| `--include-attributes` | Copy current and historical person and organization attribute values | +| `--include-profiles` | Copy profiles, profile history and media, relationships, employment history, and referenced organizations | +| `--include-vcard-resources` | Copy complete native vCards and retired UID aliases; requires `--include-profiles` | + +The command is SQLite-only. The optional identity, attribute, profile, and +vCard flags can copy personal records that have no message in the subset; read +the command's warning before sharing its output. + +--- + ## export-token Export a browser-created OAuth refresh token to a remote msgvault instance. @@ -2163,6 +2216,21 @@ For automatic cache rebuilds after daemon-owned syncs, configure --- +## activity + +Refresh contact activity and last-contact dates from archived messages. + +```bash +msgvault activity build +msgvault activity build --backstop +``` + +The normal build resumes from its watermark. `--backstop` rescans the complete +archive. `msgvault serve` also runs this projection on the schedule configured +under `[activity]`. + +--- + ## rebuild-fts Rebuild the SQLite FTS5 search index. @@ -2190,6 +2258,7 @@ msgvault embeddings [flags] | `list` | List index generations with their state, model, dimension, and pending count. | | `activate ` | Activate a completed building generation, retiring the current active one. | | `retire ` | Retire a generation. | +| `prune` | Remove embeddings whose messages were hard-deleted. | ### embeddings build @@ -2262,10 +2331,106 @@ Mark a generation as retired. Retiring the active generation requires `--force-a | `--yes` | Skip the confirmation prompt. | | `--force-active` | Allow retiring the generation that is currently active. | +### embeddings prune + +```bash +msgvault embeddings prune +``` + +Remove vector rows whose source messages no longer exist. The configured +vector backend must support orphan pruning. + `msgvault build-embeddings` remains as a deprecated alias for `msgvault embeddings build` (same `--full-rebuild` and `--yes` flags). --- +## multimodal + +Build, inspect, and search the optional visual attachment index. The workflow +requires `[vector.multimodal]` configuration, a probed capability manifest, +and explicit consent for hosted processing. See [Visual attachment +search](usage/vector-search.md#visual-attachment-search). + +| Subcommand | Purpose | +|---|---| +| `probe --seeds --out --yes` | Send synthetic fixtures to the configured provider and write a capability manifest. `--fixtures` keeps the generated fixtures instead of using a temporary directory. | +| `build --yes` | Record consent for the configured capability profile and build the visual index. | +| `resume` | Continue a consented build. | +| `status [--json]` | Report generation state and coverage. | +| `retry --message --hash ` | Retry one attachment occurrence. | +| `retire --yes` | Retire one generation and delete its vectors. Original attachments remain archived. | + +Search by text or by one local JPEG, PNG, or WebP image: + +```bash +msgvault multimodal search "a whiteboard timeline" +msgvault multimodal search --image ./reference.png +``` + +| Search flag | Default | Description | +|---|---|---| +| `--image ` | — | Use a local query image of at most 20 MiB instead of text | +| `--limit ` | `20` | Results to return; must be 1–100 | +| `--cursor ` | — | Continue from an opaque result cursor | +| `--sender-person ` | — | Attachments sent by one durable person | +| `--person ` | — | Attachments related to one durable person | +| `--participant ` | — | Attachments related to one observed participant | +| `--direction ` | — | `from_person`, `to_person`, or `group`; requires `--person` or `--participant` | +| `--source ` | — | Restrict to one source | +| `--message ` | — | Restrict to one owning message | +| `--filename ` | — | Case-insensitive filename substring | +| `--mime-prefix ` | — | Case-insensitive MIME prefix | +| `--after`, `--before` | — | `YYYY-MM-DD` sent-date bounds | +| `--json` | `false` | Emit JSON | + +Supply exactly one text query or `--image`. `--person` and `--participant` are +mutually exclusive. `--sender-person` cannot be combined with either of those +or with `--direction`. + +--- + +## eval + +Compare how well search modes find messages you have rated. The command runs +the same full-text, vector, and hybrid retrieval paths used by production +search. It reports ranking quality, recall, latency, configuration, and input +diagnostics. + +```text +# topics.tsv: qidqueryoptional-category +q1 quarterly planning pointed +``` + +```text +# qrels.txt: qid iteration document-id relevance (1 or greater means relevant) +# Replace example-message-001 with a real source_message_id, not a local numeric ID. +q1 0 example-message-001 2 +``` + +```bash +msgvault eval \ + --topics topics.tsv \ + --qrels qrels.txt \ + --modes fts,vector,hybrid \ + --limit 100 +``` + +| Flag | Default | Description | +|---|---|---| +| `--topics ` | required | Tab-separated topics: `qid`, query, and optional category | +| `--qrels ` | required | Whitespace-separated judgments: `qid iteration docid relevance`; relevance of 1 or greater means relevant | +| `--modes ` | `fts,vector,hybrid` | Comma-separated modes to evaluate | +| `--doc-key ` | `message` | Match judgments to `message` source IDs or `conversation` source IDs | +| `-n`, `--limit ` | `100` | Distinct documents retrieved per query | +| `--json` | `false` | Emit the report as JSON | + +`eval` opens the archive selected by local configuration directly; it does not +use `[remote]`. Vector and hybrid evaluation currently require a SQLite archive, +an `sqlite_vec` build, enabled vector configuration, and a compatible active +generation. On PostgreSQL, run `--modes fts`. + +--- + ## cache-stats Show statistics about the analytics cache. @@ -2518,6 +2683,9 @@ msgvault show-message [flags] |---|---| | `--json` | Output as JSON | +JSON output includes `web_url` when the selected daemon can provide a browser +link for the message. + --- ## list-accounts diff --git a/docs/development.md b/docs/development.md index 57e856560..525a0d236 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,15 +28,19 @@ make install ### Windows -Use the PowerShell build helper from the repository root. It provides the same -debug and release builds as the Make targets and selects the host architecture -automatically: +Use the PowerShell helper from the repository root to compile the Go binary. +It selects the host architecture automatically and embeds assets already in +`internal/web/dist`; it does not build the browser application. + +For a binary with the Web UI, first run `make web-embed` in an MSYS2 shell +with GNU Make, Bun, and Node.js available. This builds and validates the browser +assets. Then run the PowerShell helper: ```powershell -# Debug build (equivalent to make build) +# Debug Go build .\scripts\build.ps1 -# Optimized, stripped build (equivalent to make build-release) +# Optimized, stripped Go build .\scripts\build.ps1 -Release ``` @@ -189,6 +193,13 @@ make lint go vet ./... ``` +## Evaluate search quality + +Use [`msgvault eval`](cli-reference.md#eval) to compare keyword, semantic, and +hybrid results against queries and relevance ratings you supply. Keep the +archive, topics, and ratings the same when comparing runs. The command reports +ranking quality and query timings; it does not create the ratings for you. + ## vCard registry maintenance The lossless vCard 2.1/3.0/4.0 codec vendors the IANA vCard Elements registry diff --git a/docs/faq.md b/docs/faq.md index f776618f5..07f42b253 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,16 +1,20 @@ --- -last_edited: 2026-09-08 +last_edited: 2026-09-22 title: Frequently Asked Questions description: Common questions about msgvault, Gmail API safety, and what the tool can and cannot do. --- -

Can msgvault send email?

+

Can msgvault send email or prepare a reply?

-No. msgvault archives and analyzes messages; it does not compose, send, forward, -or reply to mail. Gmail authorization requests `gmail.modify` by default for -archive and deletion workflows. `add-account --readonly` requests read-only -access instead. See [read-only Gmail access](guides/oauth-setup.md#read-only-access) -for existing-account restrictions. +msgvault never sends email. For an IMAP source, an operator can grant access to +one exact Drafts folder; `draft-reply` can then create a plain-text reply draft +for review and sending in your usual mail application. The grant is disabled by +default. See [IMAP reply drafts](usage/imap.md#reply-drafts). + +Gmail authorization requests `gmail.modify` by default for archive and deletion +workflows. `add-account --readonly` requests read-only access instead. See +[read-only Gmail access](guides/oauth-setup.md#read-only-access) for +existing-account restrictions.

What can an AI assistant do through MCP?

@@ -40,10 +44,10 @@ features. See [recommended configuration](usage/recommended-configuration.md).

Why is a documented feature missing from my binary?

-The documentation follows current `main`, including work after 0.19.3 that is -not yet released. Check `msgvault version` and the installed command's `--help`, -then consult [the changelog](changelog.md#unreleased). A configured remote daemon also -needs a compatible version. +The documentation follows current `main`; your installed binary may be older. +Check `msgvault version` and the installed command's `--help`, then consult the +[0.20.0 changelog](changelog.md#0200). Upgrade the CLI and a configured remote +daemon together.

What is the web server for?

diff --git a/docs/guides/remote-deployment.md b/docs/guides/remote-deployment.md index 6494991c1..6bec931ea 100644 --- a/docs/guides/remote-deployment.md +++ b/docs/guides/remote-deployment.md @@ -1,9 +1,12 @@ --- +last_edited: "2026-09-22" title: Remote Deployment description: Run msgvault in Docker on a remote host and provision it from a machine with a browser. --- -msgvault supports a remote-first workflow where you configure a remote instance using a local browser session, then deploy and sync on headless hardware. This works with any always-on host: a NAS (a good choice for RAID fault tolerance), a cloud VM, a Raspberry Pi, or any Linux server with Docker. +Run your archive on an always-on server and use it from your own computer. +Set up source credentials on a machine with a browser, then copy a deployment +bundle to a Linux host with Docker, such as a NAS, cloud VM, or Raspberry Pi. The flow is built on three capabilities: @@ -25,19 +28,15 @@ The flow is built on three capabilities: ## Docker Image -The image is published to GitHub Container Registry: +Choose a published image from the [container package](https://github.com/kenn-io/msgvault/pkgs/container/msgvault) +and set its full name in your Compose file. Version tags omit the leading `v`: +for example, release 0.19.3 uses `ghcr.io/kenn-io/msgvault:0.19.3`. -```bash -docker pull ghcr.io/kenn-io/msgvault:latest -``` - -| Tag | Description | -|-----|-------------| -| `latest` | Latest stable release from main branch | -| `v1.2.3` | Specific version | -| `1.2` | Latest patch of minor version | -| `1` | Latest minor/patch of major version | -| `sha-abc1234` | Specific commit (for debugging) | +`latest` is not a stable-release guarantee and can refer to an older development +snapshot. Repository-owned image publishing has been removed, so a new commit +or release tag does not automatically update GHCR. Check that the image you +choose contains the version you need. To deploy current source before a matching +image is published, follow [Container builds](../development.md#container-builds). **Architectures:** `linux/amd64` (Intel/AMD NAS, standard servers) and `linux/arm64` (Raspberry Pi 4/5, newer NAS). Docker selects the correct one automatically. @@ -116,6 +115,10 @@ services: start_period: 10s ``` +The wizard currently writes `:latest` as shown above. Before deployment, change +`image` to your chosen published version or digest. For an image built and +loaded locally, use its local tag and remove `pull_policy: always`. + ## 2) Deploy to Remote Host Copy the bundle and start services via SSH: @@ -327,10 +330,10 @@ docker exec -it msgvault msgvault tui # Interactive TUI # Restart using the currently installed image docker-compose restart -# Reconcile the service; generated bundles check for a newer latest image +# Reconcile the service with the image configured in Compose docker-compose up -d -# Explicitly update to the latest image +# Pull the configured image after choosing the version to deploy docker-compose pull docker-compose up -d @@ -339,9 +342,10 @@ docker-compose down ``` `restart` does not check the registry or replace the image. Generated bundles -set `pull_policy: always`, so `up -d` reconciles against GHCR. The explicit -`pull` followed by `up -d` sequence remains the clearest update procedure -across NAS Compose implementations. +set `pull_policy: always`, so `up -d` checks the configured tag in GHCR. A +version-pinned service stays on that version until you edit `image`. Back up +before upgrading and keep the client and daemon versions compatible. Then use +`pull` followed by `up -d` to deploy the selected image. Bundles generated before `pull_policy: always` was added are not rewritten automatically. Existing installations should either regenerate the bundle, diff --git a/docs/guides/sources.md b/docs/guides/sources.md index 8bad57667..373cf6472 100644 --- a/docs/guides/sources.md +++ b/docs/guides/sources.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-22" title: Choose a Source description: Find the right sync or import path for mail, chat, meetings, calendars, and contacts. --- @@ -72,4 +72,6 @@ review its conflict and consent workflow before enabling that direction. For records already stored, keyword search and analytics use the archive. Optional [semantic search and profile automation](../usage/recommended-configuration.md) -have their own provider configuration and consent steps. +have their own provider configuration. Message embeddings are enabled through +configuration; people, visual, and document features also require their stated +consent steps. diff --git a/docs/guides/verification.md b/docs/guides/verification.md index 1d8160556..ede63fd0f 100644 --- a/docs/guides/verification.md +++ b/docs/guides/verification.md @@ -1,9 +1,18 @@ --- +last_edited: "2026-09-22" title: Verify Integrity -description: Verify your archive against Gmail. +description: Check a Gmail archive's database, raw-message coverage, and sampled MIME data. --- -## Usage +Check whether a Gmail archive is structurally readable and whether sampled raw +messages can be decompressed. The command also reports Gmail's current message +count beside the archive count. Those counts are a comparison, not proof that +the two systems contain the same messages. + +`verify` is Gmail-specific. It contacts Gmail with the selected account's +authorization, even when the archive itself is local. + +## Run a check ```bash # Default: sample 100 messages @@ -13,16 +22,22 @@ msgvault verify you@gmail.com msgvault verify you@gmail.com --sample 500 ``` -## What It Checks +## What it checks -The verify command compares your archive against Gmail through the configured -remote server or local daemon: +The command goes through the configured remote server or local daemon and +reports four checks: | Check | Description | |---|---| -| Message count | Compares local count vs Gmail message count | -| Raw MIME presence | Verifies sampled messages have raw MIME data stored | -| FTS index entries | Confirms sampled messages are indexed for full-text search | +| Database integrity | Runs SQLite `PRAGMA integrity_check` unless skipped. PostgreSQL archives must use `pg_amcheck` separately. | +| Message counts | Reports Gmail's profile total, the archive account total, and their signed difference. Gmail's total and msgvault's archive policy can cover different sets. | +| Raw MIME coverage | Counts archived messages that have stored raw MIME data and reports the percentage. | +| MIME sample | Selects up to `--sample` archived raw messages and checks that each stored MIME value can be decompressed. | + +This command does not compare Gmail and archive message IDs, inspect +attachments for completeness, or confirm that every archived message has an entry in the full-text +search index. See [rebuilding the search index](../cli-reference.md#rebuild-fts) +for index recovery. ## Flags @@ -32,9 +47,9 @@ remote server or local daemon: | `--skip-db-check` | `false` | Skip SQLite integrity check | | `--json` | `false` | Emit machine-readable JSON summary | -## When to Verify +## When to verify -- After initial full sync to confirm completeness +- After an initial full sync to check database and raw-MIME health - Before executing deletions from Gmail - Periodically to check for database corruption - After recovering from interrupted syncs diff --git a/docs/index.md b/docs/index.md index bbe3a38f8..a19455b28 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-22" title: Documentation description: Set up your archive, find messages and files, maintain people, and operate msgvault. --- @@ -12,14 +12,13 @@ the archive through the browser, terminal, CLI, or an agent.

Get started - Changelog + Changelog

!!! note "Returning after 0.19?" - These docs follow current `main`, including unreleased work after 0.19.3. - The [changelog](changelog.md#unreleased) lists new capabilities and - [upgrade notes](changelog.md#upgrade-and-compatibility), with released and - unreleased changes kept separate. + The [0.20.0 changelog](changelog.md#0200) lists the new capabilities. + Read the [upgrade notes](changelog.md#upgrade-and-compatibility) before + opening an existing archive with the new version. ## Start an archive @@ -60,6 +59,7 @@ limit an archive view. | Verify stored mail | [Archive verification](guides/verification.md) | | Hide duplicate copies | [Deduplication](usage/deduplication.md) | | Remove mail from a provider | [Deletion staging and execution](usage/deletion.md) | +| Prepare an email for review in a mail client | [Managed IMAP drafts](usage/imap.md#reply-drafts) | | Keep a recoverable copy | [Backup and restore](usage/backup.md) | | Take data elsewhere | [Exporting](usage/exporting.md) | | Diagnose a problem | [Troubleshooting](troubleshooting.md) and [FAQ](faq.md) | diff --git a/docs/setup.md b/docs/setup.md index b2f04ac9c..cd1badba6 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -77,7 +77,14 @@ On macOS and Linux this installs to `~/.local/bin` or `$GOPATH/bin`. For a debug build use `make build`, or `make build-release` for an optimized binary with stripped debug symbols. -On Windows, use the native PowerShell helper: +On Windows, first build the embedded Web UI from an MSYS2 shell with GNU Make, +Bun, and Node available: + +```bash +make web-embed +``` + +Then use the native PowerShell helper for the Go binary: ```powershell .\scripts\build.ps1 # Debug build @@ -85,8 +92,9 @@ On Windows, use the native PowerShell helper: ``` It detects AMD64 or ARM64 automatically and writes `msgvault.exe` in the -repository root. See [Development](/docs/development/#windows) for the -one-time MSYS2 compiler prerequisites. +repository root. The PowerShell helper does not build the Web UI itself. See +[Development](/docs/development/#windows) for the UI build step and one-time +MSYS2 compiler prerequisites. Verify the installation: @@ -211,7 +219,10 @@ To start with only part of a large account, see 4. Use this password when `add-imap` prompts for your credentials !!! note - IMAP sync always performs a full scan of the mailbox. The `sync` (incremental) command falls back to a full sync for IMAP accounts because IMAP does not provide a change-tracking API like Gmail's History API. Messages already in the database are skipped efficiently. + After the first scan, `msgvault sync` uses QRESYNC when the server supports + it. Otherwise it uses CONDSTORE or a UID-based scan and skips unchanged + messages. See [IMAP Sync and Repair](usage/imap.md#how-later-syncs-find-changes) + for the fallback order and deletion limits. ## Sync Email @@ -296,7 +307,10 @@ Reduce this value if you encounter rate limit errors during large syncs. ### Safety -Sync operations are **read-only**. They use only `messages.list` and `messages.get` Gmail APIs. No write operations are performed. Your Gmail data remains untouched. +Sync operations are **read-only**. They read the Gmail profile, labels, +message lists, raw messages, and history as needed. They do not call Gmail +write APIs or change provider data. Remote deletion is a separate, +explicitly enabled workflow. ## Explore @@ -329,12 +343,16 @@ the rest: export VOYAGE_API_KEY="..." # text, people, and visual search export MISTRAL_API_KEY="..." # document attachments export OPENAI_API_KEY="..." # people sweep (and text search without a Voyage key) -msgvault setup providers # one consent per provider, then config.toml is written +msgvault setup providers # review provider disclosures and write opt-in config msgvault setup status # what is on, what is off, and why ``` -The people sweep additionally requires `msgvault setup providers --allow-sensitive` -to permit sensitive archive excerpts and sensitive personal inferences. +Message embeddings are authorized by enabling their configuration and have no +separate stored consent record. Semantic people search, visual processing, +document extraction and vectors, and people sweeps have separate consent gates; +`setup status` names any command still required. The people sweep additionally +requires `msgvault setup providers --allow-sensitive` to permit sensitive +archive excerpts and sensitive personal inferences. See [Recommended Configuration](/docs/usage/recommended-configuration/) for the values it writes and the probe steps the hosted lanes still need. @@ -355,14 +373,14 @@ full workflow, scheduled sync, and headless-server setup. ## Open the Web UI -Build the analytical cache and start the daemon: +Start the daemon: ```bash -msgvault build-cache msgvault serve ``` -Open the `API server` URL printed at startup. With the default loopback bind, +The daemon builds a missing or stale analytical cache during startup. Open the +`API server` URL it prints. With the default loopback bind, the browser is trusted locally. A daemon bound to another interface must use an API key; the browser presents a login screen and stores only an in-memory daemon session. The release binary contains the complete UI, so no frontend diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 0b1d75eee..d598cc521 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -236,15 +236,20 @@ If you hit Gmail API rate limits during large syncs: ## Database Corruption -If the database is corrupted: +Do not delete a corrupt database. An archive can contain local imports and +provider-deleted messages that a new sync cannot recover. -1. Back up your database: `cp ~/.msgvault/msgvault.db ~/.msgvault/msgvault.db.bak` -2. Delete and re-sync: - ```bash - rm ~/.msgvault/msgvault.db - msgvault init-db - msgvault sync-full you@gmail.com - ``` +1. Stop the daemon with `msgvault daemon stop` or stop its Docker/systemd + service. +2. Preserve the complete data directory, including the database, WAL and SHM + files, attachments, configuration, and tokens. +3. Restore a verified snapshot into an empty directory. Follow + [Restoring to a New Machine](/docs/usage/backup/#restoring-to-a-new-machine); + the same empty-target rule applies on the original machine. + +If no verified backup exists, keep the damaged directory unchanged and ask for +recovery help before initializing another archive. Re-syncing is only a +reconstruction option when every source record is still available upstream. ## Interrupted Syncs @@ -315,7 +320,10 @@ api_port = 9090 ### HTTP 429 Too Many Requests -The API server enforces a rate limit of 10 requests per second per client IP. If you are hitting this limit, space out your requests or check the `Retry-After` response header for the wait duration. +The general API limit allows 10 requests per second per client IP with a burst +of 20. Trusted, authenticated loopback requests are exempt, except session +login. Some expensive endpoints have tighter limits. If you receive 429, space +out requests and follow the `Retry-After` header. ## Using Logs for Troubleshooting diff --git a/docs/usage/analytics.md b/docs/usage/analytics.md index c4bde079e..0a07d74d0 100644 --- a/docs/usage/analytics.md +++ b/docs/usage/analytics.md @@ -1,8 +1,11 @@ --- +last_edited: "2026-09-22" title: Analytics & Stats description: Archive statistics, top senders, domains, and labels. --- +Use the built-in aggregate commands for a quick archive summary. They query the +configured remote server when one is set; otherwise they use the local daemon. ## Stats @@ -12,7 +15,14 @@ Show overall archive statistics: msgvault stats ``` -Displays total message count, account breakdown, date range, storage size, and attachment count. +The output includes message, thread, attachment, label, and account counts plus +the database size. Source-deleted messages are reported separately when any +exist. Scope the counts to one account or collection when needed: + +```bash +msgvault stats --account you@example.com +msgvault stats --collection work +```
msgvault stats command output
@@ -39,7 +49,6 @@ msgvault list-labels
msgvault list-senders command output
-These commands query the configured daemon or remote server. For interactive -exploration, use the [Web UI](/docs/web-ui/) to combine search, filters, grouping, -and modality-aware drill-down in a shareable URL-backed context. The -[TUI](/docs/usage/tui/) provides a terminal-native analytical workflow. +For interactive exploration, use the [Web UI](/docs/web-ui/) to combine search, +filters, and grouping, then share a link to that view. Use the +[TUI](/docs/usage/tui/) to explore the same archive from the terminal. diff --git a/docs/usage/querying.md b/docs/usage/querying.md index e65dc3f0e..29aaf2a41 100644 --- a/docs/usage/querying.md +++ b/docs/usage/querying.md @@ -1,9 +1,12 @@ --- +last_edited: "2026-09-22" title: SQL Queries -description: Run arbitrary SQL against your archived email using DuckDB. +description: Run read-only DuckDB queries against the analytics cache. --- -The `msgvault query` command lets you run SQL directly against your archive's Parquet analytics cache using an in-memory DuckDB engine. This gives you full SQL expressiveness for ad-hoc analysis, custom reports, and data exploration beyond what the built-in analytics commands provide. +Use `msgvault query` for ad-hoc analysis across email, chat, calendar, and +meeting data in the Parquet analytics cache. The daemon runs one read-only +DuckDB statement. It rejects writes, session changes, and multiple statements. ## Basic Usage @@ -145,10 +148,10 @@ msgvault query --format table " " ``` -Known values are `email`, `calendar_event`, `meeting_transcript`, `beeper`, -`teams`, `discord`, `sms`, `mms`, `whatsapp`, `imessage`, `fbmessenger`, -`synctech_sms_call`, `google_voice_text`, `google_voice_call`, and -`google_voice_voicemail`. +Known values are `email`, `google_chat`, `calendar_event`, +`meeting_transcript`, `beeper`, `teams`, `discord`, `slack`, `sms`, `mms`, +`rcs`, `whatsapp`, `imessage`, `fbmessenger`, `synctech_sms_call`, +`google_voice_text`, `google_voice_call`, and `google_voice_voicemail`. ### Label statistics diff --git a/docs/usage/vector-search.md b/docs/usage/vector-search.md index 086edeb79..8d2f35939 100644 --- a/docs/usage/vector-search.md +++ b/docs/usage/vector-search.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-22" title: Vector Search description: Find messages by meaning and set up separate people, visual, and document search indexes. --- @@ -305,6 +305,12 @@ messages, embed it: msgvault embeddings build --full-rebuild --yes ``` +For message embeddings, enabling `[vector]` is the opt-in. The command does not +create a separate stored provider-consent record; `--yes` only skips the full +rebuild confirmation. If the endpoint is hosted, both indexed message text and +semantic query text leave the archive as described above. People, visual, and +document vectors use separate consent gates. + This creates a new **building generation**, scans every non-deleted message in the configured scope, embeds missing rows in batches through your configured embedder, and atomically activates the generation once @@ -385,7 +391,7 @@ trigger). |---|---| | Manual `sync-full` / `sync` (Gmail, IMAP) | No. Run `msgvault embeddings build` afterward | | Manual `sync-calendar` / `sync-teams` / `sync-discord` | No. Run `msgvault embeddings build` afterward | -| Manual `sync-slack` / `sync-beeper` / `sync-granola` / `sync-circleback` / `sync-notion` | No. Run `msgvault embeddings build` afterward | +| Manual `sync-slack` / `sync-beeper` / `sync-granola` / `sync-circleback` / `sync-notion-meetings` | No. Run `msgvault embeddings build` afterward | | Scheduled account syncs in `msgvault serve` (Gmail, IMAP, Teams, Discord) | Yes, when `[vector.embed.schedule].run_after_sync = true` | | Scheduled calendar, Slack, Beeper, Granola, Circleback, and Notion syncs in `msgvault serve` | No immediate post-sync run. Picked up by the embed worker's `[vector.embed.schedule].cron` schedule | | Local email imports (`import-pst`, `import-emlx`, `import-eml`, `import-mbox`) | No. Run `msgvault embeddings build` afterward | diff --git a/docs/web-ui.md b/docs/web-ui.md index af930ce00..02cac442c 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-09-22" title: Web UI description: Browse messages and files, maintain people, and monitor archive work from your browser. --- @@ -23,31 +23,6 @@ you do not need a separate web application process. | What is staged for deletion? | Deletions | | How do I change the daemon's configuration? | Settings | -
- Experimental Relationships workspace in dark theme with ranked people and activity timeline -
Relationships ranked view and selected activity timeline.
-
- -
- Experimental Relationships workspace in light theme with compact density -
Relationships workspace in light theme with compact density.
-
- -
- Experimental analytical web UI in dark theme with comfortable density -
Dark theme with comfortable density.
-
- -
- Experimental analytical web UI in light theme with compact density -
Light theme with compact density.
-
- -The screenshots use a curated public Enron research-data fixture. Authentic -names and message text are intentional; the repository's `docs-fixtures` -branch records provenance, attribution, and the content review. Screenshots -illustrate the workflows; newer controls may differ from these captures. - ## Start and discover the URL ```bash @@ -104,6 +79,16 @@ warns that its session cookie travels without TLS. `HttpOnly` and ## Explore and search +
+ Everything workspace showing archived email in light theme with compact rows +
Browse archived email in Everything. Select the image to view it at full size.
+
+ +The screenshots use a curated public Enron research-data fixture. Authentic +names and message text are intentional; the repository's `docs-fixtures` +branch records provenance, attribution, and the content review. This fixture +contains email only; it does not illustrate chat, calendar, or attachment content. + Everything opens as a compact, sortable table of logical entries: one row per email, calendar event, meeting note, other durable item, or chat conversation. Raw chat fragments appear only after drilling into a conversation. Filter, @@ -116,18 +101,18 @@ them. Search mode is always explicit: -- **Full text** searches the complete lexical index. +- **Full text** matches words in the text index. - **Semantic** ranks only content covered by the current embedding generation. -- **Hybrid** combines complete lexical matching with semantic ranking where it - is available. +- **Hybrid** combines keyword matches with semantic ranking where it is + available. The context strip reports semantic coverage. Disabled, building, stale, incomplete, unavailable, and ready are different states; msgvault never silently changes the requested mode. Semantic-only results cannot include unembedded content. Hybrid retains full-text coverage and labels the semantic contribution. -Search execution also has explicit terminal states. **Timed out** means the -selected search backend did not finish within the request budget; it is an +When a search fails, the UI explains what happened and keeps your query. +**Timed out** means the selected search backend did not finish within the request budget; it is an error, not an empty result, and the query and filters remain available to retry. **Incompatible mode** means the daemon, browser contract, or current index cannot safely honor the selected search mode. Update or rebuild the named @@ -136,6 +121,13 @@ substitute full-text search for either state. ## Read messages +Open a `/messages/` link to go directly to an archived message. CLI +`search --json` and `show-message --json` responses include a `web_url` when +the selected daemon has an HTTP address. The link uses that daemon's address; +the person opening it still needs access to the archive. Links to chat messages +open a bounded part of the conversation around the selected message, with +controls to load earlier or later messages. + Click an entry in Everything to open its preview below the results. On wide windows, choose **Preview position → Right** to read beside the results. Drag the divider to resize either layout, or focus it and use the arrow keys. @@ -152,9 +144,8 @@ applies to the open message. Light mode preserves designed email colors. ## Cache states The web tables share one analytical cache across message types. When it is missing, -building, stale, or unavailable, the UI names that state instead of quietly -switching selected modalities to a different read path. Run `msgvault -build-cache` for an explicit rebuild, or leave `analytics.auto_build_cache = +building, stale, or unavailable, the UI names that state and offers the +available recovery steps. Run `msgvault build-cache` for an explicit rebuild, or leave `analytics.auto_build_cache = true` for daemon startup to build a stale cache. With `analytics.engine = "duckdb"`, startup fails if no usable cache can be produced. @@ -190,6 +181,11 @@ separate choices. ## People and domains +
+ Relationships workspace showing a selected person's activity calendar and email timeline in dark theme +
Select a person to explore their activity and messages.
+
+ People combines identifiers backed by explicit archive identity evidence; it does not merge records merely because their display names match. Select a person to inspect contextual activity across email, chat, calendar events, and @@ -278,7 +274,7 @@ backoff to show the run and live progress; it opens no streaming connection. If the accepted run never appears, the UI reports `sync_start_not_observed` rather than claiming success. Conflicting runs and unavailable capabilities retain their explicit errors or reasons. Full resync, pause/resume, schedule editing, -and source add/remove are outside this workspace's initial scope. +and source add/remove are not available in Sources. ## Operations diff --git a/web/tests/docs-fixture-screenshots.spec.ts b/web/tests/docs-fixture-screenshots.spec.ts index 3b69aa6c6..28e6060ab 100644 --- a/web/tests/docs-fixture-screenshots.spec.ts +++ b/web/tests/docs-fixture-screenshots.spec.ts @@ -8,7 +8,7 @@ const outputDir = process.env.MSGVAULT_DOCS_SCREENSHOT_OUTPUT ?? ''; const platform = process.env.MSGVAULT_DOCS_SCREENSHOT_PLATFORM ?? 'darwin'; const exploreURL = (workspace: 'everything' | 'relationships') => - `/?explore=${encodeURIComponent(JSON.stringify({ workspace }))}`; + `/?workspace=${workspace}`; async function waitForOverview(page: import('@playwright/test').Page) { const grid = page.getByRole('grid', { name: 'Everything results' }); @@ -90,6 +90,16 @@ test.describe('documentation fixture capture', () => { await expect.poll(async () => await timeline.getByRole('row').count()).toBeGreaterThan(0); await expect(timeline.locator('[role="row"]').first()).toContainText(/\S/); await expect(page.getByLabel('Relationship activity intensity from less to more')).toBeVisible(); + // Show activity from the fixture rather than an empty current year. + const calendar = page.getByRole('region', { name: 'Relationship activity calendar' }); + const year = calendar.locator('.year'); + while (Number(await year.textContent()) > 2001) { + const previousYear = Number(await year.textContent()) - 1; + await page.getByRole('button', { name: 'Previous relationship year' }).click(); + await expect(year).toHaveText(String(previousYear)); + } + await expect(year).toHaveText('2001'); + await expect(calendar.getByText('No interactions in 2001.')).toHaveCount(0); await captureEvidence(page, filename); } } diff --git a/website/guide.md b/website/guide.md index c9af7a782..ceec688c3 100644 --- a/website/guide.md +++ b/website/guide.md @@ -4,6 +4,9 @@ Follow your archive from first capture to long-term ownership. Source access and media policies determine what is captured. Optional hosted processing sends selected data to the providers you configure. +See the [0.20.0 changelog](/docs/changelog/#0200) for new features and upgrade +steps. + 1. [Capture](#capture) 2. [Preserve](#preserve) 3. [Resolve](#resolve) @@ -16,83 +19,84 @@ sends selected data to the providers you configure. ## Capture -Live sources sync on a schedule — Gmail, IMAP, Slack, Teams, Discord, Beeper, -Google Calendar, CardDAV, meeting notes. Local exports import on demand — MBOX, Maildir, -Apple Mail, PST, EML, Slackdump, WhatsApp, iMessage, Messenger, and SMS backups. -Interrupted syncs resume from checkpoints. +Choose a connected source for recurring sync or import a local export. +Supported sources include email, chat, calendars, meeting notes, and contacts. +Each source guide explains setup, captured history, media limits, and how +interrupted work resumes. -[Importing local email](/docs/usage/importing/) +[Choose a source](/docs/guides/sources/) ## Preserve -Raw provider payloads are retained compressed beside the parsed record. -Attachments are content-addressed by SHA-256, deduplicated, and sealed into -immutable packs. Cross-account duplicates hide behind a reversible safety -ladder — msgvault checks source preference, raw message evidence, and attachment -completeness under defined rules. +Keep original message data alongside the records used for browsing. Downloaded +attachments share storage when their contents match. They start as individual +files and can be grouped into packs. Preview duplicate messages before hiding +extra copies; hidden copies remain available to restore. [Data storage](/docs/architecture/storage/) ## Resolve -Every source knows you and your contacts by different addresses and handles. -Identity discovery classifies the evidence; observed people cluster from -explicit archive links, never from matching display names. Nothing merges -without proof. +Connect the addresses and handles that belong to the same person. msgvault +groups identities using explicit links in the archive; matching display names +alone do not merge people. Review suggested identity matches before accepting +them. [People, profiles, and identities](/docs/usage/people/) ## Curate -Promote the people who matter into durable profiles with stable IDs and vCard -UIDs. Attach typed attributes, organizations, employment history, and -relationships over a fact ledger with evidence and reversible merges. Watch -each relationship's activity calendar and temperature across every channel. +Save a profile to keep contact details, notes, employment, and relationships +together. Inspect the evidence behind profile facts and correct them when +needed. Activity calendars show when you were in contact; profile history +records supported merges and reversals. [Curating people](/docs/usage/people/) ## Understand -Opt in to semantic search by pointing msgvault at an embedding server you -choose — local ones included. The embedded Docbank document engine extracts -and indexes attachment text and images behind explicit, fail-closed consent. -Search indexes can be rebuilt from the archive. Stored evidence and curated -profiles remain part of the record. +Enable search by meaning with a local or hosted embedding service. Separately +configure document and image processing, and approve uploads before sending +attachment content to a provider. You can rebuild search indexes from the +archive; stored evidence and saved profiles remain part of the record. [Vector search](/docs/usage/vector-search/) ## Search -Full-text search with Gmail-style operators answers instantly and offline. -Semantic mode finds results by meaning. Hybrid mode combines keyword and -vector rankings, with explicit coverage and ranking details; msgvault never quietly -substitutes one mode for another. +Find messages offline with keywords and filters such as sender, subject, and +date. Semantic search finds related meanings using your configured service. +Hybrid search combines both. Coverage and ranking details show which content +the selected mode can find. [Searching](/docs/usage/searching/) ## Analyze -A DuckDB-over-Parquet analytics cache answers aggregate questions across -hundreds of thousands of messages in milliseconds: senders, domains, labels, -time. Drill down from a decade to a single message in the TUI or the browser. +See which people, domains, labels, and periods account for your messages and +storage. Drill down from a group to individual messages in the terminal or +browser. SQLite archives use a separate analytics cache so these summaries do +not scan message bodies. [Analytics and stats](/docs/usage/analytics/) ## Act -Stage a deletion manifest from the CLI, browser, TUI, or MCP and review it -before execution. The separate CLI execution step requires client consent. -Gmail and IMAP default to moving messages to Trash; permanent deletion requires -explicit opt-in. Archived content remains searchable unless you separately -purge it locally. +Select messages and create a deletion manifest: a saved list you can review +before removing mail from a provider. A separate CLI command requires your +consent to execute it. Gmail and IMAP move messages to Trash by default; +permanent deletion needs an explicit option. Archived content remains +available unless you separately purge it locally. [Deleting email](/docs/usage/deletion/) ## Own -Run it on a laptop or serve it from your own NAS: the daemon carries the Web -UI, HTTP API, scheduler, and MCP server in one binary. Verifiable backup -snapshots restore the archive with no provider in the loop. +Run msgvault on your laptop or your own server. One binary provides the browser +interface, API, scheduled work, and tools for assistants. For SQLite archives, +backup snapshots include the database and attachments and restore without +contacting the original providers. PostgreSQL archives require separate +database backups. [Backup and restore](/docs/usage/backup/) diff --git a/website/guide/index.html b/website/guide/index.html index 0b9d0daa6..b6771f1a4 100644 --- a/website/guide/index.html +++ b/website/guide/index.html @@ -99,7 +99,8 @@

Lifecycle guide

The archive lifecycle

-

One archive moves through nine stages. Source access and media policies determine what is captured. Optional hosted processing sends selected data to the providers you configure.

+

Follow your archive from first capture to long-term ownership. Source access and media policies determine what is captured. Optional hosted processing sends selected data to the providers you configure.

+

See the 0.20.0 changelog for new features and upgrade steps.

  1. 01 Capture
  2. 02 Preserve
    3. @@ -117,8 +118,8 @@

    The archive lifecycle

  3. Capture every channel

    -

    Live sources sync on a schedule — Gmail, IMAP, Slack, Teams, Discord, Beeper, Google Calendar, CardDAV, meeting notes. Local exports import on demand — MBOX, Maildir, Apple Mail, PST, EML, Slackdump, WhatsApp, iMessage, Messenger, SMS backups. Interrupted syncs resume from checkpoints.

    - Importing local email +

    Choose a connected source for recurring sync or import a local export. Supported sources include email, chat, calendars, meeting notes, and contacts. Each source guide explains setup, captured history, media limits, and how interrupted work resumes.

    + Choose a source
    @@ -130,7 +131,7 @@

    Capture every channel

  4. Preserve the original

    -

    Raw provider payloads are retained compressed beside the parsed record. Attachments are content-addressed by SHA-256, deduplicated, and sealed into immutable packs. Cross-account duplicates hide behind a reversible safety ladder — msgvault checks source preference, raw message evidence, and attachment completeness under defined rules.

    +

    Keep original message data alongside the records used for browsing. Downloaded attachments share storage when their contents match. They start as individual files and can be grouped into packs. Preview duplicate messages before hiding extra copies; hidden copies remain available to restore.

    Data storage
    @@ -145,7 +146,7 @@

    Preserve the original

  5. Resolve identities into people

    -

    Every source knows you and your contacts by different addresses and handles. Identity discovery classifies the evidence; observed people cluster from explicit archive links, never from matching display names. Nothing merges without proof.

    +

    Connect the addresses and handles that belong to the same person. msgvault groups identities using explicit links in the archive; matching display names alone do not merge people. Review suggested identity matches before accepting them.

    People, profiles, and identities
    @@ -157,8 +158,8 @@

    Resolve identities into people

  6. -

    Curate durable profiles

    -

    Promote the people who matter into durable profiles with stable IDs and vCard UIDs. Attach typed attributes, organizations, employment history, and relationships over a fact ledger with evidence and reversible merges. Watch each relationship's activity calendar and temperature across every channel.

    +

    Maintain saved profiles

    +

    Save a profile to keep contact details, notes, employment, and relationships together. Inspect the evidence behind profile facts and correct them when needed. Activity calendars show when you were in contact; profile history records supported merges and reversals.

    Curating people
    @@ -174,7 +175,7 @@

    Curate durable profiles

  7. Understand with your models

    -

    Opt in to semantic search by pointing msgvault at an embedding server you choose — local ones included. The embedded Docbank document engine extracts and indexes attachment text and images behind explicit, fail-closed consent. Search indexes can be rebuilt from the archive. Stored evidence and curated profiles remain part of the record.

    +

    Enable search by meaning with a local or hosted embedding service. Separately configure document and image processing, and approve uploads before sending attachment content to a provider. You can rebuild search indexes from the archive; stored evidence and saved profiles remain part of the record.

    Vector search
    @@ -187,23 +188,23 @@

    Understand with your models

  8. -

    Analyze decades in milliseconds

    -

    A DuckDB-over-Parquet analytics cache answers aggregate questions across hundreds of thousands of messages in milliseconds: senders, domains, labels, time. Drill down from a decade to a single message in the TUI or the browser.

    +

    Understand patterns over time

    +

    See which people, domains, labels, and periods account for your messages and storage. Drill down from a group to individual messages in the terminal or browser. SQLite archives use a separate analytics cache so these summaries do not scan message bodies.

    Analytics and stats
    @@ -218,8 +219,8 @@

    Analyze decades in milliseconds

  9. -

    Act with a safety ladder

    -

    Stage a deletion manifest from the CLI, browser, TUI, or MCP and review it before execution. The separate CLI execution step requires client consent. Gmail and IMAP default to moving messages to Trash; permanent deletion requires explicit opt-in. Archived content remains searchable unless you separately purge it locally.

    +

    Review before deleting

    +

    Select messages and create a deletion manifest: a saved list you can review before removing mail from a provider. A separate CLI command requires your consent to execute it. Gmail and IMAP move messages to Trash by default; permanent deletion needs an explicit option. Archived content remains available unless you separately purge it locally.

    Deleting email
    @@ -235,7 +236,7 @@

    Act with a safety ladder

  10. Own the whole system

    -

    Run it on a laptop or serve it from your own NAS: the daemon carries the Web UI, HTTP API, scheduler, and MCP server in one binary. Verifiable backup snapshots restore the archive with no provider in the loop.

    +

    Run msgvault on your laptop or your own server. One binary provides the browser interface, API, scheduled work, and tools for assistants. For SQLite archives, backup snapshots include the database and attachments and restore without contacting the original providers. PostgreSQL archives require separate database backups.

    Backup and restore
    diff --git a/website/index.html b/website/index.html index c0123e41d..1da057eae 100644 --- a/website/index.html +++ b/website/index.html @@ -132,7 +132,7 @@

    Your communications.
    Your relationships.
    One archive.<

    11. Installers fetch the latest release and verify its SHA-256 checksum. Also on conda-forge, or build from source.

    -

    This site describes current main, including unreleased work. Read the changelog for new features and upgrade notes.

    +

    See what changed in 0.20.0 and read the upgrade notes.

    @@ -141,14 +141,14 @@

    Your communications.
    Your relationships.
    One archive.<

    01 / Record

    Every channel. One archive.

-

Twenty years of correspondence should not be scattered across a dozen walled gardens. msgvault syncs live sources and imports local exports into one schema, keeping raw payloads and content-addressed attachments intact.

+

Bring history from several providers into one searchable archive. Sync connected accounts or import local exports. Keep original message data and downloaded attachments alongside the records you browse.

Mail, chat, meetings, calendar, contacts, and offline imports converging into one msgvault archive of SQLite, attachments, and Parquet
  1. MailGmail, IMAP, and Microsoft 365 sync; MBOX, Maildir, Apple Mail, PST, and EML imports.
    2. -
  2. ChatSlack, Teams, Discord, and every network behind Beeper; WhatsApp, iMessage, Messenger, and SMS imports.
    3. +
  3. ChatSlack, Teams, Discord, and chats available through Beeper Desktop; WhatsApp, iMessage, Google Voice, Messenger, and SMS imports.
  4. MeetingsGranola, Circleback, and Notion AI Meeting Notes in the same searchable record.
  5. CalendarGoogle Calendar events, organizers, and attendees, read-only.
  6. ContactsBidirectional CardDAV: pull address books, publish curated people back.
    7. @@ -161,28 +161,28 @@

    Every channel. One archive.

    02 / People

    Messages come from addresses. Relationships come from people.

    -

    The people layer resolves decades of addresses, handles, and phone numbers into the people behind them — with archive evidence and user curation kept strictly apart.

    +

    Connect the addresses, handles, and phone numbers that belong to one person. Keep information found in the archive separate from the profile details you choose to save.

    -

    Observed, not guessed

    -

    Cluster identities on evidence.

    -

    Observed people are assembled from explicit archive links across sources. Equal display names alone never merge two people.

    +

    Identity evidence

    +

    Connect identities with evidence.

    +

    msgvault groups identities using explicit links in the archive. Matching display names alone do not merge two people.

    -

    Durable profiles

    -

    Promote the people who matter.

    -

    A promoted profile gets a stable ID and vCard UID, so names, notes, and typed attributes survive later identity changes. Merge profiles with history and explicit reversal controls.

    +

    Saved profiles

    +

    Keep the details that matter.

    +

    Save a profile to keep names, notes, and contact details when linked identities change. Review profile history, merge duplicates, and reverse supported merges.

    -

    Fact ledger

    -

    Curate facts with provenance.

    -

    Organizations, employment history, typed relationships, and custom attributes rest on immutable evidence, deterministic decisions, and per-person pins.

    +

    Profile history

    +

    See where a profile fact came from.

    +

    Maintain organizations, employment, relationships, and custom fields. Inspect the evidence behind a fact and pin a correction so automated updates keep your choice.

    Activity

    Watch each relationship over time.

    -

    An activity calendar tracks interaction with each person across email, chat, calendar, and meetings, year by year, including current and peak relationship temperature.

    +

    See when you exchanged messages or shared events and meetings. Activity calendars and relationship scores help you find frequent contacts and people you have not heard from recently.

    @@ -193,7 +193,7 @@

    Watch each relationship over time.

    03 / Operation

    Work the archive in the browser.

    -

    The daemon serves a dense, keyboard-driven browser application: relationships, a unified Everything table, files, saved views, Directory, operation history, source status, deletion staging, and settings. Every analytical slice is URL-addressable, so Back and Forward restore exact views.

    +

    Search messages, browse files, and maintain contacts in the browser. Save a useful view or share its URL with someone who has access to your archive. Browser Back and Forward restore your browsing context.

    @@ -214,7 +214,7 @@

    The Directory workspace

    04 / Intelligence

    Semantic search and document understanding.

    -

    Keyword search reads your archive offline. Semantic search, document extraction, and visual search are opt-in, with explicit consent recording exactly what leaves your machine and where it goes.

    +

    Keyword search reads your archive offline. Optional search by meaning sends message and query text to the embedding service you configure, which can run locally. Document, image, and profile processing have separate settings and consent steps.

    The msgvault archive feeding an FTS5 keyword index, vector embeddings, and Docbank document extraction, fused by hybrid retrieval for the CLI, Web UI, TUI, and MCP agents @@ -222,22 +222,22 @@

    Semantic search and document understanding.

    Hybrid search

    -

    Fuse keywords and meaning.

    -

    FTS5 with Gmail-style operators, pure semantic search, or hybrid BM25-plus-vector fusion via reciprocal rank fusion, with an explain mode that shows why each result ranked.

    +

    Find messages by words or meaning.

    +

    Search with familiar filters such as sender, subject, and date. Use semantic search to find related meanings, or hybrid search to combine words and meaning. Ranking details explain each result.

    Local models

    Choose your embedding server.

    -

    Use a supported local or hosted embedding endpoint, such as Ollama with an embedding model. Scope a text index to selected accounts; document, image, and profile processing have separate configuration and consent.

    +

    An embedding service turns text into numbers used to compare meaning. Choose a supported local or hosted service and select which accounts to index.

    Attachments

    Read the attachments too.

    -

    The embedded Docbank document engine handles OCR extraction, normalized chunks, lexical and semantic document search, and visual search over images. Consent-gated and fail-closed.

    +

    Find text inside supported attachments or search images by their content. The embedded Docbank engine manages this processing. Review provider access and approve uploads before sending attachment content.

    Agents

    -

    Give your AI the whole archive.

    +

    Connect an assistant to your archive.

    An MCP server exposes search, people, files, and analytics tools to Claude Desktop and other agents; bundled agent skills install into Claude Code and Codex. Profile writes stay behind explicit flags.

    @@ -248,7 +248,7 @@

    Give your AI the whole archive.

    05 / Interfaces

    One archive across every surface.

    -

    The daemon coordinates archive access and background work. People, scripts, and agents work through the interface suited to the task, against the same record.

    +

    One background service, the daemon, coordinates archive access and scheduled work. Use the browser, terminal, scripts, or an assistant to work with the same archive.

    • CLIScriptable sync, search, and repair.
    • WebAnalytical workspaces in the browser.
      • @@ -274,13 +274,13 @@

      One archive across every surface.

      06 / Ownership

      Archive everything. Then delete upstream.

    -

    Once the archive is complete and verified, you can start deleting from the provider. Every step is explicit and reviewed, and nothing is irreversible until the last one.

    +

    Back up your archive and check the messages you intend to remove before deleting from a provider. Review the selection, then run a separate command to move mail to Trash or permanently delete it.

    Verify

    -

    Prove the copy is complete.

    -

    Integrity verification checks the archive against the mailbox before you trust it with anything irreversible.

    +

    Check the archive before deleting.

    +

    For Gmail, verification compares message counts and checks a sample of stored messages. It does not prove every message or attachment was captured. Review the items you plan to delete and keep a backup.

    Stage

    @@ -304,25 +304,25 @@

    Back up the vault itself.

    07 / Boundary

    -

    Not a mail client. Not a takeout file.

    +

    Keep a record beyond the provider.

    -

    msgvault is a data warehouse for your communications: a system of record you operate, query, and extend. Not a viewport, and not cold storage.

    +

    Choose how you use your communications history. msgvault keeps a searchable archive that you operate, query, and extend.

    Mail client

    -

    The provider is the record.

    -

    A client renders whatever the server still holds. Identity, search, and history live and die with the account.

    +

    Handle day-to-day mail.

    +

    Use a mail client to read, compose, and send mail. msgvault can prepare managed IMAP drafts for review there; it does not send mail.

    Export archive

    The zip is a snapshot.

    -

    A takeout captures one moment in one format. It does not sync, resolve people, answer questions, or talk to agents.

    +

    An export gives you a snapshot in a provider's format. Import supported exports into msgvault to browse and search them alongside other sources.

    msgvault

    The archive is the record.

    -

    Providers become replaceable feeds around a database you own — continuously synced, people-resolved, searchable by meaning, and open to your tools.

    +

    Keep captured history available after it leaves the provider. Sync supported accounts, connect identities, search across sources, and use your own tools.

    diff --git a/website/index.md b/website/index.md index b2bd5a639..01b35fb51 100644 --- a/website/index.md +++ b/website/index.md @@ -10,8 +10,8 @@ behind it, and use your history from the terminal, browser, or an AI assistant. msgvault is usable through the CLI, browser application, terminal interface, HTTP API, MCP server, and bundled agent skills. It is alpha software — back up -your data. This site describes current main, including unreleased work. -[Read the changelog for new features and upgrade notes](/docs/changelog/#unreleased). +your data. +[See what changed in 0.20.0 and read the upgrade notes](/docs/changelog/#0200). ## Install @@ -42,14 +42,14 @@ Then [follow the archive lifecycle](/guide/). ## Every channel. One archive. -Twenty years of correspondence should not be scattered across a dozen walled -gardens. msgvault syncs live sources and imports local exports into one schema, -keeping raw payloads and content-addressed attachments intact. +Bring history from several providers into one searchable archive. Sync connected +accounts or import local exports. Keep original message data and downloaded +attachments alongside the records you browse. - **Mail** — Gmail, IMAP, and Microsoft 365 sync; MBOX, Maildir, Apple Mail, PST, and EML imports. -- **Chat** — Slack, Teams, Discord, and every network behind Beeper; WhatsApp, - iMessage, Messenger, and SMS imports. +- **Chat** — Slack, Teams, Discord, and chats available through Beeper Desktop; + WhatsApp, iMessage, Google Voice, Messenger, and SMS imports. - **Meetings** — Granola, Circleback, and Notion AI Meeting Notes in the same searchable record. - **Calendar** — Google Calendar events, organizers, and attendees, read-only. @@ -58,37 +58,37 @@ keeping raw payloads and content-addressed attachments intact. ## Messages come from addresses. Relationships come from people. -The people layer resolves decades of addresses, handles, and phone numbers into -the people behind them — with archive evidence and user curation kept strictly -apart. +Connect the addresses, handles, and phone numbers that belong to one person. +Keep information found in the archive separate from the profile details you +choose to save. -### Observed, not guessed +### Identity evidence -Observed people are assembled from explicit archive links across sources. Equal -display names alone never merge two people. +msgvault groups identities using explicit links in the archive. Matching display +names alone do not merge two people. -### Durable profiles +### Saved profiles -A promoted profile gets a stable ID and vCard UID, so names, notes, and typed -attributes survive later identity changes. Merge profiles with history and explicit reversal controls. +Save a profile to keep names, notes, and contact details when linked identities +change. Review profile history, merge duplicates, and reverse supported merges. -### Fact ledger +### Profile history -Organizations, employment history, typed relationships, and custom attributes -rest on immutable evidence, deterministic decisions, and per-person pins. +Maintain organizations, employment, relationships, and custom fields. Inspect +the evidence behind a fact and pin a correction so automated updates keep your +choice. ### Activity -An activity calendar tracks interaction with each person across email, chat, -calendar, and meetings, year by year, including current and peak relationship -temperature. +See when you exchanged messages or shared events and meetings. Activity +calendars and relationship scores help you find frequent contacts and people you +have not heard from recently. ## Work the archive in the browser -The daemon serves a dense, keyboard-driven browser application: relationships, -a unified Everything table, files, saved views, Directory, operation history, -source status, deletion staging, and settings. Every analytical slice is URL-addressable, so Back and -Forward restore exact views. +Search messages, browse files, and maintain contacts in the browser. Save a +useful view or share its URL with someone who has access to your archive. +Browser Back and Forward restore your browsing context. Maintain profiles, review identities and merges, publish contacts through CardDAV, and catch up with a saved conversation brief in **Directory**. @@ -97,28 +97,30 @@ credentials and restart-pending changes. [Explore the workspaces](/docs/web-ui/) ## Semantic search and document understanding -Keyword search reads your archive offline. Semantic search, document extraction, -and visual search are opt-in, with explicit consent recording exactly what -leaves your machine and where it goes. - -- **Hybrid search:** FTS5 with Gmail-style operators, pure semantic search, - or hybrid BM25-plus-vector fusion via reciprocal rank fusion, with an - explain mode that shows why each result ranked. -- **Local models:** use a supported local or hosted embedding endpoint, such - as Ollama with an embedding model. Scope a text index to selected accounts; - document, image, and profile processing have separate configuration and consent. -- **Attachments:** the embedded - [Docbank](https://github.com/kenn-io/docbank) document engine handles OCR - extraction, normalized chunks, lexical and semantic document search, and - visual search over images. Consent-gated and fail-closed. +Keyword search reads your archive offline. Optional search by meaning sends +message and query text to the embedding service you configure, which can run +locally. Document, image, and profile processing have separate settings and +consent steps. + +- **Hybrid search:** search with familiar filters such as sender, subject, and + date. Use semantic search to find related meanings, or hybrid search to + combine words and meaning. Ranking details explain each result. +- **Local models:** an embedding service turns text into numbers used to compare + meaning. Choose a supported local or hosted service and select which accounts + to index. +- **Attachments:** find text inside supported attachments or search images by + their content. The embedded [Docbank](https://github.com/kenn-io/docbank) engine + manages this processing. Review provider access and approve uploads before + sending attachment content. - **Agents:** an MCP server exposes search, people, files, and analytics tools to Claude Desktop and other agents; bundled agent skills install into Claude Code and Codex. Profile writes stay behind explicit flags. ## One archive across every surface -The daemon coordinates archive access and background work. People, scripts, and -agents work through the interface suited to the task, against the same record. +One background service, the daemon, coordinates archive access and scheduled +work. Use the browser, terminal, scripts, or an assistant to work with the same +archive. - **CLI:** scriptable sync, search, and repair. - **Web:** analytical workspaces in the browser. @@ -131,12 +133,13 @@ agents work through the interface suited to the task, against the same record. ## Archive everything. Then delete upstream. -Once the archive is complete and verified, you can start deleting from the -provider. Every step is explicit and reviewed, and nothing is irreversible -until the last one. +Back up your archive and check the messages you intend to remove before deleting +from a provider. Review the selection, then run a separate command to move mail +to Trash or permanently delete it. -- **Verify:** integrity verification checks the archive against the mailbox - before you trust it with anything irreversible. +- **Verify:** for Gmail, compare message counts and check a sample of stored + messages. This does not prove every message or attachment was captured. + Review the items you plan to delete and keep a backup. - **Stage:** create a deletion manifest from the CLI, Web UI, TUI, or MCP, then inspect it. Staging does not remove provider messages; execution is a separate CLI command. @@ -147,20 +150,18 @@ until the last one. - **Restore:** append-only, verifiable backup snapshots cover the database and attachments, with restore paths that need no provider at all. -## Not a mail client. Not a takeout file. +## Keep a record beyond the provider. -msgvault is a data warehouse for your communications: a system of record you -operate, query, and extend. Not a viewport, and not cold storage. +Choose how you use your communications history. msgvault keeps a searchable +archive that you operate, query, and extend. -- **Mail client:** the provider is the record. A client renders whatever the - server still holds; identity, search, and history live and die with the - account. -- **Export archive:** the zip is a snapshot. A takeout captures one moment in - one format; it does not sync, resolve people, answer questions, or talk to - agents. -- **msgvault:** the archive is the record. Providers become replaceable feeds - around a database you own — continuously synced, people-resolved, searchable - by meaning, and open to your tools. +- **Mail client:** read, compose, and send mail. msgvault can prepare managed + IMAP drafts for review there; it does not send mail. +- **Export archive:** keep a snapshot in a provider's format. Import supported + exports into msgvault to browse and search them alongside other sources. +- **msgvault:** keep captured history available after it leaves the provider. + Sync supported accounts, connect identities, search across sources, and use + your own tools. ## Follow one archive through the system diff --git a/website/llms.txt b/website/llms.txt index 73d31bc53..e59884900 100644 --- a/website/llms.txt +++ b/website/llms.txt @@ -35,6 +35,7 @@ companions for machine reading. - [External enrichment](https://msgvault.io/docs/usage/people-enrichment/): provider policies and suppression - [CardDAV contacts](https://msgvault.io/docs/usage/people-carddav/): address-book import, publication, and conflicts - [Remote email images](https://msgvault.io/docs/usage/remote-images/): opt-in downloads and offline display +- [IMAP mail and drafts](https://msgvault.io/docs/usage/imap/): sync folders and prepare, edit, recover, or delete managed reply drafts - [MCP server](https://msgvault.io/docs/usage/chat/): AI assistant integration - [Deleting email](https://msgvault.io/docs/usage/deletion/): staging and gated execution - [Backup](https://msgvault.io/docs/usage/backup/): snapshot repositories From e3289099a8e7ab5a929a03d5c3dcae333cf1e442 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 22 Sep 2026 11:11:37 -0500 Subject: [PATCH 014/125] docs: use plain language on the website and move Changelog up (#893) Replace vague website claims with concrete descriptions of saving messages, managing contacts, searching attachments, and reviewing deletions. Update the homepage, guide, page descriptions, and Markdown companions together, including the SQLite backup limit. Move Changelog directly below Setup in the documentation navigation so release changes and upgrade notes are easier to find. This PR changes copy and navigation only; the deployment and Markdown publishing fixes are in #892. Co-authored-by: Wes McKinney --- docs/zensical.toml | 2 +- website/guide.md | 13 +++--- website/guide/index.html | 36 ++++++++-------- website/index.html | 88 ++++++++++++++++++++-------------------- website/index.md | 76 ++++++++++++++++++---------------- website/llms.txt | 16 ++++---- 6 files changed, 117 insertions(+), 114 deletions(-) diff --git a/docs/zensical.toml b/docs/zensical.toml index 4c401c511..297fac066 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -15,6 +15,7 @@ nav = [ {"Start Here" = [ {"Documentation" = "index.md"}, {"Setup" = "setup.md"}, + {"Changelog" = "changelog.md"}, {"Choose a Source" = "guides/sources.md"}, ]}, {"Explore Your Archive" = [ @@ -65,7 +66,6 @@ nav = [ {"MCP Server" = "usage/chat.md"}, {"Agent Skills" = "guides/agent-skills.md"}, {"FAQ" = "faq.md"}, - {"Changelog" = "changelog.md"}, ]}, {"Architecture" = [ {"Overview" = "architecture/overview.md"}, diff --git a/website/guide.md b/website/guide.md index ceec688c3..e3c7222e3 100644 --- a/website/guide.md +++ b/website/guide.md @@ -1,8 +1,9 @@ -# The archive lifecycle +# How msgvault works -Follow your archive from first capture to long-term ownership. Source access -and media policies determine what is captured. Optional hosted processing -sends selected data to the providers you configure. +Sync or import your messages, link them to people, search your history, and +back up your archive. What gets saved depends on source access and your media +settings. Optional hosted processing sends selected data to the providers you +configure. See the [0.20.0 changelog](/docs/changelog/#0200) for new features and upgrade steps. @@ -102,5 +103,5 @@ database backups. ## Next -Move from the lifecycle model to [installation and setup](/docs/setup/) or -[all documentation](/docs/). +[Install msgvault and connect your first account](/docs/setup/). The +[docs](/docs/) cover commands, configuration, and how msgvault stores your data. diff --git a/website/guide/index.html b/website/guide/index.html index b6771f1a4..20a74cc24 100644 --- a/website/guide/index.html +++ b/website/guide/index.html @@ -3,20 +3,20 @@ - The archive lifecycle — msgvault - + How msgvault works + - - + + - - + + @@ -97,9 +97,9 @@
    -

    Lifecycle guide

    -

    The archive lifecycle

    -

    Follow your archive from first capture to long-term ownership. Source access and media policies determine what is captured. Optional hosted processing sends selected data to the providers you configure.

    +

    Guide

    +

    How msgvault works

    +

    Sync or import your messages, link them to people, search your history, and back up your archive. What gets saved depends on source access and your media settings. Optional hosted processing sends selected data to the providers you configure.

    See the 0.20.0 changelog for new features and upgrade steps.

    1. 01 Capture
      2. @@ -117,7 +117,7 @@

      The archive lifecycle

      1. -

        Capture every channel

        +

        Sync an account or import an export

        Choose a connected source for recurring sync or import a local export. Supported sources include email, chat, calendars, meeting notes, and contacts. Each source guide explains setup, captured history, media limits, and how interrupted work resumes.

        Choose a source
        @@ -136,7 +136,7 @@

        Preserve the original

      -
    1. PreviewReview duplicate groups and the proposed survivor.
      2. +
    2. PreviewReview duplicates and which copy will stay visible.
    3. HideKeep one visible copy; retain hidden copies and source evidence.
    4. UndoRestore hidden copies while they remain in the archive.
    @@ -145,7 +145,7 @@

    Preserve the original

  7. -

    Resolve identities into people

    +

    Link addresses to people

    Connect the addresses and handles that belong to the same person. msgvault groups identities using explicit links in the archive; matching display names alone do not merge people. Review suggested identity matches before accepting them.

    People, profiles, and identities
    @@ -174,7 +174,7 @@

    Maintain saved profiles

  8. -

    Understand with your models

    +

    Choose local or hosted models

    Enable search by meaning with a local or hosted embedding service. Separately configure document and image processing, and approve uploads before sending attachment content to a provider. You can rebuild search indexes from the archive; stored evidence and saved profiles remain part of the record.

    Vector search
    @@ -212,7 +212,7 @@

    Understand patterns over time

    msgvault TUI showing sender aggregates with counts and sizes over the whole archive -
    The TUI's sender aggregates, backed by DuckDB over Parquet.
    +
    Message counts and storage by sender in the terminal.
    9. @@ -225,7 +225,7 @@

    Review before deleting

      -
    1. StageSelect one source and create a reviewable manifest.
      2. +
    2. StageSelect messages from one source and save the list for review.
    3. ReviewInspect the count and contents. Cancel if needed.
    4. ExecuteGrant client consent, then move to Trash or explicitly delete permanently.
    5. RetainKeep the archived messages and attachments. Local purging is separate.
      6. @@ -235,7 +235,7 @@

      Review before deleting

    6. -

      Own the whole system

      +

      Run and back up your archive

      Run msgvault on your laptop or your own server. One binary provides the browser interface, API, scheduled work, and tools for assistants. For SQLite archives, backup snapshots include the database and attachments and restore without contacting the original providers. PostgreSQL archives require separate database backups.

      Backup and restore
      @@ -254,9 +254,9 @@

      Own the whole system

      Build your archive.

    -

    Move from the lifecycle model to installation, OAuth setup, exact command behavior, configuration, and architecture.

    +

    Install msgvault and connect your first account. The docs cover commands, configuration, and how msgvault stores your data.

    - Run the setup guide + Set up msgvault Open all documentation
    diff --git a/website/index.html b/website/index.html index 1da057eae..35063180a 100644 --- a/website/index.html +++ b/website/index.html @@ -3,20 +3,20 @@ - msgvault — the system of record for your communications - + msgvault — archive and search your messages + - - + + - - + + @@ -99,9 +99,9 @@
    -

    Local-first. Open source. Yours.

    -

    Your communications.
    Your relationships.
    One archive.

    -

    Bring email, chat, meetings, and contacts together on your own hardware. Find what matters, connect the people behind it, and use your history from the terminal, browser, or an AI assistant.

    +

    Open source. Runs on your hardware.

    +

    Keep and search
    your message history.

    +

    Save email, chats, meetings, calendars, and contacts on your own computer or server. Search across accounts, look up a person's history, and work from a browser, terminal, or AI assistant.

    See how it works Read the docs @@ -139,7 +139,7 @@

    Your communications.
    Your relationships.
    One archive.<

    01 / Record

    -

    Every channel. One archive.

    +

    Bring your accounts together.

    Bring history from several providers into one searchable archive. Sync connected accounts or import local exports. Keep original message data and downloaded attachments alongside the records you browse.

    @@ -151,7 +151,7 @@

    Every channel. One archive.

  9. ChatSlack, Teams, Discord, and chats available through Beeper Desktop; WhatsApp, iMessage, Google Voice, Messenger, and SMS imports.
  10. MeetingsGranola, Circleback, and Notion AI Meeting Notes in the same searchable record.
  11. CalendarGoogle Calendar events, organizers, and attendees, read-only.
    12. -
  12. ContactsBidirectional CardDAV: pull address books, publish curated people back.
    13. +
  13. ContactsImport contacts from a CardDAV address book and choose which saved profiles to sync back.
@@ -159,19 +159,19 @@

Every channel. One archive.

02 / People

-

Messages come from addresses. Relationships come from people.

+

Keep contact details and message history together.

Connect the addresses, handles, and phone numbers that belong to one person. Keep information found in the archive separate from the profile details you choose to save.

Identity evidence

-

Connect identities with evidence.

+

Link addresses to the same person.

msgvault groups identities using explicit links in the archive. Matching display names alone do not merge two people.

Saved profiles

-

Keep the details that matter.

+

Save names, notes, and contact details.

Save a profile to keep names, notes, and contact details when linked identities change. Review profile history, merge duplicates, and reverse supported merges.

@@ -181,7 +181,7 @@

See where a profile fact came from.

Activity

-

Watch each relationship over time.

+

See when you were last in contact.

See when you exchanged messages or shared events and meetings. Activity calendars and relationship scores help you find frequent contacts and people you have not heard from recently.

@@ -190,10 +190,10 @@

Watch each relationship over time.

-

03 / Operation

-

Work the archive in the browser.

+

03 / Browser

+

Browse your archive.

-

Search messages, browse files, and maintain contacts in the browser. Save a useful view or share its URL with someone who has access to your archive. Browser Back and Forward restore your browsing context.

+

Search messages, browse files, and maintain contacts in the browser. Save a useful view or share its URL with someone who has access to your archive. Use Back and Forward to return to earlier views.

@@ -204,15 +204,15 @@

Work the archive in the browser.

Directory and Operations

The Directory workspace

-

Maintain profiles, review identities and merges, publish contacts through CardDAV, and catch up with a saved conversation brief in Directory. Operations tracks syncs and background work; Settings manages provider credentials and restart-pending changes. Explore the workspaces.

+

Maintain profiles, review identities and merges, publish contacts through CardDAV, and catch up with a saved conversation brief in Directory. Check syncs and background jobs in Operations. Update provider credentials in Settings, which shows changes that take effect after a restart. Explore the workspaces.

-

04 / Intelligence

-

Semantic search and document understanding.

+

04 / Search

+

Search messages and attachments.

Keyword search reads your archive offline. Optional search by meaning sends message and query text to the embedding service you configure, which can run locally. Document, image, and profile processing have separate settings and consent steps.

@@ -226,19 +226,19 @@

Find messages by words or meaning.

Search with familiar filters such as sender, subject, and date. Use semantic search to find related meanings, or hybrid search to combine words and meaning. Ranking details explain each result.

-

Local models

-

Choose your embedding server.

+

Search models

+

Choose where search models run.

An embedding service turns text into numbers used to compare meaning. Choose a supported local or hosted service and select which accounts to index.

Attachments

-

Read the attachments too.

+

Search inside attachments.

Find text inside supported attachments or search images by their content. The embedded Docbank engine manages this processing. Review provider access and approve uploads before sending attachment content.

Agents

Connect an assistant to your archive.

-

An MCP server exposes search, people, files, and analytics tools to Claude Desktop and other agents; bundled agent skills install into Claude Code and Codex. Profile writes stay behind explicit flags.

+

An MCP server exposes search, people, files, and analytics tools to Claude Desktop and other agents; bundled agent skills install into Claude Code and Codex. Allowing an agent to edit profiles requires explicit flags.

@@ -247,12 +247,12 @@

Connect an assistant to your archive.

05 / Interfaces

-

One archive across every surface.

+

Use the browser, terminal, or an assistant.

One background service, the daemon, coordinates archive access and scheduled work. Use the browser, terminal, scripts, or an assistant to work with the same archive.

  • CLIScriptable sync, search, and repair.
    • -
  • WebAnalytical workspaces in the browser.
    • -
  • TUIKeyboard drill-down analytics.
    • +
  • WebSearch, browse, and manage contacts.
    • +
  • TUIExplore message counts and storage with the keyboard.
  • HTTPAn authenticated, versioned API.
  • MCPArchive tools for AI assistants.
  • SkillsWorkflows for Claude Code and Codex.
    • @@ -271,8 +271,8 @@

    One archive across every surface.

    -

    06 / Ownership

    -

    Archive everything. Then delete upstream.

    +

    06 / Deletion and backups

    +

    Review mail before deleting provider copies.

    Back up your archive and check the messages you intend to remove before deleting from a provider. Review the selection, then run a separate command to move mail to Trash or permanently delete it.

    @@ -284,18 +284,18 @@

    Check the archive before deleting.

    Stage

    -

    Review before anything moves.

    -

    Create a deletion manifest from the CLI, Web UI, TUI, or MCP, then inspect it. Staging does not remove provider messages; execution is a separate CLI command.

    +

    Review the messages you selected.

    +

    Create a deletion manifest: a saved list of messages to review. You can create it from the CLI, Web UI, TUI, or MCP. Staging does not remove provider messages; execution is a separate CLI command.

    Execute

    -

    Delete upstream, keep the record.

    -

    Execution is a separate CLI step with explicit client consent. Gmail and IMAP default to moving messages to Trash; permanent deletion requires explicit opt-in. Archived messages and attachments remain available; msgvault records their source-deletion state.

    +

    Delete provider copies.

    +

    Execution is a separate CLI step with explicit client consent. Gmail and IMAP default to moving messages to Trash; permanent deletion requires explicit opt-in. Archived messages and attachments remain available; msgvault records that they were deleted from their source.

    Restore

    -

    Back up the vault itself.

    -

    Append-only, verifiable backup snapshots cover the database and attachments, with restore paths that need no provider at all.

    +

    Back up your archive.

    +

    Back up SQLite archives with snapshots of the database and attachments. New snapshots leave earlier ones intact, and you can verify them before restoring. Restoring does not require the original providers. PostgreSQL databases need separate backups.

@@ -303,10 +303,10 @@

Back up the vault itself.

-

07 / Boundary

+

07 / Your mail tools

Keep a record beyond the provider.

-

Choose how you use your communications history. msgvault keeps a searchable archive that you operate, query, and extend.

+

Use msgvault alongside your mail client and provider exports. Keep searching your saved history after messages leave the provider.

@@ -316,12 +316,12 @@

Handle day-to-day mail.

Export archive

-

The zip is a snapshot.

+

Import a provider's export.

An export gives you a snapshot in a provider's format. Import supported exports into msgvault to browse and search them alongside other sources.

msgvault

-

The archive is the record.

+

Search your saved history.

Keep captured history available after it leaves the provider. Sync supported accounts, connect identities, search across sources, and use your own tools.

@@ -331,13 +331,13 @@

The archive is the record.

08 / Start

-

Follow one archive through the system.

+

Set up your first archive.

-

The guide walks the archive lifecycle from capture to ownership. The documentation carries setup, exact command behavior, configuration, and architecture.

+

Read the guide for an overview, or go straight to setup. The docs cover commands, configuration, and how msgvault stores your data.

- Read the lifecycle guide - Open the documentation + Read the guide + Set up msgvault
diff --git a/website/index.md b/website/index.md index 01b35fb51..2f79519ba 100644 --- a/website/index.md +++ b/website/index.md @@ -1,10 +1,10 @@ # msgvault -**Your communications. Your relationships. One archive.** +**Keep and search your message history.** -msgvault is a local-first, open-source archive. Bring email, chat, meetings, and -contacts together on your own hardware. Find what matters, connect the people -behind it, and use your history from the terminal, browser, or an AI assistant. +msgvault is open source and runs on your hardware. Save email, chats, meetings, +calendars, and contacts on your own computer or server. Search across accounts, +look up a person's history, and work from a browser, terminal, or AI assistant. ![People and communications flow into one msgvault archive, accessible through the TUI, Web UI, and MCP for AI assistants.](/assets/archive-flow.svg) @@ -38,9 +38,9 @@ checksum. msgvault is also on [conda-forge](https://prefix.dev/channels/conda-forge/packages/msgvault), and the [setup documentation](/docs/setup/) covers building from source. -Then [follow the archive lifecycle](/guide/). +Then [read how msgvault works](/guide/). -## Every channel. One archive. +## Bring your accounts together Bring history from several providers into one searchable archive. Sync connected accounts or import local exports. Keep original message data and downloaded @@ -53,49 +53,50 @@ attachments alongside the records you browse. - **Meetings** — Granola, Circleback, and Notion AI Meeting Notes in the same searchable record. - **Calendar** — Google Calendar events, organizers, and attendees, read-only. -- **Contacts** — bidirectional CardDAV: pull address books, publish curated - people back. +- **Contacts** — import contacts from a CardDAV address book and choose which + saved profiles to sync back. -## Messages come from addresses. Relationships come from people. +## Keep contact details and message history together Connect the addresses, handles, and phone numbers that belong to one person. Keep information found in the archive separate from the profile details you choose to save. -### Identity evidence +### Link addresses to the same person msgvault groups identities using explicit links in the archive. Matching display names alone do not merge two people. -### Saved profiles +### Save names, notes, and contact details Save a profile to keep names, notes, and contact details when linked identities change. Review profile history, merge duplicates, and reverse supported merges. -### Profile history +### See where a profile fact came from Maintain organizations, employment, relationships, and custom fields. Inspect the evidence behind a fact and pin a correction so automated updates keep your choice. -### Activity +### See when you were last in contact See when you exchanged messages or shared events and meetings. Activity calendars and relationship scores help you find frequent contacts and people you have not heard from recently. -## Work the archive in the browser +## Browse your archive Search messages, browse files, and maintain contacts in the browser. Save a useful view or share its URL with someone who has access to your archive. -Browser Back and Forward restore your browsing context. +Use Back and Forward to return to earlier views. Maintain profiles, review identities and merges, publish contacts through CardDAV, and catch up with a saved conversation brief in **Directory**. -**Operations** tracks syncs and background work; **Settings** manages provider -credentials and restart-pending changes. [Explore the workspaces](/docs/web-ui/). +Check syncs and background jobs in **Operations**. Update provider credentials +in **Settings**, which shows changes that take effect after a restart. +[Explore the workspaces](/docs/web-ui/). -## Semantic search and document understanding +## Search messages and attachments Keyword search reads your archive offline. Optional search by meaning sends message and query text to the embedding service you configure, which can run @@ -105,7 +106,7 @@ consent steps. - **Hybrid search:** search with familiar filters such as sender, subject, and date. Use semantic search to find related meanings, or hybrid search to combine words and meaning. Ranking details explain each result. -- **Local models:** an embedding service turns text into numbers used to compare +- **Search models:** an embedding service turns text into numbers used to compare meaning. Choose a supported local or hosted service and select which accounts to index. - **Attachments:** find text inside supported attachments or search images by @@ -114,24 +115,25 @@ consent steps. sending attachment content. - **Agents:** an MCP server exposes search, people, files, and analytics tools to Claude Desktop and other agents; bundled agent skills install into - Claude Code and Codex. Profile writes stay behind explicit flags. + Claude Code and Codex. Allowing an agent to edit profiles requires explicit + flags. -## One archive across every surface +## Use the browser, terminal, or an assistant One background service, the daemon, coordinates archive access and scheduled work. Use the browser, terminal, scripts, or an assistant to work with the same archive. - **CLI:** scriptable sync, search, and repair. -- **Web:** analytical workspaces in the browser. -- **TUI:** keyboard drill-down analytics. +- **Web:** search, browse, and manage contacts. +- **TUI:** explore message counts and storage with the keyboard. - **HTTP:** an authenticated, versioned API. - **MCP:** archive tools for AI assistants. - **Skills:** workflows for Claude Code and Codex. [Connect an agent](/docs/usage/chat/) or [inspect the API](/docs/api-server/). -## Archive everything. Then delete upstream. +## Review mail before deleting provider copies Back up your archive and check the messages you intend to remove before deleting from a provider. Review the selection, then run a separate command to move mail @@ -140,20 +142,22 @@ to Trash or permanently delete it. - **Verify:** for Gmail, compare message counts and check a sample of stored messages. This does not prove every message or attachment was captured. Review the items you plan to delete and keep a backup. -- **Stage:** create a deletion manifest from the CLI, Web UI, TUI, or MCP, - then inspect it. Staging does not remove provider messages; execution is a - separate CLI command. +- **Stage:** create a deletion manifest: a saved list of messages to review. + You can create it from the CLI, Web UI, TUI, or MCP. Staging does not remove + provider messages; execution is a separate CLI command. - **Execute:** the CLI requires explicit client consent. Gmail and IMAP default to moving messages to Trash; permanent deletion requires explicit opt-in. Archived messages and attachments remain available; msgvault records their - source-deletion state. -- **Restore:** append-only, verifiable backup snapshots cover the database and - attachments, with restore paths that need no provider at all. + deletion from their source. +- **Restore:** back up SQLite archives with snapshots of the database and + attachments. New snapshots leave earlier ones intact, and you can verify + them before restoring. Restoring does not require the original providers. + PostgreSQL databases need separate backups. ## Keep a record beyond the provider. -Choose how you use your communications history. msgvault keeps a searchable -archive that you operate, query, and extend. +Use msgvault alongside your mail client and provider exports. Keep searching +your saved history after messages leave the provider. - **Mail client:** read, compose, and send mail. msgvault can prepare managed IMAP drafts for review there; it does not send mail. @@ -163,8 +167,8 @@ archive that you operate, query, and extend. Sync supported accounts, connect identities, search across sources, and use your own tools. -## Follow one archive through the system +## Set up your first archive -The [lifecycle guide](/guide/) walks the archive from capture to ownership. -The [documentation](/docs/) carries setup, exact command behavior, -configuration, and architecture. +Read the [guide](/guide/) for an overview, or go straight to +[setup](/docs/setup/). The [docs](/docs/) cover commands, configuration, and how +msgvault stores your data. diff --git a/website/llms.txt b/website/llms.txt index e59884900..9fdc75723 100644 --- a/website/llms.txt +++ b/website/llms.txt @@ -1,9 +1,8 @@ # msgvault -> msgvault is a local-first, open-source archive for a lifetime of email, -> chat, meetings, calendars, and contacts — one database with a durable people -> layer, keyword and semantic search, an MCP server, and agent skills, owned -> and operated by you. +> msgvault is open source and runs on your hardware. Save email, chats, meetings, +> calendars, and contacts on your own computer or server. Search across accounts, +> look up a person's history, and work from a browser, terminal, or AI assistant. The product and guide pages have Markdown companions with the same core information; documentation pages are served as HTML. Use the Markdown @@ -11,11 +10,10 @@ companions for machine reading. ## Product -- [Product overview](https://msgvault.io/index.md): what msgvault is, every - supported source, the people layer, search modes, interfaces, and the - deletion safety model -- [Archive lifecycle guide](https://msgvault.io/guide.md): capture, preserve, - resolve, curate, understand, search, analyze, act, own +- [Product overview](https://msgvault.io/index.md): supported sources, contact + profiles, search, interfaces, and how deletion works +- [How msgvault works](https://msgvault.io/guide.md): sync and import messages, + manage contacts, search, review deletions, and back up your archive ## Docs From e27d0a467c89d56346f8cda464c0802ea1ba5055 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 23 Sep 2026 07:28:02 -0500 Subject: [PATCH 015/125] fix(docs): repair deployment and publish Markdown pages (#892) Repair the Vercel deployment that failed because uploading only `docs/` omitted the sibling `website/` sources. Preview and production targets now build locally and deploy the complete output. Legacy documentation redirects also match trailing slashes, so links such as `/setup/` reach `/docs/setup/`. Publish Markdown companions for all public content pages using the same sanitized sources as the HTML build. `llms.txt` lists those pages, and each HTML page advertises its Markdown URL. The existing built-site check now verifies coverage, alternate links, and source content. These changes are already deployed on [msgvault.io](https://msgvault.io). Co-authored-by: Wes McKinney --- Makefile | 9 ++- docs/README.md | 12 +++ docs/overrides/main.html | 7 ++ docs/scripts/check_built_site.py | 27 +++++++ docs/scripts/check_vercel_redirects.py | 33 ++++---- docs/vercel.json | 32 ++++---- docs/zensical-docs.sh | 8 ++ website/guide/index.html | 1 + website/index.html | 1 + website/llms.txt | 107 +++++++++++++++++++------ 10 files changed, 177 insertions(+), 60 deletions(-) diff --git a/Makefile b/Makefile index ec47697b7..e493491a6 100644 --- a/Makefile +++ b/Makefile @@ -461,13 +461,18 @@ docs-assets-branch: docs-generated-assets-branch: bash docs/screenshots/update-generated-assets-branch.sh +# Build locally so the sibling website/ directory is available to both deploys. # Deploy docs to Vercel staging docs-deploy-staging: - cd docs && vercel + cd docs && vercel pull --yes --environment=preview + cd docs && vercel build + cd docs && vercel deploy --prebuilt # Deploy docs to Vercel production docs-deploy: - cd docs && vercel --prod + cd docs && vercel pull --yes --environment=production + cd docs && vercel build --prod + cd docs && vercel deploy --prebuilt --prod # Build the MIME shootout tool shootout: diff --git a/docs/README.md b/docs/README.md index 2b642815c..1179bd592 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,18 @@ and copies the static marketing site to `/`. `make docs-check` runs source validation, builds that actual layout, and checks the output and redirects. Use `make docs-serve` to inspect it at `http://127.0.0.1:8000`. +Every public page has a Markdown companion. The build copies documentation +sources to matching paths such as `/docs/setup.md` and `/docs/usage/searching.md`; +the documentation index is `/docs/index.md`. HTML pages link to their Markdown +companions with `rel="alternate"`. Keep `website/llms.txt` linked to every public +Markdown page. `make docs-check` verifies this coverage against the built site. + +Deploy with `make docs-deploy-staging` for a Vercel preview or +`make docs-deploy` for production. Both targets pull the matching Vercel +environment, build locally with access to `docs/` and `website/`, and upload +the built site. Running `vercel` directly from `docs/` uploads only that +directory, so the remote build cannot find the sibling `website/` sources. + Check a representative rendered page after changing structure, navigation, tables, diagrams, or HTML. Inspect links, headings, and mobile line wrapping. For prose changes, run the existing docs checks; do not add tests that search diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 4579b4b8d..0e205d378 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -9,4 +9,11 @@ + {% if page and page.canonical_url %} + {% if page.canonical_url == config.site_url %} + + {% elif page.canonical_url[-1:] == "/" %} + + {% endif %} + {% endif %} {% endblock %} diff --git a/docs/scripts/check_built_site.py b/docs/scripts/check_built_site.py index 7851eb1d2..dc9e1af9b 100755 --- a/docs/scripts/check_built_site.py +++ b/docs/scripts/check_built_site.py @@ -259,6 +259,7 @@ def __init__(self) -> None: super().__init__() self.ids: set[str] = set() self.links: list[str] = [] + self.markdown_links: list[str] = [] self.nav_links: list[str] = [] self.assets: list[str] = [] self.style_attrs: list[str] = [] @@ -284,6 +285,12 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._nav_label_href = attr["href"] self._nav_label_text = [] self._nav_label_depth = 1 + if ( + tag == "link" + and "alternate" in rel_tokens(attr.get("rel", "")) + and attr.get("type") == "text/markdown" + ): + self.markdown_links.append(attr.get("href", "")) if tag in {"img", "script", "source"} and "src" in attr: self.assets.append(attr["src"]) if tag in {"img", "source"} and "srcset" in attr: @@ -478,6 +485,26 @@ def main() -> None: fail(f"forbidden generated marker found: {pattern}") parsed_by_file = {path.resolve(): parse_html(path) for path in html_files} + llms_text = (SITE / "llms.txt").read_text(encoding="utf-8") + for current, parser in parsed_by_file.items(): + if current.name != "index.html": + continue + route = "/" + current.relative_to(SITE.resolve()).as_posix().removesuffix("index.html") + markdown_route = ( + route + "index.md" if route in {"/", "/docs/"} else route.rstrip("/") + ".md" + ) + markdown_file = SITE / markdown_route.lstrip("/") + if not markdown_file.is_file(): + fail(f"missing Markdown companion for {route}: {markdown_route}") + markdown_url = "https://msgvault.io" + markdown_route + if markdown_url not in parser.markdown_links: + fail(f"missing Markdown alternate link on {route}: {markdown_url}") + if f"]({markdown_url})" not in llms_text: + fail(f"llms.txt is missing Markdown page {markdown_url}") + if markdown_route.startswith("/docs/"): + source = ROOT / markdown_route.removeprefix("/docs/") + if markdown_file.read_bytes() != source.read_bytes(): + fail(f"published Markdown differs from its source: {markdown_route}") docs_index = SITE / "docs" / "index.html" index_parser = parsed_by_file[docs_index.resolve()] web_ui_route = route_to_file("/docs/web-ui/").resolve() diff --git a/docs/scripts/check_vercel_redirects.py b/docs/scripts/check_vercel_redirects.py index 25aeed247..6bd8c8f2e 100755 --- a/docs/scripts/check_vercel_redirects.py +++ b/docs/scripts/check_vercel_redirects.py @@ -17,23 +17,24 @@ # Legacy root docs URLs permanently redirect into the /docs/ tier so links # published before the tiered site keep resolving. +# Capture trailing slashes: Vercel adds them to directory URLs before redirects. PERMANENT = { - "/introduction/:path*": "/docs/introduction/:path*", - "/setup/:path*": "/docs/setup/:path*", - "/web-ui/:path*": "/docs/web-ui/:path*", - "/configuration/:path*": "/docs/configuration/:path*", - "/cli-reference/:path*": "/docs/cli-reference/:path*", - "/api-server/:path*": "/docs/api-server/:path*", - "/changelog/:path*": "/docs/changelog/:path*", - "/troubleshooting/:path*": "/docs/troubleshooting/:path*", - "/development/:path*": "/docs/development/:path*", - "/faq/:path*": "/docs/faq/:path*", - "/usage/:path*": "/docs/usage/:path*", - "/guides/:path*": "/docs/guides/:path*", - "/architecture/:path*": "/docs/architecture/:path*", - "/assets/static/:path*": "/docs/assets/static/:path*", - "/assets/generated/:path*": "/docs/assets/generated/:path*", - "/search/:path*": "/docs/search/:path*", + "/introduction/:path(.*)": "/docs/introduction/:path*", + "/setup/:path(.*)": "/docs/setup/:path*", + "/web-ui/:path(.*)": "/docs/web-ui/:path*", + "/configuration/:path(.*)": "/docs/configuration/:path*", + "/cli-reference/:path(.*)": "/docs/cli-reference/:path*", + "/api-server/:path(.*)": "/docs/api-server/:path*", + "/changelog/:path(.*)": "/docs/changelog/:path*", + "/troubleshooting/:path(.*)": "/docs/troubleshooting/:path*", + "/development/:path(.*)": "/docs/development/:path*", + "/faq/:path(.*)": "/docs/faq/:path*", + "/usage/:path(.*)": "/docs/usage/:path*", + "/guides/:path(.*)": "/docs/guides/:path*", + "/architecture/:path(.*)": "/docs/architecture/:path*", + "/assets/static/:path(.*)": "/docs/assets/static/:path*", + "/assets/generated/:path(.*)": "/docs/assets/generated/:path*", + "/search/:path(.*)": "/docs/search/:path*", } diff --git a/docs/vercel.json b/docs/vercel.json index 40436d5a7..d6803f0d6 100644 --- a/docs/vercel.json +++ b/docs/vercel.json @@ -17,82 +17,82 @@ "permanent": false }, { - "source": "/introduction/:path*", + "source": "/introduction/:path(.*)", "destination": "/docs/introduction/:path*", "permanent": true }, { - "source": "/setup/:path*", + "source": "/setup/:path(.*)", "destination": "/docs/setup/:path*", "permanent": true }, { - "source": "/web-ui/:path*", + "source": "/web-ui/:path(.*)", "destination": "/docs/web-ui/:path*", "permanent": true }, { - "source": "/configuration/:path*", + "source": "/configuration/:path(.*)", "destination": "/docs/configuration/:path*", "permanent": true }, { - "source": "/cli-reference/:path*", + "source": "/cli-reference/:path(.*)", "destination": "/docs/cli-reference/:path*", "permanent": true }, { - "source": "/api-server/:path*", + "source": "/api-server/:path(.*)", "destination": "/docs/api-server/:path*", "permanent": true }, { - "source": "/changelog/:path*", + "source": "/changelog/:path(.*)", "destination": "/docs/changelog/:path*", "permanent": true }, { - "source": "/troubleshooting/:path*", + "source": "/troubleshooting/:path(.*)", "destination": "/docs/troubleshooting/:path*", "permanent": true }, { - "source": "/development/:path*", + "source": "/development/:path(.*)", "destination": "/docs/development/:path*", "permanent": true }, { - "source": "/faq/:path*", + "source": "/faq/:path(.*)", "destination": "/docs/faq/:path*", "permanent": true }, { - "source": "/usage/:path*", + "source": "/usage/:path(.*)", "destination": "/docs/usage/:path*", "permanent": true }, { - "source": "/guides/:path*", + "source": "/guides/:path(.*)", "destination": "/docs/guides/:path*", "permanent": true }, { - "source": "/architecture/:path*", + "source": "/architecture/:path(.*)", "destination": "/docs/architecture/:path*", "permanent": true }, { - "source": "/assets/static/:path*", + "source": "/assets/static/:path(.*)", "destination": "/docs/assets/static/:path*", "permanent": true }, { - "source": "/assets/generated/:path*", + "source": "/assets/generated/:path(.*)", "destination": "/docs/assets/generated/:path*", "permanent": true }, { - "source": "/search/:path*", + "source": "/search/:path(.*)", "destination": "/docs/search/:path*", "permanent": true } diff --git a/docs/zensical-docs.sh b/docs/zensical-docs.sh index 710586797..eb32f7c04 100755 --- a/docs/zensical-docs.sh +++ b/docs/zensical-docs.sh @@ -143,6 +143,14 @@ awk -v docs_dir="$tmp_docs_name" -v site_dir="$site_dir" ' case "$command_name" in build) (cd "$docs_root" && "$zensical_bin" build --strict --config-file "$tmp_config_name" "$@") + # Publish the same sanitized Markdown sources alongside the rendered pages. + site_output_dir="$(cd "$docs_root" && cd "$site_dir" && pwd)" + find "$tmp_docs" -type f -name '*.md' -print0 | + while IFS= read -r -d '' source; do + target="$site_output_dir/${source#"$tmp_docs/"}" + mkdir -p "$(dirname "$target")" + cp "$source" "$target" + done ;; serve) (cd "$docs_root" && "$zensical_bin" serve --config-file "$tmp_config_name" "$@") diff --git a/website/guide/index.html b/website/guide/index.html index 20a74cc24..505f64588 100644 --- a/website/guide/index.html +++ b/website/guide/index.html @@ -6,6 +6,7 @@ How msgvault works + diff --git a/website/index.html b/website/index.html index 35063180a..6089b4aff 100644 --- a/website/index.html +++ b/website/index.html @@ -6,6 +6,7 @@ msgvault — archive and search your messages + diff --git a/website/llms.txt b/website/llms.txt index 9fdc75723..cd3c8fed9 100644 --- a/website/llms.txt +++ b/website/llms.txt @@ -4,9 +4,9 @@ > calendars, and contacts on your own computer or server. Search across accounts, > look up a person's history, and work from a browser, terminal, or AI assistant. -The product and guide pages have Markdown companions with the same core -information; documentation pages are served as HTML. Use the Markdown -companions for machine reading. +Every public page has a Markdown companion for machine reading. Use the links +below, or replace a page's trailing slash with `.md`: `/docs/setup/` becomes +`/docs/setup.md`. The documentation index is `/docs/index.md`. ## Product @@ -15,29 +15,84 @@ companions for machine reading. - [How msgvault works](https://msgvault.io/guide.md): sync and import messages, manage contacts, search, review deletions, and back up your archive -## Docs - -- [Documentation index](https://msgvault.io/docs/): all operating documentation -- [Changelog](https://msgvault.io/docs/changelog/): released and unreleased changes, with upgrade notes -- [Choose a source](https://msgvault.io/docs/guides/sources/): sync and import routes by provider and format -- [Setup guide](https://msgvault.io/docs/setup/): installation and first sync -- [CLI reference](https://msgvault.io/docs/cli-reference/): every command and flag -- [Configuration](https://msgvault.io/docs/configuration/): config.toml reference -- [Web UI](https://msgvault.io/docs/web-ui/): the browser application -- [Searching](https://msgvault.io/docs/usage/searching/): query syntax and modes -- [Vector search](https://msgvault.io/docs/usage/vector-search/): semantic and hybrid setup -- [Document attachment indexing](https://msgvault.io/docs/usage/document-indexing/): Docbank-backed extraction -- [People, profiles, and identities](https://msgvault.io/docs/usage/people/): the people layer -- [Profile automation](https://msgvault.io/docs/usage/people-automation/): tracking, fact resolution, and provider consent -- [Conversation briefs](https://msgvault.io/docs/usage/people-briefs/): saved summaries, citations, and input limits -- [External enrichment](https://msgvault.io/docs/usage/people-enrichment/): provider policies and suppression -- [CardDAV contacts](https://msgvault.io/docs/usage/people-carddav/): address-book import, publication, and conflicts -- [Remote email images](https://msgvault.io/docs/usage/remote-images/): opt-in downloads and offline display -- [IMAP mail and drafts](https://msgvault.io/docs/usage/imap/): sync folders and prepare, edit, recover, or delete managed reply drafts -- [MCP server](https://msgvault.io/docs/usage/chat/): AI assistant integration -- [Deleting email](https://msgvault.io/docs/usage/deletion/): staging and gated execution -- [Backup](https://msgvault.io/docs/usage/backup/): snapshot repositories -- [Architecture overview](https://msgvault.io/docs/architecture/overview/): design decisions +## Start Here + +- [Documentation](https://msgvault.io/docs/index.md): all operating documentation +- [Setup](https://msgvault.io/docs/setup.md): installation and first sync +- [Choose a Source](https://msgvault.io/docs/guides/sources.md): sync and import routes by provider and format + +## Explore Your Archive + +- [Web UI](https://msgvault.io/docs/web-ui.md): the browser application +- [Terminal UI](https://msgvault.io/docs/usage/tui.md) +- [Searching](https://msgvault.io/docs/usage/searching.md): query syntax and modes +- [Semantic and Visual Search](https://msgvault.io/docs/usage/vector-search.md): semantic and hybrid setup +- [Document Attachment Indexing](https://msgvault.io/docs/usage/document-indexing.md): Docbank-backed extraction +- [Recommended Configuration](https://msgvault.io/docs/usage/recommended-configuration.md) +- [Analytics and Stats](https://msgvault.io/docs/usage/analytics.md) +- [SQL Queries](https://msgvault.io/docs/usage/querying.md) +- [Exporting](https://msgvault.io/docs/usage/exporting.md) + +## Sync and Import + +- [OAuth Setup](https://msgvault.io/docs/guides/oauth-setup.md) +- [IMAP Mail](https://msgvault.io/docs/usage/imap.md): sync folders and prepare, edit, recover, or delete managed reply drafts +- [Local Email](https://msgvault.io/docs/usage/importing.md) +- [Remote Email Images](https://msgvault.io/docs/usage/remote-images.md): opt-in downloads and offline display +- [Text Messages](https://msgvault.io/docs/usage/text-messages.md) +- [Google Calendar](https://msgvault.io/docs/usage/calendar.md) +- [Microsoft Teams](https://msgvault.io/docs/usage/teams.md) +- [Discord](https://msgvault.io/docs/usage/discord.md) +- [Beeper](https://msgvault.io/docs/usage/beeper.md) +- [Slack](https://msgvault.io/docs/usage/slack.md) +- [Meeting Notes and Transcripts](https://msgvault.io/docs/usage/meetings.md) + +## People and Accounts + +- [People and Profiles](https://msgvault.io/docs/usage/people.md): the people layer +- [Conversation Briefs](https://msgvault.io/docs/usage/people-briefs.md): saved summaries, citations, and input limits +- [Profile Automation](https://msgvault.io/docs/usage/people-automation.md): tracking, fact resolution, and provider consent +- [External Enrichment](https://msgvault.io/docs/usage/people-enrichment.md): provider policies and suppression +- [CardDAV Contacts](https://msgvault.io/docs/usage/people-carddav.md): address-book import, publication, and conflicts +- [Accounts and Collections](https://msgvault.io/docs/usage/multi-account.md) + +## Maintain and Operate + +- [Backup and Restore](https://msgvault.io/docs/usage/backup.md): snapshot repositories +- [Verify Integrity](https://msgvault.io/docs/guides/verification.md) +- [Deduplication](https://msgvault.io/docs/usage/deduplication.md) +- [Deleting Email](https://msgvault.io/docs/usage/deletion.md): staging and gated execution +- [Remote Deployment](https://msgvault.io/docs/guides/remote-deployment.md) +- [Daemon Lifecycle](https://msgvault.io/docs/guides/daemon-migration.md) +- [Troubleshooting](https://msgvault.io/docs/troubleshooting.md) + +## Reference and Integrations + +- [CLI Reference](https://msgvault.io/docs/cli-reference.md): every command and flag +- [Configuration](https://msgvault.io/docs/configuration.md): config.toml reference +- [HTTP API](https://msgvault.io/docs/api-server.md) +- [MCP Server](https://msgvault.io/docs/usage/chat.md): AI assistant integration +- [Agent Skills](https://msgvault.io/docs/guides/agent-skills.md) +- [FAQ](https://msgvault.io/docs/faq.md) +- [Changelog](https://msgvault.io/docs/changelog.md): released and unreleased changes, with upgrade notes + +## Architecture + +- [Overview](https://msgvault.io/docs/architecture/overview.md): design decisions +- [Data Storage](https://msgvault.io/docs/architecture/storage.md) +- [PostgreSQL Backend](https://msgvault.io/docs/architecture/postgresql.md) +- [Search Ranking](https://msgvault.io/docs/architecture/search-ranking.md) +- [Backup Repository Format](https://msgvault.io/docs/architecture/backup-format.md) + +## Project + +- [Development](https://msgvault.io/docs/development.md) +- [Why msgvault?](https://msgvault.io/docs/introduction.md) + +## Optional + +- [Controlled web UI fixture](https://msgvault.io/docs/fixtures/README.template.md) +- [Documentation fixture format](https://msgvault.io/docs/fixtures/fixture-format.md) ## Other From b5969b2f311d12ed52cde64c0cddd06bf5275cbc Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:28:14 -0700 Subject: [PATCH 016/125] web: make Directory detail scroll and summarize attributes (#902) ## What changed - Give Directory results and person details their own scroll areas so long records remain reachable. - Show populated attributes near the top of the person overview, with typed values, choice labels, and sensitive values concealed. - Hide empty attribute fields by default, with a control to reveal them when adding values. The summary's Edit attributes control jumps to the full field section. ## Why Long person records can extend past the visible detail pane, and useful attributes sit below several other sections. This makes the Directory harder to use for people with more profile data. ## Usage Open a person in Directory. Use Edit attributes from the overview to reach the full attribute section, and Show empty fields when adding a value to an unused field. Refs #901 Co-authored-by: Rusty Shackleford --- .../AttributeDefinitionDialog.test.ts | 13 ++- .../directory/AttributeSection.svelte | 62 ++++++------- .../directory/AttributeSection.test.ts | 22 ++++- .../directory/AttributeSummary.svelte | 40 +++++++++ .../directory/AttributeSummary.test.ts | 53 +++++++++++ .../directory/DirectoryWorkspace.svelte | 8 +- .../components/directory/PersonDetail.svelte | 12 ++- .../components/directory/PersonDetail.test.ts | 75 +++++++++++++++- .../components/directory/attribute-value.ts | 29 +++++++ web/tests/e2e/directory.spec.ts | 87 +++++++++++++++++++ 10 files changed, 357 insertions(+), 44 deletions(-) create mode 100644 web/src/lib/components/directory/AttributeSummary.svelte create mode 100644 web/src/lib/components/directory/AttributeSummary.test.ts create mode 100644 web/src/lib/components/directory/attribute-value.ts diff --git a/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts b/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts index ffc118cac..4f1a035ee 100644 --- a/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts +++ b/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts @@ -978,13 +978,20 @@ describe('AttributeDefinitionDialog', () => { is_sensitive: false, options, }); + const unrelated = definition({ + id: 8, + universal_id: 'unrelated-empty', + slug: 'unrelated_empty', + label: 'Unrelated empty field', + is_sensitive: false, + }); const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); return request.method === 'POST' ? Response.json(created, { status: 201 }) - : Response.json({ definitions: [created] }); + : Response.json({ definitions: [unrelated, created] }); }); - render(AttributeSection, { controller: controller(fetchFn) }); + render(AttributeSection, { controller: controller(fetchFn, [unrelated]) }); await fireEvent.click(screen.getByRole('button', { name: 'Create attribute field' })); await fireEvent.input(screen.getByLabelText('Label'), { target: { value: 'Option field' } }); await chooseSelectOption(screen.getByLabelText('Value type'), option); @@ -992,6 +999,7 @@ describe('AttributeDefinitionDialog', () => { await fireEvent.click(screen.getByRole('button', { name: 'Create field' })); await screen.findByRole('status'); await fireEvent.click(screen.getByRole('button', { name: 'Done' })); + expect(screen.queryByRole('heading', { name: 'Unrelated empty field' })).toBeNull(); await fireEvent.click(screen.getByRole('button', { name: 'Add Option field value' })); expect(screen.getByRole('form', { name: 'Add Option field value' }).textContent).toContain(visible); @@ -1018,6 +1026,7 @@ describe('AttributeDefinitionDialog', () => { }); const profile = controller(fetchFn, [existing]); render(AttributeSection, { controller: profile }); + await fireEvent.click(screen.getByRole('button', { name: 'Show empty fields (1)' })); await fireEvent.click(screen.getByRole('button', { name: 'Add Existing field value' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Existing field' }), { target: { value: 'retained local draft' }, diff --git a/web/src/lib/components/directory/AttributeSection.svelte b/web/src/lib/components/directory/AttributeSection.svelte index b2fa05eca..4339391bc 100644 --- a/web/src/lib/components/directory/AttributeSection.svelte +++ b/web/src/lib/components/directory/AttributeSection.svelte @@ -4,17 +4,16 @@ import type { AttributeDefinition as GeneratedAttributeDefinition, - AttributeValue as GeneratedAttributeValue, PersonAttributeGroup as GeneratedPersonAttributeGroup, PersonAttributeValue as GeneratedPersonAttributeValue, } from '../../api/generated/models'; import type { DirectoryProfileController } from '../../directory/profile-controller.svelte'; import AttributeDefinitionDialog from './AttributeDefinitionDialog.svelte'; import AttributeEditor from './AttributeEditor.svelte'; + import { displayAttributeValue } from './attribute-value'; type AttributeDefinition = GeneratedAttributeDefinition; type AttributeGroup = GeneratedPersonAttributeGroup; - type AttributeValue = GeneratedAttributeValue; type PersonAttributeValue = GeneratedPersonAttributeValue; interface Props { @@ -41,9 +40,22 @@ confirming = undefined; revealed = {}; creatingDefinition = false; + showEmpty = false; + recentlyCreatedID = null; }); const fields = $derived.by(() => joinDefinitions(controller.definitions, controller.attributes?.attributes ?? [])); + let showEmpty = $state(false); + let recentlyCreatedID = $state(null); + const emptyCount = $derived(fields.filter((field) => field.current.length === 0).length); + const visibleFields = $derived(showEmpty ? fields : fields.filter((field) => + field.current.length > 0 || editing?.universalID === field.definition.universal_id || recentlyCreatedID === field.definition.universal_id + )); + + // Keep a newly created field actionable after its confirmation dialog closes. + $effect(() => { + if (controller.createdDefinition) recentlyCreatedID = controller.createdDefinition.universal_id; + }); function joinDefinitions(definitions: AttributeDefinition[], groups: AttributeGroup[]): JoinedAttribute[] { const byUniversalID = new Map(groups.map((group) => [group.definition.universal_id, group])); @@ -91,33 +103,6 @@ if (confirming?.universalID === definition.universal_id) confirming = undefined; } - function displayValue(definition: AttributeDefinition, value: AttributeValue): string { - const canonical = rawValue(value); - const choice = definition.options?.choices?.find((candidate) => candidate.value === canonical); - return choice?.label ?? canonical; - } - - function rawValue(value: AttributeValue): string { - switch (value.type) { - case 'text': - return value.text ?? '—'; - case 'integer': - return value.integer?.toString() ?? '—'; - case 'real': - return value.real?.toString() ?? '—'; - case 'boolean': - return value.boolean === undefined ? '—' : value.boolean ? 'Yes' : 'No'; - case 'date': - return value.date ?? '—'; - case 'timestamp': - return value.timestamp ?? '—'; - case 'record_reference': - return value.record_type === 'person' && value.record_id ? `Person ${value.record_id}` : '—'; - default: - return value.json === undefined ? '—' : JSON.stringify(value.json); - } - } - function provenance(value: PersonAttributeValue): string { return [ `Source: ${value.source}`, @@ -241,7 +226,7 @@ } -
+

Attributes

diff --git a/web/src/lib/components/directory/AttributeSummary.test.ts b/web/src/lib/components/directory/AttributeSummary.test.ts new file mode 100644 index 000000000..6e09134ed --- /dev/null +++ b/web/src/lib/components/directory/AttributeSummary.test.ts @@ -0,0 +1,53 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { AttributeValue, PersonAttributeGroup } from '../../api/generated/models'; +import AttributeSummary from './AttributeSummary.svelte'; + +afterEach(() => cleanup()); + +function group(label: string, value: AttributeValue | undefined, sensitive = false): PersonAttributeGroup { + return { + definition: { universal_id: `u-${label}`, slug: label.toLowerCase(), label, is_sensitive: sensitive, display_order: 0 }, + current: value === undefined ? [] : [{ value }], + history: [] + } as unknown as PersonAttributeGroup; +} + +describe('AttributeSummary', () => { + it('lists only current values, formats typed values and choice labels, and conceals sensitive values', async () => { + const onEdit = vi.fn(); + const status = group('Status', { type: 'text', text: 'active' }); + status.definition.options = { choices: [{ value: 'active', label: 'Active contact' }] }; + const subscribed = group('Subscribed', { type: 'boolean', boolean: true }); + subscribed.definition.options = { choices: [{ value: 'true', label: 'Opted in' }] }; + render(AttributeSummary, { + groups: [ + group('Birthday', { type: 'date', date: '1990-01-01' }), + subscribed, + status, + group('Employer', undefined), + group('Health', { type: 'text', text: 'synthetic private value' }, true) + ], + onEdit + }); + const region = screen.getByRole('region', { name: 'Attributes summary' }); + expect(region.textContent).toContain('Birthday'); + expect(region.textContent).toContain('1990-01-01'); + expect(region.textContent).toContain('Subscribed'); + expect(region.textContent).toContain('Opted in'); + expect(region.textContent).toContain('Active contact'); + expect(region.textContent).not.toContain('active'); + expect(region.textContent).not.toContain('Employer'); + expect(region.textContent).toContain('Health'); + expect(region.textContent).toContain('concealed'); + expect(region.textContent).not.toContain('synthetic private value'); + await fireEvent.click(screen.getByRole('button', { name: 'Edit attributes' })); + expect(onEdit).toHaveBeenCalledOnce(); + }); + + it('renders nothing when no attribute has a current value', () => { + const { container } = render(AttributeSummary, { groups: [group('Employer', undefined)] }); + expect(container.querySelector('section')).toBeNull(); + }); +}); diff --git a/web/src/lib/components/directory/DirectoryWorkspace.svelte b/web/src/lib/components/directory/DirectoryWorkspace.svelte index 30a38181c..3bdc893f0 100644 --- a/web/src/lib/components/directory/DirectoryWorkspace.svelte +++ b/web/src/lib/components/directory/DirectoryWorkspace.svelte @@ -183,12 +183,14 @@ diff --git a/web/src/lib/components/directory/PersonDetail.svelte b/web/src/lib/components/directory/PersonDetail.svelte index ea9e616b0..09ff97916 100644 --- a/web/src/lib/components/directory/PersonDetail.svelte +++ b/web/src/lib/components/directory/PersonDetail.svelte @@ -5,6 +5,7 @@ import type { DirectoryEntityController } from '../../directory/entity-controller.svelte'; import FilesWorkspace from '../files/FilesWorkspace.svelte'; import AttributeSection from './AttributeSection.svelte'; + import AttributeSummary from './AttributeSummary.svelte'; import StructuredProfileSection from './StructuredProfileSection.svelte'; import OrganizationEmploymentTab from './OrganizationEmploymentTab.svelte'; import PersonNetwork from './PersonNetwork.svelte'; @@ -173,6 +174,14 @@ {#if bundle.person || profile}

{bundle.person?.display_name ?? profile?.person?.display_name ?? `Person ${personID}`}

{/if} + { + const section = document.getElementById('person-attributes'); + section?.scrollIntoView({ block: 'start', behavior: 'smooth' }); + section?.focus({ preventScroll: true }); + } : undefined} + /> - {:else if bundle.attributes?.attributes?.length} -

Attributes

    {#each bundle.attributes.attributes as group}
  • {group.definition.label}{#if group.definition.is_sensitive} Sensitive: concealed{:else}: {group.current?.map((value) => valueText(value.value)).join(', ')}{/if}
    • {/each}
{/if} {#if entityController?.employments.length}

Organizations and employment

    {#each entityController.employments as employment}
  • {employment.title ?? employment.role ?? 'Employment'} · {employmentOrganization(employment.id) ?? `Organization ${employment.organization_id}`}{#if employment.is_current} Current{/if}
    • {/each}
@@ -232,5 +239,4 @@ h3 { font-size: var(--font-size-md); } h4, small { color: var(--text-muted); font-size: var(--font-size-sm); } ul { padding-left: var(--space-5); } .section-error { margin: 0; padding: var(--space-2); background: var(--bg-inset); color: var(--text-secondary); } - .sensitive { display: inline-block; padding: 1px 5px; border-radius: var(--radius-sm); background: var(--bg-warning); color: var(--text-primary); font-size: var(--font-size-sm); } diff --git a/web/src/lib/components/directory/PersonDetail.test.ts b/web/src/lib/components/directory/PersonDetail.test.ts index 1456300a9..3062d11ea 100644 --- a/web/src/lib/components/directory/PersonDetail.test.ts +++ b/web/src/lib/components/directory/PersonDetail.test.ts @@ -2,10 +2,44 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { describe, expect, it, vi } from 'vitest'; import { createAPIClient } from '../../api/client'; +import type { AttributeDefinition as GeneratedAttributeDefinition } from '../../api/generated/models'; import { DirectoryEntityController } from '../../directory/entity-controller.svelte'; +import { DirectoryProfileController } from '../../directory/profile-controller.svelte'; import type { DirectoryReadBundle } from '../../directory/models'; import PersonDetail from './PersonDetail.svelte'; +const when = '2026-01-01T00:00:00Z'; + +function nicknameDefinition(): GeneratedAttributeDefinition { + return { + id: 1, slug: 'nickname', label: 'Nickname', value_type: 'text', field_type: 'text', + api_mutable: true, cardinality: 'single', display_order: 0, history_exempt: false, + is_sensitive: false, is_active: true, is_audited: true, is_deletable: true, is_required: false, + is_searchable: false, object_type: 'person', ownership: 'user', revision: 1, + ui_creatable: true, ui_editable: true, universal_id: 'synthetic-nickname', + created_at: when, updated_at: when + }; +} + +function nicknameCurrent() { + return [{ id: 1, person_id: 7, definition_id: 1, definition_slug: 'nickname', ordinal: 0, source: 'user', active_from: when, created_at: when, value: { type: 'text', text: 'Synthetic nickname' } }]; +} + +// The Overview panel's self-loading cards must never decide one of these +// tests; answer their mount reads quietly and deterministically. +function quietOverviewFetch() { + return vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + const overview = overviewCardResponse(request); + if (overview) return overview; + if (path === '/api/v1/carddav/publications/7') { + return Response.json({ error: 'carddav_unavailable', message: 'not rendered' }, { status: 503 }); + } + return Response.json({ merges: [], limit: 100, offset: 0 }); + }); +} + // The Overview panel mounts three self-loading cards. This answers the reads // they make on mount so each test only has to state what it is actually about. // An unenrolled brief enrollment is the quiet default: the brief card then @@ -70,7 +104,7 @@ describe('PersonDetail', () => { expect(screen.getByText('Names')).toBeDefined(); expect(screen.getByText('person@example.test')).toBeDefined(); - expect(screen.getByText('Sensitive')).toBeDefined(); + expect(document.querySelector('.attribute-summary .sensitive')?.textContent).toBe('concealed'); expect(document.body.innerHTML).not.toContain('Synthetic value'); expect(await screen.findByText(/Example Org/)).toBeDefined(); expect(screen.getByText('Synthetic Child · child')).toBeDefined(); @@ -387,4 +421,43 @@ describe('PersonDetail', () => { await fireEvent.click(screen.getByRole('button', { name: 'Open CardDAV settings' })); expect(onOpenCardDAVSettings).toHaveBeenCalledOnce(); }); + + it('renders read-only bundle attributes once through the summary without a duplicate legacy list', () => { + const definition = nicknameDefinition(); + const client = createAPIClient(quietOverviewFetch()); + const bundle = { + person: { id: 7, revision: 2, display_name: 'Synthetic Person', participant_ids: [], vcard_uid: '', created_at: when, updated_at: when }, + attributes: { person_id: 7, attributes: [{ definition, current: nicknameCurrent() }] }, + etags: {}, errors: {} + } satisfies DirectoryReadBundle; + + render(PersonDetail, { client, personID: 7, bundle }); + + expect(document.querySelector('.attribute-summary')).not.toBeNull(); + expect(screen.getAllByText('Nickname')).toHaveLength(1); + expect(screen.getAllByText('Synthetic nickname')).toHaveLength(1); + expect(screen.queryByRole('heading', { name: 'Attributes' })).toBeNull(); + }); + + it('moves focus to the attributes section when Edit attributes is pressed', async () => { + const definition = nicknameDefinition(); + const client = createAPIClient(quietOverviewFetch()); + const bundle = { + person: { id: 7, revision: 2, display_name: 'Synthetic Person', participant_ids: [], vcard_uid: '', created_at: when, updated_at: when }, + attributes: { person_id: 7, attributes: [{ definition, current: nicknameCurrent() }] }, + definitions: { definitions: [definition] }, + etags: {}, errors: {} + } satisfies DirectoryReadBundle; + const profileController = new DirectoryProfileController(client, 7, bundle); + + render(PersonDetail, { client, personID: 7, bundle, profileController }); + + const button = await screen.findByRole('button', { name: 'Edit attributes' }); + await fireEvent.click(button); + + const section = document.getElementById('person-attributes'); + expect(section).not.toBeNull(); + expect(section!.getAttribute('tabindex')).toBe('-1'); + expect(document.activeElement).toBe(section); + }); }); diff --git a/web/src/lib/components/directory/attribute-value.ts b/web/src/lib/components/directory/attribute-value.ts new file mode 100644 index 000000000..28ce8d6a6 --- /dev/null +++ b/web/src/lib/components/directory/attribute-value.ts @@ -0,0 +1,29 @@ +import type { AttributeDefinition, AttributeValue } from '../../api/generated/models'; + +function rawValue(value: AttributeValue): string { + switch (value.type) { + case 'text': + return value.text ?? '—'; + case 'integer': + return value.integer?.toString() ?? '—'; + case 'real': + return value.real?.toString() ?? '—'; + case 'boolean': + return value.boolean === undefined ? '—' : value.boolean ? 'Yes' : 'No'; + case 'date': + return value.date ?? '—'; + case 'timestamp': + return value.timestamp ?? '—'; + case 'record_reference': + return value.record_type === 'person' && value.record_id ? `Person ${value.record_id}` : '—'; + default: + return value.json === undefined ? '—' : JSON.stringify(value.json); + } +} + +export function displayAttributeValue(definition: AttributeDefinition, value: AttributeValue): string { + const display = rawValue(value); + const canonical = value.type === 'boolean' && value.boolean !== undefined ? String(value.boolean) : display; + const choice = definition.options?.choices?.find((candidate) => candidate.value === canonical); + return choice?.label ?? display; +} diff --git a/web/tests/e2e/directory.spec.ts b/web/tests/e2e/directory.spec.ts index 89b74812a..9cb252072 100644 --- a/web/tests/e2e/directory.spec.ts +++ b/web/tests/e2e/directory.spec.ts @@ -15,6 +15,23 @@ async function expectNoAxeViolations(page: Page, label: string): Promise { .toEqual([]); } +async function installTallDirectory(page: Page): Promise { + await page.unroute('**/api/v1/people/directory*'); + await page.route('**/api/v1/people/directory*', (route) => route.fulfill({ + json: { + people: Array.from({ length: 40 }, (_, index) => ({ + id: index === 0 ? 42 : 100 + index, + revision: 1, + display_name: index === 0 ? 'Archive Person' : `Synthetic Person ${index}`, + primary_channel: 'email', + contact_state: 'active', + categories: [], + organizations: [] + })) + } + })); +} + test('Directory lists durable people, opens split detail, and scopes Media & Files to the durable person', async ({ page }) => { const requests: string[] = []; page.on('request', (request) => requests.push(new URL(request.url()).pathname)); @@ -45,6 +62,76 @@ test('Directory lists durable people, opens split detail, and scopes Media & Fil await expectNoAxeViolations(page, 'Directory split media detail'); }); +test('Directory person detail scrolls independently at desktop width', async ({ page }) => { + await installMixedArchive(page); + await installTallDirectory(page); + await page.setViewportSize({ width: 1280, height: 600 }); + await page.goto(directoryURL(42)); + const pane = page.getByRole('complementary', { name: 'Person detail' }); + await expect(pane.getByRole('heading', { name: 'Archive Person' })).toBeVisible(); + const metrics = await pane.evaluate((el) => ({ + scroll: el.scrollHeight, + client: el.clientHeight, + overflow: getComputedStyle(el).overflowY + })); + expect(metrics.scroll).toBeGreaterThan(metrics.client); + expect(metrics.overflow).toBe('auto'); + await pane.evaluate((el) => el.scrollTo(0, el.scrollHeight)); + expect(await pane.evaluate((el) => el.scrollTop)).toBeGreaterThan(0); + await expect(page.getByRole('region', { name: 'Person merge history' }).or(pane.locator('section').last())).toBeInViewport(); + + const directory = page.getByRole('main', { name: 'Directory' }); + const list = directory.getByRole('region', { name: 'Directory results' }); + const toolbar = directory.locator('.directory-toolbar'); + const filters = directory.locator('.filters'); + const before = { toolbar: await toolbar.boundingBox(), filters: await filters.boundingBox() }; + const listMetrics = await list.evaluate((el) => ({ scroll: el.scrollHeight, client: el.clientHeight, overflow: getComputedStyle(el).overflowY })); + expect(listMetrics.scroll).toBeGreaterThan(listMetrics.client); + expect(listMetrics.overflow).toBe('auto'); + await list.evaluate((el) => el.scrollTo(0, el.scrollHeight)); + expect(await list.evaluate((el) => el.scrollTop)).toBeGreaterThan(0); + expect((await toolbar.boundingBox())?.y).toBe(before.toolbar?.y); + expect((await filters.boundingBox())?.y).toBe(before.filters?.y); +}); + +test('Directory list stays in its flexible row when a promotion alert appears', async ({ page }) => { + await installMixedArchive(page); + await installTallDirectory(page); + await page.setViewportSize({ width: 1280, height: 720 }); + await page.goto('/'); + await page.getByRole('grid', { name: 'Relationship results' }).getByText('Archive Person').click(); + await page.getByRole('button', { name: 'Open in Directory' }).click(); + const directory = page.getByRole('main', { name: 'Directory' }); + await expect(directory.getByRole('button', { name: 'Promote to person' })).toBeVisible(); + + async function expectContentContained(): Promise { + const metrics = await directory.evaluate((root) => { + const content = root.querySelector('.directory-content'); + if (!(content instanceof HTMLElement)) throw new Error('Directory content missing'); + return { + rootBottom: root.getBoundingClientRect().bottom, + contentBottom: content.getBoundingClientRect().bottom, + contentHeight: content.getBoundingClientRect().height + }; + }); + expect(metrics.contentHeight).toBeGreaterThan(0); + expect(metrics.contentBottom).toBeLessThanOrEqual(metrics.rootBottom + 1); + const list = directory.getByRole('region', { name: 'Directory results' }); + const listMetrics = await list.evaluate((el) => ({ scroll: el.scrollHeight, client: el.clientHeight, overflow: getComputedStyle(el).overflowY })); + expect(listMetrics.scroll).toBeGreaterThan(listMetrics.client); + expect(listMetrics.overflow).toBe('auto'); + } + + await expectContentContained(); + await page.route('**/api/v1/people', (route) => route.fulfill({ + status: 409, + json: { error: 'person_binding_conflict', message: 'Synthetic promotion conflict.' } + })); + await directory.getByRole('button', { name: 'Promote to person' }).click(); + await expect(directory.getByRole('alert')).toContainText('Synthetic promotion conflict.'); + await expectContentContained(); +}); + test('Directory opens selected detail in an accessible narrow drawer', async ({ page }) => { await installMixedArchive(page); await page.setViewportSize({ width: 640, height: 900 }); From 06e9f7d80e3ca5688511ac6738ce3c03b16d1abf Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Wed, 23 Sep 2026 08:29:31 -0400 Subject: [PATCH 017/125] fix(analytics): skip pre-1970 years in annual relationship temperatures (#907) Archives with qualifying messages dated before 1970 can fail analytics cache builds because annual relationship temperature rows violate the validator's year floor. The annual rollup now filters its input at the same 1970 floor used by validation, while current scores and daily activity still include older messages. A shared constant keeps the two checks aligned. Regression coverage proves cache builds succeed with pre-1970 messages, keeps 1970 annual data, retains older activity in current and daily projections, and rejects invalid stored annual rows. Closes https://github.com/kenn-io/msgvault/issues/897 Co-authored-by: Rod Boev --- internal/identityindex/build_test.go | 145 ++++++++++++++++++ internal/identityindex/temperature_sql.go | 8 + .../identityindex/temperature_sql_test.go | 73 +++++++++ internal/identityindex/validate.go | 2 +- 4 files changed, 227 insertions(+), 1 deletion(-) diff --git a/internal/identityindex/build_test.go b/internal/identityindex/build_test.go index dff6dc070..556ffd258 100644 --- a/internal/identityindex/build_test.go +++ b/internal/identityindex/build_test.go @@ -383,6 +383,110 @@ func TestBuildPeoplePublishesMessageGrainRelationshipTemperatures(t *testing.T) assert.InDelta(1.0, receivedVolume, 0) } +func TestBuildPeopleSkipsPreEpochYearsInAnnualTemperatures(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + root, db := writeRelationshipBaseFixture(t, false) + replaceRelationshipParquet(t, db, root, "messages", ` + SELECT * FROM (VALUES + (100::BIGINT, 1::BIGINT, 'm-100'::VARCHAR, 10::BIGINT, + 'Sent'::VARCHAR, ''::VARCHAR, TIMESTAMP '2026-07-20 10:30:00', + 10::BIGINT, false, 0::INTEGER, NULL::TIMESTAMP, + 1::BIGINT, 1::BIGINT, 'email'::VARCHAR, true, 2026::INTEGER, 7::INTEGER), + (101::BIGINT, 1::BIGINT, 'm-101'::VARCHAR, 10::BIGINT, + 'Received'::VARCHAR, ''::VARCHAR, TIMESTAMP '2026-07-21 10:30:00', + 10::BIGINT, false, 0::INTEGER, NULL::TIMESTAMP, + 2::BIGINT, 1::BIGINT, 'email'::VARCHAR, false, 2026::INTEGER, 7::INTEGER), + (102::BIGINT, 1::BIGINT, 'm-102'::VARCHAR, 10::BIGINT, + 'Pre-epoch 1899'::VARCHAR, ''::VARCHAR, TIMESTAMP '1899-12-29 13:00:00', + 10::BIGINT, false, 0::INTEGER, NULL::TIMESTAMP, + 2::BIGINT, 1::BIGINT, 'email'::VARCHAR, false, 1899::INTEGER, 12::INTEGER), + (103::BIGINT, 1::BIGINT, 'm-103'::VARCHAR, 10::BIGINT, + 'Pre-epoch 1904'::VARCHAR, ''::VARCHAR, TIMESTAMP '1904-01-04 09:00:00', + 10::BIGINT, false, 0::INTEGER, NULL::TIMESTAMP, + 4::BIGINT, 1::BIGINT, 'email'::VARCHAR, false, 1904::INTEGER, 1::INTEGER) + ) AS t(id, source_id, source_message_id, conversation_id, subject, + snippet, sent_at, size_estimate, has_attachments, attachment_count, + deleted_from_source_at, sender_id, owner_participant_id, message_type, is_from_me, year, month)`) + replaceRelationshipParquet(t, db, root, "message_recipients", ` + SELECT * FROM (VALUES + (100::BIGINT, 1::BIGINT, 'from'::VARCHAR, 'Owner'::VARCHAR), + (100::BIGINT, 2::BIGINT, 'to'::VARCHAR, 'Bob'::VARCHAR), + (100::BIGINT, 3::BIGINT, 'cc'::VARCHAR, 'Bob Alias'::VARCHAR), + (101::BIGINT, 2::BIGINT, 'from'::VARCHAR, 'Bob'::VARCHAR), + (101::BIGINT, 1::BIGINT, 'to'::VARCHAR, 'Owner'::VARCHAR), + (102::BIGINT, 2::BIGINT, 'from'::VARCHAR, 'Bob'::VARCHAR), + (102::BIGINT, 1::BIGINT, 'to'::VARCHAR, 'Owner'::VARCHAR), + (103::BIGINT, 4::BIGINT, 'from'::VARCHAR, 'Member'::VARCHAR), + (103::BIGINT, 1::BIGINT, 'to'::VARCHAR, 'Owner'::VARCHAR) + ) AS t(message_id, participant_id, recipient_type, display_name)`) + + effectiveAt := time.Date(2026, time.July, 22, 12, 34, 56, 0, time.UTC) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: root, OutputRoot: root, + EffectiveAt: effectiveAt, + }) + require.NoError(err) + + var annualYears string + var current, currentPopulation, peakYear int + require.NoError(db.QueryRow(` + SELECT CAST(to_json(list_transform(annual_temperatures, x -> x.year)) AS VARCHAR), + current_temperature, current_temperature_population, peak_year + FROM read_parquet(?) WHERE canonical_id = 2 + `, relationshipParquetGlob(root, DatasetPeople)).Scan( + &annualYears, ¤t, ¤tPopulation, &peakYear, + )) + assert.JSONEq(`[2026]`, annualYears) + assert.Equal(100, current) + assert.Equal(2, currentPopulation) + assert.Equal(2026, peakYear) + + var annualTemperature int + var annualRank, annualPopulation int64 + var annualRaw, sentSignal, receivedVolume float64 + require.NoError(db.QueryRow(` + SELECT annual.item.temperature, annual.item.rank, annual.item.population, + annual.item.raw_score, annual.item.sent_signal, + annual.item.received_volume + FROM read_parquet(?) p, + unnest(p.annual_temperatures) AS annual(item) + WHERE p.canonical_id = 2 AND annual.item.year = 2026 + `, relationshipParquetGlob(root, DatasetPeople)).Scan( + &annualTemperature, &annualRank, &annualPopulation, + &annualRaw, &sentSignal, &receivedVolume, + )) + assert.Equal(100, annualTemperature) + assert.Equal(int64(1), annualRank) + assert.Equal(int64(1), annualPopulation) + assert.InDelta(3*math.Log(2), annualRaw, 1e-9) + assert.InDelta(math.Log(2), sentSignal, 1e-9) + assert.InDelta(1.0, receivedVolume, 0) + + var annualCount, canonical4Population, canonical4Peak, canonical4PeakYear, firstYear int + require.NoError(db.QueryRow(` + SELECT len(annual_temperatures), current_temperature_population, + peak_temperature, peak_year, year(first_at) + FROM read_parquet(?) WHERE canonical_id = 4 + `, relationshipParquetGlob(root, DatasetPeople)).Scan( + &annualCount, &canonical4Population, &canonical4Peak, + &canonical4PeakYear, &firstYear, + )) + assert.Equal(0, annualCount) + assert.Equal(2, canonical4Population) + assert.Equal(0, canonical4Peak) + assert.Equal(0, canonical4PeakYear) + assert.Equal(1904, firstYear) + + var dailyCount int + require.NoError(db.QueryRow(` + SELECT count(*) + FROM read_parquet(?) + WHERE canonical_id = 2 AND event_date = DATE '1899-12-29' + `, relationshipParquetGlob(root, DatasetRelationshipDaily)).Scan(&dailyCount)) + assert.Equal(1, dailyCount) +} + func TestValidateRejectsDuplicateActivityGrain(t *testing.T) { root, db := writeRelationshipBaseFixture(t, false) _, err := Build(context.Background(), db, BuildOptions{ @@ -428,6 +532,47 @@ func TestValidateRejectsInvalidRelationshipTemperatureSummary(t *testing.T) { require.ErrorContains(t, err, "invalid relationship temperature summary") } +func TestValidateRejectsPreEpochAnnualTemperature(t *testing.T) { + root, db := writeRelationshipBaseFixture(t, false) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: root, OutputRoot: root, + }) + require.NoError(t, err) + + oldRoot := moveRelationshipDatasetAside(t, root, DatasetPeople) + source := quoteSQLString(relationshipParquetGlob(oldRoot, DatasetPeople)) + writeRelationshipParquet(t, db, root, DatasetPeople, ` + SELECT * REPLACE ([struct_pack( + year := 1969, + temperature := 100, + rank := 1::BIGINT, + population := 1::BIGINT, + raw_score := 1.0::DOUBLE, + sent_signal := 0.0::DOUBLE, + received_volume := 1.0::DOUBLE, + meeting_signal := 0.0::DOUBLE, + modalities := 1 + )]::STRUCT( + year INTEGER, + temperature INTEGER, + rank BIGINT, + population BIGINT, + raw_score DOUBLE, + sent_signal DOUBLE, + received_volume DOUBLE, + meeting_signal DOUBLE, + modalities INTEGER + )[] AS annual_temperatures) + FROM read_parquet('`+source+`') + `) + + err = Validate(context.Background(), db, ValidationOptions{ + OutputRoot: root, + RequiredOutputDatasets: RequiredDatasets, + }) + require.ErrorContains(t, err, "invalid relationship temperature summary") +} + func writeRelationshipBaseFixture(t *testing.T, empty bool) (string, *sql.DB) { t.Helper() root := t.TempDir() diff --git a/internal/identityindex/temperature_sql.go b/internal/identityindex/temperature_sql.go index 3d3315e7d..b6b98a46f 100644 --- a/internal/identityindex/temperature_sql.go +++ b/internal/identityindex/temperature_sql.go @@ -7,6 +7,12 @@ import ( const temperatureBuildRelation = "relationship_build_temperature_daily" +// relationshipTemperatureFirstYear is the first UTC calendar year that +// receives an annual temperature, matching the relationship calendar's year +// floor. Older qualifying activity still decays into the current score and +// stays in the daily datasets. +const relationshipTemperatureFirstYear = 1970 + // RelationshipTemperatureFactsSQL selects one qualifying row per stored // message and canonical person. activity is a trusted DuckDB relation rendered // by the cache/query packages; it is not user input. The shared SQL keeps cache @@ -160,6 +166,7 @@ func relationshipTemperatureCTEs(effectiveAt time.Time) string { sum(meeting_count)::DOUBLE AS meeting_signal, bit_count(bit_or(modality_mask))::INTEGER AS modalities FROM %[3]s + WHERE year(event_date) >= %[8]d GROUP BY canonical_id, score_year ), annual_scored AS ( SELECT *, @@ -213,5 +220,6 @@ func relationshipTemperatureCTEs(effectiveAt time.Time) string { temperatureWeightReceived, temperatureWeightMeetings, temperatureBreadthStep, + relationshipTemperatureFirstYear, ) } diff --git a/internal/identityindex/temperature_sql_test.go b/internal/identityindex/temperature_sql_test.go index 83df60c10..f5470613c 100644 --- a/internal/identityindex/temperature_sql_test.go +++ b/internal/identityindex/temperature_sql_test.go @@ -105,3 +105,76 @@ func TestRelationshipTemperatureCurrentHalfLifeKeepsOlderSignalsInWholeGraph(t * assert.False(rows.Next()) require.NoError(rows.Err()) } + +func TestRelationshipTemperatureAnnualRollupsStartAtFirstYear(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + _, db := writeRelationshipBaseFixture(t, true) + _, err := db.Exec(` + CREATE TEMP TABLE ` + temperatureBuildRelation + ` ( + canonical_id BIGINT, event_date DATE, sent_count BIGINT, + received_count BIGINT, meeting_count BIGINT, email_count BIGINT, + chat_count BIGINT, total_count BIGINT, modality_mask UTINYINT, + last_at TIMESTAMP + ); + INSERT INTO ` + temperatureBuildRelation + ` VALUES + (2, DATE '1969-12-31', 1, 0, 0, 1, 0, 1, 1, TIMESTAMP '1969-12-31 23:59:59'), + (2, DATE '1970-01-01', 1, 0, 0, 1, 0, 1, 1, TIMESTAMP '1970-01-01 00:00:00'), + (3, DATE '1904-01-04', 0, 1, 0, 1, 0, 1, 1, TIMESTAMP '1904-01-04 09:00:00') + `) + require.NoError(err) + + query := `WITH anchor AS (SELECT 1)` + relationshipTemperatureCTEs( + time.Date(1970, time.January, 2, 0, 0, 0, 0, time.UTC), + ) + + func() { + rows, err := db.Query(query + ` SELECT canonical_id, + CAST(to_json(list_transform(annual_temperatures, x -> x.year)) AS VARCHAR) + FROM annual_rollups ORDER BY canonical_id`) + require.NoError(err) + defer func() { require.NoError(rows.Close()) }() + require.True(rows.Next()) + var canonicalID int64 + var years string + require.NoError(rows.Scan(&canonicalID, &years)) + assert.Equal(int64(2), canonicalID) + assert.JSONEq(`[1970]`, years) + assert.False(rows.Next()) + require.NoError(rows.Err()) + }() + + func() { + rows, err := db.Query(query + ` SELECT canonical_id, peak_year + FROM peaks ORDER BY canonical_id`) + require.NoError(err) + defer func() { require.NoError(rows.Close()) }() + require.True(rows.Next()) + var canonicalID, peakYear int64 + require.NoError(rows.Scan(&canonicalID, &peakYear)) + assert.Equal(int64(2), canonicalID) + assert.Equal(int64(1970), peakYear) + assert.False(rows.Next()) + require.NoError(rows.Err()) + }() + + func() { + rows, err := db.Query(query + ` SELECT canonical_id, population, raw_score + FROM current_ranked ORDER BY canonical_id`) + require.NoError(err) + defer func() { require.NoError(rows.Close()) }() + require.True(rows.Next()) + var canonicalID, population int64 + var rawScore float64 + require.NoError(rows.Scan(&canonicalID, &population, &rawScore)) + assert.Equal(int64(2), canonicalID) + assert.Equal(int64(2), population) + assert.InDelta(2*(RelationshipDayWeight(2)+RelationshipDayWeight(1))*math.Log(2), rawScore, 1e-9) + require.True(rows.Next()) + require.NoError(rows.Scan(&canonicalID, &population, &rawScore)) + assert.Equal(int64(3), canonicalID) + assert.Equal(int64(2), population) + assert.False(rows.Next()) + require.NoError(rows.Err()) + }() +} diff --git a/internal/identityindex/validate.go b/internal/identityindex/validate.go index cefa35f3e..489b37002 100644 --- a/internal/identityindex/validate.go +++ b/internal/identityindex/validate.go @@ -154,7 +154,7 @@ func Validate( OR p.peak_year < 0 OR EXISTS ( SELECT 1 FROM unnest(p.annual_temperatures) AS annual(item) - WHERE annual.item.year < 1970 + WHERE annual.item.year < ` + strconv.Itoa(relationshipTemperatureFirstYear) + ` OR annual.item.temperature NOT BETWEEN 0 AND 100 OR annual.item.rank < 1 OR annual.item.population < annual.item.rank From 57f16e37d70cb46562347c5af812dd2d4356f2c6 Mon Sep 17 00:00:00 2001 From: Elie BRUNO <72560989+eliemada@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:02:29 +0000 Subject: [PATCH 018/125] fix(imap): preserve legacy Message-IDs during membership refresh (#913) IMAP synchronization can lose the identity of already archived messages whose `Message-ID` uses historical, non-RFC syntax, such as `123456789` or `<[legacy-token==@example.test]>`. MIME import preserves these identifiers, but `rawMIMEMessageID` returned an empty string when the stricter mail-header parser rejected them. After a filtered Inbox import, a full sync can consequently report `inconclusive IMAP identity returned a dedup stub` and leave folder state unpublished. Keep the existing strict parser as the first choice and fall back to the existing `mime.ParseMessageIDs` normalizer when it rejects an identifier. This keeps standard identifiers and comment handling intact while allowing header enumeration, label refresh, and raw-message deduplication to recognize the same legacy IDs. Malformed angle-bracket structures and identifier-shaped body text remain rejected. The regression tests cover legacy IDs across IMAP fetch paths and repeated full syncs after a filtered Inbox import, checking that one archived message retains both folder memberships, both saved folder cursors, and its original raw MIME bytes. Co-authored-by: Elie BRUNO --- .../cmd/imap_legacy_message_id_test.go | 78 +++++++++++++++++++ internal/imap/batch_fetch.go | 6 +- internal/imap/batch_fetch_test.go | 2 +- internal/imap/client_test.go | 2 +- internal/imap/message_id_legacy_test.go | 58 ++++++++++++++ 5 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 cmd/msgvault/cmd/imap_legacy_message_id_test.go create mode 100644 internal/imap/message_id_legacy_test.go diff --git a/cmd/msgvault/cmd/imap_legacy_message_id_test.go b/cmd/msgvault/cmd/imap_legacy_message_id_test.go new file mode 100644 index 000000000..a99ea55e5 --- /dev/null +++ b/cmd/msgvault/cmd/imap_legacy_message_id_test.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "fmt" + "testing" + + imapapi "github.com/emersion/go-imap/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestIMAPLegacyMessageIDFullSyncPersistsMemberships(t *testing.T) { + for _, header := range []string{"123456789", "<[legacy-token==@example.test]>"} { + t.Run(header, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + addr, user := testutil.StartIMAPMemServerWithSpecialUse(t, + map[string]int{"All Mail": 0, "INBOX": 0}, + map[string][]imapapi.MailboxAttr{"All Mail": {imapapi.MailboxAttrAll}}, + ) + raw := []byte(fmt.Sprintf("From: sender@example.test\r\nTo: recipient@example.test\r\n"+ + "Date: Mon, 1 Jan 2024 00:00:00 +0000\r\nMessage-ID: %s\r\n"+ + "Subject: Synthetic legacy identifier\r\n\r\nSynthetic body.\r\n", header)) + for _, mailbox := range []string{"All Mail", "INBOX"} { + testutil.AppendIMAPRawMessage(t, user, mailbox, raw) + } + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource(sourceTypeIMAP, "imap://legacy@example.test") + require.NoError(err) + initial := newScriptedRFC7162Client(t, addr, imaplib.WithFolderFilter([]string{"INBOX"}, nil)) + initialOptions := msgsync.DefaultOptions() + initialOptions.SourceType = sourceTypeIMAP + initialSummary, err := newMessageSyncer(initial, st, initialOptions).Full(t.Context(), source.Identifier) + require.NoError(err) + require.Zero(initialSummary.Errors) + require.NoError(initial.Close()) + + // After a filtered import, both a full scan and a repeated refresh must + // publish the same two mailbox memberships for a single archive row. + for attempt := range 2 { + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, true)...) + options := msgsync.DefaultOptions() + options.SourceType = sourceTypeIMAP + options.NoResume = true + summary, err := newMessageSyncer(client, st, options). + FullWithFinalizer(t.Context(), source, func(summary *gmail.SyncSummary) error { + return saveIMAPFolderStates(t.Context(), st, source, client, summary, 0) + }) + require.NoError(err, "sync attempt %d", attempt+1) + require.Zero(summary.Errors) + assert.Equal(2, imapMembershipRowCount(t, st, source.ID)) + + var count int + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT COUNT(*) FROM messages WHERE source_id = ?`), source.ID).Scan(&count)) + assert.Equal(1, count) + var allMailID, inboxID int64 + for mailbox, target := range map[string]*int64{"All Mail": &allMailID, "INBOX": &inboxID} { + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT message_id FROM imap_message_memberships WHERE source_id = ? AND mailbox = ? AND uid = 1`, + ), source.ID, mailbox).Scan(target)) + } + assert.Equal(allMailID, inboxID) + storedRaw, err := st.GetMessageRaw(allMailID) + require.NoError(err) + assert.Equal(raw, storedRaw) + states, err := loadIMAPFolderStates(st, source.ID) + require.NoError(err) + assert.Len(states, 2) + require.NoError(client.Close()) + } + }) + } +} diff --git a/internal/imap/batch_fetch.go b/internal/imap/batch_fetch.go index da14e28db..8b0ccc2f3 100644 --- a/internal/imap/batch_fetch.go +++ b/internal/imap/batch_fetch.go @@ -84,7 +84,11 @@ func rawMIMEMessageID(rawMIME []byte) string { header := gomail.Header{Header: entity.Header} msgID, err := header.MessageID() if err != nil { - return "" + // Historical messages can carry bare IDs or other non-RFC syntax + // that the archive's MIME parser preserves. Use that same fallback + // for folder membership and deduplication, otherwise an archived + // message can become unidentifiable when its headers are refreshed. + msgID, _ = mime.ParseMessageIDs(rawMIME) } return msgID } diff --git a/internal/imap/batch_fetch_test.go b/internal/imap/batch_fetch_test.go index 4eba260f4..facc5cd72 100644 --- a/internal/imap/batch_fetch_test.go +++ b/internal/imap/batch_fetch_test.go @@ -658,7 +658,7 @@ func TestApplyFetchResultsImportsWhenRawMessageIDMissingOrInvalid(t *testing.T) }, { name: "invalid message id value", - raw: []byte("Message-ID: not a message id\r\n\r\nbody"), + raw: []byte("Message-ID: ", "[legacy-token==@example.test]"}, + {"standard ID", "", "standard@example.test"}, + {"comment", " (comment)", "comment@example.test"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + header := []byte("Message-ID: " + tt.header + "\r\n\r\n") + raw := append(append([]byte{}, header...), []byte("body")...) + assert.Equal(tt.want, rawMIMEMessageID(header)) + assert.Equal(tt.want, rawMIMEMessageID(raw)) + + client := Client{selectedUIDValidity: 1} + message := fetchMessageBufferWithoutEnvelope(header) + var unidentified []imapapi.UID + identities := make(map[string]bool) + client.recordMessageIDResults("INBOX", identities, &unidentified, []*imapclient.FetchMessageBuffer{message}) + assert.Empty(unidentified) + assert.True(identities[tt.want]) + require.Len(client.observedMemberships, 1) + assert.Equal(tt.want, client.observedMemberships[0].RFC822MessageID) + + labels := newLabelBatchResults([]string{"INBOX|10"}) + client.applyLabelFetchResults(labels, map[imapapi.UID]int{10: 0}, "INBOX", nil, []*imapclient.FetchMessageBuffer{message}) + require.NoError(labels[0].Err) + assert.Equal(tt.want, labels[0].RFC822MessageID) + }) + } +} + +func TestLegacyMessageIDFallbackRejectsMalformedBracketsAndBodyText(t *testing.T) { + for _, raw := range []string{ + "Message-ID: >\r\n\r\nbody", + "Subject: no identifier\r\n\r\nMessage-ID: 123456789\r\n", + } { + assert.Empty(t, rawMIMEMessageID([]byte(raw))) + } +} From 94981b9182fed6f496b5afa1bf4a4987ff2aac3c Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:22:07 -0700 Subject: [PATCH 019/125] web: improve relationship calendar layout and navigation (#904) ## What changed - Let the relationship calendar use the available card width while keeping narrow layouts reachable. - Show each day's activity summary on pointer hover or keyboard focus, with edge tooltips kept inside the calendar. - Hide year arrows for single-year history. For multi-year history, the unavailable direction is visibly disabled. ## Why The calendar's fixed cell sizing wastes room in wider cards and can clip in narrow ones. Day details and year controls are also easy to miss or misread. ## Usage Hover over or focus a calendar day to read its summary. Use the year arrows when the relationship spans multiple years. Refs #901 Co-authored-by: Rusty Shackleford --- .../relationships/RelationshipCalendar.svelte | 208 ++++++++++++++---- .../RelationshipCalendar.test.ts | 132 ++++++++++- .../RelationshipsWorkspace.test.ts | 4 +- .../relationships/calendar-tooltip.test.ts | 25 +++ web/src/lib/relationships/calendar-tooltip.ts | 16 ++ .../lib/relationships/controller.svelte.ts | 6 +- web/tests/relationship-calendar.spec.ts | 188 ++++++++++++++++ 7 files changed, 530 insertions(+), 49 deletions(-) create mode 100644 web/src/lib/relationships/calendar-tooltip.test.ts create mode 100644 web/src/lib/relationships/calendar-tooltip.ts create mode 100644 web/tests/relationship-calendar.spec.ts diff --git a/web/src/lib/components/relationships/RelationshipCalendar.svelte b/web/src/lib/components/relationships/RelationshipCalendar.svelte index 3313dfd84..a4cea5201 100644 --- a/web/src/lib/components/relationships/RelationshipCalendar.svelte +++ b/web/src/lib/components/relationships/RelationshipCalendar.svelte @@ -7,13 +7,14 @@ RelationshipCalendar as RelationshipCalendarModel, RelationshipCalendarDay } from '../../relationships/models'; + import { dayTooltipText } from '../../relationships/calendar-tooltip'; interface Props { calendar: RelationshipCalendarModel | null; loading: boolean; error: string | null; year?: number; - firstYear: number; + firstYear: number | null; currentYear: number; onYearChange: (year: number) => void; } @@ -47,6 +48,16 @@ buildCalendarPanel(calendar, 6, 11, 'second-half') ] : []); const hasActivity = $derived(Boolean(calendar?.days?.some((day) => day.total > 0))); + const multiYear = $derived(firstYear === null || firstYear < currentYear); + let root = $state(); + const tooltipID = $props.id(); + let tooltip = $state(null); + let tooltipNode = $state(); + let activeCell: HTMLElement | null = null; + let pointer: { x: number; y: number } | null = null; + // Survives the pointer being cleared (focus moves, hide) so scroll + // dismissal still knows what kind of pointer was last seen. + let lastPointerType: string | null = null; const weekdays = ['Sun', '', 'Tue', '', 'Thu', '', 'Sat']; const levels = ['none', 'first-quartile', 'second-quartile', 'third-quartile', 'fourth-quartile']; @@ -93,9 +104,98 @@ return date.toISOString().slice(0, 10); } - function dayLabel(day: RelationshipCalendarDay): string { - return `${day.date}: ${day.total} interactions; ${day.sent} sent, ${day.received} received, ` + - `${day.email} email, ${day.chat} chat, ${day.meetings} meetings`; + $effect(() => { + calendar; + hideTooltip(); + }); + + $effect(() => { + const cell = tooltip; + if (!cell || !tooltipNode) return; + positionTooltip(cell); + cell.setAttribute('aria-describedby', tooltipID); + document.addEventListener('keydown', dismissTooltip, true); + return () => { + cell.removeAttribute('aria-describedby'); + document.removeEventListener('keydown', dismissTooltip, true); + }; + }); + + function positionTooltip(target: HTMLElement): void { + if (!root || !tooltipNode) return; + const cell = target.getBoundingClientRect(); + const box = root.getBoundingClientRect(); + const halfWidth = tooltipNode.getBoundingClientRect().width / 2; + const center = cell.left - box.left + cell.width / 2; + tooltipNode.style.left = `${Math.max(halfWidth + 4, Math.min(center, box.width - halfWidth - 4))}px`; + tooltipNode.style.top = `${cell.top - box.top}px`; + } + + function activateTooltip(target: HTMLElement): void { + if (target === activeCell) return; + activeCell = target; + tooltip = target; + } + + function dismissTooltip(event: KeyboardEvent): void { + if (event.key !== 'Escape') return; + event.preventDefault(); + event.stopPropagation(); + tooltip = null; + pointer = null; + // Keep activeCell until the pointer leaves so movement within a dismissed + // day does not reopen it. A different day can open immediately. + } + + function showTooltip(event: Event): void { + if (event.type.startsWith('pointer')) { + const movement = event as PointerEvent; + pointer = { x: movement.clientX, y: movement.clientY }; + lastPointerType = movement.pointerType; + } else if (event.type === 'focusin') { + pointer = null; + } + const target = (event.target as HTMLElement | null)?.closest('button.heat-cell'); + if (target) activateTooltip(target); + else hideTooltip(); + } + + function hideTooltip(): void { + activeCell = null; + tooltip = null; + // Drop captured coordinates so a later scroll cannot resurrect the tip + // from wherever the pointer last was. + pointer = null; + } + + function tooltipPointerLeave(event: PointerEvent): void { + // Touch pointers fire pointerleave right after pointerup; keep a tapped + // day's tooltip up until the next tap or scroll elsewhere. + if (event.pointerType === 'touch') return; + hideTooltip(); + } + + function scrollTooltip(): void { + if (!root || !tooltip) return; + // A touch drag leaves the tap behind (its coordinates may already be + // cleared by focus); scrolling must dismiss the tooltip rather than + // re-pin it to an unrelated cell. + if (lastPointerType === 'touch') { + hideTooltip(); + return; + } + const pointedCell = pointer + ? document.elementFromPoint(pointer.x, pointer.y)?.closest('button.heat-cell') + : activeCell; + if (!pointedCell || !root.contains(pointedCell)) { + hideTooltip(); + return; + } + if (pointedCell !== activeCell) { + activateTooltip(pointedCell); + return; + } + positionTooltip(pointedCell); } function levelClass(day: RelationshipCalendarDay | undefined): string { @@ -104,25 +204,24 @@ {#snippet panel(panel: CalendarPanel, variant: 'full' | 'half')} -
-
+ + diff --git a/web/src/lib/components/settings/PeopleInferenceSettings.test.ts b/web/src/lib/components/settings/PeopleInferenceSettings.test.ts new file mode 100644 index 000000000..00943944c --- /dev/null +++ b/web/src/lib/components/settings/PeopleInferenceSettings.test.ts @@ -0,0 +1,292 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; +import { createAPIClient } from '../../api/client'; +import type { PeopleInferenceProfileSetting, PeopleInferenceSettingsResponse } from '../../api/generated/models'; +import { PeopleInferenceController } from '../../settings/people-inference-controller.svelte'; +import PeopleInferenceSettings from './PeopleInferenceSettings.svelte'; + +const profile: PeopleInferenceProfileSetting = { + name: 'routed', preset_id: 'openrouter', protocol: 'openai-chat', model: 'model-one', + endpoint: 'https://openrouter.example.test/api/v1', credential_source: 'stored', + credential_configured: true, credential_revision: '"credential-a"', checked: false, + consent_active: false, fingerprint: 'fingerprint-routed', selected: false, + output_mode: 'strict_schema', allowed_sources: ['conversation_text'], source_since: '2025-01-01', + allow_sensitive: true, retention_posture: 'No retention', training_posture: 'No training', +}; +const status: PeopleInferenceSettingsResponse = { + stored_credentials_supported: true, profiles: [profile], configured_enabled: false, running_enabled: false, pending_restart: false, +}; +const preset = { + preset_id: 'openrouter' as const, model: 'model-one', allowed_sources: ['conversation_text'], + source_since: '2025-01-01', allow_sensitive: true, retention_posture: 'No retention', training_posture: 'No training', +}; +function fixture(initial = status, respond?: (request: Request) => Response | Promise | undefined) { + let current = structuredClone(initial); + const requests: Request[] = []; + const client = createAPIClient(vi.fn(async (input) => { + const request = input as Request; + requests.push(request.clone()); + const custom = await respond?.(request.clone()); + if (custom) return custom; + const path = new URL(request.url).pathname; + if (request.method === 'GET') return Response.json(current, { headers: { ETag: '"config-a"' } }); + if (path.endsWith('/check')) { + current = { ...current, profiles: current.profiles.map((item) => ({ ...item, checked: true })) }; + return Response.json({ ok: true, fingerprint: 'fingerprint-routed' }); + } + if (path.endsWith('/consent')) { + current = { ...current, profiles: current.profiles.map((item) => ({ ...item, consent_active: true })) }; + } else if (path.endsWith('/revoke')) { + current = { ...current, profiles: current.profiles.map((item) => ({ ...item, consent_active: false })) }; + } else if (path.endsWith('/select')) { + current = { ...current, configured_name: 'routed', configured_enabled: true, pending_restart: true }; + } else if (path.endsWith('/disable')) { + current = { ...current, configured_enabled: false, pending_restart: true }; + } else if (path.endsWith('/key')) { + current = { ...current, profiles: current.profiles.map((item) => ({ ...item, + credential_revision: '"credential-b"', credential_configured: true, checked: false, consent_active: false })) }; + } else if (request.method === 'DELETE') { + current = { ...current, profiles: current.profiles.filter((item) => !path.endsWith(`/${item.name}`)) }; + } else if (request.method === 'PUT' && path.endsWith('/routed')) { + current = { ...current, profiles: [{ ...profile, credential_configured: false }] }; + } else throw new Error(`Unexpected request: ${request.method} ${path}`); + return Response.json(current, { headers: { ETag: '"config-b"' } }); + })); + return { client, requests }; +} + +async function checkAndConsent(controller: PeopleInferenceController): Promise { + await controller.check(); + controller.disclosureConfirmed = true; + await controller.consent(); +} + +describe('PeopleInferenceSettings', () => { + it('hides stored-key enrollment on unsupported hosts while retaining host-managed profiles', async () => { + const { client, requests } = fixture({ ...status, stored_credentials_supported: false, + profiles: [{ ...profile, credential_source: 'env', credential_env: 'PEOPLE_API_KEY' }] }); + render(PeopleInferenceSettings, { client }); + await screen.findByText(/Configure an environment credential with/); + expect(screen.getByText('msgvault person provider add')).toBeDefined(); + expect(screen.getByText('--credential-env')).toBeDefined(); + expect(screen.getByText(/on the daemon host/)).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Create profile' })).toBeNull(); + expect(screen.queryByLabelText('API key')).toBeNull(); + expect(screen.queryByLabelText('Replacement API key')).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Check provider' })); + await screen.findByRole('region', { name: 'Archive disclosure' }); + expect(requests.some((request) => request.url.endsWith('/check'))).toBe(true); + expect(requests.some((request) => request.method === 'PUT')).toBe(false); + }); + + it('hides replacement keys for saved stored-key profiles on unsupported hosts', async () => { + const { client } = fixture({ ...status, stored_credentials_supported: false, + profiles: [{ ...profile, credential_configured: false, credential_revision: undefined }] }); + render(PeopleInferenceSettings, { client }); + await screen.findByText(/Configure an environment credential with/); + expect(screen.queryByLabelText('Replacement API key')).toBeNull(); + expect((screen.getByRole('button', { name: 'Check provider' }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('keeps a newly saved profile visible when saving its key fails and lets the user retry', async () => { + let failKey = true; + const { client, requests } = fixture({ ...status, profiles: [] }, (request) => { + if (request.url.endsWith('/key') && failKey) return Response.json({ message: 'Credential store unavailable' }, { status: 500 }); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await controller.create('routed', preset, 'synthetic-secret'); + expect(controller.error).toBe('Credential store unavailable'); + expect(controller.selectedProfile?.name).toBe('routed'); + expect(controller.selectedProfile?.credential_configured).toBe(false); + expect(requests[1]!.headers.get('If-Match')).toBe('"config-a"'); + expect(requests[2]!.headers.get('If-Match')).toBe('"credential-a"'); + await expect(requests[1]!.json()).resolves.toEqual(preset); + failKey = false; + await controller.saveKey('synthetic-replacement'); + expect(controller.error).toBe(''); + expect(controller.selectedProfile?.credential_configured).toBe(true); + }); + + it('requires the checked disclosure and consent before enabling, and shows saved versus running state', async () => { + const { client, requests } = fixture(); + render(PeopleInferenceSettings, { client }); + await screen.findByText('Stored key'); + expect((screen.getByRole('button', { name: 'Select and enable' }) as HTMLButtonElement).disabled).toBe(true); + await fireEvent.click(screen.getByRole('button', { name: 'Check provider' })); + await screen.findByText('Retention: No retention'); + expect((screen.getByRole('button', { name: 'Grant consent' }) as HTMLButtonElement).disabled).toBe(true); + await fireEvent.click(screen.getByLabelText('I confirm this exact disclosure')); + await fireEvent.click(screen.getByRole('button', { name: 'Grant consent' })); + await waitFor(() => expect((screen.getByRole('button', { name: 'Select and enable' }) as HTMLButtonElement).disabled).toBe(false)); + await fireEvent.click(screen.getByRole('button', { name: 'Select and enable' })); + expect((await screen.findByRole('status')).textContent).toContain('Restart the daemon'); + const consent = requests.find((request) => request.url.endsWith('/consent'))!; + await expect(consent.json()).resolves.toEqual({ fingerprint: 'fingerprint-routed', confirmed: true }); + expect(consent.headers.get('If-Match')).toBe('"config-a"'); + expect(requests.at(-1)!.headers.get('If-Match')).toBe('"config-b"'); + }); + + it('submits explicit archive policy with a write-only API key through the generated client', async () => { + const { client, requests } = fixture({ ...status, profiles: [] }); + render(PeopleInferenceSettings, { client }); + await screen.findByLabelText('Profile name'); + await fireEvent.click(screen.getByRole('combobox', { name: /^Provider:/ })); + await fireEvent.click(screen.getByRole('option', { name: 'OpenRouter' })); + await fireEvent.input(screen.getByLabelText('Profile name'), { target: { value: 'routed' } }); + await fireEvent.input(screen.getByLabelText('Model ID'), { target: { value: 'model-one' } }); + await fireEvent.input(screen.getByLabelText('API key'), { target: { value: 'synthetic-secret' } }); + await fireEvent.click(screen.getByLabelText('Conversation text')); + await fireEvent.input(screen.getByLabelText('Archive data since (YYYY-MM-DD)'), { target: { value: '2025-01-01' } }); + await fireEvent.input(screen.getByLabelText('Retention statement'), { target: { value: 'No retention' } }); + await fireEvent.input(screen.getByLabelText('Training statement'), { target: { value: 'No training' } }); + await fireEvent.click(screen.getByLabelText('Allow sensitive content')); + await fireEvent.click(screen.getByRole('button', { name: 'Create profile' })); + await screen.findByText('Stored key'); + await expect(requests[1]!.json()).resolves.toEqual(preset); + await expect(requests[2]!.json()).resolves.toEqual({ value: 'synthetic-secret' }); + expect((screen.getByLabelText('API key') as HTMLInputElement).value).toBe(''); + }); + + it('clears the checked disclosure when a repeat check fails and shows the daemon error', async () => { + let failCheck = false; + const { client } = fixture(status, (request) => { + if (request.url.endsWith('/check') && failCheck) return Response.json({ message: 'Provider rejected the model' }, { status: 422 }); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await checkAndConsent(controller); + expect(controller.canSelect).toBe(true); + failCheck = true; + await controller.check(); + expect(controller.canSelect).toBe(false); + expect(controller.checkedProfile).toBeUndefined(); + expect(controller.error).toBe('Provider rejected the model'); + }); + + it('requires renewed disclosure confirmation even when saved consent is active', async () => { + const { client } = fixture({ ...status, profiles: [{ ...profile, consent_active: true }] }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await controller.check(); + expect(controller.canSelect).toBe(false); + controller.disclosureConfirmed = true; + await controller.consent(); + expect(controller.canSelect).toBe(true); + }); + + it('rejects a check when the refreshed fingerprint differs', async () => { + let reads = 0; + const { client } = fixture(status, (request) => { + if (request.method === 'GET' && ++reads > 1) return Response.json({ ...status, + profiles: [{ ...profile, fingerprint: 'changed' }] }, { headers: { ETag: '"config-b"' } }); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await controller.check(); + expect(controller.checkedProfile).toBeUndefined(); + expect(controller.error).toContain('changed after the check'); + }); + + it('drops a disclosure when selection changes while the check is running', async () => { + let finish!: (response: Response) => void; + const checked = new Promise((resolve) => { finish = resolve; }); + const { client } = fixture({ ...status, profiles: [profile, { ...profile, name: 'spare' }] }, (request) => + request.url.endsWith('/check') ? checked : undefined); + const controller = new PeopleInferenceController(client); + await controller.load(); + const checking = controller.check(); + controller.choose('spare'); + finish(Response.json({ ok: true, fingerprint: 'fingerprint-routed' })); + await checking; + expect(controller.selectedName).toBe('spare'); + expect(controller.checkedProfile).toBeUndefined(); + }); + + it('requires reload after a config conflict or a missing ETag', async () => { + let missingETag = false; + const { client, requests } = fixture(status, (request) => { + if (request.method !== 'GET') return Response.json({ message: 'Stale settings' }, { status: 412 }); + if (missingETag) return Response.json(status); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await controller.disable(); + expect(controller.error).toContain('changed on disk'); + await controller.disable(); + expect(controller.error).toContain('Reload'); + expect(requests).toHaveLength(2); + missingETag = true; + await controller.load(); + expect(controller.error).toContain('config ETag'); + await controller.disable(); + expect(requests).toHaveLength(3); + }); + + it('uses credential revisions for key replacement and clears checked consent', async () => { + const { client, requests } = fixture(); + const controller = new PeopleInferenceController(client); + await controller.load(); + await checkAndConsent(controller); + await controller.saveKey('synthetic-secret'); + expect(controller.canSelect).toBe(false); + await controller.saveKey('synthetic-replacement'); + const keys = requests.filter((request) => request.url.endsWith('/key')); + expect(keys.map((request) => request.headers.get('If-Match'))).toEqual(['"credential-a"', '"credential-b"']); + }); + + it('requires reload after a credential conflict without invalidating config edits', async () => { + const { client, requests } = fixture(status, (request) => { + if (request.url.endsWith('/key')) return Response.json({ message: 'Stale key' }, { status: 412 }); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await controller.saveKey('synthetic-secret'); + expect(controller.error).toContain('credential changed'); + await controller.saveKey('synthetic-replacement'); + expect(requests).toHaveLength(2); + await controller.disable(); + expect(requests.at(-1)!.headers.get('If-Match')).toBe('"config-a"'); + }); + + it('does not enable a profile if consent is missing from the server response', async () => { + const { client } = fixture(status, (request) => { + if (request.url.endsWith('/consent')) return Response.json(status, { headers: { ETag: '"config-a"' } }); + }); + const controller = new PeopleInferenceController(client); + await controller.load(); + await checkAndConsent(controller); + expect(controller.canSelect).toBe(false); + expect(controller.error).toContain('Consent was not recorded'); + }); + + it('shows host-managed credentials and unavailable Codex profiles without actionable checks', async () => { + const { client } = fixture({ ...status, profiles: [{ ...profile, + credential_source: 'env', credential_env: 'PEOPLE_API_KEY', credential_configured: false }] }); + const rendered = render(PeopleInferenceSettings, { client }); + await screen.findByText('Set PEOPLE_API_KEY on daemon host'); + expect((screen.getByRole('button', { name: 'Check provider' }) as HTMLButtonElement).disabled).toBe(true); + rendered.unmount(); + const codex = fixture({ ...status, profiles: [{ ...profile, protocol: 'codex_app_server' }] }); + render(PeopleInferenceSettings, { client: codex.client }); + await screen.findByText('Codex is unavailable in this release'); + expect((screen.getByRole('button', { name: 'Check provider' }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('revokes consent, disables sweeps, and confirms removal when another profile remains', async () => { + const { client, requests } = fixture({ ...status, configured_name: 'routed', configured_enabled: true, + running_name: 'routed', running_enabled: true, + profiles: [{ ...profile, consent_active: true }, { ...profile, name: 'backup' }] }); + render(PeopleInferenceSettings, { client }); + await screen.findByText('Granted for this profile'); + await fireEvent.click(screen.getByRole('button', { name: 'Revoke consent' })); + await screen.findByText('Not granted'); + await fireEvent.click(screen.getByRole('button', { name: 'Disable people sweep' })); + await waitFor(() => expect((screen.getByRole('button', { name: 'Remove profile' }) as HTMLButtonElement).disabled).toBe(false)); + await fireEvent.click(screen.getByRole('button', { name: 'Remove profile' })); + expect(requests.some((request) => request.method === 'DELETE')).toBe(false); + await fireEvent.click(screen.getByRole('button', { name: 'Confirm removal' })); + await waitFor(() => expect(screen.getByRole('combobox', { name: /^Profile:/ }).textContent).toContain('backup')); + expect(requests.at(-1)!.method).toBe('DELETE'); + }); +}); diff --git a/web/src/lib/components/settings/SettingsWorkspace.svelte b/web/src/lib/components/settings/SettingsWorkspace.svelte index 88463ddfc..f9633bf73 100644 --- a/web/src/lib/components/settings/SettingsWorkspace.svelte +++ b/web/src/lib/components/settings/SettingsWorkspace.svelte @@ -47,6 +47,7 @@ } from '../../api/generated/models'; import type { CardDAVSettingsRequest, SettingsNavigationTarget } from '../../carddav/navigation'; import CardDAVSettingsWorkspace from './CardDAVSettingsWorkspace.svelte'; + import PeopleInferenceSettings from './PeopleInferenceSettings.svelte'; import CronField from './CronField.svelte'; import PersonEnrichmentProviderCard from './PersonEnrichmentProviderCard.svelte'; import PersonEnrichmentProviderCreator from './PersonEnrichmentProviderCreator.svelte'; @@ -109,6 +110,7 @@ const categories: SettingsCategory[] = $derived([ ...settingsGroups.map((group) => ({ id: group.id, label: group.label })), { id: 'carddav', label: 'CardDAV account' }, + { id: 'people', label: 'People sweep' }, ]); const dirtyCount = $derived(Object.keys(drafts).length + Object.keys(secretUpdates).length); // An emptied number field is a draft in progress, not a value: it keeps the @@ -612,7 +614,7 @@ {categories} bind:active={activeCategory} title="Settings" - footer={activeCategory === 'carddav' ? undefined : settingsFooter} + footer={activeCategory === 'carddav' || activeCategory === 'people' ? undefined : settingsFooter} > {#snippet panel(activeId)}
@@ -638,6 +640,8 @@ {onCardDAVRequestConsumed} onSettingsRefresh={() => loadSettings(true)} /> + {:else if activeId === 'people'} + {:else} {#each settingsGroups.filter((candidate) => candidate.id === activeId) as group (group.id)} {@const posture = restartPosture(group.settings)} diff --git a/web/src/lib/components/settings/SettingsWorkspace.test.ts b/web/src/lib/components/settings/SettingsWorkspace.test.ts index cc491e7c2..9ad59408e 100644 --- a/web/src/lib/components/settings/SettingsWorkspace.test.ts +++ b/web/src/lib/components/settings/SettingsWorkspace.test.ts @@ -49,6 +49,179 @@ const initialSettings = { afterEach(() => vi.useRealTimers()); describe('SettingsWorkspace', () => { + it('reads a host-configured environment profile and waits for daemon credentials before checking', async () => { + const requests: Request[] = []; + const created = true; + let configured = false; + const revision = 1; + const status = () => Response.json({ + stored_credentials_supported: true, + profiles: created ? [{ + name: 'from-env', preset_id: 'openrouter', protocol: 'openai-chat', model: 'model-one', + endpoint: 'https://openrouter.example.test/api/v1', credential_source: 'env', + credential_env: 'PEOPLE_API_KEY', credential_configured: configured, + checked: false, consent_active: false, fingerprint: 'env-fingerprint', selected: false, + output_mode: 'strict_schema', allowed_sources: ['conversation_text'], source_since: '2025-01-01', + allow_sensitive: false, retention_posture: 'Operator assertion: no retention', + training_posture: 'Operator assertion: no training', + }] : [], configured_enabled: false, running_enabled: false, pending_restart: created, + }, { headers: { ETag: `"env-config-${revision}"` } }); + const fetchFn = vi.fn(async (input) => { + const request = input as Request; + requests.push(request); + const path = new URL(request.url).pathname; + if (path === '/api/v1/settings') return settingsResponse(initialSettings, '"settings-a"'); + if (request.method === 'GET' && path === '/api/v1/settings/people-inference') return status(); + if (request.method === 'POST' && path.endsWith('/providers/from-env/check')) { + return Response.json({ ok: true, fingerprint: 'env-fingerprint', model: 'model-one', usage: {} }); + } + throw new Error(`Unexpected request: ${request.method} ${path}`); + }); + render(SettingsWorkspace, { client: createAPIClient(fetchFn) }); + await openSettingsCategory('People sweep'); + await screen.findByRole('heading', { name: 'Add a profile' }); + expect(await screen.findByText('Set PEOPLE_API_KEY on daemon host')).toBeDefined(); + expect(screen.queryByLabelText('Replacement API key')).toBeNull(); + expect((screen.getByRole('button', { name: 'Check provider' }) as HTMLButtonElement).disabled).toBe(true); + expect(requests.some((request) => request.url.endsWith('/providers/from-env/check'))).toBe(false); + configured = true; + await fireEvent.click(screen.getByRole('button', { name: 'Reload people sweep settings' })); + expect(await screen.findByText('Environment PEOPLE_API_KEY ready')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Check provider' })); + expect(await screen.findByText('Retention: Operator assertion: no retention')).toBeDefined(); + expect(requests.some((request) => request.method === 'PUT' && request.url.endsWith('/key'))).toBe(false); + expect(document.body.textContent).not.toContain('synthetic-secret'); + }, 15000); + + it('shows a failed people settings read without an actionable setup form', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input as Request; + if (new URL(request.url).pathname === '/api/v1/settings') return settingsResponse(initialSettings, '"settings-a"'); + return Response.json({ message: 'People inference unavailable on this daemon' }, { status: 404 }); + }); + render(SettingsWorkspace, { client: createAPIClient(fetchFn) }); + await openSettingsCategory('People sweep'); + expect(await screen.findByRole('alert')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Create profile' })).toBeNull(); + }); + + it('creates, checks, consents, and selects an HTTP profile through generated settings requests', async () => { + const requests: Request[] = []; + let revision = 1; + let stored = false; + let checked = false; + let consented = false; + let selected = false; + const profile = () => ({ + name: 'routed', preset_id: 'openrouter', protocol: 'openai-chat', model: 'model-one', + endpoint: 'https://openrouter.example.test/api/v1', credential_source: 'stored', + credential_configured: stored, credential_revision: '"credential-a"', + checked, consent_active: consented, fingerprint: stored ? 'fingerprint-key' : 'fingerprint-new', + selected, output_mode: 'strict_schema', allowed_sources: ['conversation_text'], + source_since: '2025-01-01', allow_sensitive: true, + retention_posture: 'No retention', training_posture: 'No training', + }); + const status = () => Response.json({ + stored_credentials_supported: true, + profiles: revision > 1 ? [profile()] : [], configured_enabled: selected, + configured_name: selected ? 'routed' : undefined, running_enabled: false, + pending_restart: selected, + }, { headers: { ETag: `"config-${revision}"` } }); + const fetchFn = vi.fn(async (input) => { + const request = input as Request; + requests.push(request); + const path = new URL(request.url).pathname; + if (path === '/api/v1/settings') return settingsResponse(initialSettings, '"settings-a"'); + if (path === '/api/v1/settings/people-inference') return status(); + if (request.method === 'PUT' && path === '/api/v1/settings/people-inference/providers/routed') { + revision += 1; + return status(); + } + if (request.method === 'PUT' && path.endsWith('/providers/routed/key')) { + stored = true; + return status(); + } + if (request.method === 'POST' && path.endsWith('/providers/routed/check')) { + checked = true; + return Response.json({ ok: true, fingerprint: 'fingerprint-key', model: 'model-one', usage: {} }); + } + if (request.method === 'POST' && path.endsWith('/providers/routed/consent')) { + consented = true; + revision += 1; + return status(); + } + if (request.method === 'POST' && path.endsWith('/people-inference/select')) { + selected = true; + revision += 1; + return status(); + } + throw new Error(`Unexpected request: ${request.method} ${path}`); + }); + render(SettingsWorkspace, { client: createAPIClient(fetchFn) }); + await openSettingsCategory('People sweep'); + await screen.findByRole('heading', { name: 'Add a profile' }); + await chooseSelectOption(screen.getByRole('combobox', { name: 'Provider: OpenAI Platform' }), 'OpenRouter'); + await fireEvent.input(screen.getByLabelText('Profile name'), { target: { value: 'routed' } }); + await fireEvent.input(screen.getByLabelText('Model ID'), { target: { value: 'model-one' } }); + await fireEvent.input(screen.getByLabelText('API key'), { target: { value: 'synthetic-secret' } }); + await fireEvent.click(screen.getByLabelText('Conversation text')); + await fireEvent.input(screen.getByLabelText(/^Archive data since/), { target: { value: '2025-01-01' } }); + await fireEvent.input(screen.getByLabelText('Retention statement'), { target: { value: 'No retention' } }); + await fireEvent.input(screen.getByLabelText('Training statement'), { target: { value: 'No training' } }); + await fireEvent.click(screen.getByLabelText('Allow sensitive content')); + await fireEvent.click(screen.getByRole('button', { name: 'Create profile' })); + expect(await screen.findByText('Stored key')).toBeDefined(); + await waitFor(() => expect((screen.getByRole('button', { name: 'Check provider' }) as HTMLButtonElement).disabled).toBe(false)); + await fireEvent.click(screen.getByRole('button', { name: 'Check provider' })); + expect(await screen.findByText('Retention: No retention')).toBeDefined(); + await fireEvent.click(screen.getByLabelText('I confirm this exact disclosure')); + await fireEvent.click(screen.getByRole('button', { name: 'Grant consent' })); + await waitFor(() => expect((screen.getByRole('button', { name: 'Select and enable' }) as HTMLButtonElement).disabled).toBe(false)); + await fireEvent.click(screen.getByRole('button', { name: 'Select and enable' })); + expect(await screen.findByText(/Restart the daemon to use the saved/)).toBeDefined(); + expect(document.body.textContent).not.toContain('synthetic-secret'); + const create = requests.find((item) => item.method === 'PUT' && item.url.endsWith('/providers/routed'))!; + expect(create.headers.get('If-Match')).toBe('"config-1"'); + await expect(create.clone().json()).resolves.toMatchObject({ preset_id: 'openrouter', + allowed_sources: ['conversation_text'], source_since: '2025-01-01', allow_sensitive: true }); + const key = requests.find((item) => item.url.endsWith('/providers/routed/key'))!; + expect(key.headers.get('If-Match')).toBe('"credential-a"'); + expect(requests.find((item) => item.url.endsWith('/providers/routed/check'))?.headers.get('If-Match')).toBe('"config-2"'); + expect(requests.find((item) => item.url.endsWith('/providers/routed/consent'))?.headers.get('If-Match')).toBe('"config-2"'); + expect(requests.find((item) => item.url.endsWith('/people-inference/select'))?.headers.get('If-Match')).toBe('"config-3"'); + }, 15000); + + it('opens server-backed people sweep setup with supported provider presets', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + if (path === '/api/v1/settings') return settingsResponse(initialSettings, '"settings-a"'); + if (path === '/api/v1/settings/people-inference') return Response.json({ + stored_credentials_supported: true, + profiles: [{ + name: 'routed', preset_id: 'openrouter', protocol: 'openai_chat', + model: 'model-one', endpoint: 'https://openrouter.example.test/api/v1', + credential_source: 'stored', credential_configured: true, + credential_revision: '"credential-a"', checked: false, consent_active: false, + fingerprint: 'fingerprint-routed', selected: false, output_mode: 'strict_schema', + retention_posture: 'Operator assertion: no retention', + training_posture: 'Operator assertion: no training', + allowed_sources: ['conversation_text'], source_since: '2025-01-01', allow_sensitive: true, + }], + configured_enabled: false, running_enabled: false, pending_restart: false, + }, { headers: { ETag: '"people-a"' } }); + throw new Error(`Unexpected request: ${request.method} ${path}`); + }); + render(SettingsWorkspace, { client: createAPIClient(fetchFn) }); + + await openSettingsCategory('People sweep'); + expect(await screen.findByRole('heading', { name: 'People sweep' })).toBeDefined(); + expect(await screen.findByText('Stored key')).toBeDefined(); + await fireEvent.click(screen.getByRole('combobox', { name: /^Provider:/ })); + expect(screen.getAllByRole('option').map((option) => option.textContent?.trim())).toEqual(['OpenAI Platform', 'OpenRouter', 'Venice']); + expect(fetchFn.mock.calls.some(([input]) => new URL((input as Request).url).pathname === '/api/v1/settings/people-inference')).toBe(true); + }); + it.each([ [{ authority: 'document_index', categoryID: 'archive', settingKey: 'analytics.auto_build_cache' }, 'Archive'], [{ authority: 'document_vector', categoryID: 'search', settingKey: 'vector.enabled' }, 'Search'], diff --git a/web/src/lib/settings/people-inference-controller.svelte.ts b/web/src/lib/settings/people-inference-controller.svelte.ts new file mode 100644 index 000000000..34c351f72 --- /dev/null +++ b/web/src/lib/settings/people-inference-controller.svelte.ts @@ -0,0 +1,214 @@ +import type { APIClient } from '../api/client'; +import type { APIResponse } from '../api/runtime'; +import { + checkSettingsPeopleInferenceProvider, + consentSettingsPeopleInferenceProvider, + deleteSettingsPeopleInferenceProvider, + disableSettingsPeopleInference, + getSettingsPeopleInference, + putSettingsPeopleInferenceKey, + putSettingsPeopleInferencePreset, + revokeSettingsPeopleInferenceProvider, + selectSettingsPeopleInference, +} from '../api/generated/api/api'; +import type { + PeopleInferencePresetCreateRequest, + PeopleInferenceProfileSetting, + PeopleInferenceSettingsResponse, +} from '../api/generated/models'; + +export class PeopleInferenceController { + status = $state(); + selectedName = $state(''); + loading = $state(true); + busy = $state(false); + error = $state(''); + checkedProfile = $state(); + disclosureConfirmed = $state(false); + private configETag = ''; + private consentedFingerprint = $state(''); + private destroyed = false; + private selectionEpoch = 0; + + constructor(private readonly client: APIClient) {} + + get selectedProfile(): PeopleInferenceProfileSetting | undefined { + return this.status?.profiles.find((profile) => profile.name === this.selectedName); + } + + get canConsent(): boolean { + return Boolean(this.disclosureConfirmed && this.checkedProfile?.fingerprint && + this.checkedProfile.fingerprint === this.selectedProfile?.fingerprint); + } + + get canSelect(): boolean { + return Boolean(this.checkedProfile?.fingerprint && + this.checkedProfile.fingerprint === this.selectedProfile?.fingerprint && + this.selectedProfile?.consent_active && this.consentedFingerprint === this.checkedProfile.fingerprint); + } + + async load(): Promise { + this.loading = true; + this.clearCheck(); + await this.run(async () => { + this.capture(await getSettingsPeopleInference(this.client)); + if (!this.status?.profiles.some((profile) => profile.name === this.selectedName)) { + this.choose(this.status?.configured_name ?? this.status?.profiles[0]?.name ?? ''); + } + }); + if (!this.destroyed) this.loading = false; + } + + choose(name: string): void { + this.selectionEpoch += 1; + this.selectedName = name; + this.clearCheck(); + } + + async create(name: string, request: PeopleInferencePresetCreateRequest, key: string): Promise { + await this.run(async () => { + this.capture(await putSettingsPeopleInferencePreset({ name }, request, this.configOptions())); + if (this.destroyed) return; + // Creating a profile and storing its key are separate writes. Keep the + // saved profile visible so a failed key write can be retried in place. + this.choose(name); + if (key) await this.writeKey(name, key); + }); + } + + async saveKey(key: string): Promise { + if (!this.selectedName || !key) return; + this.clearCheck(); + await this.run(() => this.writeKey(this.selectedName, key)); + } + + private async writeKey(name: string, key: string): Promise { + const profile = this.status?.profiles.find((item) => item.name === name); + if (!profile?.credential_revision) throw new Error('Reload people sweep settings before replacing the key.'); + const result = await putSettingsPeopleInferenceKey({ name }, { value: key }, { + ...this.client, headers: { 'If-Match': profile.credential_revision }, + }); + if (result.response.status === 412) { + profile.credential_revision = undefined; + throw new Error('People provider credential changed. Reload people sweep settings before replacing its key.'); + } + this.capture(result); + } + + async check(): Promise { + const profile = this.selectedProfile; + const selectionEpoch = this.selectionEpoch; + if (!profile) return; + this.clearCheck(); + await this.run(async () => { + const result = this.read(await checkSettingsPeopleInferenceProvider({ name: profile.name }, this.configOptions())); + const refreshed = await getSettingsPeopleInference(this.client); + this.capture(refreshed); + if (this.destroyed || this.selectionEpoch !== selectionEpoch) return; + if (!result.ok || !result.fingerprint || result.fingerprint !== profile.fingerprint || + result.fingerprint !== this.selectedProfile?.fingerprint) { + throw new Error('This profile changed after the check. Reload and check again.'); + } + this.checkedProfile = this.selectedProfile; + }); + } + + async consent(): Promise { + if (!this.canConsent) return; + const name = this.selectedName; + const fingerprint = this.checkedProfile!.fingerprint!; + await this.run(async () => { + this.capture(await consentSettingsPeopleInferenceProvider({ name }, { + fingerprint, confirmed: true, + }, this.configOptions())); + const profile = this.status?.profiles.find((item) => item.name === name); + if (profile?.fingerprint !== fingerprint || !profile.consent_active) { + this.clearCheck(); + throw new Error('Consent was not recorded for the checked profile. Reload and try again.'); + } + this.consentedFingerprint = fingerprint; + }); + } + + async select(): Promise { + if (!this.canSelect) return; + await this.run(async () => { + this.capture(await selectSettingsPeopleInference({ name: this.selectedName }, this.configOptions())); + }); + } + + async revoke(): Promise { + if (!this.selectedProfile?.consent_active) return; + this.clearCheck(); + await this.run(async () => { + this.capture(await revokeSettingsPeopleInferenceProvider({ name: this.selectedName }, this.configOptions())); + }); + } + + async disable(): Promise { + this.clearCheck(); + await this.run(async () => { + this.capture(await disableSettingsPeopleInference(this.configOptions())); + }); + } + + async remove(name: string): Promise { + this.clearCheck(); + await this.run(async () => { + this.capture(await deleteSettingsPeopleInferenceProvider({ name }, this.configOptions())); + if (!this.destroyed) this.choose(this.status?.profiles[0]?.name ?? ''); + }); + } + + destroy(): void { this.destroyed = true; } + + private clearCheck(): void { + this.checkedProfile = undefined; + this.disclosureConfirmed = false; + this.consentedFingerprint = ''; + } + + private configOptions() { + if (!this.configETag) throw new Error('Reload people sweep settings before editing.'); + return { ...this.client, headers: { 'If-Match': this.configETag } }; + } + + private read(result: APIResponse): T { + if (result.response.status === 412) { + this.configETag = ''; + this.clearCheck(); + throw new Error('People sweep settings changed on disk. Reload and review your changes before trying again.'); + } + if (!result.data) throw new Error(result.error?.message || 'Unable to update people sweep settings.'); + return result.data; + } + + private capture(result: APIResponse): void { + const status = this.read(result); + if (this.destroyed) return; + this.status = status; + this.configETag = result.response.headers.get('ETag') ?? ''; + if (!this.configETag) throw new Error('People sweep settings response omitted its config ETag. Reload before editing.'); + } + + private async run(action: () => Promise): Promise { + if (this.busy) return; + this.busy = true; + this.error = ''; + try { await action(); } + catch (cause) { + if (!this.destroyed) this.error = cause instanceof Error ? cause.message : 'Unable to update people sweep settings.'; + } finally { + if (!this.destroyed) this.busy = false; + } + } +} + +export function credentialStatus(profile: PeopleInferenceProfileSetting): string { + if (profile.protocol === 'codex_app_server') return 'Codex is unavailable in this release'; + if (profile.credential_source === 'env') { + const variable = profile.credential_env || 'the provider variable'; + return profile.credential_configured ? `Environment ${variable} ready` : `Set ${variable} on daemon host`; + } + return profile.credential_configured ? 'Stored key' : 'Key needed'; +} diff --git a/web/tests/people-inference-viewport.spec.ts b/web/tests/people-inference-viewport.spec.ts new file mode 100644 index 000000000..70eca2972 --- /dev/null +++ b/web/tests/people-inference-viewport.spec.ts @@ -0,0 +1,94 @@ +import { expect, test } from '@playwright/test'; +import { installMixedArchive } from './e2e/fixtures/mixed-archive'; + +test('People sweep keeps removal unavailable until another profile exists', async ({ page }) => { + await installMixedArchive(page); + const profile = { + name: 'router', protocol: 'openai_chat', model: 'model-one', + credential_source: 'env', credential_configured: true, + checked: true, consent_active: false, fingerprint: 'router-fingerprint', + }; + const status = { + profiles: [profile], configured_name: 'router', configured_enabled: true, + running_enabled: false, pending_restart: false, stored_credentials_supported: true, + }; + let removals = 0; + await page.route('**/api/v1/settings/people-inference', (route) => route.fulfill({ + headers: { ETag: '"config-1"' }, json: status, + })); + await page.route('**/api/v1/settings/people-inference/disable', (route) => { + status.configured_enabled = false; + return route.fulfill({ headers: { ETag: '"config-2"' }, json: status }); + }); + await page.route('**/api/v1/settings/people-inference/providers/router', (route) => { + expect(route.request().method()).toBe('DELETE'); + removals += 1; + status.profiles = status.profiles.filter((item) => item.name !== 'router'); + status.configured_name = 'backup'; + return route.fulfill({ headers: { ETag: '"config-3"' }, json: status }); + }); + await page.goto(`/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'settings' }))}`); + await page.getByRole('button', { name: /^People sweep/ }).click(); + const remove = page.getByRole('button', { name: 'Remove profile', exact: true }); + await expect(remove).toBeDisabled(); + await page.getByRole('button', { name: 'Disable people sweep', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Disable people sweep', exact: true })).toHaveCount(0); + await expect(remove).toBeDisabled(); + await expect(page.getByText('Add another profile before removing this one.')).toBeVisible(); + expect(removals).toBe(0); + + status.profiles.push({ ...profile, name: 'backup', fingerprint: 'backup-fingerprint' }); + await page.getByRole('button', { name: 'Reload people sweep settings' }).click(); + await expect(remove).toBeEnabled(); + await remove.click(); + await page.getByRole('button', { name: 'Confirm removal' }).click(); + await expect(page.getByText('Add another profile before removing this one.')).toBeVisible(); + await expect(remove).toBeDisabled(); + expect(removals).toBe(1); +}); + +for (const width of [390, 320]) { + test(`People sweep setup fits a ${width}px viewport with disclosure open`, async ({ page }) => { + await installMixedArchive(page); + const name = 'routed-profile-with-a-long-visible-name'; + const profile = { + name, preset_id: 'openrouter', protocol: 'openai-chat', model: 'model-one', + endpoint: 'https://openrouter.example.test/a/long/path/to/the/selected/endpoint/for/this/model', + credential_source: 'stored', credential_configured: true, checked: true, + consent_active: false, fingerprint: 'synthetic-fingerprint', + allowed_sources: ['conversation_text', 'meeting_text'], source_since: '2025-01-01', + allow_sensitive: false, retention_posture: 'No retention', training_posture: 'No training', + }; + await page.route('**/api/v1/settings/people-inference', (route) => route.fulfill({ + headers: { ETag: '"config-a"' }, json: { + stored_credentials_supported: true, + profiles: [profile, { ...profile, name: 'spare-profile', fingerprint: 'spare-fingerprint' }], + configured_name: name, running_name: 'previous-profile', + configured_enabled: true, running_enabled: true, pending_restart: true, + }, + })); + await page.route('**/api/v1/settings/people-inference/providers/*/check', (route) => route.fulfill({ + json: { ok: true, fingerprint: 'synthetic-fingerprint' }, + })); + await page.setViewportSize({ width, height: 844 }); + await page.goto(`/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'settings' }))}`); + await page.getByRole('button', { name: /^People sweep/ }).click(); + await expect(page.getByRole('heading', { name: 'People sweep' })).toBeVisible(); + await page.getByRole('button', { name: 'Check provider' }).click(); + await expect(page.getByRole('region', { name: 'Archive disclosure' })).toBeVisible(); + + const overflow = await page.evaluate(() => { + const component = document.querySelector('[aria-label="People sweep settings"]')!; + return { viewport: window.innerWidth, document: document.documentElement.scrollWidth, + component: component.scrollWidth, componentWidth: component.clientWidth }; + }); + expect(overflow.document).toBeLessThanOrEqual(overflow.viewport); + expect(overflow.component).toBeLessThanOrEqual(overflow.componentWidth); + await expect(page.getByRole('button', { name: 'Select and enable' })).toBeDisabled(); + await page.getByRole('combobox', { name: /^Profile:/ }).click(); + await page.getByRole('option', { name: 'spare-profile' }).click(); + await page.getByRole('button', { name: 'Remove profile' }).click(); + await expect(page.getByRole('button', { name: 'Confirm removal' })).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); + }); +} From b377dbf6a9fdb7d3b54beb4ed2c908704abbcbbd Mon Sep 17 00:00:00 2001 From: Chad Metcalf Date: Mon, 28 Sep 2026 09:27:52 -0700 Subject: [PATCH 069/125] fix(gmail): recover from quota throttling in the shared client (#969) Gmail syncs now recover when a quota pause outlasts the request timeout. The shared Gmail client waits under the caller's context and starts a fresh request budget, with at most five quota retries. This covers profile, labels, history, message listings, and raw downloads. Quota pauses honor `Retry-After` seconds or HTTP dates, with minimums of 30 seconds for 429 responses and 60 seconds for quota 403 responses. Cancellation and earlier caller deadlines still stop the operation. Ordinary network and server retries retain one 30-second budget, or five minutes for raw downloads. Draft writes are not replayed. Deadline errors preserve the last Gmail response and its reason. The configuration and troubleshooting guides explain Gmail's local quota budget, its cap, and the shared setting's effect on Teams imports. Quota costs and defaults are unchanged. Refs #803 and #806. Co-authored-by: Chad Metcalf --- docs/configuration.md | 2 +- docs/troubleshooting.md | 14 +- internal/gmail/client.go | 128 +++++++++-- internal/gmail/client_deadline_test.go | 285 +++++++++++++++++++++++-- 4 files changed, 394 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 92e32c381..c73f63d7e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -565,7 +565,7 @@ Use `msgvault logs` to view and tail log files from the selected local or remote | Key | Default | Description | |---|---|---| -| `rate_limit_qps` | `5` | Gmail API requests per second | +| `rate_limit_qps` | `5` | Scales Gmail's local quota-unit refill rate: `5` allows 250 units/second; `3` allows 150. Gmail values above `5` are capped at `5`. Also sets Microsoft Teams Graph requests/second, without that cap, so lowering it slows Teams imports too. Reduce it if Gmail reports quota errors; Google's [project quotas](https://developers.google.com/workspace/gmail/api/reference/quota) can be lower than this local budget. | | `archive_remote_images` | `false` | Download remote email images during Gmail/IMAP sync and EML, EMLX, MBOX, and PST imports | | `trusted_imap_sent_mailboxes` | `{}` | Per-IMAP-account Sent-folder names (keyed by the ACCOUNT identifier from `msgvault list-accounts`) that enable edited-copy snapshot refresh for servers without advertised special-use roles | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d598cc521..388cee3ea 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -228,9 +228,17 @@ docker exec msgvault msgvault add-account you@gmail.com ## Rate Limiting -If you hit Gmail API rate limits during large syncs: - -1. Reduce `rate_limit_qps` in config (default: 5) +If you hit Gmail API rate limits during large syncs, the sync error names +the refusal, for example `quota exceeded (403): rateLimitExceeded; Quota +exceeded for quota metric 'Total Query Cost' and limit 'Units per minute per +user'`. The Gmail client waits out each quota pause and retries up to five +times. This applies to profile, labels, history, message listings, and raw +downloads; draft writes are not replayed. Caller cancellation or an earlier +deadline stops the wait. A failed full sync resumes from its checkpoint. + +1. Reduce [`rate_limit_qps`](configuration.md#sync) in config (default: `5`). + The local budget can exceed your project's Gmail quota. Lowering this + setting also slows Teams imports. 2. Use `--limit` during initial testing 3. Wait and retry. Rate limits reset over time diff --git a/internal/gmail/client.go b/internal/gmail/client.go index 0ca84c718..9d1adf127 100644 --- a/internal/gmail/client.go +++ b/internal/gmail/client.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "log/slog" + "math" "math/rand" "net/http" "net/http/httptrace" @@ -18,15 +19,18 @@ import ( "sync/atomic" "time" + "go.kenn.io/msgvault/internal/httpretry" + "golang.org/x/oauth2" "golang.org/x/sync/errgroup" ) const ( - baseURL = "https://gmail.googleapis.com/gmail/v1" - maxRetries = 12 // Upper bound; the request deadline also limits retries - maxBackoff = 600 // Max backoff in seconds - defaultTimeout = 30 * time.Second + baseURL = "https://gmail.googleapis.com/gmail/v1" + maxRetries = 12 // Upper bound; the request deadline also limits retries + maxQuotaRetries = 5 // Quota waits use the caller's context, outside the request budget + maxBackoff = 600 // Max backoff in seconds + defaultTimeout = 30 * time.Second // Raw MIME includes attachments and needs more time on slow connections. rawRequestTimeout = 5 * time.Minute ) @@ -97,12 +101,35 @@ func (c *Client) Close() error { // request makes an HTTP request with rate limiting and retry logic. // bodyBytes can be nil for requests without a body. func (c *Client) request(ctx context.Context, op Operation, method, path string, bodyBytes []byte) ([]byte, error) { - // Quota pauses can exceed the request timeout. Wait using the caller's - // context so a previous request's throttle does not exhaust this budget. - if err := c.rateLimiter.Acquire(ctx, op); err != nil { - return nil, fmt.Errorf("rate limit: %w", err) + var lastErr error + for quotaRetries := 0; ; quotaRetries++ { + // Quota pauses can exceed the request timeout. Wait under the caller's + // context, then start a fresh HTTP/retry budget after tokens are available. + if err := c.rateLimiter.Acquire(ctx, op); err != nil { + return nil, fmt.Errorf("rate limit: %w", retryBudgetError(err, quotaRetries, lastErr)) + } + data, err := c.requestWithRetryBudget(ctx, op, method, path, bodyBytes, lastErr) + // A deadline can carry an earlier quota response for diagnostics; + // only a fresh throttle response starts another quota retry. + if _, throttled := errors.AsType[*ThrottledError](err); !throttled || + errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return data, err + } + if ctx.Err() != nil { + return nil, retryBudgetError(ctx.Err(), quotaRetries+1, err) + } + if quotaRetries >= maxQuotaRetries { + return nil, fmt.Errorf("quota retries exhausted after %d retries: %w", quotaRetries, err) + } + lastErr = err + c.logger.Info("Gmail throttled request; retrying after quota pause", + "path", path, "attempt", quotaRetries+1, "max", maxQuotaRetries, "error", err) } +} +// requestWithRetryBudget retries transient failures within one I/O budget. +// A quota response ends this budget so request can wait out the shared pause. +func (c *Client) requestWithRetryBudget(ctx context.Context, op Operation, method, path string, bodyBytes []byte, lastErr error) ([]byte, error) { // Share one budget across HTTP I/O and retry backoff after acquiring tokens. // It bounds the Gmail request path once a token is available; tokens are // fetched beforehand via the contextless TokenSource.Token(), so sources @@ -118,7 +145,6 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, reqURL := baseURL + path - var lastErr error remoteMutation := op.remoteMutation() for attempt := 0; attempt <= maxRetries; attempt++ { if attempt > 0 { @@ -127,9 +153,12 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, retryBudgetError(ctx.Err(), attempt, lastErr) case <-time.After(backoff): } + if err := c.rateLimiter.Acquire(ctx, op); err != nil { + return nil, retryBudgetError(err, attempt, lastErr) + } } // Create a new reader for each attempt to ensure body can be re-read on retry @@ -171,6 +200,11 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, } return nil, fmt.Errorf("%w: http request: %w", errWriteOutcomeUnknown, err) } + if ctx.Err() != nil { + // The budget expired mid-request; report the response that + // drove the retries rather than this interrupted attempt. + return nil, retryBudgetError(ctx.Err(), attempt+1, lastErr) + } lastErr = fmt.Errorf("http request: %w", err) continue // Retry on network errors } @@ -181,6 +215,9 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, if remoteMutation { return nil, fmt.Errorf("%w: read response: %w", errWriteOutcomeUnknown, err) } + if ctx.Err() != nil { + return nil, retryBudgetError(ctx.Err(), attempt+1, lastErr) + } lastErr = fmt.Errorf("read response: %w", err) continue } @@ -195,28 +232,31 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, case http.StatusTooManyRequests: // Rate limited // Log at Debug level since rate limiting is expected during high-volume syncs // and the retry logic handles it automatically - c.logger.Debug("rate limited, backing off 30s", "path", path, "attempt", attempt) + detail := gmailErrorDetail(resp.Header, respBody) + // The caller's context bounds the wait; don't shorten Gmail's Retry-After. + pause := max(30*time.Second, httpretry.RetryAfter(resp.Header.Get("Retry-After"), 0, math.MaxInt64)) + c.logger.Debug("rate limited, backing off", "pause", pause, "path", path, "attempt", attempt, "detail", detail) // Throttle the rate limiter to back off - c.rateLimiter.Throttle(30 * time.Second) + c.rateLimiter.Throttle(pause) if remoteMutation { return nil, newStatusError(resp.StatusCode, respBody) } - lastErr = errors.New("rate limited (429)") - continue + return nil, &ThrottledError{Summary: "rate limited (429)", Detail: detail} case http.StatusForbidden: // Could be rate limit or permission error // Gmail returns 403 for quota exceeded with "rateLimitExceeded" reason if isRateLimitError(respBody) { // Log at Debug level since quota throttling is expected during high-volume syncs // and the retry logic handles it automatically - c.logger.Debug("quota exceeded, backing off 60s", "path", path, "attempt", attempt) + detail := gmailErrorDetail(resp.Header, respBody) + pause := max(time.Minute, httpretry.RetryAfter(resp.Header.Get("Retry-After"), 0, math.MaxInt64)) + c.logger.Debug("quota exceeded, backing off", "pause", pause, "path", path, "attempt", attempt, "detail", detail) // Throttle the rate limiter - quota errors need longer backoff - c.rateLimiter.Throttle(60 * time.Second) + c.rateLimiter.Throttle(pause) if remoteMutation { return nil, newStatusError(resp.StatusCode, respBody) } - lastErr = errors.New("quota exceeded (403)") - continue // Retry with backoff + return nil, &ThrottledError{Summary: "quota exceeded (403)", Detail: detail} } // Actual permission error - don't retry return nil, newStatusError(resp.StatusCode, respBody) @@ -243,6 +283,58 @@ func (c *Client) request(ctx context.Context, op Operation, method, path string, return nil, fmt.Errorf("max retries exceeded: %w", lastErr) } +// retryBudgetError reports why a request ran out of retry budget. The context +// error stays in the chain for errors.Is checks; the last upstream response is +// attached so callers see Gmail's reason instead of only the deadline. +func retryBudgetError(ctxErr error, attempts int, lastErr error) error { + if lastErr == nil || errors.Is(lastErr, ctxErr) { + return ctxErr + } + return fmt.Errorf("%w after %d attempt(s); last response: %w", ctxErr, attempts, lastErr) +} + +// ThrottledError is a Gmail 429 or quota 403 response. The client retries it +// after the shared quota pause, up to maxQuotaRetries times. +type ThrottledError struct { + Summary string // e.g. "quota exceeded (403)" + Detail string // Gmail's stated reason, message and Retry-After, if any +} + +func (e *ThrottledError) Error() string { + if e.Detail == "" { + return e.Summary + } + return e.Summary + ": " + e.Detail +} + +// gmailErrorDetail extracts the reason from a Gmail API error response so a +// quota refusal says why Google refused the call. It reads the standard +// {"error":{"message","errors":[{"reason"}]}} shape and reports a Retry-After +// header; an unfamiliar body contributes nothing. +func gmailErrorDetail(header http.Header, body []byte) string { + var parsed struct { + Error struct { + Message string `json:"message"` + Errors []struct { + Reason string `json:"reason"` + } `json:"errors"` + } `json:"error"` + } + var parts []string + if err := json.Unmarshal(body, &parsed); err == nil { + if len(parsed.Error.Errors) > 0 && parsed.Error.Errors[0].Reason != "" { + parts = append(parts, parsed.Error.Errors[0].Reason) + } + if parsed.Error.Message != "" { + parts = append(parts, parsed.Error.Message) + } + } + if retryAfter := header.Get("Retry-After"); retryAfter != "" { + parts = append(parts, "Retry-After "+retryAfter) + } + return strings.Join(parts, "; ") +} + func newStatusError(statusCode int, body []byte) *StatusError { var msg string switch statusCode { diff --git a/internal/gmail/client_deadline_test.go b/internal/gmail/client_deadline_test.go index 40bf9e0ef..27a4629ba 100644 --- a/internal/gmail/client_deadline_test.go +++ b/internal/gmail/client_deadline_test.go @@ -2,6 +2,7 @@ package gmail import ( "context" + "errors" "io" "net/http" "strings" @@ -72,7 +73,7 @@ func TestGetProfileRetryDeadline(t *testing.T) { return nil, r.Context().Err() case <-time.After(3 * time.Second): } - return deadlineResponse(http.StatusTooManyRequests, `{"error":{"code":429}}`), nil + return deadlineResponse(http.StatusServiceUnavailable, `{"error":{"code":503}}`), nil }) profile, err := c.GetProfile(ctx) require.ErrorIs(err, context.DeadlineExceeded) @@ -246,26 +247,141 @@ func TestGetProfileCancellationInterruptsRetry(t *testing.T) { }) } -func TestGetProfileRecoversWithinDeadline(t *testing.T) { +func TestReadRequestsRecoverAfterQuotaPause(t *testing.T) { + for _, op := range []struct { + name string + call func(*Client, context.Context) error + }{ + {"profile", func(c *Client, ctx context.Context) error { _, err := c.GetProfile(ctx); return err }}, + {"labels", func(c *Client, ctx context.Context) error { _, err := c.ListLabels(ctx); return err }}, + {"messages", func(c *Client, ctx context.Context) error { _, err := c.ListMessages(ctx, "", "page-2"); return err }}, + {"snapshot", func(c *Client, ctx context.Context) error { + _, err := c.ListCompleteMessageSnapshot(ctx, "page-2") + return err + }}, + {"history", func(c *Client, ctx context.Context) error { _, err := c.ListHistory(ctx, 42, "page-2"); return err }}, + } { + for _, quota := range []struct { + status int + pause time.Duration + }{ + {http.StatusForbidden, time.Minute}, + {http.StatusTooManyRequests, 30 * time.Second}, + } { + t.Run(op.name+"/"+http.StatusText(quota.status), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + requests := 0 + var firstURL string + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + if requests == 1 { + firstURL = r.URL.String() + return deadlineResponse(quota.status, `{"error":{"errors":[{"reason":"rateLimitExceeded"}]}}`), nil + } + assert.Equal(t, firstURL, r.URL.String(), "retry the same page and query") + assert.GreaterOrEqual(t, time.Since(start), quota.pause, "do not send requests during the quota pause") + deadline, ok := r.Context().Deadline() + require.True(t, ok) + assert.Equal(t, 30*time.Second, time.Until(deadline), "start a fresh budget after the pause") + select { + case <-r.Context().Done(): + return nil, r.Context().Err() + case <-time.After(20 * time.Second): + } + return deadlineResponse(http.StatusOK, `{}`), nil + }) + require.NoError(t, op.call(c, context.Background())) + assert.Equal(t, 2, requests) + }) + }) + } + } +} + +func TestGetProfileHonorsRetryAfter(t *testing.T) { + for _, status := range []int{http.StatusForbidden, http.StatusTooManyRequests} { + for _, tc := range []struct { + name, header string + httpDate bool + pause time.Duration + }{ + {name: "seconds", header: "1800", pause: 30 * time.Minute}, + {name: "HTTP date", httpDate: true, pause: 30 * time.Minute}, + {name: "short", header: "1"}, + {name: "invalid", header: "invalid"}, + } { + t.Run(http.StatusText(status)+"/"+tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() + pause := 30 * time.Second + if status == http.StatusForbidden { + pause = time.Minute + } + pause = max(pause, tc.pause) + requests := 0 + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + if requests == 1 { + resp := deadlineResponse(status, `{"error":{"errors":[{"reason":"rateLimitExceeded"}]}}`) + resp.Header.Set("Retry-After", tc.header) + if tc.httpDate { + resp.Header.Set("Retry-After", start.Add(tc.pause).UTC().Format(http.TimeFormat)) + } + return resp, nil + } + assert.GreaterOrEqual(t, time.Since(start), pause, "wait for both Retry-After and the minimum quota pause") + return deadlineResponse(http.StatusOK, `{}`), nil + }) + _, err := c.GetProfile(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, requests) + }) + }) + } + } +} + +func TestGetProfileCallerDeadlineInterruptsRetryAfter(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + requests := 0 + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + resp := deadlineResponse(http.StatusTooManyRequests, `{}`) + resp.Header.Set("Retry-After", "600") + return resp, nil + }) + start := time.Now() + _, err := c.GetProfile(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, time.Minute, time.Since(start)) + assert.Equal(t, 1, requests, "the caller deadline must stop the wait before another request") + }) +} + +func TestGetProfileQuotaRetriesAreBounded(t *testing.T) { synctest.Test(t, func(t *testing.T) { assert := assert.New(t) require := require.New(t) requests := 0 + var lastRequest time.Time c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { requests++ - if requests == 1 { - return deadlineResponse(http.StatusTooManyRequests, ""), nil + if !lastRequest.IsZero() { + assert.GreaterOrEqual(time.Since(lastRequest), 30*time.Second) } - return deadlineResponse(http.StatusOK, `{"emailAddress":"user@example.com","historyId":"42"}`), nil + lastRequest = time.Now() + return deadlineResponse(http.StatusTooManyRequests, `{"error":{"message":"Too many concurrent requests"}}`), nil }) - start := time.Now() - profile, err := c.GetProfile(context.Background()) - require.NoError(err) - require.NotNil(profile) - assert.Equal("user@example.com", profile.EmailAddress) - assert.Equal(uint64(42), profile.HistoryID) - assert.Equal(2, requests) - assert.Less(time.Since(start), 30*time.Second) + _, err := c.GetProfile(context.Background()) + require.Error(err) + throttled, ok := errors.AsType[*ThrottledError](err) + require.True(ok) + assert.Equal("Too many concurrent requests", throttled.Detail) + assert.Equal(6, requests, "one attempt and five quota retries") + assert.NotErrorIs(err, context.DeadlineExceeded, "quota exhaustion is not a context deadline") }) } @@ -359,3 +475,146 @@ func TestGetDraftAllowsSlowRawTransfer(t *testing.T) { assert.Equal(t, []byte("test"), draft.Message.Raw) }) } + +func TestListMessagesDeadlineReportsLastResponse(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + requests := 0 + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + // The caller's deadline expires during the following quota pause. + select { + case <-r.Context().Done(): + return nil, r.Context().Err() + case <-time.After(3 * time.Second): + } + resp := deadlineResponse(http.StatusTooManyRequests, + `{"error":{"code":429,"message":"User-rate limit exceeded. Retry after 2026-09-26T22:23:39Z",`+ + `"errors":[{"reason":"rateLimitExceeded","domain":"usageLimits"}],"status":"RESOURCE_EXHAUSTED"}}`) + resp.Header.Set("Retry-After", "7") + return resp, nil + }) + list, err := c.ListMessages(ctx, "", "") + require.ErrorIs(err, context.DeadlineExceeded, "the deadline must stay matchable") + assert.Nil(list) + assert.Equal(1, requests, "caller deadline interrupts the quota pause before another request") + _, throttled := errors.AsType[*ThrottledError](err) + assert.True(throttled, "callers must be able to recognise the throttle") + assert.Contains(err.Error(), "context deadline exceeded after ") + assert.Contains(err.Error(), "last response: rate limited (429): rateLimitExceeded; "+ + "User-rate limit exceeded. Retry after 2026-09-26T22:23:39Z; Retry-After 7") + }) +} + +func TestGetProfileBodyDeadlinePreservesLastResponse(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + requests := 0 + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + if requests == 1 { + return deadlineResponse(http.StatusServiceUnavailable, ""), nil + } + return &http.Response{StatusCode: http.StatusOK, Body: &deadlineBody{ctx: r.Context()}}, nil + }) + _, err := c.GetProfile(context.Background()) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Contains(t, err.Error(), "last response: server error (503)") + assert.Equal(t, 2, requests) + }) +} + +func TestGetProfileRetryHonorsSharedThrottle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + requests := 0 + var c *Client + c = newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + // Another request can pause the shared limiter during this retry budget. + c.rateLimiter.Throttle(time.Minute) + return deadlineResponse(http.StatusServiceUnavailable, ""), nil + }) + start := time.Now() + _, err := c.GetProfile(context.Background()) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Contains(err.Error(), "last response: server error (503)") + assert.Equal(1, requests) + assert.Equal(30*time.Second, time.Since(start)) + }) +} + +func TestGetProfileDeadlineAfterQuotaPausePreservesLastResponse(t *testing.T) { + for _, bodyRead := range []bool{false, true} { + name := "transport" + if bodyRead { + name = "body read" + } + t.Run(name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + requests := 0 + var retryStart time.Time + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + requests++ + if requests == 1 { + return deadlineResponse(http.StatusTooManyRequests, `{"error":{"message":"Too many concurrent requests"}}`), nil + } + retryStart = time.Now() + if bodyRead { + return &http.Response{StatusCode: http.StatusOK, Body: &deadlineBody{ctx: r.Context()}}, nil + } + <-r.Context().Done() + return nil, r.Context().Err() + }) + _, err := c.GetProfile(context.Background()) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Contains(err.Error(), "last response: rate limited (429): Too many concurrent requests") + assert.Equal(2, requests, "an I/O deadline must not start another quota retry") + assert.Equal(30*time.Second, time.Since(retryStart)) + }) + }) + } +} + +func TestGetProfileDeadlineWithoutResponseStaysBare(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newDeadlineClient(t, func(r *http.Request) (*http.Response, error) { + <-r.Context().Done() + return nil, r.Context().Err() + }) + profile, err := c.GetProfile(context.Background()) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Nil(t, profile) + assert.NotContains(t, err.Error(), "last response", + "a stalled request has no upstream response to report") + }) +} + +func TestGmailErrorDetail(t *testing.T) { + withRetryAfter := http.Header{"Retry-After": []string{"30"}} + for _, tc := range []struct { + name string + header http.Header + body string + want string + }{ + {"reason and message", nil, + `{"error":{"message":"Quota exceeded for quota metric","errors":[{"reason":"userRateLimitExceeded"}]}}`, + "userRateLimitExceeded; Quota exceeded for quota metric"}, + {"message only", nil, `{"error":{"message":"Too many concurrent requests"}}`, "Too many concurrent requests"}, + {"unfamiliar body", withRetryAfter, "upstream proxy failure", "Retry-After 30"}, + {"empty body with header", withRetryAfter, "", "Retry-After 30"}, + {"empty", nil, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + header := tc.header + if header == nil { + header = http.Header{} + } + assert.Equal(t, tc.want, gmailErrorDetail(header, []byte(tc.body))) + }) + } +} From 452562252e4fc1b4c927df992aaacdae3b3121b3 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:28:57 -0700 Subject: [PATCH 070/125] perf(search): check FTS coverage without reading stored content (#978) ## What changed - Check FTS completeness through the small indexed-ID table, with a fallback for FTS tables that omit it. - Keep detecting holes anywhere in the index. The quick tail check uses the same table and preserves cancellation. - Show the incomplete-results warning when the index needs rebuilding or is rebuilding. An unfinished check alone stays quiet; the API still reports its state. ## Why A search after a daemon restart could trigger minutes of index-check I/O on a large archive. Every search then warned that results might be incomplete before the check had found a gap. Looking up indexed IDs avoids reading stored FTS content. ## Usage No usage change. Refs #977. Co-authored-by: Rusty Shackleford --- cmd/msgvault/cmd/search.go | 8 +- cmd/msgvault/cmd/search_test.go | 17 +- docs/changelog.md | 5 + docs/cli-reference.md | 6 + docs/usage/searching.md | 9 +- internal/api/handlers_test.go | 11 + internal/store/dialect_sqlite.go | 53 +++-- .../store/dialect_sqlite_fts_probe_test.go | 189 ++++++++++++++++++ internal/store/messages_test.go | 1 + 9 files changed, 251 insertions(+), 48 deletions(-) create mode 100644 internal/store/dialect_sqlite_fts_probe_test.go diff --git a/cmd/msgvault/cmd/search.go b/cmd/msgvault/cmd/search.go index e438b395c..0e7308458 100644 --- a/cmd/msgvault/cmd/search.go +++ b/cmd/msgvault/cmd/search.go @@ -207,13 +207,9 @@ func runHTTPSearch(cmd *cobra.Command, queryStr string) error { // Pre-0.18 daemons built the index synchronously inside the request. fmt.Fprintf(os.Stderr, "Built search index (%d messages indexed).\n", resp.IndexedMessages) } - switch resp.IndexState { - case "building": + if resp.IndexState == "building" { fmt.Fprintln(os.Stderr, - "Note: the search index is being rebuilt in the background; results may be incomplete until it finishes.") - case "checking": - fmt.Fprintln(os.Stderr, - "Note: search index completeness is still being verified in the background; results may be incomplete until it finishes.") + "Note: the search index is rebuilding or awaiting a rebuild in the background; results may be incomplete until it finishes.") } if searchCollection != "" { label := resp.ScopeLabel diff --git a/cmd/msgvault/cmd/search_test.go b/cmd/msgvault/cmd/search_test.go index a11dd5f26..7462c5163 100644 --- a/cmd/msgvault/cmd/search_test.go +++ b/cmd/msgvault/cmd/search_test.go @@ -519,11 +519,8 @@ func searchHTTPDaemon(t *testing.T) (*httptest.Server, *atomic.Int32) { return server, searchRequests } -// TestSearchCmd_PrintsBackgroundIndexNote verifies the CLI caveats results -// whenever the daemon reports the FTS index is not yet known complete: a -// rebuild in progress (index_state="building") and an unfinished completeness -// probe (index_state="checking") get distinct notes; a complete index gets -// none. +// TestSearchCmd_PrintsBackgroundIndexNote verifies only known index gaps +// produce a caveat. An unfinished completeness probe alone stays silent. func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { cfg := testConfigValue() useLocal := false @@ -534,14 +531,14 @@ func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { wantNote string }{ { - name: "building warns about the rebuild", + name: "building warns about rebuilding or awaiting rebuild", indexState: "building", - wantNote: "the search index is being rebuilt in the background; results may be incomplete", + wantNote: "the search index is rebuilding or awaiting a rebuild in the background; results may be incomplete", }, { - name: "checking warns the probe has not finished", + name: "checking alone prints no note", indexState: "checking", - wantNote: "search index completeness is still being verified in the background; results may be incomplete", + wantNote: "", }, { name: "complete index prints no note", @@ -606,7 +603,7 @@ func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { assert.Contains(out, "Lunch", "results still print") if tt.wantNote == "" { - assert.NotContains(errOut, "Note:", "no index note for a complete index") + assert.NotContains(errOut, "Note:", "no index note without a known gap") } else { assert.Contains(errOut, tt.wantNote, "index state note") } diff --git a/docs/changelog.md b/docs/changelog.md index 83befc1b8..f7c207541 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,11 @@ All notable changes to msgvault, grouped by release. CSV field, such as a display name with a comma, first appears deep in a large archive. The CSV snapshot now tells DuckDB the quote character instead of letting it guess from a sample. +- Check default SQLite full-text indices for completeness through row IDs without + reading stored search content. Search warns about incomplete results only + for a known index gap or rebuild, including rebuilds awaiting other daemon + work; an unfinished completeness check alone stays silent. + - Query published analytics while the daemon refreshes the cache in the background. `query --fresh` waits for current results; HTTP and MCP callers can track refresh jobs. The new `query_sql` MCP tool restricts SQL to archive diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 1582c1f07..852d3fd25 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1567,6 +1567,12 @@ terminals may wrap. An unknown message size appears as `-`. With `--json`, each result also includes `web_url` when the selected daemon can provide a browser link for that message. +The daemon checks full-text index completeness in the background. The CLI +warns that results may be incomplete when the daemon finds an index gap or +is rebuilding the index. The warning also applies while a rebuild waits for +other daemon work to finish. A completeness check alone prints no warning. +API clients can still observe `index_state="checking"` while that check runs. + --- ## repair-list-ids diff --git a/docs/usage/searching.md b/docs/usage/searching.md index b94d21859..07f6b3a2b 100644 --- a/docs/usage/searching.md +++ b/docs/usage/searching.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-27" title: Searching description: Find archived messages by words, meaning, account, conversation, or message type. --- @@ -29,9 +29,10 @@ msgvault search !!! note The full-text search index (FTS5) is populated automatically during sync. If an older archive needs an index backfill, msgvault checks and rebuilds - the index in the background. Search returns immediately from the index as - it exists now and reports when results may be incomplete while the daemon - is still checking or building. + the index in the background. Search uses the current index and warns about + incomplete results only for a known gap or rebuild, including a queued + rebuild. A completeness check alone stays quiet. See the + [CLI reference](../cli-reference.md#search) for details. ## Search Operators diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go index 4850cd87b..da56af325 100644 --- a/internal/api/handlers_test.go +++ b/internal/api/handlers_test.go @@ -2546,6 +2546,17 @@ func TestHandleCLISearchBackfillUsesOperationGate(t *testing.T) { synctest.Wait() assert.Empty(backfillStarted, "backfill started while operation gate was occupied") + // The full probe found a gap, but rebuilding still waits for sync. + queuedReq := httptest.NewRequest(http.MethodGet, "/api/v1/cli/search?q=hello", nil) + queuedResp := httptest.NewRecorder() + srv.Router().ServeHTTP(queuedResp, queuedReq) + require.Equal(http.StatusOK, queuedResp.Code) + var queued struct { + IndexState string `json:"index_state"` + } + require.NoError(json.NewDecoder(queuedResp.Body).Decode(&queued)) + assert.Equal("building", queued.IndexState, "known gaps must be visible while rebuilding is queued") + releaseGate() synctest.Wait() assert.NotEmpty(backfillStarted, "backfill did not start after gate release") diff --git a/internal/store/dialect_sqlite.go b/internal/store/dialect_sqlite.go index 86ba5008d..01d4d5510 100644 --- a/internal/store/dialect_sqlite.go +++ b/internal/store/dialect_sqlite.go @@ -489,34 +489,28 @@ func (d *SQLiteDialect) FTSAvailable(ctx context.Context, db *sql.DB) (bool, err return err == nil || errors.Is(err, sql.ErrNoRows), nil } -// FTSNeedsBackfill reports whether the FTS5 table needs population. -// Probes for the existence of ANY message lacking an FTS entry, matching the -// PostgreSQL EXISTS(search_fts IS NULL) semantics. The previous MAX(rowid) -// vs MAX(id) heuristic missed a hole left at a LOW id while later ids were -// indexed — reachable because UpsertFTS failures during sync are -// warn-and-continue (sync.go) while the message row still commits, so id N can -// be unindexed while N+1.. are indexed. messages_fts.rowid == messages.id and -// there are no triggers, so the NOT EXISTS join is rowid-served and cheap on -// FTS5 (no full body scan). +const sqliteFTSNeedsBackfillDocsizeSQL = `SELECT EXISTS (SELECT 1 FROM messages m WHERE NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id))` +const sqliteFTSNeedsBackfillVirtualSQL = `SELECT EXISTS (SELECT 1 FROM messages m WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.rowid = m.id))` + +// FTSNeedsBackfill reports whether any message lacks an FTS5 entry. Probe +// indexed row IDs through the docsize shadow table's integer primary key to +// avoid reading stored FTS content. columnsize=0 indices have no docsize table +// and fall back to the virtual table. Checking every message catches interior +// holes left when indexing fails during sync but later messages are indexed. func (d *SQLiteDialect) FTSNeedsBackfill(db *sql.DB) bool { var exists bool - if err := db.QueryRowContext(context.Background(), - `SELECT EXISTS ( - SELECT 1 FROM messages m - WHERE NOT EXISTS ( - SELECT 1 FROM messages_fts f WHERE f.rowid = m.id - ) - )`, - ).Scan(&exists); err != nil { - return false + err := db.QueryRowContext(context.Background(), sqliteFTSNeedsBackfillDocsizeSQL).Scan(&exists) + if d.IsNoSuchTableError(err) { + err = db.QueryRowContext(context.Background(), sqliteFTSNeedsBackfillVirtualSQL).Scan(&exists) } - return exists + return err == nil && exists } -// FTSNeedsBackfillQuick compares MAX(id) against MAX(rowid) — two B-tree -// lookups, instant at any archive size. It catches the dominant staleness -// (tail of the messages table not yet indexed: fresh import, interrupted -// backfill) but misses interior holes; FTSNeedsBackfill stays authoritative. +// FTSNeedsBackfillQuick compares maximum message and indexed row IDs. The +// docsize shadow table avoids stored-content reads; columnsize=0 indices fall +// back to the virtual table. This catches unindexed tails after imports or +// interrupted backfills but misses interior holes; FTSNeedsBackfill remains +// authoritative. func (d *SQLiteDialect) FTSNeedsBackfillQuick(ctx context.Context, db *sql.DB) bool { var msgMax int64 if err := db.QueryRowContext(ctx, @@ -525,12 +519,15 @@ func (d *SQLiteDialect) FTSNeedsBackfillQuick(ctx context.Context, db *sql.DB) b return false } var ftsMax int64 - if err := db.QueryRowContext(ctx, - "SELECT COALESCE(MAX(rowid), 0) FROM messages_fts", - ).Scan(&ftsMax); err != nil { - return false + err := db.QueryRowContext(ctx, + "SELECT COALESCE(MAX(id), 0) FROM messages_fts_docsize", + ).Scan(&ftsMax) + if d.IsNoSuchTableError(err) { + err = db.QueryRowContext(ctx, + "SELECT COALESCE(MAX(rowid), 0) FROM messages_fts", + ).Scan(&ftsMax) } - return ftsMax < msgMax + return err == nil && ftsMax < msgMax } // FTSClearSQL returns the SQL to clear all FTS5 data. diff --git a/internal/store/dialect_sqlite_fts_probe_test.go b/internal/store/dialect_sqlite_fts_probe_test.go new file mode 100644 index 000000000..54f926ecc --- /dev/null +++ b/internal/store/dialect_sqlite_fts_probe_test.go @@ -0,0 +1,189 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newFTSProbeDB(tb testing.TB, columnsizeZero bool) *sql.DB { + tb.Helper() + req := require.New(tb) + db, err := sql.Open("sqlite3", filepath.Join(tb.TempDir(), "probe.db")) + req.NoError(err) + tb.Cleanup(func() { req.NoError(db.Close()) }) + _, err = db.Exec("CREATE TABLE messages (id INTEGER PRIMARY KEY)") + req.NoError(err) + options := "" + if columnsizeZero { + options = ", columnsize=0" + } + _, err = db.Exec("CREATE VIRTUAL TABLE messages_fts USING fts5(text" + options + ")") + req.NoError(err) + return db +} + +func TestSQLiteFTSProbePresence(t *testing.T) { + for _, zero := range []bool{false, true} { + name := "docsize" + if zero { + name = "columnsize_zero" + } + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + req := require.New(t) + db := newFTSProbeDB(t, zero) + d := &SQLiteDialect{} + ctx := context.Background() + assert.False(d.FTSNeedsBackfill(db)) + assert.False(d.FTSNeedsBackfillQuick(ctx, db)) + _, err := db.Exec("INSERT INTO messages VALUES (1), (2), (3)") + req.NoError(err) + assert.True(d.FTSNeedsBackfill(db)) + assert.True(d.FTSNeedsBackfillQuick(ctx, db)) + _, err = db.Exec("INSERT INTO messages_fts(rowid, text) VALUES (1, ''), (3, 'synthetic'), (4, 'extra')") + req.NoError(err) + assert.True(d.FTSNeedsBackfill(db), "equal counts must not hide the interior hole") + assert.False(d.FTSNeedsBackfillQuick(ctx, db), "quick probe only checks the tail") + _, err = db.Exec("INSERT INTO messages_fts(rowid, text) VALUES (2, '')") + req.NoError(err) + assert.False(d.FTSNeedsBackfill(db), "empty text still counts as indexed") + assert.False(d.FTSNeedsBackfillQuick(ctx, db)) + _, err = db.Exec("DELETE FROM messages_fts WHERE rowid IN (3, 4)") + req.NoError(err) + assert.True(d.FTSNeedsBackfill(db)) + assert.True(d.FTSNeedsBackfillQuick(ctx, db)) + _, err = db.Exec("DELETE FROM messages WHERE id = 3") + req.NoError(err) + assert.False(d.FTSNeedsBackfill(db), "deleted messages need no index entry") + _, err = db.Exec("DELETE FROM messages_fts WHERE rowid = 1") + req.NoError(err) + assert.True(d.FTSNeedsBackfill(db), "lowest ID hole must be found") + assert.False(d.FTSNeedsBackfillQuick(ctx, db)) + _, err = db.Exec("INSERT INTO messages VALUES (4)") + req.NoError(err) + assert.True(d.FTSNeedsBackfillQuick(ctx, db), "uncancelled probe must find the tail") + cancelled, cancel := context.WithCancel(ctx) + cancel() + assert.False(d.FTSNeedsBackfillQuick(cancelled, db)) + }) + } +} + +func TestSQLiteFTSProbeAvoidsStoredContent(t *testing.T) { + assert := assert.New(t) + req := require.New(t) + db := newFTSProbeDB(t, false) + _, err := db.Exec("INSERT INTO messages VALUES (1), (2), (3)") + req.NoError(err) + _, err = db.Exec("INSERT INTO messages_fts(rowid, text) VALUES (1, 'synthetic')") + req.NoError(err) + // The row-ID metadata remains usable even if stored content cannot be read. + _, err = db.Exec("DROP TABLE messages_fts_content") + req.NoError(err) + d := &SQLiteDialect{} + assert.True(d.FTSNeedsBackfill(db)) + assert.True(d.FTSNeedsBackfillQuick(context.Background(), db)) +} + +func TestSQLiteFTSProbeErrors(t *testing.T) { + for _, damage := range []string{"DROP TABLE messages_fts", "DROP TABLE messages_fts_docsize; CREATE TABLE messages_fts_docsize (wrong INTEGER)"} { + t.Run(damage, func(t *testing.T) { + assert := assert.New(t) + req := require.New(t) + db := newFTSProbeDB(t, false) + _, err := db.Exec("INSERT INTO messages VALUES (1)") + req.NoError(err) + _, err = db.Exec(damage) + req.NoError(err) + d := &SQLiteDialect{} + assert.False(d.FTSNeedsBackfill(db), "errors must not be treated as gaps") + assert.False(d.FTSNeedsBackfillQuick(context.Background(), db)) + }) + } +} + +func TestSQLiteFTSProbeQueryPlan(t *testing.T) { + assert := assert.New(t) + req := require.New(t) + db := newFTSProbeDB(t, false) + rows, err := db.Query("EXPLAIN QUERY PLAN " + sqliteFTSNeedsBackfillDocsizeSQL) + req.NoError(err) + defer func() { req.NoError(rows.Close()) }() + var details []string + for rows.Next() { + var id, parent, unused int + var detail string + req.NoError(rows.Scan(&id, &parent, &unused, &detail)) + details = append(details, detail) + } + req.NoError(rows.Err()) + plan := strings.Join(details, "\n") + assert.Contains(plan, "SEARCH d USING INTEGER PRIMARY KEY (rowid=?)") + assert.NotContains(plan, "VIRTUAL TABLE") +} + +func FuzzSQLiteFTSProbePresence(f *testing.F) { + f.Add([]byte{}, false) + f.Add([]byte{1, 1, 1}, false) + f.Add([]byte{1, 0, 1}, false) + f.Add([]byte{0, 1, 1}, true) + f.Add([]byte{1, 1, 0}, true) + f.Fuzz(func(t *testing.T, indexed []byte, zero bool) { + assert := assert.New(t) + req := require.New(t) + db := newFTSProbeDB(t, zero) + wantGap := false + // Bound SQLite fixture work at materialization, retaining arbitrary fuzz inputs. + if len(indexed) > 64 { + indexed = indexed[:64] + } + for offset, present := range indexed { + id := offset + 1 + _, err := db.Exec("INSERT INTO messages VALUES (?)", id) + req.NoError(err) + if present&1 == 0 { + wantGap = true + continue + } + _, err = db.Exec("INSERT INTO messages_fts(rowid, text) VALUES (?, '')", id) + req.NoError(err) + } + assert.Equal(wantGap, (&SQLiteDialect{}).FTSNeedsBackfill(db)) + }) +} + +func BenchmarkSQLiteFTSProbe(b *testing.B) { + req := require.New(b) + db := newFTSProbeDB(b, false) + tx, err := db.Begin() + req.NoError(err) + text := strings.Repeat("synthetic ", 409) + "sample" + for id := 1; id <= 10000; id++ { + _, err = tx.Exec("INSERT INTO messages VALUES (?)", id) + req.NoError(err) + _, err = tx.Exec("INSERT INTO messages_fts(rowid, text) VALUES (?, ?)", id, text) + req.NoError(err) + } + req.NoError(tx.Commit()) + b.Run("production", func(b *testing.B) { + req := require.New(b) + for range b.N { + req.False((&SQLiteDialect{}).FTSNeedsBackfill(db)) + } + }) + b.Run("virtual", func(b *testing.B) { + req := require.New(b) + for range b.N { + var gap bool + err := db.QueryRow(sqliteFTSNeedsBackfillVirtualSQL).Scan(&gap) + req.NoError(err) + req.False(gap) + } + }) +} diff --git a/internal/store/messages_test.go b/internal/store/messages_test.go index d739168f7..4c9b0dd43 100644 --- a/internal/store/messages_test.go +++ b/internal/store/messages_test.go @@ -1083,6 +1083,7 @@ func TestPersistRepairMessageReplacesCompleteSnapshotAtomically(t *testing.T) { _, oldHits, err := fixture.Store.SearchMessages("original-target-token", 0, 10) require.NoError(err) assert.Zero(oldHits, "repair must remove the old FTS document") + waitForFeedPast(t, fixture.Store, after.ContentChangedAt.Time) page, err := fixture.Store.ListChangedMessages( t.Context(), store.ChangedMessagesFrom(fixture.Baseline.Add(time.Second)), 20) require.NoError(err) From 20cf096cdbacdc94ad99ca1787f4ae75da6f3bed Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 28 Sep 2026 15:16:03 -0500 Subject: [PATCH 071/125] test(peoplesweep): skip Linux-only Codex proxy tests on macOS (#982) Skip the four Codex auth-home and refresh tests on non-Linux platforms. They require the Linux-only service proxy, so their existing Windows-only skips let macOS fail before reaching the behavior under test ([failing CI job](https://github.com/kenn-io/msgvault/actions/runs/36451346874/job/109026607886)). Linux keeps this coverage, and the remaining Codex tests continue to run on macOS. The [branch CI run](https://github.com/kenn-io/msgvault/actions/runs/36474336216) includes the macOS job, which PR-triggered CI skips. Co-authored-by: Wes McKinney --- internal/peoplesweep/codex_app_server_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/peoplesweep/codex_app_server_test.go b/internal/peoplesweep/codex_app_server_test.go index f39169525..013c67e84 100644 --- a/internal/peoplesweep/codex_app_server_test.go +++ b/internal/peoplesweep/codex_app_server_test.go @@ -366,8 +366,8 @@ func newSuccessfulCodexTransport( func TestCodexRegistryUsesOnlyExplicitAuthHome(t *testing.T) { assertChecks := assert.New(t) requireChecks := require.New(t) - if runtime.GOOS == "windows" { - t.Skip("codex auth home permission gates require Unix permission bits") + if runtime.GOOS != "linux" { + t.Skip("codex service proxy requires Linux") } authHome := t.TempDir() requireChecks.NoError(os.Chmod(authHome, 0o700)) @@ -426,8 +426,8 @@ func TestCodexInferenceCopiesBackRefreshForSameAccount(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assertChecks := assert.New(t) requireChecks := require.New(t) - if runtime.GOOS == "windows" { - t.Skip("codex auth home permission gates require Unix permission bits") + if runtime.GOOS != "linux" { + t.Skip("codex service proxy requires Linux") } authHome := t.TempDir() requireChecks.NoError(os.Chmod(authHome, 0o700)) @@ -482,8 +482,8 @@ func TestCodexInferenceCopiesBackRefreshForSameAccount(t *testing.T) { func TestCodexModelListingCopiesBackRefresh(t *testing.T) { assert := assert.New(t) require := require.New(t) - if runtime.GOOS == "windows" { - t.Skip("codex auth home permission gates require Unix permission bits") + if runtime.GOOS != "linux" { + t.Skip("codex service proxy requires Linux") } authHome := t.TempDir() require.NoError(os.Chmod(authHome, 0o700)) @@ -536,8 +536,8 @@ func TestCodexModelListingCopiesBackRefresh(t *testing.T) { func TestCodexInferenceRejectsChangedAccountDuringRefresh(t *testing.T) { assertChecks := assert.New(t) requireChecks := require.New(t) - if runtime.GOOS == "windows" { - t.Skip("codex auth home permission gates require Unix permission bits") + if runtime.GOOS != "linux" { + t.Skip("codex service proxy requires Linux") } authHome := t.TempDir() requireChecks.NoError(os.Chmod(authHome, 0o700)) From 7ba0ec48a8100895277e7e9822875e8d797f1914 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:11:46 -0700 Subject: [PATCH 072/125] perf(cache): normalize relationship activity and keep appends incremental (#916) Relationship cache appends now reuse historical contributions instead of rescanning the archive for every new batch. Message facts and direct links are stored once; queries expand conversation membership from the shared roster. Label and attachment metadata edits replace only the affected datasets and preserve existing relationship activity. Recipient repairs read committed and staged message files directly, without hard links. Sender-recipient changes rebuild message facts so cached owner flags stay current. Logical and daily temperature contributions preserve chat anchors, attachment totals, and the future-message scoring window. Existing caches need one full rebuild for schema 30. Depends on #915. Refs #912. Co-authored-by: Rusty Shackleford --- cmd/msgvault/cmd/build_cache.go | 249 ++++++----------- cmd/msgvault/cmd/build_cache_identity_test.go | 47 ++++ cmd/msgvault/cmd/build_cache_test.go | 2 +- cmd/msgvault/cmd/cache_derived.go | 115 +++++++- cmd/msgvault/cmd/cache_publication.go | 2 + cmd/msgvault/cmd/cache_refresh_test.go | 11 +- cmd/msgvault/cmd/cache_related_repair.go | 191 +++++++++++++ cmd/msgvault/cmd/cache_staleness.go | 69 ++++- cmd/msgvault/cmd/cache_staleness_test.go | 248 +++++++++++++++++ internal/identityindex/build.go | 115 ++++---- internal/identityindex/build_test.go | 219 ++++++++++++++- internal/identityindex/compact_sql.go | 3 +- internal/identityindex/contributions.go | 148 +++++++++++ internal/identityindex/normalized_activity.go | 156 +++++++++++ .../identityindex/normalized_activity_test.go | 74 ++++++ internal/identityindex/relationship_sql.go | 188 +------------ .../identityindex/relationship_sql_test.go | 250 +++++++++++++++++- internal/identityindex/schema.go | 12 +- internal/identityindex/validate.go | 88 ++++-- internal/query/cache_state.go | 19 +- internal/query/cache_state_test.go | 7 +- internal/query/duckdb.go | 12 + internal/query/explore_analysis.go | 16 +- internal/query/files.go | 10 +- internal/query/identity_activity.go | 39 +-- internal/query/identity_activity_test.go | 2 +- internal/query/people.go | 10 +- internal/query/people_inboxes.go | 4 +- internal/query/relationship_calendar.go | 4 +- .../relationship_scale_benchmark_test.go | 157 +++++++++++ internal/query/relationship_timeline.go | 11 +- internal/query/views.go | 36 +++ internal/store/attachments.go | 2 +- .../cache_related_change_journal_test.go | 140 ++++++++++ internal/store/derived_data_revision.go | 23 +- internal/store/messages.go | 2 +- internal/store/migrate_recipient_envelope.go | 37 +++ .../store/migrate_recipient_envelope_test.go | 12 + internal/store/schema.sql | 105 ++++++++ 39 files changed, 2330 insertions(+), 505 deletions(-) create mode 100644 cmd/msgvault/cmd/cache_related_repair.go create mode 100644 internal/identityindex/contributions.go create mode 100644 internal/identityindex/normalized_activity.go create mode 100644 internal/identityindex/normalized_activity_test.go create mode 100644 internal/query/relationship_scale_benchmark_test.go create mode 100644 internal/store/cache_related_change_journal_test.go diff --git a/cmd/msgvault/cmd/build_cache.go b/cmd/msgvault/cmd/build_cache.go index 2b8497a3f..a6a06643a 100644 --- a/cmd/msgvault/cmd/build_cache.go +++ b/cmd/msgvault/cmd/build_cache.go @@ -566,6 +566,7 @@ func buildCacheDerivedOnly( dbPath, analyticsDir, acquirePublishLock, + false, builderOverrides..., ) } @@ -583,6 +584,7 @@ func isDaemonBuildCacheChild() bool { type buildResult struct { ExportedCount int64 + StagedCount int64 MaxMessageID int64 OutputDir string Skipped bool @@ -808,6 +810,10 @@ func buildCacheScheduled( builderOverrides..., ) } + if relatedDriftOnly(staleness) { + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + acquirePublishLock, true, builderOverrides...) + } return buildCacheLocked( dbPath, analyticsDir, @@ -937,7 +943,12 @@ func derivedDriftOnly(staleness cacheStaleness) bool { staleness.HasParticipantDisplayNameDrift || staleness.HasPersonDisplayNameDrift) && !staleness.HasNew && !staleness.HasDeleted && !staleness.HasUpdated && !staleness.HasAccountIdentityDrift && - !staleness.HasDerivedDataDrift + !staleness.HasDerivedDataDrift && !staleness.HasRelatedRowDrift +} + +func relatedDriftOnly(staleness cacheStaleness) bool { + return staleness.HasUsablePublication && staleness.HasRelatedRowDrift && + staleness.Reason == "related rows changed" } // refreshIdentityDatasetsOnly rebuilds every identity-derived dataset while @@ -953,6 +964,7 @@ func refreshIdentityDatasetsOnly( dbPath, analyticsDir, locking, + false, builderOverrides..., ) } @@ -985,6 +997,10 @@ func buildCacheLocked( if derivedDriftOnly(staleness) { return refreshIdentityDatasetsOnly(dbPath, analyticsDir, locking, builderOverrides...) } + if relatedDriftOnly(staleness) { + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + locking, true, builderOverrides...) + } fullRebuild = staleness.FullRebuild } @@ -1114,6 +1130,7 @@ func buildCacheLocked( var maxMessageID sql.NullInt64 var lastCompletedSyncRunID int64 + var relatedChangeSeq int64 var syncCounters cacheSyncCounters // Use indexed query: id is PRIMARY KEY, sent_at has an index maxIDQuery := `SELECT MAX(id) FROM messages WHERE sent_at IS NOT NULL` @@ -1142,6 +1159,21 @@ func buildCacheLocked( return nil, fmt.Errorf("get cache sync counters: %w", err) } } + var hasRelatedChangeJournal int + if err := sourceSnapshot.QueryRow(` + SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' AND name = 'cache_related_change_journal' + `).Scan(&hasRelatedChangeJournal); err != nil { + return nil, fmt.Errorf("check cache related-change journal: %w", err) + } + if hasRelatedChangeJournal > 0 { + if err := sourceSnapshot.QueryRow(` + SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0) + `).Scan(&relatedChangeSeq); err != nil { + return nil, fmt.Errorf("read cache related-change sequence: %w", err) + } + } if !fullRebuild && hasPreviousState && hasSyncRunsTable > 0 { updatesChanged := syncCounters.updates != previousState.LastCacheUpdateCount coveredAdditionsChanged := syncCounters.additions != previousState.LastCacheAdditionCount && @@ -1155,6 +1187,27 @@ func buildCacheLocked( } } + var relatedKinds relatedChangeKinds + if hasPreviousState && !fullRebuild && relatedChangeSeq > previousState.LastRelatedChangeSeq { + relatedKinds, err = inspectRelatedChangeKinds(sourceSnapshot, + previousState.LastRelatedChangeSeq, relatedChangeSeq, previousState.LastMessageID) + if err != nil { + return nil, err + } + if relatedKinds.other { + fullRebuild = true + lastMessageID = 0 + relatedKinds = relatedChangeKinds{} + } + } + repairRelated := relatedKinds.recipients || relatedKinds.labels || relatedKinds.attachments + if hasPreviousState && maxID <= lastMessageID && !fullRebuild && repairRelated { + if err := sourceSnapshot.Close(); err != nil { + return nil, fmt.Errorf("close SQLite snapshot before related repair: %w", err) + } + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + locking, true, builderOverrides...) + } if hasPreviousState && maxID <= lastMessageID && !fullRebuild { if err := sourceSnapshot.Close(); err != nil { return nil, fmt.Errorf("close SQLite snapshot after metadata check: %w", err) @@ -1241,28 +1294,6 @@ func buildCacheLocked( idFilter += fmt.Sprintf(" AND TRY_CAST(m.id AS BIGINT) > %d", lastMessageID) } - // Junction rows are searchable exactly when their parent message is - // exportable. This includes calendar invitees and meeting attendees while - // excluding hidden rows and messages without a timestamp. - exportableJunctionWhereFor := func(messageIDColumn string) string { - return fmt.Sprintf( - "TRY_CAST(%s AS BIGINT) IN (SELECT CAST(m.id AS BIGINT) FROM sqlite_db.messages m WHERE %s AND TRY_CAST(m.id AS BIGINT) <= %d)", - messageIDColumn, exportableMessageWhere("m"), maxID, - ) - } - junctionFilterFor := func(messageIDColumn, incremental string) string { - where := exportableJunctionWhereFor(messageIDColumn) - if incremental != "" { - return incremental + " AND " + where - } - return " WHERE " + where - } - junctionFilter := func(incremental string) string { - return junctionFilterFor("message_id", incremental) - } - - junctionFile := "data.parquet" - // runExport executes a COPY query and prints timing info. runExport := func(label, copyQuery string) error { start := time.Now() @@ -1278,116 +1309,24 @@ func buildCacheLocked( // Export each table separately - this is MUCH faster than joining during export // because DuckDB can use SQLite indexes efficiently for simple queries - // 1. Export message_recipients (large junction table) - recipientsDir := filepath.Join(staging.root, "message_recipients") - escapedRecipientsDir := strings.ReplaceAll(recipientsDir, "'", "''") - // This export joins participants, so every column reference is alias - // qualified and the incremental predicate names mr.message_id rather - // than the bare column the shared junctionFilter helper produces. - recipientsFilter := "" - if !replaceAll && lastMessageID > 0 { - recipientsFilter = fmt.Sprintf(" WHERE mr.message_id > %d", lastMessageID) - } - recipientsFilter = junctionFilterFor("mr.message_id", recipientsFilter) - // Two address columns leave here. envelope_address is the header address - // exactly as the store recorded it (NULL when none was — chat, calendar, - // and mail ingested before the column existed; invalid UTF-8 also exports - // NULL so repaired identity keys cannot collide); identity filters key on - // its presence. email_address is the resolved recipient address: the - // envelope when present, otherwise the participant's current address, so - // an address filter over this dataset finds pre-upgrade mail too. - // Presence is byte-level, not validity-level: an envelope whose bytes are - // invalid UTF-8 is present but unusable, so both columns export NULL - // rather than silently substituting the participant's current address, - // which would attribute the message to whoever the participant is today. - // Only a participant with no email address at all (phone or handle only) - // leaves email_address NULL. Databases from before the envelope column - // export NULL envelopes for every row. - recipientEnvelopeKey := "NULL::VARCHAR" - recipientPresence := "FALSE" - // NULLIF can evaluate its first argument twice. Check emptiness with the - // pure key so each unknown output value increments the counter only once. - recipientParticipantKey := "CASE WHEN " + cacheIdentityTextSQL("p.email_address") + - " = '' THEN NULL ELSE " + sourceSnapshot.identityExportSQL("p.email_address") + " END" - if sourceSnapshot.hasRecipientEnvelope { - recipientEnvelopeKey = "CASE WHEN " + cacheIdentityTextSQL("mr.email_address") + - " = '' THEN NULL ELSE " + sourceSnapshot.identityExportSQL("mr.email_address") + " END" - recipientPresence = sourceSnapshot.identityPresenceSQL("mr.email_address", "mr.envelope_present") - } - if err := runExport("message_recipients", fmt.Sprintf(` - COPY ( - SELECT - mr.message_id, - mr.participant_id, - %[2]s AS recipient_type, - COALESCE(%[3]s, '') as display_name, - CASE WHEN %[8]s THEN %[1]s ELSE %[4]s END as email_address, - %[1]s as envelope_address - FROM sqlite_db.message_recipients mr - LEFT JOIN sqlite_db.participants p ON p.id = mr.participant_id%[5]s - ) TO '%[6]s/%[7]s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, recipientEnvelopeKey, sourceSnapshot.identityExportSQL("mr.recipient_type"), sourceSnapshot.textSQL("mr.display_name"), recipientParticipantKey, recipientsFilter, escapedRecipientsDir, junctionFile, recipientPresence)); err != nil { - return nil, fmt.Errorf("export message_recipients: %w", err) - } - - // 2. Export message_labels (large junction table) - messageLabelsDir := filepath.Join(staging.root, "message_labels") - escapedMessageLabelsDir := strings.ReplaceAll(messageLabelsDir, "'", "''") - messageLabelsFilter := "" - if !replaceAll && lastMessageID > 0 { - messageLabelsFilter = fmt.Sprintf(" WHERE message_id > %d", lastMessageID) - } - messageLabelsFilter = junctionFilter(messageLabelsFilter) - if err := runExport("message_labels", fmt.Sprintf(` - COPY ( - SELECT - message_id, - label_id - FROM sqlite_db.message_labels%s - ) TO '%s/%s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, messageLabelsFilter, escapedMessageLabelsDir, junctionFile)); err != nil { - return nil, fmt.Errorf("export message_labels: %w", err) - } - - // 3. Export attachments - attachmentsDir := filepath.Join(staging.root, tableAttachments) - escapedAttachmentsDir := strings.ReplaceAll(attachmentsDir, "'", "''") - attachmentsFilter := "" - if !replaceAll && lastMessageID > 0 { - attachmentsFilter = fmt.Sprintf(" WHERE message_id > %d", lastMessageID) - } - attachmentsFilter = junctionFilter(attachmentsFilter) - attachmentMIMEExpression := "'' AS mime_type" - if sourceSnapshot.hasAttachmentMIME { - attachmentMIMEExpression = "COALESCE(" + sourceSnapshot.textSQL("mime_type") + ", '') AS mime_type" + publicationPlan := cachePublishPlanForMode(replaceAll) + for dataset, changed := range relatedKinds.datasets() { + if changed { + delete(publicationPlan.Append, dataset) + publicationPlan.Replace[dataset] = true + } } - attachmentMetadataExpression := "NULL::VARCHAR AS attachment_metadata" - if sourceSnapshot.hasAttachmentMetadata { - attachmentMetadataExpression = sourceSnapshot.textSQL("attachment_metadata") + " AS attachment_metadata" + afterMessageIDs := make(map[string]int64) + for _, dataset := range []string{"message_recipients", "message_labels", tableAttachments, tableLabels} { + if publicationPlan.Append[dataset] { + afterMessageIDs[dataset] = lastMessageID + } else { + afterMessageIDs[dataset] = 0 + } } - if err := runExport(tableAttachments, fmt.Sprintf(` - COPY ( - SELECT - id AS attachment_id, - message_id, - size, - COALESCE(%s, '') as filename, - %s, - %s - FROM sqlite_db.attachments%s - ) TO '%s/%s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, sourceSnapshot.textSQL("filename"), attachmentMIMEExpression, attachmentMetadataExpression, - attachmentsFilter, escapedAttachmentsDir, junctionFile)); err != nil { - return nil, fmt.Errorf("export attachments: %w", err) + if err := exportRelatedDatasets(context.Background(), exportDB, sourceSnapshot, + maxID, staging.root, afterMessageIDs); err != nil { + return nil, err } // 4. Export participants @@ -1502,23 +1441,6 @@ func buildCacheLocked( return nil, fmt.Errorf("export conversation participants: %w", err) } - // 5. Export labels - labelsDir := filepath.Join(staging.root, tableLabels) - escapedLabelsDir := strings.ReplaceAll(labelsDir, "'", "''") - if err := runExport(tableLabels, fmt.Sprintf(` - COPY ( - SELECT - id, - COALESCE(%s, '') as name - FROM sqlite_db.labels - ) TO '%s/labels.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, sourceSnapshot.textSQL("name"), escapedLabelsDir)); err != nil { - return nil, fmt.Errorf("export labels: %w", err) - } - // 6. Export sources sourcesDir := filepath.Join(staging.root, "sources") escapedSourcesDir := strings.ReplaceAll(sourcesDir, "'", "''") @@ -1658,19 +1580,24 @@ func buildCacheLocked( if replaceAll { buildMode = identityindex.ModeFull } + if relatedKinds.recipients && !replaceAll { + buildMode = identityindex.ModeAppendRepair + delete(publicationPlan.Append, identityindex.DatasetActivity) + publicationPlan.Replace[identityindex.DatasetActivity] = true + } derived, err := identityindex.Build(context.Background(), exportDB, identityindex.BuildOptions{ - Mode: buildMode, - CommittedRoot: analyticsDir, - StagedBaseRoot: staging.root, - OutputRoot: staging.root, - EffectiveAt: cacheWatermark, - Progress: reportIdentityBuildProgress, + Mode: buildMode, + CommittedRoot: analyticsDir, + StagedBaseRoot: staging.root, + ReplacedBaseDatasets: publicationPlan.Replace, + OutputRoot: staging.root, + EffectiveAt: cacheWatermark, + Progress: reportIdentityBuildProgress, }) if err != nil { return nil, fmt.Errorf("build identity index: %w", err) } reportRelationshipActivityStats(derived.Activity) - publicationPlan := cachePublishPlanForMode(replaceAll) fmt.Printf(" %-25s %s\n", "Total:", time.Since(buildStart).Round(time.Millisecond)) @@ -1740,6 +1667,7 @@ func buildCacheLocked( LastCompletedSyncRunID: lastCompletedSyncRunID, LastCacheAdditionCount: syncCounters.additions, LastCacheUpdateCount: syncCounters.updates, + LastRelatedChangeSeq: relatedChangeSeq, LastFailedSyncRunCount: syncCounters.failedRunCount, LastFailedSyncRunIDSum: syncCounters.failedRunIDSum, IdentityRevision: identityRevision, @@ -1761,9 +1689,13 @@ func buildCacheLocked( return nil, err } reportCacheTextRepairs(os.Stderr, textRepairs) + if hasRelatedChangeJournal > 0 { + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + } return &buildResult{ ExportedCount: expectedTotalCount, + StagedCount: stagedCount, MaxMessageID: maxID, OutputDir: analyticsDir, }, nil @@ -1783,13 +1715,6 @@ func reportRelationshipActivityStats(stats identityindex.ActivityStats) { stats.FinalRows, stats.ExpansionRatio, ) - if stats.ExpansionRatio > 4 { - fmt.Printf( - " Warning: relationship membership fan-out is %.2fx; "+ - "consider a normalized conversation-member index if this archive keeps growing\n", - stats.ExpansionRatio, - ) - } } func countStagedMessages(db sqlRowQuerier, messagesDir string, requireShard bool) (int64, error) { diff --git a/cmd/msgvault/cmd/build_cache_identity_test.go b/cmd/msgvault/cmd/build_cache_identity_test.go index 045d9f0c6..c5b7a25df 100644 --- a/cmd/msgvault/cmd/build_cache_identity_test.go +++ b/cmd/msgvault/cmd/build_cache_identity_test.go @@ -71,6 +71,53 @@ func TestBuildCache_SlackDefaultIdentityResolvesOwnerParticipant(t *testing.T) { assert.True(t, isFromMe, "owner resolution must feed Slack relationship analytics") } +func TestBuildCache_FromRecipientChangeRefreshesOwner(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dbPath := filepath.Join(t.TempDir(), "msgvault.db") + analyticsDir := filepath.Join(t.TempDir(), "analytics") + st, err := store.Open(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("slack", "synthetic-owner") + require.NoError(err) + conv, err := st.EnsureConversationWithType(src.ID, "thread", "direct_chat", "Synthetic") + require.NoError(err) + first, err := st.EnsureParticipant("first@example.test", "First", "example.test") + require.NoError(err) + second, err := st.EnsureParticipant("second@example.test", "Second", "example.test") + require.NoError(err) + id, err := st.UpsertMessage(&store.Message{ + ConversationID: conv, SourceID: src.ID, SourceMessageID: "message", MessageType: "slack", + IsFromMe: true, SentAt: sql.NullTime{Time: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(id, "from", []int64{first}, []string{"First"})) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(id, "from", []int64{second}, []string{"Second"})) + var isFromMe bool + require.NoError(st.DB().QueryRow(`SELECT is_from_me FROM messages WHERE id = ?`, id).Scan(&isFromMe)) + require.True(isFromMe) + assert.True(cacheNeedsBuild(dbPath, analyticsDir).FullRebuild) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + db, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + var owner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&owner)) + assert.Equal(second, owner) + var isSender, isOwner bool + require.NoError(db.QueryRow(`SELECT is_sender, is_owner FROM read_parquet(?, hive_partitioning=true) + WHERE message_id = ? AND canonical_id = ?`, + filepath.Join(analyticsDir, "relationship_activity", "**", "*.parquet"), id, second).Scan(&isSender, &isOwner)) + assert.True(isSender) + assert.True(isOwner) +} + // TestBuildCache_DerivesIsFromMeAndIdentityDatasets verifies that: // - messages Parquet gains a derived is_from_me column: true when the // sender's participant email case-insensitively matches a confirmed diff --git a/cmd/msgvault/cmd/build_cache_test.go b/cmd/msgvault/cmd/build_cache_test.go index 5594cb4d8..1c9ea6866 100644 --- a/cmd/msgvault/cmd/build_cache_test.go +++ b/cmd/msgvault/cmd/build_cache_test.go @@ -4759,7 +4759,7 @@ func TestBuildCacheWarningExportScope(t *testing.T) { _, err = db.Exec(`UPDATE conversations SET title=CAST(? AS TEXT) WHERE id=101`, []byte(title)) require.NoError(err) stderr = captureStderrDuring(t, func() { - result, err = refreshDerivedDatasetsOnly(context.Background(), dbPath, cache, acquirePublishLock) + result, err = refreshDerivedDatasetsOnly(context.Background(), dbPath, cache, acquirePublishLock, false) }) require.NoError(err) assert.True(result.IdentityOnly) diff --git a/cmd/msgvault/cmd/cache_derived.go b/cmd/msgvault/cmd/cache_derived.go index bdf038cd0..dadd4c3ea 100644 --- a/cmd/msgvault/cmd/cache_derived.go +++ b/cmd/msgvault/cmd/cache_derived.go @@ -27,6 +27,7 @@ func refreshDerivedDatasetsOnly( ctx context.Context, dbPath, analyticsDir string, locking cachePublishLocking, + repairRelated bool, builderOverrides ...duckdbutil.BuilderOverrides, ) (*buildResult, error) { readiness, err := query.InspectCacheReadiness(analyticsDir) @@ -75,9 +76,17 @@ func refreshDerivedDatasetsOnly( return nil, fmt.Errorf("read derived-data revision: %w", err) } if derivedDataRevision != state.DerivedDataRevision { - _ = st.Close() - return nil, fmt.Errorf("%w: derived-data revision changed", - ErrDerivedRefreshRequiresFullBuild) + relatedOnly, relatedErr := st.RelatedDerivedRevisionsOnly(ctx, + state.DerivedDataRevision, derivedDataRevision) + if relatedErr != nil { + _ = st.Close() + return nil, fmt.Errorf("classify derived-data revision: %w", relatedErr) + } + if !repairRelated || !relatedOnly { + _ = st.Close() + return nil, fmt.Errorf("%w: derived-data revision changed", + ErrDerivedRefreshRequiresFullBuild) + } } accountIdentityRevision, err := st.AccountIdentityRevision() if err != nil { @@ -132,7 +141,7 @@ func refreshDerivedDatasetsOnly( return nil, err } defer func() { _ = sourceSnapshot.Close() }() - if err := sourceSnapshot.PrepareDatasets( + datasets := []string{ tableMessages, tableConversations, tableConversationParticipants, @@ -140,7 +149,40 @@ func refreshDerivedDatasetsOnly( tableParticipants, tableParticipantIdentifiers, "persons", "person_participants", - ); err != nil { + } + var relatedChangeSeq int64 + var relatedKinds relatedChangeKinds + if repairRelated { + if err := sourceSnapshot.QueryRow(`SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0)`).Scan(&relatedChangeSeq); err != nil { + return nil, fmt.Errorf("read related-change boundary: %w", err) + } + if err := inspectRelatedSnapshotColumns(sourceSnapshot); err != nil { + return nil, err + } + // The CSV fallback closes its SQLite transaction during preparation. + // Read journal metadata while that snapshot is still available. + var err error + relatedKinds, err = inspectRelatedChangeKinds(sourceSnapshot, + state.LastRelatedChangeSeq, relatedChangeSeq, state.LastMessageID) + if err != nil { + return nil, err + } + if relatedKinds.other { + return nil, fmt.Errorf("%w: unsupported related-row journal dataset", + ErrDerivedRefreshRequiresFullBuild) + } + } + relatedExports := make(map[string]int64) + relatedReplacements := make(map[string]bool) + for dataset, changed := range relatedKinds.datasets() { + if changed { + relatedExports[dataset] = 0 + datasets = append(datasets, dataset) + relatedReplacements[dataset] = true + } + } + if err := sourceSnapshot.PrepareDatasets(datasets...); err != nil { return nil, err } exportDB := sourceSnapshot.DuckDB() @@ -162,7 +204,7 @@ func refreshDerivedDatasetsOnly( return nil, err } - if identityRevision == state.IdentityRevision && + if !repairRelated && identityRevision == state.IdentityRevision && participantIdentifierRevision == state.ParticipantIdentifierRevision && participantDisplayNameRevision == state.ParticipantDisplayNameRevision && personDisplayNameRevision == state.PersonDisplayNameRevision && @@ -174,6 +216,39 @@ func refreshDerivedDatasetsOnly( // it active pagination cursors — for no analytical difference. return &buildResult{OutputDir: analyticsDir, IdentityOnly: true, Skipped: true}, nil } + if repairRelated && !relatedKinds.recipients && + identityRevision == state.IdentityRevision && + participantIdentifierRevision == state.ParticipantIdentifierRevision && + participantDisplayNameRevision == state.ParticipantDisplayNameRevision && + personDisplayNameRevision == state.PersonDisplayNameRevision && + conversationFingerprint == state.ConversationParticipantsFingerprint && + typesFingerprint == state.ConversationTypesFingerprint { + // Labels and attachment metadata do not enter relationship_activity. + // Publish their child rows and marker directly, avoiding a scan of the + // expanded relationship population for a small metadata correction. + if err := exportRelatedDatasets(ctx, exportDB, sourceSnapshot, + state.LastMessageID, staging.root, relatedExports); err != nil { + return nil, err + } + if err := refreshRelatedCacheStats(ctx, exportDB, &state, relatedKinds); err != nil { + return nil, err + } + if err := sourceSnapshot.Close(); err != nil { + return nil, fmt.Errorf("close SQLite related-refresh snapshot: %w", err) + } + state.DerivedDataRevision = derivedDataRevision + state.LastRelatedChangeSeq = relatedChangeSeq + plan := cachePublishPlan{ + Append: map[string]bool{}, + Replace: relatedReplacements, + } + if err := publishDerivedCache(staging, analyticsDir, plan, state, locking); err != nil { + return nil, err + } + reportCacheTextRepairs(os.Stderr, textRepairs) + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + return &buildResult{OutputDir: analyticsDir, IdentityOnly: true}, nil + } if err := exportDerivedOwnerParticipants(ctx, exportDB, staging.root, sourceSnapshot.identityPresenceSQL("email_address", "primary_email_present")); err != nil { @@ -237,6 +312,14 @@ func refreshDerivedDatasetsOnly( return nil, err } } + if repairRelated { + if err := exportRelatedDatasets(ctx, exportDB, sourceSnapshot, state.LastMessageID, staging.root, relatedExports); err != nil { + return nil, err + } + if err := refreshRelatedCacheStats(ctx, exportDB, &state, relatedKinds); err != nil { + return nil, err + } + } derived, err := identityindex.Build(ctx, exportDB, identityindex.BuildOptions{ Mode: identityindex.ModeIndexOnly, @@ -258,13 +341,19 @@ func refreshDerivedDatasetsOnly( } state.IdentityRevision = identityRevision + if repairRelated { + state.DerivedDataRevision = derivedDataRevision + } state.ParticipantIdentifierRevision = participantIdentifierRevision state.ParticipantDisplayNameRevision = participantDisplayNameRevision state.PersonDisplayNameRevision = personDisplayNameRevision state.ConversationParticipantsFingerprint = conversationFingerprint state.ConversationTypesFingerprint = typesFingerprint - // Stats describe the unchanged committed raw snapshot. Preserve them - // byte-for-byte instead of scanning Parquet again. + if repairRelated { + state.LastRelatedChangeSeq = relatedChangeSeq + } + // The message snapshot remains unchanged; child-row statistics were + // refreshed above when their source rows changed. plan := derivedCachePublishPlan( conversationChanged, typesChanged, @@ -272,10 +361,18 @@ func refreshDerivedDatasetsOnly( identifiersChanged || displayNamesChanged, personDisplayNamesChanged || identityChanged, ) + if repairRelated { + for dataset := range relatedReplacements { + plan.Replace[dataset] = true + } + } if err := publishDerivedCache(staging, analyticsDir, plan, state, locking); err != nil { return nil, err } reportCacheTextRepairs(os.Stderr, textRepairs) + if repairRelated { + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + } return &buildResult{OutputDir: analyticsDir, IdentityOnly: true}, nil } @@ -549,6 +646,8 @@ func derivedCachePublishPlan( identityindex.DatasetPeople, identityindex.DatasetDomains, identityindex.DatasetRelationshipDaily, + identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, } { plan.Replace[dataset] = true } diff --git a/cmd/msgvault/cmd/cache_publication.go b/cmd/msgvault/cmd/cache_publication.go index bd70bbe3f..23d4f3f8e 100644 --- a/cmd/msgvault/cmd/cache_publication.go +++ b/cmd/msgvault/cmd/cache_publication.go @@ -141,6 +141,8 @@ func cachePublishPlanForMode(replaceAll bool) cachePublishPlan { identityindex.DatasetPeople, identityindex.DatasetDomains, identityindex.DatasetRelationshipDaily, + identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, } { plan.Replace[dataset] = true } diff --git a/cmd/msgvault/cmd/cache_refresh_test.go b/cmd/msgvault/cmd/cache_refresh_test.go index 2b8811bb1..c2315a9b2 100644 --- a/cmd/msgvault/cmd/cache_refresh_test.go +++ b/cmd/msgvault/cmd/cache_refresh_test.go @@ -148,17 +148,16 @@ func TestDerivedOnlyRefreshCarriesStatsAndRefreshesMembershipRollups(t *testing. defer func() { require.NoError(t, duckDB.Close()) }() var membershipRows int64 requirementsForTest.NoError(duckDB.QueryRow(` - SELECT count(DISTINCT message_id) - FROM read_parquet(?, hive_partitioning = true) - WHERE conversation_id = 102 - AND canonical_id = 3 - AND is_conversation_member + SELECT count(DISTINCT a.message_id) + FROM read_parquet(?, hive_partitioning = true) a + JOIN read_parquet(?) cp ON cp.conversation_id = a.conversation_id + WHERE a.conversation_id = 102 AND cp.participant_id = 3 `, filepath.Join( analyticsDir, identityindex.DatasetActivity, "**", "*.parquet", - )).Scan(&membershipRows)) + ), filepath.Join(analyticsDir, "conversation_participants", "*.parquet")).Scan(&membershipRows)) assertionsForTest.Positive(membershipRows) } diff --git a/cmd/msgvault/cmd/cache_related_repair.go b/cmd/msgvault/cmd/cache_related_repair.go new file mode 100644 index 000000000..03c0e72e5 --- /dev/null +++ b/cmd/msgvault/cmd/cache_related_repair.go @@ -0,0 +1,191 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +type relatedChangeKinds struct { + recipients bool + labels bool + attachments bool + other bool +} + +func inspectRelatedChangeKinds(snapshot *cacheSourceSnapshot, after, through, lastMessageID int64) (relatedChangeKinds, error) { + var kinds relatedChangeKinds + err := snapshot.QueryRow(`SELECT + count(*) FILTER (WHERE dataset = 'message_recipients') > 0, + count(*) FILTER (WHERE dataset IN ('message_labels', 'labels')) > 0, + count(*) FILTER (WHERE dataset = 'attachments') > 0, + count(*) FILTER (WHERE dataset NOT IN + ('message_recipients', 'message_labels', 'labels', 'attachments')) > 0 + FROM cache_related_change_journal WHERE seq > ? AND seq <= ? AND message_id <= ?`, + after, through, lastMessageID).Scan(&kinds.recipients, &kinds.labels, &kinds.attachments, &kinds.other) + if err != nil { + return relatedChangeKinds{}, fmt.Errorf("inspect related change kinds: %w", err) + } + return kinds, nil +} + +func refreshRelatedCacheStats(ctx context.Context, db sqlRunner, state *query.CacheSyncState, kinds relatedChangeKinds) error { + if kinds.recipients { + statement := `SELECT count(DISTINCT p.email_address), count(DISTINCT p.domain) + FROM sqlite_db.message_recipients mr + JOIN sqlite_db.messages m ON m.id = mr.message_id + JOIN sqlite_db.participants p ON p.id = mr.participant_id + WHERE mr.recipient_type = 'from' AND m.id <= ? AND ` + exportableMessageWhere("m") + if err := db.QueryRowContext(ctx, statement, state.LastMessageID). + Scan(&state.Stats.UniqueSenders, &state.Stats.UniqueDomains); err != nil { + return fmt.Errorf("refresh related sender statistics: %w", err) + } + } + if kinds.attachments { + statement := `SELECT coalesce(sum(try_cast(a.size AS BIGINT)), 0) + FROM sqlite_db.attachments a JOIN sqlite_db.messages m ON m.id = a.message_id + WHERE m.id <= ? AND ` + exportableMessageWhere("m") + if err := db.QueryRowContext(ctx, statement, state.LastMessageID). + Scan(&state.Stats.AttachmentSizeBytes); err != nil { + return fmt.Errorf("refresh related attachment statistics: %w", err) + } + } + return nil +} + +// The marker is published before pruning. A failed publication keeps all +// journal entries available for replay; a failed prune only leaves redundant +// entries, since sqlite_sequence preserves the acknowledged high watermark. +func pruneAcknowledgedRelatedChanges(dbPath string, relatedSeq, derivedRevision int64) error { + st, err := store.Open(dbPath) + if err != nil { + return fmt.Errorf("open store to prune related changes: %w", err) + } + defer func() { _ = st.Close() }() + tx, err := st.DB().Begin() + if err != nil { + return fmt.Errorf("begin related-change prune: %w", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.Exec(`DELETE FROM cache_related_change_journal WHERE seq <= ?`, relatedSeq); err != nil { + return fmt.Errorf("prune related changes: %w", err) + } + if _, err := tx.Exec(`DELETE FROM cache_related_revision_journal WHERE revision <= ?`, derivedRevision); err != nil { + return fmt.Errorf("prune related revisions: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit related-change prune: %w", err) + } + return nil +} + +func warnRelatedChangePrune(dbPath string, relatedSeq, derivedRevision int64) { + if err := pruneAcknowledgedRelatedChanges(dbPath, relatedSeq, derivedRevision); err != nil { + fmt.Fprintf(os.Stderr, "Warning: %v\n", err) + } +} + +func inspectRelatedSnapshotColumns(snapshot *cacheSourceSnapshot) error { + for _, column := range []struct { + table string + name string + present *bool + }{ + {"message_recipients", "email_address", &snapshot.hasRecipientEnvelope}, + {"attachments", "mime_type", &snapshot.hasAttachmentMIME}, + {"attachments", "attachment_metadata", &snapshot.hasAttachmentMetadata}, + } { + var count int + statement := fmt.Sprintf("SELECT COUNT(*) FROM pragma_table_info('%s') WHERE name = '%s'", + column.table, column.name) + if err := snapshot.QueryRow(statement).Scan(&count); err != nil { + return fmt.Errorf("inspect %s.%s for related repair: %w", column.table, column.name, err) + } + *column.present = count > 0 + } + return nil +} + +// datasets lists replacements needed for edits to already-cached messages. +func (k relatedChangeKinds) datasets() map[string]bool { + return map[string]bool{ + "message_recipients": k.recipients, + tableLabels: k.labels, + "message_labels": k.labels, + tableAttachments: k.attachments, + } +} + +// exportRelatedDatasets exports the requested child datasets from one snapshot. +// Each lower message-ID boundary is zero for a replacement, or the committed +// watermark for an append. Label definitions are always replaced. +func exportRelatedDatasets( + ctx context.Context, db sqlRunner, snapshot *cacheSourceSnapshot, + lastMessageID int64, stagingRoot string, afterMessageIDs map[string]int64, +) error { + parentFilter := func(dataset string) string { + return fmt.Sprintf(`SELECT CAST(m.id AS BIGINT) FROM sqlite_db.messages m + WHERE %s AND TRY_CAST(m.id AS BIGINT) <= %d AND TRY_CAST(m.id AS BIGINT) > %d`, + exportableMessageWhere("m"), lastMessageID, afterMessageIDs[dataset]) + } + // Missing envelopes fall back to the participant address. A damaged but + // present envelope stays unknown and must still suppress that fallback. + envelope := "NULL::VARCHAR" + presence := "FALSE" + participant := "CASE WHEN " + cacheIdentityTextSQL("p.email_address") + + " = '' THEN NULL ELSE " + snapshot.identityExportSQL("p.email_address") + " END" + if snapshot.hasRecipientEnvelope { + envelope = "CASE WHEN " + cacheIdentityTextSQL("mr.email_address") + + " = '' THEN NULL ELSE " + snapshot.identityExportSQL("mr.email_address") + " END" + presence = snapshot.identityPresenceSQL("mr.email_address", "mr.envelope_present") + } + mimeType := "'' AS mime_type" + if snapshot.hasAttachmentMIME { + mimeType = "COALESCE(" + snapshot.textSQL("mime_type") + ", '') AS mime_type" + } + metadata := "NULL::VARCHAR AS attachment_metadata" + if snapshot.hasAttachmentMetadata { + metadata = snapshot.textSQL("attachment_metadata") + " AS attachment_metadata" + } + exports := []struct { + dataset string + selectSQL string + }{ + {tableLabels, `SELECT id, COALESCE(` + snapshot.textSQL("name") + `, '') AS name + FROM sqlite_db.labels`}, + {"message_recipients", fmt.Sprintf(`SELECT mr.message_id, mr.participant_id, + %[4]s AS recipient_type, + COALESCE(%[5]s, '') AS display_name, + CASE WHEN %[6]s THEN %[1]s ELSE %[7]s END AS email_address, + %[1]s AS envelope_address + FROM sqlite_db.message_recipients mr + LEFT JOIN sqlite_db.participants p ON p.id = mr.participant_id + WHERE mr.message_id > %[3]d AND TRY_CAST(mr.message_id AS BIGINT) IN (%[2]s)`, envelope, parentFilter("message_recipients"), afterMessageIDs["message_recipients"], snapshot.identityExportSQL("mr.recipient_type"), snapshot.textSQL("mr.display_name"), presence, participant)}, + {"message_labels", fmt.Sprintf(`SELECT message_id, label_id + FROM sqlite_db.message_labels WHERE message_id > %d AND TRY_CAST(message_id AS BIGINT) IN (%s)`, afterMessageIDs["message_labels"], parentFilter("message_labels"))}, + {tableAttachments, fmt.Sprintf(`SELECT id AS attachment_id, message_id, size, + COALESCE(%s, '') AS filename, + %s, %s FROM sqlite_db.attachments + WHERE message_id > %d AND TRY_CAST(message_id AS BIGINT) IN (%s)`, snapshot.textSQL("filename"), mimeType, metadata, afterMessageIDs[tableAttachments], parentFilter(tableAttachments))}, + } + for _, item := range exports { + if _, ok := afterMessageIDs[item.dataset]; !ok { + continue + } + dir := filepath.Join(stagingRoot, item.dataset) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create related repair dataset %s: %w", item.dataset, err) + } + path := filepath.Join(dir, "data.parquet") + statement := fmt.Sprintf("COPY (%s) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd')", + item.selectSQL, quoteCacheSQL(path)) + if _, err := db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("export related dataset %s: %w", item.dataset, query.HintRepairEncoding(err)) + } + } + return nil +} diff --git a/cmd/msgvault/cmd/cache_staleness.go b/cmd/msgvault/cmd/cache_staleness.go index 94a2c7a93..02ce7bac5 100644 --- a/cmd/msgvault/cmd/cache_staleness.go +++ b/cmd/msgvault/cmd/cache_staleness.go @@ -25,6 +25,7 @@ type cacheStaleness struct { HasNew bool // new messages since last build HasDeleted bool // deletions since last build HasUpdated bool // updates or additions within the cached ID boundary require repair + HasRelatedRowDrift bool // journaled child rows changed within the committed message boundary // HasIdentityDrift signals participant_links or account_identities // changed since the last build. Also set whenever // HasAccountIdentityDrift is set (AddAccountIdentity/RemoveAccountIdentity @@ -379,6 +380,61 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an } } + if ctx.Err() != nil { + return cacheStaleness{} + } + var hasRelatedChangeJournal int + err = db.DB().QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' AND name = 'cache_related_change_journal' + `).Scan(&hasRelatedChangeJournal) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect related-change journal") + } + if hasRelatedChangeJournal == 0 && state.LastRelatedChangeSeq != 0 { + return cacheStaleness{NeedsBuild: true, FullRebuild: true, Reason: "related-change journal is missing"} + } + if hasRelatedChangeJournal > 0 { + var latestSeq int64 + err = db.DB().QueryRowContext(ctx, ` + SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0) + `).Scan(&latestSeq) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect related-change sequence") + } + if latestSeq < state.LastRelatedChangeSeq { + result.FullRebuild = true + reasons = append(reasons, "related-change journal moved backwards") + } else if latestSeq > state.LastRelatedChangeSeq { + var coveredChanges bool + err = db.DB().QueryRowContext(ctx, ` + SELECT EXISTS(SELECT 1 FROM cache_related_change_journal + WHERE seq > ? AND message_id <= ?) + `, state.LastRelatedChangeSeq, state.LastMessageID).Scan(&coveredChanges) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect related-row changes") + } + if coveredChanges { + result.HasRelatedRowDrift = true + reasons = append(reasons, "related rows changed") + } + var messageFactsChanged bool + err = db.DB().QueryRowContext(ctx, ` + SELECT EXISTS(SELECT 1 FROM cache_related_change_journal + WHERE seq > ? AND message_id <= ? AND dataset = 'message_facts') + `, state.LastRelatedChangeSeq, state.LastMessageID).Scan(&messageFactsChanged) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect message fact changes") + } + if messageFactsChanged { + result.HasDerivedDataDrift = true + result.FullRebuild = true + reasons = append(reasons, "cached message facts changed") + } + } + } + if ctx.Err() != nil { return cacheStaleness{} } @@ -387,9 +443,16 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an return cacheStalenessFailure(ctx, "cannot verify derived-data revision") } if derivedDataRevision != state.DerivedDataRevision { - result.HasDerivedDataDrift = true - result.FullRebuild = true - reasons = append(reasons, "derived message data changed") + relatedOnly, relatedErr := db.RelatedDerivedRevisionsOnly(ctx, + state.DerivedDataRevision, derivedDataRevision) + if relatedErr != nil { + return cacheStalenessFailure(ctx, "cannot classify derived-data revision") + } + if !relatedOnly || !result.HasRelatedRowDrift { + result.HasDerivedDataDrift = true + result.FullRebuild = true + reasons = append(reasons, "derived message data changed") + } } // Account-identity drift covers identity mutations that invalidate baked diff --git a/cmd/msgvault/cmd/cache_staleness_test.go b/cmd/msgvault/cmd/cache_staleness_test.go index 7544523fe..dee672083 100644 --- a/cmd/msgvault/cmd/cache_staleness_test.go +++ b/cmd/msgvault/cmd/cache_staleness_test.go @@ -3,7 +3,9 @@ package cmd import ( "context" "database/sql" + "errors" "log/slog" + "os" "path/filepath" "strings" "testing" @@ -11,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -94,6 +97,251 @@ func TestCacheNeedsBuildInterruptedStateOnlyCache(t *testing.T) { assert.Contains(t, got.Reason, "interrupted") } +func TestCacheNeedsBuildTracksCoveredRelatedRows(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "msgvault.db") + analyticsDir := filepath.Join(tmp, "analytics") + st, err := store.Open(dbPath) + require.NoError(err) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("test", "synthetic@example.com") + require.NoError(err) + conversationID, err := st.EnsureConversationWithType(src.ID, "thread", "email_thread", "Synthetic") + require.NoError(err) + messageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "message", + MessageType: "email", SentAt: sql.NullTime{ + Time: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC), Valid: true, + }, + }) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (1, 'synthetic'), (2, 'changed')`) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO participants (id, email_address, domain) + VALUES (1, 'one@example.com', 'example.com'), (2, 'two@example.com', 'example.com')`) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO message_recipients + (message_id, participant_id, recipient_type) VALUES (?, 1, 'to')`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO attachments + (id, message_id, storage_path, filename, size) VALUES (1, ?, 'synthetic', 'old.txt', 1)`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE messages SET has_attachments = TRUE, attachment_count = 1 WHERE id = ?`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO message_labels (message_id, label_id) VALUES (?, 1)`, messageID) + require.NoError(err) + require.NoError(st.Close()) + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + state, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Positive(state.LastRelatedChangeSeq) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + + st, err = store.Open(dbPath) + require.NoError(err) + var acknowledgedRows int64 + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal`).Scan(&acknowledgedRows)) + assert.Zero(acknowledgedRows, "published journal entries should be pruned") + require.NoError(st.AddMessageLabels(messageID, []int64{2})) + require.NoError(st.ReplaceMessageRecipients(messageID, "to", []int64{2}, []string{"Recipient Two"})) + _, err = st.DB().Exec(`UPDATE attachments SET filename = 'new.txt', size = 2 WHERE id = 1`) + require.NoError(err) + require.NoError(st.Close()) + got := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.NeedsBuild) + assert.True(got.HasRelatedRowDrift) + assert.Contains(got.Reason, "related rows changed") + st, err = store.Open(dbPath) + require.NoError(err) + var seq int64 + require.NoError(st.DB().QueryRow(`SELECT MAX(seq) FROM cache_related_change_journal`).Scan(&seq)) + require.NoError(st.Close()) + assert.Greater(seq, state.LastRelatedChangeSeq) + previousHook := buildCacheBeforeMessagesExportHook + buildCacheBeforeMessagesExportHook = func() error { + return errors.New("related-row repair attempted a message export") + } + t.Cleanup(func() { buildCacheBeforeMessagesExportHook = previousHook }) + result, err := buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.True(result.IdentityOnly) + assert.Zero(result.ExportedCount) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + repairedState, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(seq, repairedState.LastRelatedChangeSeq) + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + labelsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "message_labels", "*.parquet"), "'", "''") + var labelCount int64 + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + labelsPath + "')").Scan(&labelCount)) + assert.Equal(int64(2), labelCount) + recipientsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "message_recipients", "*.parquet"), "'", "''") + var recipientID int64 + require.NoError(duckDB.QueryRow("SELECT participant_id FROM read_parquet('" + recipientsPath + "')").Scan(&recipientID)) + assert.Equal(int64(2), recipientID) + attachmentsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "attachments", "*.parquet"), "'", "''") + var filename string + var attachmentSize int64 + require.NoError(duckDB.QueryRow("SELECT filename, size FROM read_parquet('"+attachmentsPath+"')").Scan(&filename, &attachmentSize)) + assert.Equal("new.txt", filename) + assert.Equal(int64(2), attachmentSize) + assert.Equal(int64(2), repairedState.Stats.AttachmentSizeBytes) + + buildCacheBeforeMessagesExportHook = nil + unchangedFiles := make(map[string]os.FileInfo) + for _, pattern := range []string{ + "message_recipients/*.parquet", "attachments/*.parquet", "relationship_activity/occurred_year=*/*.parquet", + } { + paths, err := filepath.Glob(filepath.Join(analyticsDir, filepath.FromSlash(pattern))) + require.NoError(err) + require.NotEmpty(paths) + for _, path := range paths { + unchangedFiles[path], err = os.Stat(path) + require.NoError(err) + } + } + st, err = store.Open(dbPath) + require.NoError(err) + newMessageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "new-message", + MessageType: "email", SentAt: sql.NullTime{ + Time: time.Date(2025, 1, 2, 10, 0, 0, 0, time.UTC), Valid: true, + }, + }) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(newMessageID, "from", []int64{1}, []string{"New Sender"})) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (3, 'second-change')`) + require.NoError(err) + require.NoError(st.AddMessageLabels(messageID, []int64{3})) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + labelsPath + "')").Scan(&labelCount)) + assert.Equal(int64(3), labelCount) + for path, before := range unchangedFiles { + after, err := os.Stat(path) + require.NoError(err) + assert.True(os.SameFile(before, after), "label repair with an append must retain %s", path) + } + + activityFiles, err := filepath.Glob(filepath.Join(analyticsDir, + "relationship_activity", "occurred_year=*", "*.parquet")) + require.NoError(err) + require.NotEmpty(activityFiles) + activityBefore, err := os.Stat(activityFiles[0]) + require.NoError(err) + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (4, 'label-only')`) + require.NoError(err) + require.NoError(st.AddMessageLabels(messageID, []int64{4})) + require.NoError(st.Close()) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + activityAfter, err := os.Stat(activityFiles[0]) + require.NoError(err) + assert.Equal(activityBefore.ModTime(), activityAfter.ModTime(), + "label-only repair must reuse relationship activity") + assert.Equal(activityBefore.Size(), activityAfter.Size()) + var labelDefinitionCount int64 + definitionsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "labels", "*.parquet"), "'", "''") + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + definitionsPath + "')").Scan(&labelDefinitionCount)) + assert.Equal(int64(4), labelDefinitionCount) + + // A display-name change has no message_labels mutation, but must still + // republish the label definitions used by analytical views. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE labels SET name = 'renamed' WHERE id = 4`) + require.NoError(err) + require.NoError(st.Close()) + got = cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.HasRelatedRowDrift) + assert.False(got.FullRebuild) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + var labelName string + require.NoError(duckDB.QueryRow("SELECT name FROM read_parquet('" + definitionsPath + "') WHERE id = 4").Scan(&labelName)) + assert.Equal("renamed", labelName) + for path, before := range unchangedFiles { + after, err := os.Stat(path) + require.NoError(err) + assert.True(os.SameFile(before, after), "label-only repair must retain %s", path) + } + + // Attachment metadata repairs must replace their totals, not add the old + // size again, while keeping old relationship activity in place. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE attachments SET size = 3 WHERE id = 1`) + require.NoError(err) + _, err = st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "attachment-append", + MessageType: "email", SentAt: sql.NullTime{Time: time.Date(2025, 1, 3, 10, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + repairedState, err = query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(int64(3), repairedState.Stats.AttachmentSizeBytes) + activityAfter, err = os.Stat(activityFiles[0]) + require.NoError(err) + assert.True(os.SameFile(activityBefore, activityAfter)) + + // An analytics directory may be a link to a separate storage location. + // Repair old recipients while appending messages through that link. + realAnalytics := filepath.Join(t.TempDir(), "analytics") + require.NoError(os.Rename(analyticsDir, realAnalytics)) + if err := os.Symlink(realAnalytics, analyticsDir); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + st, err = store.Open(dbPath) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(messageID, "to", []int64{1}, []string{"Recipient One"})) + _, err = st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "third-message", + MessageType: "email", SentAt: sql.NullTime{Time: time.Date(2025, 1, 3, 10, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + require.NoError(duckDB.QueryRow("SELECT participant_id FROM read_parquet('"+recipientsPath+"') WHERE message_id = ?", messageID).Scan(&recipientID)) + assert.Equal(int64(1), recipientID) + repairedState, err = query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(int64(4), repairedState.Stats.TotalMessages) + assert.Equal(int64(3), repairedState.Stats.AttachmentSizeBytes) + var canonicalID int64 + require.NoError(duckDB.QueryRow(`SELECT canonical_id FROM read_parquet(?, hive_partitioning=true) + WHERE message_id = ? AND canonical_id IS NOT NULL`, + filepath.Join(analyticsDir, "relationship_activity", "**", "*.parquet"), messageID).Scan(&canonicalID)) + assert.Equal(int64(1), canonicalID) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + + // Attachment or From-recipient edits may also alter facts baked into old + // message shards; those cannot use the child-only repair path. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE messages SET attachment_count = attachment_count + 1 WHERE id = ?`, messageID) + require.NoError(err) + require.NoError(st.Close()) + got = cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.FullRebuild) + assert.True(got.HasDerivedDataDrift) +} + func TestCacheNeedsBuild_MeetingMutation(t *testing.T) { tests := []struct { name string diff --git a/internal/identityindex/build.go b/internal/identityindex/build.go index 09675ad89..8ccfcc7c6 100644 --- a/internal/identityindex/build.go +++ b/internal/identityindex/build.go @@ -19,6 +19,9 @@ const ( ModeFull Mode = iota ModeIncremental ModeIndexOnly + // ModeAppendRepair rebuilds activity after old recipients change while + // reading committed and staged message shards together. + ModeAppendRepair ) // BuildOptions identifies committed, staged, and output cache roots. @@ -27,6 +30,8 @@ type BuildOptions struct { CommittedRoot string StagedBaseRoot string OutputRoot string + // ReplacedBaseDatasets identifies complete child exports during an append. + ReplacedBaseDatasets map[string]bool // EffectiveAt pins recency decay and future-event exclusion to the cache // publication snapshot. A zero value falls back to the newest committed // activity timestamp, which keeps direct package callers deterministic. @@ -34,7 +39,8 @@ type BuildOptions struct { Progress func(dataset string, elapsed time.Duration) } -// ActivityStats records the fan-out chosen by the flat activity grain. +// ActivityStats records the rows built in this run. Incremental builds +// report only their staged delta, so reporting does not rescan old activity. type ActivityStats struct { DirectRows int64 ConversationExpandedRows int64 @@ -55,9 +61,9 @@ type sqlExecutor interface { QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } -// Build derives the four relationship datasets from schema-correct base -// Parquet. Incremental mode writes an activity delta and rebuilds the compact -// datasets over the committed population plus that delta. +// Build derives the relationship datasets from schema-correct base Parquet. +// Incremental mode writes an activity delta, merges compact contributions, +// and refreshes the published summaries from those compact grains. func Build( ctx context.Context, db sqlExecutor, @@ -69,7 +75,7 @@ func Build( if err := validateBuildOptions(opts); err != nil { return BuildResult{}, err } - if opts.Mode > ModeIndexOnly { + if opts.Mode > ModeAppendRepair { return BuildResult{}, fmt.Errorf("build relationship index: unknown mode %d", opts.Mode) } @@ -115,20 +121,15 @@ func Build( ); err != nil { return BuildResult{}, err } - if err := b.materializeBuildTable( - ctx, - temperatureBuildRelation, - buildRelationshipTemperatureDailySQL(activity, effectiveAt), - "relationship_temperature_daily", - ); err != nil { + if err := b.materializeContributions(ctx, activity, effectiveAt); err != nil { return BuildResult{}, err } - if err := b.materializeBuildTable( - ctx, - logicalBuildRelation, - buildLogicalActivityMaterializationSQL(activity), - "logical_activity", - ); err != nil { + if err := b.copyDataset(ctx, DatasetTemperatureContributions, + "SELECT * FROM "+temperatureBuildRelation); err != nil { + return BuildResult{}, err + } + if err := b.copyDataset(ctx, DatasetLogicalContributions, + "SELECT * FROM "+logicalBuildRelation); err != nil { return BuildResult{}, err } if err := b.copyDataset(ctx, DatasetPeople, buildRelationshipPeopleSQL(effectiveAt)); err != nil { @@ -140,10 +141,16 @@ func Build( if err := b.copyDataset(ctx, DatasetRelationshipDaily, buildRelationshipDailySQL()); err != nil { return BuildResult{}, err } + validationActivity := activity + if opts.Mode == ModeIncremental { + // The committed generation was validated before publication. Appends + // use new message IDs, so only the staged edges need row validation. + validationActivity = b.deltaActivityRelation() + } if err := Validate(ctx, db, ValidationOptions{ OutputRoot: opts.OutputRoot, RequiredOutputDatasets: RequiredDatasets, - ActivityPath: activity, + ActivityRelation: validationActivity, }); err != nil { return BuildResult{}, err } @@ -156,7 +163,11 @@ func Build( if err != nil { return BuildResult{}, err } - activityStats, err := collectActivityStats(ctx, db, activity) + statsActivity := activity + if opts.Mode == ModeIncremental { + statsActivity = b.deltaActivityRelation() + } + activityStats, err := collectActivityStats(ctx, db, statsActivity) if err != nil { return BuildResult{}, err } @@ -205,40 +216,55 @@ func (b builder) committed(dataset string) string { return parquetDatasetGlob(b.opts.CommittedRoot, dataset) } -func (b builder) output(dataset string) string { - return parquetDatasetGlob(b.opts.OutputRoot, dataset) +func (b builder) outputActivity() string { + return parquetDatasetGlob(b.opts.OutputRoot, DatasetActivity) } func (b builder) activityRelation() string { - paths := []string{b.output(DatasetActivity)} + paths := []string{b.outputActivity()} if b.opts.Mode == ModeIncremental { paths = append([]string{b.committed(DatasetActivity)}, paths...) } + return b.expandedActivity(paths) +} + +func (b builder) deltaActivityRelation() string { + return b.expandedActivity([]string{b.outputActivity()}) +} + +func (b builder) expandedActivity(paths []string) string { + return ExpandedActivityRelation( + readParquetRelation(paths, true), + readParquetRelation([]string{b.base("conversation_participants")}, false), + readParquetRelation([]string{b.base("participants")}, false), + readParquetRelation([]string{b.base("participant_clusters")}, false), + readParquetRelation([]string{b.base("owner_participants")}, false), + ) +} + +func (b builder) messageRelation() string { + paths := []string{b.base("messages")} + if b.opts.Mode == ModeAppendRepair { + paths = append([]string{b.committed("messages")}, paths...) + } return readParquetRelation(paths, true) } func (b builder) statsRelations() cacheStatsRelations { - inputs := cacheStatsRelations{ - messages: readParquetRelation([]string{b.base("messages")}, true), - recipients: readParquetRelation([]string{b.base("message_recipients")}, false), + relation := func(dataset string, hivePartitioning bool) string { + paths := []string{b.base(dataset)} + if (b.opts.Mode == ModeIncremental || b.opts.Mode == ModeAppendRepair) && + !b.opts.ReplacedBaseDatasets[dataset] { + paths = append([]string{b.committed(dataset)}, paths...) + } + return readParquetRelation(paths, hivePartitioning) + } + return cacheStatsRelations{ + messages: relation("messages", true), + recipients: relation("message_recipients", false), participants: readParquetRelation([]string{b.base("participants")}, false), - attachments: readParquetRelation([]string{b.base("attachments")}, false), + attachments: relation("attachments", false), } - if b.opts.Mode == ModeIncremental { - inputs.messages = readParquetRelation( - []string{b.committed("messages"), b.base("messages")}, - true, - ) - inputs.recipients = readParquetRelation( - []string{b.committed("message_recipients"), b.base("message_recipients")}, - false, - ) - inputs.attachments = readParquetRelation( - []string{b.committed("attachments"), b.base("attachments")}, - false, - ) - } - return inputs } func (b builder) copyDataset(ctx context.Context, dataset, query string) error { @@ -267,7 +293,7 @@ func (b builder) copyRelationshipActivity(ctx context.Context) error { options := "FORMAT PARQUET, COMPRESSION 'zstd', PARTITION_BY (occurred_year), " + "WRITE_PARTITION_COLUMNS true, OVERWRITE_OR_IGNORE" for _, year := range years { - query := buildRelationshipActivitySQL(b.base, year) + query := buildSparseRelationshipActivitySQL(b.base, b.messageRelation(), year) statement := "COPY (" + query + ") TO '" + quoteSQLString(output) + "' (" + options + ")" if _, err := b.db.ExecContext(ctx, statement); err != nil { @@ -289,8 +315,7 @@ func (b builder) copyRelationshipActivity(ctx context.Context) error { func (b builder) relationshipActivityYears(ctx context.Context) ([]int64, error) { query := ` SELECT DISTINCT year(sent_at)::BIGINT AS occurred_year - FROM read_parquet('` + quoteSQLString(b.base("messages")) + `', - hive_partitioning=true, union_by_name=true) + FROM ` + b.messageRelation() + ` ORDER BY occurred_year` rows, err := b.db.QueryContext(ctx, query) if err != nil { @@ -322,7 +347,7 @@ func (b builder) copyEmptyRelationshipActivity(ctx context.Context) error { if err := os.MkdirAll(filepath.Dir(emptyOutput), 0o755); err != nil { return fmt.Errorf("create empty %s partition: %w", DatasetActivity, err) } - query := buildRelationshipActivitySQL(b.base, 0) + query := buildSparseRelationshipActivitySQL(b.base, b.messageRelation(), 0) statement := "COPY (SELECT * FROM (" + query + ") WHERE false) TO '" + quoteSQLString(emptyOutput) + "' (FORMAT PARQUET, COMPRESSION 'zstd')" if _, err := b.db.ExecContext(ctx, statement); err != nil { diff --git a/internal/identityindex/build_test.go b/internal/identityindex/build_test.go index 556ffd258..8a126f123 100644 --- a/internal/identityindex/build_test.go +++ b/internal/identityindex/build_test.go @@ -49,9 +49,9 @@ func TestBuildPublishesFourRelationshipDatasets(t *testing.T) { var direct, conversation, author, owner bool requirements.NoError(db.QueryRow(` SELECT is_direct, is_conversation_member, is_author, is_owner - FROM read_parquet(?, hive_partitioning=true, union_by_name=true) + FROM `+testExpandedActivity(root, root)+` WHERE canonical_id = 2 - `, relationshipParquetGlob(root, DatasetActivity)). + `). Scan(&direct, &conversation, &author, &owner)) assertions.True(direct) assertions.False(conversation) @@ -60,9 +60,9 @@ func TestBuildPublishesFourRelationshipDatasets(t *testing.T) { requirements.NoError(db.QueryRow(` SELECT is_direct, is_conversation_member - FROM read_parquet(?, hive_partitioning=true, union_by_name=true) + FROM `+testExpandedActivity(root, root)+` WHERE canonical_id = 4 - `, relationshipParquetGlob(root, DatasetActivity)). + `). Scan(&direct, &conversation)) assertions.False(direct) assertions.True(conversation) @@ -194,7 +194,7 @@ func TestLogicalChatReductionKeepsParticipantlessNewestMessage(t *testing.T) { var isFromMe bool var attachmentCount int64 query := logicalActivitySQL( - relationshipParquetGlob(root, DatasetActivity), + testExpandedActivity(root, root), "true", ) + ` SELECT anchor_message_id, is_from_me, attachment_count @@ -227,6 +227,16 @@ func TestBuildIncrementalWritesActivityDeltaAndRebuildsCompactPopulation(t *test stagedRoot, stagedDB := writeRelationshipBaseFixture(t, false) rewriteRelationshipFixtureIDs(t, stagedDB, stagedRoot, 200, 20) + // Incremental exports replace the roster with a complete snapshot. Keep + // the older conversation in this synthetic staged base as well. + _, err = db.Exec(`CREATE TEMP TABLE staged_roster AS + SELECT * FROM read_parquet(?) UNION + SELECT * FROM read_parquet(?)`, + relationshipParquetGlob(committedRoot, "conversation_participants"), + relationshipParquetGlob(stagedRoot, "conversation_participants")) + requirements.NoError(err) + replaceRelationshipParquet(t, db, stagedRoot, "conversation_participants", + "SELECT * FROM staged_roster") result, err := Build(context.Background(), db, BuildOptions{ Mode: ModeIncremental, CommittedRoot: committedRoot, @@ -248,6 +258,205 @@ func TestBuildIncrementalWritesActivityDeltaAndRebuildsCompactPopulation(t *test SELECT activity_count FROM read_parquet(?) WHERE canonical_id = 2 `, relationshipParquetGlob(stagedRoot, DatasetPeople)).Scan(&activityCount)) assertions.Equal(int64(2), activityCount) + assertIncrementalContributionsMatchFullReduction(t, db, committedRoot, stagedRoot, + time.Date(2027, 7, 21, 10, 30, 0, 0, time.UTC)) + assertIncrementalDatasetsMatchFullBuild(t, db, committedRoot, stagedRoot, + time.Date(2027, 7, 21, 10, 30, 0, 0, time.UTC)) +} + +func assertIncrementalDatasetsMatchFullBuild( + t *testing.T, db *sql.DB, committedRoot, stagedRoot string, effectiveAt time.Time, +) { + t.Helper() + fullRoot := t.TempDir() + appendDatasets := map[string]string{ + "messages": "UNION ALL", "message_recipients": "UNION ALL", + "attachments": "UNION ALL", "conversations": "UNION", + "conversation_participants": "UNION", + } + for _, dataset := range baseIdentityDatasets { + query := "SELECT * FROM read_parquet('" + + quoteSQLString(relationshipParquetGlob(stagedRoot, dataset)) + "')" + if union, ok := appendDatasets[dataset]; ok { + query = "SELECT * FROM read_parquet('" + + quoteSQLString(relationshipParquetGlob(committedRoot, dataset)) + + "') " + union + " " + query + } + writeRelationshipParquet(t, db, fullRoot, dataset, query) + } + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: fullRoot, OutputRoot: fullRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + for _, dataset := range []string{ + DatasetPeople, DatasetDomains, DatasetRelationshipDaily, + DatasetLogicalContributions, DatasetTemperatureContributions, + } { + t.Run(dataset+" matches full build", func(t *testing.T) { + incremental := "read_parquet('" + + quoteSQLString(relationshipParquetGlob(stagedRoot, dataset)) + "')" + full := "read_parquet('" + + quoteSQLString(relationshipParquetGlob(fullRoot, dataset)) + "')" + var differences int64 + err := db.QueryRow(`SELECT count(*) FROM ( + (SELECT * FROM ` + incremental + ` EXCEPT ALL SELECT * FROM ` + full + `) + UNION ALL + (SELECT * FROM ` + full + ` EXCEPT ALL SELECT * FROM ` + incremental + `) + )`).Scan(&differences) + require.NoError(t, err) + assert.Zero(t, differences) + }) + } +} + +func assertIncrementalContributionsMatchFullReduction( + t *testing.T, db *sql.DB, committedRoot, stagedRoot string, effectiveAt time.Time, +) { + t.Helper() + fullActivity := testExpandedActivity(stagedRoot, committedRoot, stagedRoot) + for _, check := range []struct { + name string + dataset string + expected string + }{ + {"logical", DatasetLogicalContributions, + buildLogicalActivityMaterializationSQL(fullActivity)}, + {"temperature", DatasetTemperatureContributions, + buildRelationshipTemperatureDailySQL(fullActivity, effectiveAt)}, + } { + t.Run(check.name+" contributions match full reduction", func(t *testing.T) { + expectedTable := "expected_" + check.name + _, err := db.Exec("CREATE TEMP TABLE " + expectedTable + " AS " + check.expected) + require.NoError(t, err) + actual := "read_parquet('" + quoteSQLString(relationshipParquetGlob(stagedRoot, check.dataset)) + "')" + var differences int64 + err = db.QueryRow(`SELECT count(*) FROM ( + (SELECT * FROM ` + actual + ` EXCEPT ALL SELECT * FROM ` + expectedTable + `) + UNION ALL + (SELECT * FROM ` + expectedTable + ` EXCEPT ALL SELECT * FROM ` + actual + `) + )`).Scan(&differences) + require.NoError(t, err) + assert.Zero(t, differences) + }) + } +} + +func testExpandedActivity(baseRoot string, activityRoots ...string) string { + paths := make([]string, len(activityRoots)) + for i, root := range activityRoots { + paths[i] = relationshipParquetGlob(root, DatasetActivity) + } + base := func(dataset string) string { + return readParquetRelation([]string{parquetDatasetGlob(baseRoot, dataset)}, false) + } + return ExpandedActivityRelation(readParquetRelation(paths, true), + base("conversation_participants"), base("participants"), + base("participant_clusters"), base("owner_participants")) +} + +func TestBuildIncrementalChatContributionsMatchFullReductionAcrossYears(t *testing.T) { + committedRoot, db := writeRelationshipBaseFixture(t, false) + setChatRelationshipFixture(t, db, committedRoot, 100, 2026) + effectiveAt := time.Date(2027, 7, 21, 10, 30, 0, 0, time.UTC) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: committedRoot, OutputRoot: committedRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + + stagedRoot, stagedDB := writeRelationshipBaseFixture(t, false) + rewriteRelationshipFixtureIDs(t, stagedDB, stagedRoot, 200, 10) + setChatRelationshipFixture(t, stagedDB, stagedRoot, 200, 2027) + _, err = Build(context.Background(), db, BuildOptions{ + Mode: ModeIncremental, CommittedRoot: committedRoot, + StagedBaseRoot: stagedRoot, OutputRoot: stagedRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + assertIncrementalContributionsMatchFullReduction(t, db, committedRoot, stagedRoot, effectiveAt) + assertIncrementalDatasetsMatchFullBuild(t, db, committedRoot, stagedRoot, effectiveAt) +} + +func TestBuildIncrementalRecomputesNewlyEligibleOldTemperatureFacts(t *testing.T) { + committedRoot, db := writeRelationshipBaseFixture(t, false) + setChatRelationshipFixture(t, db, committedRoot, 100, 2026) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: committedRoot, OutputRoot: committedRoot, + EffectiveAt: time.Date(2025, 7, 21, 10, 30, 0, 0, time.UTC), + }) + require.NoError(t, err) + + stagedRoot, stagedDB := writeRelationshipBaseFixture(t, false) + rewriteRelationshipFixtureIDs(t, stagedDB, stagedRoot, 201, 10) + setChatRelationshipFixture(t, stagedDB, stagedRoot, 201, 2027) + effectiveAt := time.Date(2027, 7, 21, 10, 30, 0, 0, time.UTC) + _, err = Build(context.Background(), db, BuildOptions{ + Mode: ModeIncremental, CommittedRoot: committedRoot, + StagedBaseRoot: stagedRoot, OutputRoot: stagedRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + assertIncrementalContributionsMatchFullReduction(t, db, committedRoot, stagedRoot, effectiveAt) +} + +func TestBuildIncrementalChatInheritsParticipantlessHistory(t *testing.T) { + committedRoot, db := writeRelationshipBaseFixture(t, false) + setChatRelationshipFixture(t, db, committedRoot, 100, 2026) + replaceRelationshipParquet(t, db, committedRoot, "message_recipients", ` + SELECT 0::BIGINT AS message_id, 0::BIGINT AS participant_id, + ''::VARCHAR AS recipient_type, ''::VARCHAR AS display_name WHERE false`) + replaceRelationshipParquet(t, db, committedRoot, "conversation_participants", ` + SELECT 0::BIGINT AS conversation_id, 0::BIGINT AS participant_id WHERE false`) + replaceRelationshipParquet(t, db, committedRoot, "messages", ` + SELECT 100::BIGINT AS id, 1::BIGINT AS source_id, 'm-100'::VARCHAR AS source_message_id, + 10::BIGINT AS conversation_id, 'Subject'::VARCHAR AS subject, + 'Preview'::VARCHAR AS snippet, TIMESTAMP '2026-07-21 10:30:00' AS sent_at, + 50::BIGINT AS size_estimate, true AS has_attachments, + 1::INTEGER AS attachment_count, NULL::TIMESTAMP AS deleted_from_source_at, + NULL::BIGINT AS sender_id, NULL::BIGINT AS owner_participant_id, + 'chat'::VARCHAR AS message_type, false AS is_from_me, + 2026::INTEGER AS year, 7::INTEGER AS month`) + effectiveAt := time.Date(2027, 7, 21, 10, 30, 0, 0, time.UTC) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: committedRoot, OutputRoot: committedRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + + stagedRoot, stagedDB := writeRelationshipBaseFixture(t, false) + rewriteRelationshipFixtureIDs(t, stagedDB, stagedRoot, 200, 10) + setChatRelationshipFixture(t, stagedDB, stagedRoot, 200, 2027) + replaceRelationshipParquet(t, stagedDB, stagedRoot, "conversation_participants", ` + SELECT 0::BIGINT AS conversation_id, 0::BIGINT AS participant_id WHERE false`) + _, err = Build(context.Background(), db, BuildOptions{ + Mode: ModeIncremental, CommittedRoot: committedRoot, + StagedBaseRoot: stagedRoot, OutputRoot: stagedRoot, + EffectiveAt: effectiveAt, + }) + require.NoError(t, err) + assertIncrementalContributionsMatchFullReduction(t, db, committedRoot, stagedRoot, effectiveAt) +} + +func setChatRelationshipFixture(t *testing.T, db *sql.DB, root string, messageID int64, year int) { + t.Helper() + replaceRelationshipParquet(t, db, root, "conversations", ` + SELECT 10::BIGINT AS id, 'thread-10'::VARCHAR AS source_conversation_id, + 'Thread'::VARCHAR AS title, 'group_chat'::VARCHAR AS conversation_type`) + replaceRelationshipParquet(t, db, root, "messages", fmt.Sprintf(` + SELECT %d::BIGINT AS id, 1::BIGINT AS source_id, 'm-%d'::VARCHAR AS source_message_id, + 10::BIGINT AS conversation_id, 'Subject'::VARCHAR AS subject, + 'Preview'::VARCHAR AS snippet, TIMESTAMP '%d-07-21 10:30:00' AS sent_at, + 50::BIGINT AS size_estimate, true AS has_attachments, + 1::INTEGER AS attachment_count, NULL::TIMESTAMP AS deleted_from_source_at, + 2::BIGINT AS sender_id, 1::BIGINT AS owner_participant_id, + 'chat'::VARCHAR AS message_type, false AS is_from_me, + %d::INTEGER AS year, 7::INTEGER AS month`, messageID, messageID, year, year)) + replaceRelationshipParquet(t, db, root, "message_recipients", fmt.Sprintf(` + SELECT * FROM (VALUES + (%d::BIGINT, 2::BIGINT, 'from'::VARCHAR, 'Bob'::VARCHAR), + (%d::BIGINT, 1::BIGINT, 'to'::VARCHAR, 'Alice'::VARCHAR) + ) AS t(message_id, participant_id, recipient_type, display_name)`, messageID, messageID)) } func TestBuildIndexOnlyUsesCommittedBaseWithStagedIdentityDimensions(t *testing.T) { diff --git a/internal/identityindex/compact_sql.go b/internal/identityindex/compact_sql.go index 5aae16d66..6b122ae9a 100644 --- a/internal/identityindex/compact_sql.go +++ b/internal/identityindex/compact_sql.go @@ -182,7 +182,8 @@ ORDER BY canonical_id, event_date`, } func activityRelation(path string, hivePartitioning bool) string { - if strings.HasPrefix(strings.TrimSpace(path), "read_parquet(") { + trimmed := strings.TrimSpace(path) + if strings.HasPrefix(trimmed, "read_parquet(") || strings.HasPrefix(trimmed, "(") { return path } options := "" diff --git a/internal/identityindex/contributions.go b/internal/identityindex/contributions.go new file mode 100644 index 000000000..722b8c984 --- /dev/null +++ b/internal/identityindex/contributions.go @@ -0,0 +1,148 @@ +package identityindex + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// materializeContributions keeps the compact logical and daily scoring grains +// across append builds. Only new activity edges are reduced on an append; +// global rankings still read the small daily contribution table. +func (b builder) materializeContributions(ctx context.Context, activity string, effectiveAt time.Time) error { + temperatureSQL := buildRelationshipTemperatureDailySQL(activity, effectiveAt) + logicalSQL := buildLogicalActivityMaterializationSQL(activity) + if b.opts.Mode == ModeIncremental && + datasetContainsParquet(b.opts.CommittedRoot, DatasetLogicalContributions) && + datasetContainsParquet(b.opts.CommittedRoot, DatasetTemperatureContributions) { + delta := b.deltaActivityRelation() + reuseTemperature, err := b.canReuseTemperatureContributions(ctx, effectiveAt) + if err != nil { + return err + } + if reuseTemperature { + temperatureSQL = mergeTemperatureContributionsSQL( + b.committed(DatasetTemperatureContributions), + buildRelationshipTemperatureDailySQL(delta, effectiveAt), + ) + } + logicalSQL = mergeLogicalContributionsSQL( + b.committed(DatasetLogicalContributions), + buildLogicalActivityMaterializationSQL(delta), + ) + } + if err := b.materializeBuildTable(ctx, temperatureBuildRelation, + temperatureSQL, "relationship_temperature_daily"); err != nil { + return err + } + if err := b.materializeBuildTable(ctx, logicalBuildRelation, + logicalSQL, "logical_activity"); err != nil { + return err + } + return nil +} + +// A previously future-dated message can enter the score window without a +// new activity shard. In that rare case the daily score reduction must read +// the committed activity once instead of adding only the new contribution. +func (b builder) canReuseTemperatureContributions(ctx context.Context, effectiveAt time.Time) (bool, error) { + var priorCutoff sql.NullTime + query := `SELECT max(temperature_effective_at) FROM read_parquet('` + + quoteSQLString(b.committed(DatasetPeople)) + `')` + if err := b.db.QueryRowContext(ctx, query).Scan(&priorCutoff); err != nil { + return false, fmt.Errorf("inspect prior relationship score window: %w", err) + } + if !priorCutoff.Valid { + return false, nil + } + if effectiveAt.Before(priorCutoff.Time) { + return false, nil + } + var newlyEligible bool + query = `SELECT EXISTS ( + SELECT 1 FROM read_parquet('` + quoteSQLString(b.committed("messages")) + + `', hive_partitioning=true, union_by_name=true) + WHERE sent_at > ? AND sent_at <= ?)` + if err := b.db.QueryRowContext(ctx, query, priorCutoff.Time, effectiveAt).Scan(&newlyEligible); err != nil { + return false, fmt.Errorf("inspect newly eligible relationship messages: %w", err) + } + return !newlyEligible, nil +} + +func mergeTemperatureContributionsSQL(committed, delta string) string { + return fmt.Sprintf(` +WITH contributions AS ( + SELECT * FROM read_parquet('%s') + UNION ALL + SELECT * FROM (%s) +) +SELECT canonical_id, event_date, + sum(sent_count)::BIGINT AS sent_count, + sum(received_count)::BIGINT AS received_count, + sum(meeting_count)::BIGINT AS meeting_count, + sum(email_count)::BIGINT AS email_count, + sum(chat_count)::BIGINT AS chat_count, + sum(total_count)::BIGINT AS total_count, + bit_or(modality_mask)::UTINYINT AS modality_mask, + max(last_at)::TIMESTAMP AS last_at +FROM contributions +GROUP BY canonical_id, event_date`, quoteSQLString(committed), delta) +} + +func mergeLogicalContributionsSQL(committed, delta string) string { + return fmt.Sprintf(` +WITH old_rows AS ( + SELECT * FROM read_parquet('%s') +), new_rows AS ( + SELECT * FROM (%s) +), combined AS ( + SELECT * FROM old_rows UNION ALL SELECT * FROM new_rows +), unit_parts AS ( + SELECT entry_key, max(attachment_count) AS attachment_count + FROM old_rows WHERE relation_kind IN (1, 2, 4) GROUP BY entry_key + UNION ALL + SELECT entry_key, max(attachment_count) AS attachment_count + FROM new_rows WHERE relation_kind IN (1, 2, 4) GROUP BY entry_key +), units AS ( + SELECT c.entry_key, + arg_max(c.anchor_message_id, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS anchor_message_id, + arg_max(c.conversation_id, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS conversation_id, + arg_max(c.source_id, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS source_id, + arg_max(c.source_type, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS source_type, + max(c.occurred_at)::TIMESTAMP AS occurred_at, + arg_max(c.message_type, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS message_type, + arg_max(c.entry_kind, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS entry_kind, + arg_max(c.is_from_me, struct_pack(at := c.occurred_at, id := c.anchor_message_id)) AS is_from_me, + max(p.attachment_count)::BIGINT AS attachment_count, + bool_or(c.is_owner) AS with_owner + FROM combined c + LEFT JOIN (SELECT entry_key, sum(attachment_count) AS attachment_count + FROM unit_parts GROUP BY entry_key) p USING (entry_key) + WHERE c.relation_kind IN (1, 2, 4) + GROUP BY c.entry_key +), merged AS ( + SELECT relation_kind, entry_key, canonical_id, domain, + bool_or(c.is_author AND c.anchor_message_id = u.anchor_message_id) AS is_author, + bool_or(c.is_owner) AS is_owner + FROM combined c + LEFT JOIN units u USING (entry_key) + GROUP BY relation_kind, entry_key, canonical_id, domain +) +SELECT m.relation_kind, m.entry_key, + CASE WHEN m.relation_kind = 3 THEN NULL::BIGINT ELSE u.anchor_message_id END AS anchor_message_id, + CASE WHEN m.relation_kind = 3 THEN NULL::BIGINT ELSE u.conversation_id END AS conversation_id, + CASE WHEN m.relation_kind = 3 THEN NULL::BIGINT ELSE u.source_id END AS source_id, + CASE WHEN m.relation_kind = 3 THEN NULL::VARCHAR ELSE u.source_type END AS source_type, + CASE WHEN m.relation_kind = 3 THEN NULL::TIMESTAMP ELSE u.occurred_at END AS occurred_at, + CASE WHEN m.relation_kind = 3 THEN NULL::VARCHAR ELSE u.message_type END AS message_type, + CASE WHEN m.relation_kind = 3 THEN NULL::VARCHAR ELSE u.entry_kind END AS entry_kind, + CASE WHEN m.relation_kind = 3 THEN NULL::BOOLEAN ELSE u.is_from_me END AS is_from_me, + CASE WHEN m.relation_kind = 3 THEN NULL::BIGINT ELSE u.attachment_count END AS attachment_count, + m.canonical_id, + CASE WHEN m.relation_kind = 1 THEN m.is_author ELSE NULL::BOOLEAN END AS is_author, + CASE WHEN m.relation_kind = 1 THEN m.is_owner ELSE NULL::BOOLEAN END AS is_owner, + CASE WHEN m.relation_kind = 1 THEN u.with_owner ELSE NULL::BOOLEAN END AS with_owner, + m.domain +FROM merged m LEFT JOIN units u USING (entry_key)`, quoteSQLString(committed), delta) +} diff --git a/internal/identityindex/normalized_activity.go b/internal/identityindex/normalized_activity.go new file mode 100644 index 000000000..f2ca53452 --- /dev/null +++ b/internal/identityindex/normalized_activity.go @@ -0,0 +1,156 @@ +package identityindex + +import "fmt" + +// buildSparseRelationshipActivitySQL writes message facts once and keeps only +// direct canonical edges beside them. Conversation membership stays in the +// conversation_participants base dataset, where it is stored once per thread. +func buildSparseRelationshipActivitySQL(path func(string) string, messages string, occurredYear int64) string { + return fmt.Sprintf(` +WITH message_facts AS ( + SELECT m.id::BIGINT AS message_id, m.conversation_id::BIGINT AS conversation_id, + m.source_id::BIGINT AS source_id, s.source_type::VARCHAR AS source_type, + m.sent_at::TIMESTAMP AS occurred_at, m.message_type::VARCHAR AS message_type, + coalesce(c.conversation_type, '')::VARCHAR AS conversation_type, + %s AS entry_kind, (%s) AS is_chat, + m.is_from_me::BOOLEAN AS is_from_me, m.attachment_count::INTEGER AS attachment_count, + coalesce(m.has_attachments::BOOLEAN, false) AS has_attachments, + (m.deleted_from_source_at IS NOT NULL) AS deleted_from_source, + year(m.sent_at)::SMALLINT AS occurred_year, + m.sender_id::BIGINT AS sender_id, + m.owner_participant_id::BIGINT AS owner_participant_id + FROM %s m + JOIN read_parquet('%s') s ON s.id = m.source_id + LEFT JOIN read_parquet('%s') c ON c.id = m.conversation_id + WHERE year(m.sent_at) = %d +), canon AS ( + SELECT p.id::BIGINT AS participant_id, + coalesce(c.canonical_id, p.id)::BIGINT AS canonical_id, + lower(coalesce(p.domain, ''))::VARCHAR AS participant_domain + FROM read_parquet('%s') p + LEFT JOIN read_parquet('%s') c ON c.participant_id = p.id +), owner_canon AS ( + SELECT DISTINCT c.canonical_id + FROM read_parquet('%s') o + JOIN canon c ON c.participant_id = o.participant_id +), direct_edges AS ( + SELECT mr.message_id::BIGINT AS message_id, mr.participant_id::BIGINT AS participant_id, + (mr.recipient_type = 'from' AND mr.participant_id = m.owner_participant_id) AS is_sender, + (mr.recipient_type = 'from' AND mr.participant_id = m.owner_participant_id) AS is_owner_sender, + (mr.recipient_type = 'from') AS is_author + FROM read_parquet('%s') mr JOIN message_facts m ON m.message_id = mr.message_id + UNION ALL + SELECT m.message_id, m.sender_id, true, + coalesce(m.sender_id = m.owner_participant_id, false), true + FROM message_facts m WHERE m.sender_id IS NOT NULL +), direct_canon AS ( + SELECT e.*, c.canonical_id, c.participant_domain + FROM direct_edges e LEFT JOIN canon c USING (participant_id) +), direct_agg AS ( + SELECT message_id, canonical_id, participant_domain, + bool_or(is_sender) AS is_sender, + bool_or(is_owner_sender) AS is_owner_sender, + bool_or(is_author) AS is_author + FROM direct_canon WHERE canonical_id IS NOT NULL + GROUP BY message_id, canonical_id, participant_domain +) +SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, + m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, + m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, + m.deleted_from_source, d.canonical_id, d.participant_domain, + true AS is_direct, false AS is_conversation_member, + d.is_sender, d.is_author, + (o.canonical_id IS NOT NULL OR (m.is_from_me AND d.is_owner_sender)) AS is_owner, + m.occurred_year +FROM message_facts m JOIN direct_agg d USING (message_id) +LEFT JOIN owner_canon o ON o.canonical_id = d.canonical_id +UNION ALL +SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, + m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, + m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, + m.deleted_from_source, NULL::BIGINT AS canonical_id, + NULL::VARCHAR AS participant_domain, + EXISTS (SELECT 1 FROM direct_canon d + WHERE d.message_id = m.message_id AND d.canonical_id IS NULL) AS is_direct, + false AS is_conversation_member, false AS is_sender, + false AS is_author, false AS is_owner, m.occurred_year +FROM message_facts m`, EntryKindSQL("m.message_type"), + IsChatSQL("m.message_type", "coalesce(c.conversation_type, '')"), + messages, quoteSQLString(path("sources")), + quoteSQLString(path("conversations")), occurredYear, + quoteSQLString(path("participants")), quoteSQLString(path("participant_clusters")), + quoteSQLString(path("owner_participants")), quoteSQLString(path("message_recipients"))) +} + +// ExpandedActivityRelation restores the logical edge relation on demand from +// sparse message facts and the normalized roster. Its arguments are trusted +// SQL relations produced by the cache builder or query engine. Keep the CTEs +// inlined so query predicates reach Parquet instead of materializing the archive. +func ExpandedActivityRelation(sparse, roster, participants, clusters, owners string) string { + return fmt.Sprintf(`( +WITH raw_activity AS NOT MATERIALIZED (SELECT * FROM %[1]s), +facts AS NOT MATERIALIZED (SELECT * FROM raw_activity WHERE canonical_id IS NULL), +direct AS NOT MATERIALIZED (SELECT * FROM raw_activity WHERE canonical_id IS NOT NULL), +canon AS NOT MATERIALIZED ( + SELECT p.id::BIGINT AS participant_id, + coalesce(c.canonical_id, p.id)::BIGINT AS canonical_id, + lower(coalesce(p.domain, ''))::VARCHAR AS participant_domain + FROM %[3]s p LEFT JOIN %[4]s c ON c.participant_id = p.id +), owner_canon AS NOT MATERIALIZED ( + SELECT DISTINCT c.canonical_id FROM %[5]s o + JOIN canon c ON c.participant_id = o.participant_id +), members AS NOT MATERIALIZED ( + SELECT DISTINCT cp.conversation_id::BIGINT AS conversation_id, + c.canonical_id, c.participant_domain, + (o.canonical_id IS NOT NULL) AS is_owner + FROM %[2]s cp + LEFT JOIN canon c ON c.participant_id = cp.participant_id + LEFT JOIN owner_canon o ON o.canonical_id = c.canonical_id +) +SELECT f.message_id, f.conversation_id, f.source_id, f.source_type, + f.occurred_at, f.message_type, f.conversation_type, f.entry_kind, + f.is_chat, f.is_from_me, f.attachment_count, f.has_attachments, + f.deleted_from_source, cm.canonical_id, cm.participant_domain, + coalesce(d.is_direct, false) AS is_direct, + true AS is_conversation_member, coalesce(d.is_sender, false) AS is_sender, + coalesce(d.is_author, false) AS is_author, + (cm.is_owner OR coalesce(d.is_owner, false)) AS is_owner, + f.occurred_year +FROM facts f JOIN members cm ON cm.conversation_id = f.conversation_id +LEFT JOIN direct d ON d.message_id = f.message_id + AND d.occurred_year = f.occurred_year + AND d.canonical_id = cm.canonical_id + AND d.participant_domain = cm.participant_domain +WHERE cm.canonical_id IS NOT NULL +UNION ALL +SELECT d.message_id, d.conversation_id, d.source_id, d.source_type, + d.occurred_at, d.message_type, d.conversation_type, d.entry_kind, + d.is_chat, d.is_from_me, d.attachment_count, d.has_attachments, + d.deleted_from_source, d.canonical_id, d.participant_domain, + d.is_direct, false AS is_conversation_member, + d.is_sender, d.is_author, d.is_owner, d.occurred_year +FROM direct d +WHERE NOT EXISTS ( + SELECT 1 FROM members cm WHERE cm.conversation_id = d.conversation_id + AND cm.canonical_id = d.canonical_id + AND cm.participant_domain = d.participant_domain +) +UNION ALL +SELECT f.message_id, f.conversation_id, f.source_id, f.source_type, + f.occurred_at, f.message_type, f.conversation_type, f.entry_kind, + f.is_chat, f.is_from_me, f.attachment_count, f.has_attachments, + f.deleted_from_source, NULL::BIGINT AS canonical_id, + NULL::VARCHAR AS participant_domain, + false AS is_direct, false AS is_conversation_member, + false AS is_sender, false AS is_author, false AS is_owner, + f.occurred_year +FROM facts f +WHERE f.is_direct + OR EXISTS (SELECT 1 FROM members cm + WHERE cm.conversation_id = f.conversation_id AND cm.canonical_id IS NULL) + OR (NOT EXISTS (SELECT 1 FROM direct d WHERE d.message_id = f.message_id) + AND NOT EXISTS (SELECT 1 FROM members cm + WHERE cm.conversation_id = f.conversation_id + AND cm.canonical_id IS NOT NULL)) +)`, sparse, roster, participants, clusters, owners) +} diff --git a/internal/identityindex/normalized_activity_test.go b/internal/identityindex/normalized_activity_test.go new file mode 100644 index 000000000..3f1e664f2 --- /dev/null +++ b/internal/identityindex/normalized_activity_test.go @@ -0,0 +1,74 @@ +package identityindex + +import ( + "encoding/json/v2" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpandedActivityFiltersReachParquet(t *testing.T) { + root, db := writeRelationshipBaseFixture(t, true) + writeSyntheticRelationshipFanOut(t, db, root, syntheticRelationshipFanOutOptions{ + firstMessageID: 1, messageCount: 20_000, memberCount: 100, + startDate: "2023-12-31 22:00:00", messageType: "whatsapp", conversationType: "group_chat", + }) + path := func(dataset string) string { return parquetDatasetGlob(root, dataset) } + writeRelationshipParquet(t, db, root, DatasetActivity, "("+buildSparseRelationshipActivitySQL(path, readParquetRelation([]string{path("messages")}, true), 2023)+") UNION ALL ("+buildSparseRelationshipActivitySQL(path, readParquetRelation([]string{path("messages")}, true), 2024)+")") + relation := ExpandedActivityRelation( + readParquetRelation([]string{filepath.Join(root, DatasetActivity, "*.parquet")}, false), + readParquetRelation([]string{path("conversation_participants")}, false), + readParquetRelation([]string{path("participants")}, false), + readParquetRelation([]string{path("participant_clusters")}, false), + readParquetRelation([]string{path("owner_participants")}, false), + ) + for _, tc := range []struct { + predicate string + filter string + want int64 + }{ + {"occurred_year = 2024", "occurred_year=2024", 1_280_000}, + {"message_id = 7", "message_id=7", 100}, + {"occurred_year = 2024 AND canonical_id = 5", "occurred_year=2024", 12_800}, + } { + t.Run(tc.predicate, func(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + var kind, plan string + requirements.NoError(db.QueryRow("EXPLAIN (FORMAT JSON) SELECT * FROM "+relation+" WHERE "+tc.predicate).Scan(&kind, &plan)) + type node struct { + Name string `json:"name"` + ExtraInfo map[string]any `json:"extra_info"` + Children []node `json:"children"` + } + var nodes []node + requirements.NoError(json.Unmarshal([]byte(plan), &nodes)) + // Inspect scans carrying message payload. Membership and existence + // joins may separately read narrow key columns. With a person filter, + // the orphan branch disappears and all activity scans should be scoped. + scans := 0 + var visit func([]node) + visit = func(nodes []node) { + for _, n := range nodes { + projections := fmt.Sprint(n.ExtraInfo["Projections"]) + if strings.Contains(n.Name, "PARQUET") && + strings.Contains(projections, "message_id") && + (strings.Contains(projections, "source_type") || strings.Contains(tc.predicate, "canonical_id")) { + scans++ + assertions.Contains(fmt.Sprint(n.ExtraInfo["Filters"]), tc.filter) + } + visit(n.Children) + } + } + visit(nodes) + assertions.Positive(scans) + var count int64 + requirements.NoError(db.QueryRow("SELECT count(*) FROM " + relation + " WHERE " + tc.predicate).Scan(&count)) + assertions.Equal(tc.want, count) + }) + } +} diff --git a/internal/identityindex/relationship_sql.go b/internal/identityindex/relationship_sql.go index 7b790dc7e..b36dbfd61 100644 --- a/internal/identityindex/relationship_sql.go +++ b/internal/identityindex/relationship_sql.go @@ -182,179 +182,6 @@ WITH filtered_facts AS ( ) } -func buildRelationshipActivitySQL(path func(string) string, occurredYear int64) string { - return fmt.Sprintf(` -WITH message_facts AS ( - SELECT m.id::BIGINT AS message_id, - m.conversation_id::BIGINT AS conversation_id, - m.source_id::BIGINT AS source_id, - s.source_type::VARCHAR AS source_type, - m.sent_at::TIMESTAMP AS occurred_at, - m.message_type::VARCHAR AS message_type, - coalesce(c.conversation_type, '')::VARCHAR AS conversation_type, - %s AS entry_kind, - (%s) AS is_chat, - m.is_from_me::BOOLEAN AS is_from_me, - m.attachment_count::INTEGER AS attachment_count, - coalesce(m.has_attachments::BOOLEAN, false) AS has_attachments, - (m.deleted_from_source_at IS NOT NULL) AS deleted_from_source, - year(m.sent_at)::SMALLINT AS occurred_year, - m.sender_id::BIGINT AS sender_id, - m.owner_participant_id::BIGINT AS owner_participant_id - FROM read_parquet('%s', hive_partitioning=true, union_by_name=true) m - JOIN read_parquet('%s') s ON s.id = m.source_id - LEFT JOIN read_parquet('%s') c ON c.id = m.conversation_id - WHERE year(m.sent_at) = %d -), scoped_conversations AS ( - SELECT DISTINCT conversation_id - FROM message_facts - WHERE conversation_id IS NOT NULL -), canon AS ( - SELECT p.id::BIGINT AS participant_id, - coalesce(c.canonical_id, p.id)::BIGINT AS canonical_id, - lower(coalesce(p.domain, ''))::VARCHAR AS participant_domain - FROM read_parquet('%s') p - LEFT JOIN read_parquet('%s') c ON c.participant_id = p.id -), owner_canon AS ( - SELECT DISTINCT c.canonical_id - FROM read_parquet('%s') o - JOIN canon c ON c.participant_id = o.participant_id -), direct_edges AS ( - SELECT mr.message_id::BIGINT AS message_id, - mr.participant_id::BIGINT AS participant_id, - true AS is_direct, - (mr.recipient_type = 'from' - AND mr.participant_id = m.owner_participant_id) AS is_sender, - (mr.recipient_type = 'from' - AND mr.participant_id = m.owner_participant_id) AS is_owner_sender, - (mr.recipient_type = 'from') AS is_author - FROM read_parquet('%s') mr - JOIN message_facts m ON m.message_id = mr.message_id - - UNION ALL - - SELECT m.message_id, m.sender_id, true, true, - coalesce(m.sender_id = m.owner_participant_id, false), true - FROM message_facts m - WHERE m.sender_id IS NOT NULL -), direct_canon AS ( - SELECT e.message_id, e.is_direct, e.is_sender, e.is_owner_sender, e.is_author, - c.canonical_id, c.participant_domain - FROM direct_edges e - LEFT JOIN canon c USING (participant_id) -), direct_agg AS ( - SELECT message_id, canonical_id, participant_domain, - bool_or(is_direct) AS is_direct, - bool_or(is_sender) AS is_sender, - bool_or(is_owner_sender) AS is_owner_sender, - bool_or(is_author) AS is_author - FROM direct_canon - WHERE canonical_id IS NOT NULL - GROUP BY message_id, canonical_id, participant_domain -), conv_members AS ( - SELECT cp.conversation_id::BIGINT AS conversation_id, - c.canonical_id, c.participant_domain - FROM read_parquet('%s') cp - JOIN scoped_conversations sc ON sc.conversation_id = cp.conversation_id - LEFT JOIN canon c ON c.participant_id = cp.participant_id -), conv_members_canon AS ( - SELECT DISTINCT conversation_id, canonical_id, participant_domain - FROM conv_members - WHERE canonical_id IS NOT NULL -) -SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, - m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, - m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, - m.deleted_from_source, - cm.canonical_id, cm.participant_domain, - coalesce(d.is_direct, false) AS is_direct, - true AS is_conversation_member, - coalesce(d.is_sender, false) AS is_sender, - coalesce(d.is_author, false) AS is_author, - (o.canonical_id IS NOT NULL OR - (m.is_from_me AND coalesce(d.is_owner_sender, false))) AS is_owner, - m.occurred_year -FROM message_facts m - JOIN conv_members_canon cm USING (conversation_id) - LEFT JOIN direct_agg d - ON d.message_id = m.message_id - AND d.canonical_id = cm.canonical_id - AND d.participant_domain = cm.participant_domain - LEFT JOIN owner_canon o ON o.canonical_id = cm.canonical_id - -UNION ALL - -SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, - m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, - m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, - m.deleted_from_source, - d.canonical_id, d.participant_domain, - d.is_direct, - false AS is_conversation_member, - d.is_sender, - d.is_author, - (o.canonical_id IS NOT NULL OR - (m.is_from_me AND d.is_owner_sender)) AS is_owner, - m.occurred_year -FROM message_facts m - JOIN direct_agg d USING (message_id) - LEFT JOIN conv_members_canon cm - ON cm.conversation_id = m.conversation_id - AND cm.canonical_id = d.canonical_id - AND cm.participant_domain = d.participant_domain - LEFT JOIN owner_canon o ON o.canonical_id = d.canonical_id -WHERE cm.canonical_id IS NULL - -UNION ALL - -SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, - m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, - m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, - m.deleted_from_source, - NULL::BIGINT AS canonical_id, - NULL::VARCHAR AS participant_domain, - false AS is_direct, - false AS is_conversation_member, - false AS is_sender, - false AS is_author, - false AS is_owner, - m.occurred_year -FROM message_facts m -WHERE EXISTS ( - SELECT 1 - FROM direct_canon d - WHERE d.message_id = m.message_id AND d.canonical_id IS NULL -) -OR EXISTS ( - SELECT 1 - FROM conv_members cm - WHERE cm.conversation_id = m.conversation_id - AND cm.canonical_id IS NULL -) -OR ( - NOT EXISTS ( - SELECT 1 FROM direct_agg d WHERE d.message_id = m.message_id - ) - AND NOT EXISTS ( - SELECT 1 - FROM conv_members_canon cm - WHERE cm.conversation_id = m.conversation_id - ) -)`, - EntryKindSQL("m.message_type"), - IsChatSQL("m.message_type", "coalesce(c.conversation_type, '')"), - quoteSQLString(path("messages")), - quoteSQLString(path("sources")), - quoteSQLString(path("conversations")), - occurredYear, - quoteSQLString(path("participants")), - quoteSQLString(path("participant_clusters")), - quoteSQLString(path("owner_participants")), - quoteSQLString(path("message_recipients")), - quoteSQLString(path("conversation_participants")), - ) -} - func buildLogicalActivityMaterializationSQL(path string) string { return logicalActivitySQL(path, "true") + ` SELECT 1::UTINYINT AS relation_kind, @@ -388,5 +215,18 @@ SELECT 3::UTINYINT AS relation_kind, NULL::BOOLEAN AS is_author, NULL::BOOLEAN AS is_owner, NULL::BOOLEAN AS with_owner, p.domain FROM logical_person_domains p -WHERE p.domain <> ''` +WHERE p.domain <> '' + +UNION ALL + +-- Keep the unit even when a conversation has no resolved participants yet. +-- A later append can then inherit its previous attachment total and anchor. +SELECT 4::UTINYINT AS relation_kind, + u.entry_key, u.anchor_message_id, u.conversation_id, u.source_id, + u.source_type, u.occurred_at, u.message_type, u.entry_kind, u.is_from_me, + u.attachment_count, NULL::BIGINT AS canonical_id, + NULL::BOOLEAN AS is_author, NULL::BOOLEAN AS is_owner, + NULL::BOOLEAN AS with_owner, NULL::VARCHAR AS domain +FROM logical_units u +WHERE u.entry_kind = 'conversation'` } diff --git a/internal/identityindex/relationship_sql_test.go b/internal/identityindex/relationship_sql_test.go index 7d78bcc94..20571236d 100644 --- a/internal/identityindex/relationship_sql_test.go +++ b/internal/identityindex/relationship_sql_test.go @@ -15,6 +15,8 @@ import ( const stressRelationshipActivity100MEnv = "MSGVAULT_STRESS_RELATIONSHIP_ACTIVITY_100M" func TestRelationshipActivityMatchesLegacyRowSet(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) root, db := writeRelationshipEquivalenceFixture(t) path := func(dataset string) string { return parquetDatasetGlob(root, dataset) @@ -27,9 +29,26 @@ func TestRelationshipActivityMatchesLegacyRowSet(t *testing.T) { (SELECT count(*) FROM (SELECT * FROM legacy EXCEPT SELECT * FROM production)), (SELECT count(*) FROM (SELECT * FROM production EXCEPT SELECT * FROM legacy))` var legacyOnly, productionOnly int64 - require.NoError(t, db.QueryRow(query).Scan(&legacyOnly, &productionOnly)) - assert.Zero(t, legacyOnly) - assert.Zero(t, productionOnly) + requirements.NoError(db.QueryRow(query).Scan(&legacyOnly, &productionOnly)) + assertions.Zero(legacyOnly) + assertions.Zero(productionOnly) + normalized := ExpandedActivityRelation( + "("+buildSparseRelationshipActivitySQL(path, readParquetRelation([]string{path("messages")}, true), 2026)+")", + readParquetRelation([]string{path("conversation_participants")}, false), + readParquetRelation([]string{path("participants")}, false), + readParquetRelation([]string{path("participant_clusters")}, false), + readParquetRelation([]string{path("owner_participants")}, false), + ) + query = `SELECT count(*) FROM ( + (SELECT * FROM (` + buildLegacyRelationshipActivitySQL(path) + `) + EXCEPT ALL SELECT * FROM ` + normalized + `) + UNION ALL + (SELECT * FROM ` + normalized + ` + EXCEPT ALL SELECT * FROM (` + buildLegacyRelationshipActivitySQL(path) + `)) + )` + var normalizedDifferences int64 + requirements.NoError(db.QueryRow(query).Scan(&normalizedDifferences)) + assertions.Zero(normalizedDifferences) } func TestRelationshipActivityYearQueryExcludesOffYearEdgesUnderLowMemory(t *testing.T) { @@ -123,7 +142,7 @@ func TestBuildStreamsRelationshipActivityUnderLowMemory(t *testing.T) { messageType: "email", conversationType: "email_thread", }) - requirements.NoError(setRelationshipTestMemoryLimit(db, "96MB")) + requirements.NoError(setRelationshipTestMemoryLimit(db, "192MB")) result, err := Build(context.Background(), db, BuildOptions{ Mode: ModeFull, @@ -135,6 +154,43 @@ func TestBuildStreamsRelationshipActivityUnderLowMemory(t *testing.T) { assertions.Equal(int64(1_000_000), result.Activity.ConversationExpandedRows) } +func TestBuildStoresConversationMembershipOncePerConversation(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + root, db := writeRelationshipBaseFixture(t, true) + writeSyntheticRelationshipFanOut(t, db, root, syntheticRelationshipFanOutOptions{ + firstMessageID: 1, + messageCount: 40, + memberCount: 300, + startDate: "2026-01-01", + messageType: "whatsapp", + conversationType: "group_chat", + }) + _, err := Build(context.Background(), db, BuildOptions{ + Mode: ModeFull, StagedBaseRoot: root, OutputRoot: root, + }) + requirements.NoError(err) + physicalRows := relationshipParquetCount(t, db, root, DatasetActivity) + assertions.LessOrEqual(physicalRows, int64(120), + "physical message activity must not repeat the 300-member roster for every message") + oldRelation := "(" + buildRelationshipActivitySQL(func(dataset string) string { + return parquetDatasetGlob(root, dataset) + }, 2026) + ")" + newRelation := ExpandedActivityRelation( + readParquetRelation([]string{parquetDatasetGlob(root, DatasetActivity)}, true), + readParquetRelation([]string{parquetDatasetGlob(root, "conversation_participants")}, false), + readParquetRelation([]string{parquetDatasetGlob(root, "participants")}, false), + readParquetRelation([]string{parquetDatasetGlob(root, "participant_clusters")}, false), + readParquetRelation([]string{parquetDatasetGlob(root, "owner_participants")}, false), + ) + for _, pair := range [][2]string{{oldRelation, newRelation}, {newRelation, oldRelation}} { + var difference int64 + requirements.NoError(db.QueryRow("SELECT count(*) FROM (SELECT * FROM " + pair[0] + + " EXCEPT ALL SELECT * FROM " + pair[1] + ")").Scan(&difference)) + assertions.Equal(int64(0), difference) + } +} + func TestBuildIncrementalAppendsIntervalOverFanOut(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) @@ -172,12 +228,12 @@ func TestBuildIncrementalAppendsIntervalOverFanOut(t *testing.T) { }) requirements.NoError(err) - assertions.Equal(int64(15), relationshipParquetCount( + assertions.Equal(int64(6), relationshipParquetCount( t, db, stagedRoot, DatasetActivity, )) - assertions.Equal(int64(7), result.Activity.DirectRows) - assertions.Equal(int64(35), result.Activity.ConversationExpandedRows) - assertions.Equal(int64(35), result.Activity.FinalRows) + assertions.Equal(int64(3), result.Activity.DirectRows) + assertions.Equal(int64(15), result.Activity.ConversationExpandedRows) + assertions.Equal(int64(15), result.Activity.FinalRows) assertions.Equal(int64(7), result.Stats.TotalMessages) var activityCount int64 @@ -253,7 +309,7 @@ func TestLogicalChatReductionPreservesCanonicalAliasDomains(t *testing.T) { requirements.NoError(err) query := logicalActivitySQL( - relationshipParquetGlob(root, DatasetActivity), + testExpandedActivity(root, root), "f.source_id = ?", ) + ` SELECT @@ -355,7 +411,7 @@ func TestLogicalChatReductionKeepsEarlierDirectOnlyIdentity(t *testing.T) { requirements.NoError(err) query := logicalActivitySQL( - relationshipParquetGlob(root, DatasetActivity), + testExpandedActivity(root, root), "true", ) + ` SELECT @@ -810,3 +866,177 @@ GROUP BY m.message_id, m.conversation_id, m.source_id, m.source_type, quoteSQLString(path("conversation_participants")), ) } + +// Reference implementation of the pre-normalization, year-scoped flat layout. +func buildRelationshipActivitySQL(path func(string) string, occurredYear int64) string { + return fmt.Sprintf(` +WITH message_facts AS ( + SELECT m.id::BIGINT AS message_id, + m.conversation_id::BIGINT AS conversation_id, + m.source_id::BIGINT AS source_id, + s.source_type::VARCHAR AS source_type, + m.sent_at::TIMESTAMP AS occurred_at, + m.message_type::VARCHAR AS message_type, + coalesce(c.conversation_type, '')::VARCHAR AS conversation_type, + %s AS entry_kind, + (%s) AS is_chat, + m.is_from_me::BOOLEAN AS is_from_me, + m.attachment_count::INTEGER AS attachment_count, + coalesce(m.has_attachments::BOOLEAN, false) AS has_attachments, + (m.deleted_from_source_at IS NOT NULL) AS deleted_from_source, + year(m.sent_at)::SMALLINT AS occurred_year, + m.sender_id::BIGINT AS sender_id, + m.owner_participant_id::BIGINT AS owner_participant_id + FROM read_parquet('%s', hive_partitioning=true, union_by_name=true) m + JOIN read_parquet('%s') s ON s.id = m.source_id + LEFT JOIN read_parquet('%s') c ON c.id = m.conversation_id + WHERE year(m.sent_at) = %d +), scoped_conversations AS ( + SELECT DISTINCT conversation_id + FROM message_facts + WHERE conversation_id IS NOT NULL +), canon AS ( + SELECT p.id::BIGINT AS participant_id, + coalesce(c.canonical_id, p.id)::BIGINT AS canonical_id, + lower(coalesce(p.domain, ''))::VARCHAR AS participant_domain + FROM read_parquet('%s') p + LEFT JOIN read_parquet('%s') c ON c.participant_id = p.id +), owner_canon AS ( + SELECT DISTINCT c.canonical_id + FROM read_parquet('%s') o + JOIN canon c ON c.participant_id = o.participant_id +), direct_edges AS ( + SELECT mr.message_id::BIGINT AS message_id, + mr.participant_id::BIGINT AS participant_id, + true AS is_direct, + (mr.recipient_type = 'from' + AND mr.participant_id = m.owner_participant_id) AS is_sender, + (mr.recipient_type = 'from' + AND mr.participant_id = m.owner_participant_id) AS is_owner_sender, + (mr.recipient_type = 'from') AS is_author + FROM read_parquet('%s') mr + JOIN message_facts m ON m.message_id = mr.message_id + + UNION ALL + + SELECT m.message_id, m.sender_id, true, true, + coalesce(m.sender_id = m.owner_participant_id, false), true + FROM message_facts m + WHERE m.sender_id IS NOT NULL +), direct_canon AS ( + SELECT e.message_id, e.is_direct, e.is_sender, e.is_owner_sender, e.is_author, + c.canonical_id, c.participant_domain + FROM direct_edges e + LEFT JOIN canon c USING (participant_id) +), direct_agg AS ( + SELECT message_id, canonical_id, participant_domain, + bool_or(is_direct) AS is_direct, + bool_or(is_sender) AS is_sender, + bool_or(is_owner_sender) AS is_owner_sender, + bool_or(is_author) AS is_author + FROM direct_canon + WHERE canonical_id IS NOT NULL + GROUP BY message_id, canonical_id, participant_domain +), conv_members AS ( + SELECT cp.conversation_id::BIGINT AS conversation_id, + c.canonical_id, c.participant_domain + FROM read_parquet('%s') cp + JOIN scoped_conversations sc ON sc.conversation_id = cp.conversation_id + LEFT JOIN canon c ON c.participant_id = cp.participant_id +), conv_members_canon AS ( + SELECT DISTINCT conversation_id, canonical_id, participant_domain + FROM conv_members + WHERE canonical_id IS NOT NULL +) +SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, + m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, + m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, + m.deleted_from_source, + cm.canonical_id, cm.participant_domain, + coalesce(d.is_direct, false) AS is_direct, + true AS is_conversation_member, + coalesce(d.is_sender, false) AS is_sender, + coalesce(d.is_author, false) AS is_author, + (o.canonical_id IS NOT NULL OR + (m.is_from_me AND coalesce(d.is_owner_sender, false))) AS is_owner, + m.occurred_year +FROM message_facts m + JOIN conv_members_canon cm USING (conversation_id) + LEFT JOIN direct_agg d + ON d.message_id = m.message_id + AND d.canonical_id = cm.canonical_id + AND d.participant_domain = cm.participant_domain + LEFT JOIN owner_canon o ON o.canonical_id = cm.canonical_id + +UNION ALL + +SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, + m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, + m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, + m.deleted_from_source, + d.canonical_id, d.participant_domain, + d.is_direct, + false AS is_conversation_member, + d.is_sender, + d.is_author, + (o.canonical_id IS NOT NULL OR + (m.is_from_me AND d.is_owner_sender)) AS is_owner, + m.occurred_year +FROM message_facts m + JOIN direct_agg d USING (message_id) + LEFT JOIN conv_members_canon cm + ON cm.conversation_id = m.conversation_id + AND cm.canonical_id = d.canonical_id + AND cm.participant_domain = d.participant_domain + LEFT JOIN owner_canon o ON o.canonical_id = d.canonical_id +WHERE cm.canonical_id IS NULL + +UNION ALL + +SELECT m.message_id, m.conversation_id, m.source_id, m.source_type, + m.occurred_at, m.message_type, m.conversation_type, m.entry_kind, + m.is_chat, m.is_from_me, m.attachment_count, m.has_attachments, + m.deleted_from_source, + NULL::BIGINT AS canonical_id, + NULL::VARCHAR AS participant_domain, + false AS is_direct, + false AS is_conversation_member, + false AS is_sender, + false AS is_author, + false AS is_owner, + m.occurred_year +FROM message_facts m +WHERE EXISTS ( + SELECT 1 + FROM direct_canon d + WHERE d.message_id = m.message_id AND d.canonical_id IS NULL +) +OR EXISTS ( + SELECT 1 + FROM conv_members cm + WHERE cm.conversation_id = m.conversation_id + AND cm.canonical_id IS NULL +) +OR ( + NOT EXISTS ( + SELECT 1 FROM direct_agg d WHERE d.message_id = m.message_id + ) + AND NOT EXISTS ( + SELECT 1 + FROM conv_members_canon cm + WHERE cm.conversation_id = m.conversation_id + ) +)`, + EntryKindSQL("m.message_type"), + IsChatSQL("m.message_type", "coalesce(c.conversation_type, '')"), + quoteSQLString(path("messages")), + quoteSQLString(path("sources")), + quoteSQLString(path("conversations")), + occurredYear, + quoteSQLString(path("participants")), + quoteSQLString(path("participant_clusters")), + quoteSQLString(path("owner_participants")), + quoteSQLString(path("message_recipients")), + quoteSQLString(path("conversation_participants")), + ) +} diff --git a/internal/identityindex/schema.go b/internal/identityindex/schema.go index 28bf7cc0d..b7f7c69e4 100644 --- a/internal/identityindex/schema.go +++ b/internal/identityindex/schema.go @@ -9,10 +9,12 @@ import ( ) const ( - DatasetActivity = "relationship_activity" - DatasetPeople = "relationship_people" - DatasetDomains = "relationship_domains" - DatasetRelationshipDaily = "relationship_daily" + DatasetActivity = "relationship_activity" + DatasetPeople = "relationship_people" + DatasetDomains = "relationship_domains" + DatasetRelationshipDaily = "relationship_daily" + DatasetLogicalContributions = "relationship_logical_contributions" + DatasetTemperatureContributions = "relationship_temperature_contributions" ModalityEmail uint8 = 1 ModalityChat uint8 = 2 @@ -35,6 +37,8 @@ var ( DatasetPeople, DatasetDomains, DatasetRelationshipDaily, + DatasetLogicalContributions, + DatasetTemperatureContributions, } ) diff --git a/internal/identityindex/validate.go b/internal/identityindex/validate.go index 489b37002..619fdf39d 100644 --- a/internal/identityindex/validate.go +++ b/internal/identityindex/validate.go @@ -6,17 +6,15 @@ import ( "fmt" "path/filepath" "strconv" - "strings" ) // ValidationOptions describes the staged datasets that must form a complete -// relationship index generation. ActivityPath overrides the default staged -// activity glob when validating an incremental build over live plus staged -// shards. +// relationship index generation. ActivityRelation supplies the expanded +// activity rows used to validate a normalized build. type ValidationOptions struct { OutputRoot string RequiredOutputDatasets []string - ActivityPath string + ActivityRelation string } // Validate rejects malformed or internally inconsistent relationship indexes @@ -32,12 +30,8 @@ func Validate( } } - activityPath := opts.ActivityPath - if strings.TrimSpace(activityPath) == "" { - activityPath = parquetDatasetGlob(opts.OutputRoot, DatasetActivity) - } relations := map[string]string{ - DatasetActivity: activityRelation(activityPath, true), + DatasetActivity: activityRelation(parquetDatasetGlob(opts.OutputRoot, DatasetActivity), true), DatasetPeople: activityRelation( filepath.Join(opts.OutputRoot, DatasetPeople, "*.parquet"), false, @@ -50,6 +44,15 @@ func Validate( filepath.Join(opts.OutputRoot, DatasetRelationshipDaily, "*.parquet"), false, ), + DatasetLogicalContributions: activityRelation( + filepath.Join(opts.OutputRoot, DatasetLogicalContributions, "*.parquet"), false, + ), + DatasetTemperatureContributions: activityRelation( + filepath.Join(opts.OutputRoot, DatasetTemperatureContributions, "*.parquet"), false, + ), + } + if opts.ActivityRelation != "" { + relations[DatasetActivity] = opts.ActivityRelation } for _, dataset := range opts.RequiredOutputDatasets { relation, ok := relations[dataset] @@ -65,11 +68,31 @@ func Validate( people := relations[DatasetPeople] domains := relations[DatasetDomains] daily := relations[DatasetRelationshipDaily] + logicalContributions := relations[DatasetLogicalContributions] + temperatureContributions := relations[DatasetTemperatureContributions] checks := []struct { dataset string invariant string query string }{ + { + DatasetLogicalContributions, + "duplicate logical contribution keys", + `SELECT count(*) FROM ( + SELECT relation_kind, entry_key, canonical_id, domain + FROM ` + logicalContributions + ` + GROUP BY ALL HAVING count(*) > 1 + )`, + }, + { + DatasetTemperatureContributions, + "duplicate daily temperature contribution keys", + `SELECT count(*) FROM ( + SELECT canonical_id, event_date + FROM ` + temperatureContributions + ` + GROUP BY ALL HAVING count(*) > 1 + )`, + }, { DatasetActivity, "duplicate message/canonical/domain keys", @@ -230,24 +253,25 @@ type schemaColumn struct { const duckDBTypeBigInt = "BIGINT" const duckDBTypeBoolean = "BOOLEAN" +const duckDBTypeVarchar = "VARCHAR" var datasetSchemas = map[string][]schemaColumn{ DatasetActivity: { {"message_id", duckDBTypeBigInt}, {"conversation_id", duckDBTypeBigInt}, {"source_id", duckDBTypeBigInt}, - {"source_type", "VARCHAR"}, + {"source_type", duckDBTypeVarchar}, {"occurred_at", "TIMESTAMP"}, - {"message_type", "VARCHAR"}, - {"conversation_type", "VARCHAR"}, - {"entry_kind", "VARCHAR"}, + {"message_type", duckDBTypeVarchar}, + {"conversation_type", duckDBTypeVarchar}, + {"entry_kind", duckDBTypeVarchar}, {"is_chat", duckDBTypeBoolean}, {"is_from_me", duckDBTypeBoolean}, {"attachment_count", "INTEGER"}, {"has_attachments", duckDBTypeBoolean}, {"deleted_from_source", duckDBTypeBoolean}, {"canonical_id", duckDBTypeBigInt}, - {"participant_domain", "VARCHAR"}, + {"participant_domain", duckDBTypeVarchar}, {"is_direct", duckDBTypeBoolean}, {"is_conversation_member", duckDBTypeBoolean}, {"is_sender", duckDBTypeBoolean}, @@ -257,7 +281,7 @@ var datasetSchemas = map[string][]schemaColumn{ }, DatasetPeople: { {"canonical_id", duckDBTypeBigInt}, - {"display_label", "VARCHAR"}, + {"display_label", duckDBTypeVarchar}, {"partial_label", duckDBTypeBoolean}, {"member_ids", "BIGINT[]"}, {"search_values", "VARCHAR[]"}, @@ -295,7 +319,7 @@ var datasetSchemas = map[string][]schemaColumn{ {"peak_year", "INTEGER"}, }, DatasetDomains: { - {"domain", "VARCHAR"}, + {"domain", duckDBTypeVarchar}, {"activity_count", duckDBTypeBigInt}, {"person_count", duckDBTypeBigInt}, {"file_count", duckDBTypeBigInt}, @@ -312,6 +336,36 @@ var datasetSchemas = map[string][]schemaColumn{ {"modality_mask", "UTINYINT"}, {"last_at", "TIMESTAMP"}, }, + DatasetLogicalContributions: { + {"relation_kind", "UTINYINT"}, + {"entry_key", duckDBTypeVarchar}, + {"anchor_message_id", duckDBTypeBigInt}, + {"conversation_id", duckDBTypeBigInt}, + {"source_id", duckDBTypeBigInt}, + {"source_type", duckDBTypeVarchar}, + {"occurred_at", "TIMESTAMP"}, + {"message_type", duckDBTypeVarchar}, + {"entry_kind", duckDBTypeVarchar}, + {"is_from_me", duckDBTypeBoolean}, + {"attachment_count", duckDBTypeBigInt}, + {"canonical_id", duckDBTypeBigInt}, + {"is_author", duckDBTypeBoolean}, + {"is_owner", duckDBTypeBoolean}, + {"with_owner", duckDBTypeBoolean}, + {"domain", duckDBTypeVarchar}, + }, + DatasetTemperatureContributions: { + {"canonical_id", duckDBTypeBigInt}, + {"event_date", "DATE"}, + {"sent_count", duckDBTypeBigInt}, + {"received_count", duckDBTypeBigInt}, + {"meeting_count", duckDBTypeBigInt}, + {"email_count", duckDBTypeBigInt}, + {"chat_count", duckDBTypeBigInt}, + {"total_count", duckDBTypeBigInt}, + {"modality_mask", "UTINYINT"}, + {"last_at", "TIMESTAMP"}, + }, } func validateDatasetSchema( diff --git a/internal/query/cache_state.go b/internal/query/cache_state.go index bcdf97e4e..f1d90b250 100644 --- a/internal/query/cache_state.go +++ b/internal/query/cache_state.go @@ -34,9 +34,13 @@ import ( // the resolved recipient address (envelope, else participant), never an empty // string. Version 27 adds curated person display names. // Version 28 projects RFC Message-ID into every messages cache shard. +// Version 29 persists compact logical and temperature contributions so +// incremental builds can update them without rescanning expanded activity. +// Version 30 stores direct activity edges and expands conversation membership +// from the current roster when queried. // Schema bumps force a full rebuild before readers use an older publication, // so committed caches never mix shards of different shapes. -const CacheSchemaVersion = 28 +const CacheSchemaVersion = 30 // CacheSyncState is the commit marker written after a complete analytics // cache publication. SQLite remains authoritative; these watermarks only @@ -48,9 +52,13 @@ type CacheSyncState struct { LastCompletedSyncRunID int64 `json:"last_completed_sync_run_id,omitzero"` LastCacheAdditionCount int64 `json:"last_cache_addition_count,omitzero"` LastCacheUpdateCount int64 `json:"last_cache_update_count,omitzero"` - LastFailedSyncRunCount int64 `json:"last_failed_sync_run_count,omitzero"` - LastFailedSyncRunIDSum int64 `json:"last_failed_sync_run_id_sum,omitzero"` - IdentityRevision int64 `json:"identity_revision,omitzero"` + // LastRelatedChangeSeq is the highest child-row journal entry represented + // by this committed publication. The journal is written transactionally + // with SQLite mutations and advances only after marker-last publication. + LastRelatedChangeSeq int64 `json:"last_related_change_seq,omitzero"` + LastFailedSyncRunCount int64 `json:"last_failed_sync_run_count,omitzero"` + LastFailedSyncRunIDSum int64 `json:"last_failed_sync_run_id_sum,omitzero"` + IdentityRevision int64 `json:"identity_revision,omitzero"` // DerivedDataRevision tracks offline repairs that rewrite existing // message, snippet, search, or attachment facts. Those rows are already // inside the committed message ID boundary, so drift requires a full cache @@ -126,13 +134,14 @@ func (e *CacheUnavailableError) Unwrap() error { return ErrCacheUnavailable } // Revision identifies one committed cache publication. It intentionally uses // only commit-marker fields, never ambient filesystem state. func (s CacheSyncState) Revision() string { - payload := fmt.Sprintf("v=%d|message=%d|watermark=%s|run=%d|add=%d|update=%d|fail_count=%d|fail_sum=%d|identity=%d|derived_data=%d|account_identity=%d|participant_identifier=%d|participant_display_name=%d|person_display_name=%d|published=%s", + payload := fmt.Sprintf("v=%d|message=%d|watermark=%s|run=%d|add=%d|update=%d|related=%d|fail_count=%d|fail_sum=%d|identity=%d|derived_data=%d|account_identity=%d|participant_identifier=%d|participant_display_name=%d|person_display_name=%d|published=%s", s.SchemaVersion, s.LastMessageID, s.LastSyncAt.UTC().Format(time.RFC3339Nano), s.LastCompletedSyncRunID, s.LastCacheAdditionCount, s.LastCacheUpdateCount, + s.LastRelatedChangeSeq, s.LastFailedSyncRunCount, s.LastFailedSyncRunIDSum, s.IdentityRevision, diff --git a/internal/query/cache_state_test.go b/internal/query/cache_state_test.go index e418e6551..1746eb19b 100644 --- a/internal/query/cache_state_test.go +++ b/internal/query/cache_state_test.go @@ -152,8 +152,8 @@ func TestInspectCacheReadiness(t *testing.T) { } } -func TestCacheSchemaVersionIncludesPersonDisplayNames(t *testing.T) { - assert.Equal(t, 28, CacheSchemaVersion) +func TestCacheSchemaVersionIncludesRelationshipContributions(t *testing.T) { + assert.Equal(t, 30, CacheSchemaVersion) } func TestInspectCacheReadinessNamesStaleSchemaAndDrift(t *testing.T) { @@ -216,6 +216,9 @@ func TestCacheRevisionUsesOnlyCommittedStateWatermarks(t *testing.T) { changed.LastFailedSyncRunIDSum++ assert.NotEqual(revision, changed.Revision()) changed = state + changed.LastRelatedChangeSeq++ + assert.NotEqual(revision, changed.Revision()) + changed = state changed.IdentityRevision++ assert.NotEqual(revision, changed.Revision()) changed = state diff --git a/internal/query/duckdb.go b/internal/query/duckdb.go index 03731f10b..7c659872b 100644 --- a/internal/query/duckdb.go +++ b/internal/query/duckdb.go @@ -312,6 +312,11 @@ func newDuckDBEngine(ctx context.Context, analyticsDir string, sqlitePath string log.Printf("[warn] failed to register SQL views: %v", err) // Non-fatal: existing CTE-based queries still work. } + } else if err := createRelationshipActivityView(ctx, db, analyticsDir); err != nil { + if ctx.Err() != nil { + return nil, errors.Join(ctx.Err(), engine.Close()) + } + log.Printf("[warn] failed to register relationship activity view: %v", err) } return engine, nil @@ -647,6 +652,11 @@ func (e *DuckDBEngine) ensureFreshOptionalCols(ctx context.Context, fp string) e } log.Printf("[warn] re-register views after analytics cache change: %v", err) } + } else if err := createRelationshipActivityView(ctx, e.db, e.analyticsDir); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + log.Printf("[warn] re-register relationship activity view after analytics cache change: %v", err) } if err := ctx.Err(); err != nil { return err @@ -2592,6 +2602,8 @@ var RequiredParquetDirs = []string{ identityindex.DatasetPeople, identityindex.DatasetDomains, identityindex.DatasetRelationshipDaily, + identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, } // SearchFast searches message metadata in Parquet files (no body text). diff --git a/internal/query/explore_analysis.go b/internal/query/explore_analysis.go index f0ec8d57b..26ad97383 100644 --- a/internal/query/explore_analysis.go +++ b/internal/query/explore_analysis.go @@ -21,7 +21,7 @@ func (e *DuckDBEngine) ExploreGroups(ctx context.Context, request ExploreGroupRe if err != nil { return nil, err } - spec, err := exploreGroupExpressions(request.Dimension, e.identityActivityPath(), + spec, err := exploreGroupExpressions(request.Dimension, e.parquetPath(identityindex.DatasetPeople)) if err != nil { return nil, err @@ -141,26 +141,24 @@ type groupExpressions struct { // participant callers pass "a.is_direct" for messages but must widen the // conversation branch with is_conversation_member. Either predicate may be // empty. -func sqlActivityEntryEdges(activityGlob, selectExpr, messagePredicate, conversationPredicate string) string { +func sqlActivityEntryEdges(selectExpr, messagePredicate, conversationPredicate string) string { if messagePredicate != "" { messagePredicate = " AND " + messagePredicate } if conversationPredicate != "" { conversationPredicate = " AND " + conversationPredicate } - activityScan := `read_parquet('` + activityGlob + `', - hive_partitioning=true, union_by_name=true)` return ` SELECT le.entry_key, ` + selectExpr + ` FROM logical_entries le - JOIN ` + activityScan + ` a ON a.message_id = le.anchor_message_id + JOIN relationship_activity_expanded a ON a.message_id = le.anchor_message_id WHERE le.entry_kind <> 'conversation' AND a.canonical_id IS NOT NULL` + messagePredicate + ` UNION SELECT le.entry_key, ` + selectExpr + ` FROM logical_entries le JOIN classified f ON f.conversation_id = le.conversation_id AND f.is_chat - JOIN ` + activityScan + ` a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id WHERE le.entry_kind = 'conversation' AND a.canonical_id IS NOT NULL` + conversationPredicate + ` ` @@ -203,7 +201,7 @@ func sqlMessageTypeGroupExpr() string { // aliases of one person collapses to a single (entry, canonical) row, so the // entry is never double-counted (entry_key is projected only to carry // per-entry uniqueness through that DISTINCT). -func exploreGroupExpressions(dimension, activityGlob, peopleGlob string) (groupExpressions, error) { +func exploreGroupExpressions(dimension, peopleGlob string) (groupExpressions, error) { simple := func(key string) groupExpressions { return groupExpressions{key: key, label: key, groupBy: key, source: "logical_entries"} } @@ -223,7 +221,7 @@ func exploreGroupExpressions(dimension, activityGlob, peopleGlob string) (groupE noLists: true, cte: ` ), participant_entries AS (` + - sqlActivityEntryEdges(activityGlob, + sqlActivityEntryEdges( "a.canonical_id AS person_id, le.occurred_at, le.estimated_bytes", "a.is_direct", "(a.is_direct OR a.is_conversation_member)"), source: "participant_entries", @@ -237,7 +235,7 @@ func exploreGroupExpressions(dimension, activityGlob, peopleGlob string) (groupE noLists: true, cte: ` ), domain_entries AS (` + - sqlActivityEntryEdges(activityGlob, + sqlActivityEntryEdges( "a.participant_domain AS group_value, le.occurred_at, le.estimated_bytes", "a.participant_domain <> ''", "a.participant_domain <> ''"), source: "domain_entries", diff --git a/internal/query/files.go b/internal/query/files.go index ca0808be6..8f144257b 100644 --- a/internal/query/files.go +++ b/internal/query/files.go @@ -282,7 +282,7 @@ func (e *DuckDBEngine) GroupFiles(ctx context.Context, request FileGroupRequest) if err := validateFileMIMEFamilies(request.MIMEFamilies); err != nil { return nil, err } - spec, err := fileGroupExpressions(request.Dimension, e.identityActivityPath(), + spec, err := fileGroupExpressions(request.Dimension, e.parquetPath(identityindex.DatasetPeople), e.parquetPath(datasetParticipants)) if err != nil { return nil, err @@ -564,7 +564,7 @@ func buildFileConditions(filenameQuery string, mimeFamilies []FileMIMEFamily) (s // rows but still receives file attributions here — so a base-participants // lookup backstops the people-dataset label before the constant fallback. func fileGroupExpressions( - dimension, activityGlob, peopleGlob, participantsGlob string, + dimension, peopleGlob, participantsGlob string, ) (groupExpressions, error) { simple := func(key string) groupExpressions { return groupExpressions{key: key, label: key, groupBy: key, source: "file_population"} @@ -582,8 +582,7 @@ func fileGroupExpressions( , participant_files AS ( SELECT DISTINCT f.attachment_id, a.canonical_id AS person_id, f.occurred_at, f.size FROM file_population f - JOIN read_parquet('` + activityGlob + `', - hive_partitioning=true, union_by_name=true) a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id WHERE a.canonical_id IS NOT NULL AND (a.is_direct OR a.is_conversation_member) ), participant_file_labels AS ( @@ -603,8 +602,7 @@ func fileGroupExpressions( , domain_files AS ( SELECT DISTINCT f.attachment_id, a.participant_domain AS group_value, f.occurred_at, f.size FROM file_population f - JOIN read_parquet('` + activityGlob + `', - hive_partitioning=true, union_by_name=true) a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id WHERE a.participant_domain <> '' )`, source: "domain_files", diff --git a/internal/query/identity_activity.go b/internal/query/identity_activity.go index 235315760..55226d961 100644 --- a/internal/query/identity_activity.go +++ b/internal/query/identity_activity.go @@ -34,12 +34,6 @@ func identityRequestIsSourceOnly(request ExploreRequest) bool { return identityRequestIsUnfiltered(request) } -// identityActivityPath returns the relationship_activity glob escaped for -// direct embedding in trusted SQL text. -func (e *DuckDBEngine) identityActivityPath() string { - return quoteIdentitySQLPath(e.parquetPath(identityindex.DatasetActivity)) -} - // buildIdentityLogicalSQL renders the context-filtered logical-entry // population with per-(entry, canonical) relationship facts as a closed // logical_people CTE: entry facts come from analytical_entries (one row per @@ -66,8 +60,6 @@ func (e *DuckDBEngine) buildIdentityLogicalSQL( return "", nil, err } conditions, args := buildExploreConditions(request) - activityScan := `read_parquet('` + e.identityActivityPath() + `', - hive_partitioning=true, union_by_name=true)` // entry_num is a numeric logical-entry key: the anchor message ID for // message entries, the (globally unique, NOT NULL) conversation ID // negated for chat conversation entries so the two spaces cannot @@ -85,7 +77,7 @@ func (e *DuckDBEngine) buildIdentityLogicalSQL( SELECT a.message_id, a.canonical_id, bool_or(a.is_author) AS is_author, bool_or(a.is_owner) AS is_owner - FROM ` + activityScan + ` a + FROM relationship_activity_expanded a JOIN ( SELECT DISTINCT anchor_message_id FROM logical_entries WHERE entry_kind <> 'conversation' @@ -98,7 +90,7 @@ func (e *DuckDBEngine) buildIdentityLogicalSQL( bool_or(a.is_author AND a.message_id = anchors.anchor_message_id) AS is_author, bool_or(a.is_owner) AS is_owner FROM classified f - JOIN ` + activityScan + ` a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id JOIN ( SELECT conversation_id, anchor_message_id FROM logical_entries WHERE entry_kind = 'conversation' @@ -156,9 +148,6 @@ func (e *DuckDBEngine) buildIdentityDomainLogicalSQL( return "", nil, err } conditions, args := buildExploreConditions(request) - activityGlob := e.identityActivityPath() - activityScan := `read_parquet('` + activityGlob + `', - hive_partitioning=true, union_by_name=true)` // entry_num mirrors buildIdentityLogicalSQL: the per-(entry, domain) // dedup hash stays numeric-keyed so a broad filter fits the interactive // memory budget. @@ -167,7 +156,7 @@ func (e *DuckDBEngine) buildIdentityDomainLogicalSQL( SELECT le.anchor_message_id AS entry_num, a.participant_domain AS domain, le.occurred_at, le.attachment_count, le.source_type FROM logical_entries le - JOIN ` + activityScan + ` a ON a.message_id = le.anchor_message_id + JOIN relationship_activity_expanded a ON a.message_id = le.anchor_message_id WHERE le.entry_kind <> 'conversation' AND a.canonical_id IS NOT NULL AND a.participant_domain <> '' @@ -176,14 +165,14 @@ func (e *DuckDBEngine) buildIdentityDomainLogicalSQL( le.occurred_at, le.attachment_count, le.source_type FROM logical_entries le JOIN classified f ON f.conversation_id = le.conversation_id AND f.is_chat - JOIN ` + activityScan + ` a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id WHERE le.entry_kind = 'conversation' AND a.canonical_id IS NOT NULL AND a.participant_domain <> '' ), logical_person_domains AS ( SELECT DISTINCT a.participant_domain AS domain, a.canonical_id FROM logical_entries le - JOIN ` + activityScan + ` a ON a.message_id = le.anchor_message_id + JOIN relationship_activity_expanded a ON a.message_id = le.anchor_message_id WHERE le.entry_kind <> 'conversation' AND a.canonical_id IS NOT NULL AND a.participant_domain <> '' AND a.is_direct @@ -191,7 +180,7 @@ func (e *DuckDBEngine) buildIdentityDomainLogicalSQL( SELECT a.participant_domain AS domain, a.canonical_id FROM logical_entries le JOIN classified f ON f.conversation_id = le.conversation_id AND f.is_chat - JOIN ` + activityScan + ` a ON a.message_id = f.message_id + JOIN relationship_activity_expanded a ON a.message_id = f.message_id WHERE le.entry_kind = 'conversation' AND a.canonical_id IS NOT NULL AND a.participant_domain <> '' @@ -214,11 +203,10 @@ func (e *DuckDBEngine) narrowIdentityFactCandidates( !identityRequestHasEdgeFilters(request) { return request, nil } - facts := e.identityActivityPath() - conditions, args := buildIdentityFactConditions(request, facts) + conditions, args := buildIdentityFactConditions(request) queryText := ` SELECT f.message_id -FROM read_parquet('` + facts + `', hive_partitioning=true, union_by_name=true) f +FROM relationship_activity_expanded f WHERE ` + conditions + ` GROUP BY f.message_id LIMIT ?` @@ -253,9 +241,8 @@ LIMIT ?` } // buildIdentityFactConditions renders the message-level filter for the alias -// f over relationship_activity. activityPath is the SQL-escaped activity glob -// used by the participant/domain edge semi-join predicates. -func buildIdentityFactConditions(request ExploreRequest, activityPath string) (string, []any) { +// f over the expanded relationship activity view. +func buildIdentityFactConditions(request ExploreRequest) (string, []any) { var conditions []string var args []any appendIntGroup := func(values []int64, expression string) { @@ -280,15 +267,13 @@ func buildIdentityFactConditions(request ExploreRequest, activityPath string) (s } participantPredicate := `(EXISTS ( SELECT 1 - FROM read_parquet('` + activityPath + `', - hive_partitioning=true, union_by_name=true) edge + FROM relationship_activity_expanded edge WHERE edge.message_id = f.message_id AND edge.canonical_id = ? ))` domainPredicate := `(EXISTS ( SELECT 1 - FROM read_parquet('` + activityPath + `', - hive_partitioning=true, union_by_name=true) edge + FROM relationship_activity_expanded edge WHERE edge.message_id = f.message_id AND lower(edge.participant_domain) = ? ))` diff --git a/internal/query/identity_activity_test.go b/internal/query/identity_activity_test.go index 2c5c70b26..43340fa19 100644 --- a/internal/query/identity_activity_test.go +++ b/internal/query/identity_activity_test.go @@ -320,7 +320,7 @@ func TestIdentityActivityDateFiltersBindUTCWallClock(t *testing.T) { _, args := buildIdentityFactConditions(ExploreRequest{ Context: Context{After: &after, Before: &before}, - }, "unused-activity-path") + }) require.Len(t, args, 2) assertionsForTest.Equal("2026-07-20 13:30:00", args[0]) assertionsForTest.Equal("2026-07-20 15:30:00", args[1]) diff --git a/internal/query/people.go b/internal/query/people.go index ff1de22a3..7f6fc47de 100644 --- a/internal/query/people.go +++ b/internal/query/people.go @@ -344,7 +344,7 @@ func (e *DuckDBEngine) searchPeopleLegacy( } conditions, args := buildExploreConditions(request.Explore) entriesCTE, entryArgs := personEntriesCTE(exactID, clusterMemberIDs, conditions, - e.parquetPath(datasetParticipantClusters), e.identityActivityPath()) + e.parquetPath(datasetParticipantClusters)) args = append(args, entryArgs...) // bestNameExpr is the shared cluster label policy (see person_label.go). // Listing/search rows are canonical identities, so the label evaluates @@ -556,7 +556,7 @@ FROM counted ORDER BY ` + order + ` LIMIT ? OFFSET ?` // cluster (e.g. cc'ing a contact's work and personal addresses) is never // double-counted. The clusters/canon CTEs remain only for the caller's // label, search-match, and identifier subqueries. -func personEntriesCTE(exactID *int64, memberIDs []int64, conditions, clustersGlob, activityGlob string) (string, []any) { +func personEntriesCTE(exactID *int64, memberIDs []int64, conditions, clustersGlob string) (string, []any) { if exactID == nil { return fmt.Sprintf(` ), clusters AS ( @@ -565,7 +565,7 @@ func personEntriesCTE(exactID *int64, memberIDs []int64, conditions, clustersGlo SELECT p.id AS participant_id, COALESCE(c.canonical_id, p.id) AS canonical_id FROM participants p LEFT JOIN clusters c ON c.participant_id = p.id ), person_entries AS (`, clustersGlob) + - sqlActivityEntryEdges(activityGlob, + sqlActivityEntryEdges( "a.canonical_id AS person_id, le.occurred_at, le.message_type, le.attachment_count, le.source_type", "a.is_direct", "(a.is_direct OR a.is_conversation_member)"), nil } @@ -595,7 +595,7 @@ func personEntriesCTE(exactID *int64, memberIDs []int64, conditions, clustersGlo FROM logical_entries WHERE entry_key IN ( SELECT edge.entry_key FROM (` + - sqlActivityEntryEdges(activityGlob, "a.canonical_id AS person_id", + sqlActivityEntryEdges("a.canonical_id AS person_id", "a.is_direct AND a.canonical_id IN "+memberList, "(a.is_direct OR a.is_conversation_member) AND a.canonical_id IN "+memberList) + ` ) AS edge @@ -716,7 +716,7 @@ func (e *DuckDBEngine) searchDomainsLegacy(ctx context.Context, request DomainSe // as the index's domain entries do. queryText := buildExploreLogicalSQLNoLists(conditions) + ` ), domain_edges AS (` + - sqlActivityEntryEdges(e.identityActivityPath(), + sqlActivityEntryEdges( "a.participant_domain AS domain, a.canonical_id AS person_id, "+ "a.is_direct AS is_direct, "+ "(le.entry_kind = 'conversation') AS is_chat_entry, "+ diff --git a/internal/query/people_inboxes.go b/internal/query/people_inboxes.go index 4b007d40d..b7654312b 100644 --- a/internal/query/people_inboxes.go +++ b/internal/query/people_inboxes.go @@ -56,7 +56,7 @@ func (e *DuckDBEngine) ListPersonInboxes(ctx context.Context, request PersonInbo WITH contact_chat AS ( SELECT DISTINCT message_id AS entry_key, source_id, source_type, conversation_id, occurred_at, is_from_me - FROM read_parquet('%s', hive_partitioning=true, union_by_name=true) + FROM relationship_activity_expanded WHERE canonical_id = ? AND is_chat ) SELECT c.source_id, c.source_type, s.account_email, @@ -70,7 +70,7 @@ FROM contact_chat c JOIN read_parquet('%s') s ON s.id = c.source_id GROUP BY c.source_id, c.source_type, s.account_email ORDER BY MAX(c.occurred_at) DESC, c.source_id`, - e.identityActivityPath(), quoteIdentitySQLPath(e.parquetPath(datasetSources))) + quoteIdentitySQLPath(e.parquetPath(datasetSources))) rows, err := e.db.QueryContext(ctx, queryText, request.CanonicalID) if err != nil { return nil, fmt.Errorf("query person inboxes: %w", err) diff --git a/internal/query/relationship_calendar.go b/internal/query/relationship_calendar.go index 840478397..076657a19 100644 --- a/internal/query/relationship_calendar.go +++ b/internal/query/relationship_calendar.go @@ -222,13 +222,13 @@ func (e *DuckDBEngine) relationshipCalendarDays( } cutoff := effectiveAt.UTC() activity := fmt.Sprintf(`( - SELECT * FROM read_parquet('%s', hive_partitioning=true, union_by_name=true) + SELECT * FROM relationship_activity_expanded WHERE occurred_year IN (%s) AND occurred_at >= TIMESTAMP '%s' AND occurred_at < TIMESTAMP '%s' AND (canonical_id = %d OR is_owner) )`, - e.identityActivityPath(), strings.Join(years, ","), + strings.Join(years, ","), startUTC.Format("2006-01-02 15:04:05.999999999"), endUTC.Format("2006-01-02 15:04:05.999999999"), request.CanonicalID, diff --git a/internal/query/relationship_scale_benchmark_test.go b/internal/query/relationship_scale_benchmark_test.go new file mode 100644 index 000000000..9bd22f988 --- /dev/null +++ b/internal/query/relationship_scale_benchmark_test.go @@ -0,0 +1,157 @@ +package query + +import ( + "encoding/json/v2" + "flag" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/duckdbutil" + "go.kenn.io/msgvault/internal/identityindex" +) + +var relationshipScaleRoot = flag.String("relationship-bench-root", "", "Scratch cache directory for relationship scale benchmarks") + +// Run Build once, then Queries against the same scratch directory: +// +// relationship_bench_root=$(mktemp -d) +// go test -tags 'fts5 sqlite_vec' ./internal/query -run '^$' -bench '^BenchmarkRelationshipScaleBuild$' -benchtime=1x -args -relationship-bench-root="$relationship_bench_root" +// go test -tags 'fts5 sqlite_vec' ./internal/query -run '^$' -bench '^BenchmarkRelationshipScaleQueries$' -benchtime=4x -args -relationship-bench-root="$relationship_bench_root" +// +// The fixture has 2,000,000 email messages across 303 contacts, plus 2,000 +// messages in one 304-member chat. Half the email is in 2024; the rest and the +// chat are in 2025. It is synthetic, +// with no attachments, labels, aliases, deletions, or external services. +func BenchmarkRelationshipScaleBuild(b *testing.B) { + root := *relationshipScaleRoot + if root == "" { + b.Skip("set -relationship-bench-root to an empty scratch directory") + } + if _, err := os.Stat(filepath.Join(root, datasetSources)); os.IsNotExist(err) { + seed, cleanup := NewTestDataBuilder(b).Build() + defer cleanup() + require.NoError(b, os.CopyFS(root, os.DirFS(seed))) + } + db, err := duckdbutil.Open(b.Context(), duckdbutil.BuilderPolicy(b.TempDir())) + require.NoError(b, err) + defer func() { require.NoError(b, db.Close()) }() + _, err = db.Exec(`CREATE VIEW scale_messages AS + SELECT i::BIGINT AS id, 1::BIGINT AS source_id, + 'message-' || i AS source_message_id, + CASE WHEN i <= 2000000 THEN i ELSE 2000001 END::BIGINT AS conversation_id, + 'Synthetic message' AS subject, 'Synthetic preview' AS snippet, + CASE WHEN i <= 1000000 THEN TIMESTAMP '2024-01-01' ELSE TIMESTAMP '2025-01-01' END + + INTERVAL (i % 1000000) SECOND AS sent_at, + 100::BIGINT AS size_estimate, false AS has_attachments, + 0::INTEGER AS attachment_count, NULL::TIMESTAMP AS deleted_from_source_at, + (2 + i % 303)::BIGINT AS sender_id, 1::BIGINT AS owner_participant_id, + CASE WHEN i <= 2000000 THEN 'email' ELSE 'imessage' END AS message_type, + NULL::VARCHAR AS list_id, false AS is_from_me, + CASE WHEN i <= 1000000 THEN 2024 ELSE 2025 END::INTEGER AS year, 1::INTEGER AS month + FROM range(1, 2002001) t(i)`) + require.NoError(b, err) + for dataset, query := range map[string]string{ + datasetMessages: `SELECT * FROM scale_messages`, + datasetSources: `SELECT 1::BIGINT AS id, 'owner@example.test' AS account_email, 'gmail' AS source_type`, + datasetParticipants: `SELECT i::BIGINT AS id, 'person-' || i || '@example.test' AS email_address, + 'example.test' AS domain, 'Person ' || i AS display_name, '' AS phone_number FROM range(1, 305) t(i)`, + datasetConversations: `SELECT id, 'thread-' || id AS source_conversation_id, 'Synthetic thread' AS title, + CASE WHEN id = 2000001 THEN 'group_chat' ELSE 'email' END AS conversation_type FROM range(1, 2000002) t(id)`, + datasetConversationParticipants: `SELECT 2000001::BIGINT AS conversation_id, i::BIGINT AS participant_id FROM range(1, 305) t(i)`, + datasetOwnerParticipants: `SELECT 1::BIGINT AS source_id, 1::BIGINT AS participant_id`, + "message_recipients": `SELECT id AS message_id, sender_id AS participant_id, 'from' AS recipient_type, + '' AS display_name, 'person-' || sender_id || '@example.test' AS email_address, + NULL::VARCHAR AS envelope_address FROM scale_messages + UNION ALL SELECT id, 1::BIGINT, 'to', '', 'owner@example.test', NULL::VARCHAR FROM scale_messages`, + } { + dir := filepath.Join(root, dataset) + require.NoError(b, os.RemoveAll(dir)) + require.NoError(b, os.MkdirAll(dir, 0o700)) + output := filepath.Join(dir, "data.parquet") + options := "FORMAT PARQUET" + if dataset == datasetMessages { + output = dir + options += ", PARTITION_BY (year), WRITE_PARTITION_COLUMNS true" + } + _, err := db.Exec("COPY (" + query + ") TO '" + escapePath(output) + "' (" + options + ")") + require.NoError(b, err) + } + var result identityindex.BuildResult + b.ResetTimer() + for range b.N { + result, err = identityindex.Build(b.Context(), db, identityindex.BuildOptions{ + Mode: identityindex.ModeFull, StagedBaseRoot: root, OutputRoot: root, + EffectiveAt: time.Date(2025, 12, 31, 23, 59, 59, 0, time.UTC), + }) + require.NoError(b, err) + } + b.StopTimer() + fingerprint, err := CacheDatasetFingerprint(root) + require.NoError(b, err) + marker, err := json.Marshal(CacheSyncState{ + LastMessageID: 2_002_000, LastSyncAt: time.Now(), PublishedAt: time.Now(), + SchemaVersion: CacheSchemaVersion, DatasetFingerprint: fingerprint, + ConversationParticipantsFingerprint: result.ConversationParticipantsFingerprint, + Stats: result.Stats, + }) + require.NoError(b, err) + require.NoError(b, os.WriteFile(CacheStatePath(root), marker, 0o600)) +} + +func BenchmarkRelationshipScaleQueries(b *testing.B) { + root := *relationshipScaleRoot + if root == "" { + b.Skip("run BenchmarkRelationshipScaleBuild with a scratch directory first") + } + for _, name := range []string{"Calendar", "Timeline", "PersonActivity", "MessageActivity"} { + b.Run(name, func(b *testing.B) { + engine, err := NewDuckDBEngine(root, "", nil) + require.NoError(b, err) + defer func() { require.NoError(b, engine.Close()) }() + var elapsed []time.Duration + b.ResetTimer() + for range b.N { + start := time.Now() + switch name { + case "Calendar": + result, err := engine.RelationshipCalendar(b.Context(), RelationshipCalendarRequest{CanonicalID: 2, Year: 2025, Timezone: "UTC"}) + require.NoError(b, err) + var total int64 + for _, day := range result.Days { + total += day.Total + } + // Calendar counts this person's 3,300 emails and seven + // authored group messages; silent membership is not a signal. + require.Equal(b, int64(3307), total) + case "Timeline": + result, err := engine.RelationshipTimeline(b.Context(), RelationshipTimelineRequest{CanonicalID: 2, Limit: 25, Timezone: "UTC"}) + require.NoError(b, err) + require.Len(b, result.Rows, 25) + require.Equal(b, int64(6601), result.TotalCount) + case "PersonActivity", "MessageActivity": + filter := "canonical_id = 2" + want := int64(8600) + if name == "MessageActivity" { + filter = "message_id = 2002000" + want = 304 + } + result, err := engine.QuerySQL(b.Context(), "SELECT count(*) FROM relationship_activity_expanded WHERE "+filter) + require.NoError(b, err) + require.Equal(b, 1, result.RowCount) + require.Equal(b, want, result.Rows[0][0]) + } + elapsed = append(elapsed, time.Since(start)) + } + b.StopTimer() + b.ReportMetric(float64(elapsed[0].Microseconds())/1000, "first-ms") + if len(elapsed) > 1 { + slices.Sort(elapsed[1:]) + b.ReportMetric(float64(elapsed[1+len(elapsed[1:])/2].Microseconds())/1000, "warm-median-ms") + } + }) + } +} diff --git a/internal/query/relationship_timeline.go b/internal/query/relationship_timeline.go index cf2f6e715..ce09cf660 100644 --- a/internal/query/relationship_timeline.go +++ b/internal/query/relationship_timeline.go @@ -115,7 +115,7 @@ func (e *DuckDBEngine) RelationshipTimeline(ctx context.Context, request Relatio // (buildIdentityFactConditions), so participant/domain filters become // edge semi-joins — the aggregated participant list columns of the // legacy view are never touched. - conditions, factArgs := buildIdentityFactConditions(explore, e.identityActivityPath()) + conditions, factArgs := buildIdentityFactConditions(explore) args := make([]any, 0, len(factArgs)+4) args = append(args, request.CanonicalID) args = append(args, factArgs...) @@ -133,7 +133,6 @@ func (e *DuckDBEngine) RelationshipTimeline(ctx context.Context, request Relatio queryText := buildRelationshipTimelineSQL( conditions, - e.identityActivityPath(), quoteIdentitySQLPath(e.parquetGlob()), quoteIdentitySQLPath(e.parquetPath(datasetConversations)), ) @@ -260,9 +259,7 @@ func validateRelationshipTimelineRequest(request RelationshipTimelineRequest) er // (MIME structure at sync time), not attachment_count: extraction can lag // or fail, leaving the flag true with a zero count, and the indicator must // match what the message list shows for the same message. -func buildRelationshipTimelineSQL(conditions, activityGlob, messagesGlob, conversationsGlob string) string { - activityScan := `read_parquet('` + activityGlob + `', - hive_partitioning=true, union_by_name=true)` +func buildRelationshipTimelineSQL(conditions, messagesGlob, conversationsGlob string) string { // The membership IN-subquery is a semi-join whose build side is the // subject's bare message IDs (compact even for archive-scale clusters); // the outer scan then folds per-message facts and owner presence into @@ -280,10 +277,10 @@ WITH subject_facts AS ( any_value(a.is_chat) AS is_chat, any_value(a.has_attachments) AS has_attachments, bool_or(a.is_owner AND a.is_direct) AS with_owner - FROM ` + activityScan + ` a + FROM relationship_activity_expanded a WHERE a.message_id IN ( SELECT f.message_id - FROM ` + activityScan + ` f + FROM relationship_activity_expanded f WHERE f.canonical_id = ? AND (` + conditions + `) ) GROUP BY a.message_id diff --git a/internal/query/views.go b/internal/query/views.go index d8168feda..87a37eaf1 100644 --- a/internal/query/views.go +++ b/internal/query/views.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "time" + + "go.kenn.io/msgvault/internal/identityindex" ) // QueryResult holds raw SQL query results in a columnar format. @@ -169,6 +171,9 @@ func RegisterViewsWithColumns(ctx context.Context, db *sql.DB, analyticsDir stri if err := createBaseViews(ctx, db, analyticsDir, optCols); err != nil { return fmt.Errorf("create base views: %w", err) } + if err := createRelationshipActivityView(ctx, db, analyticsDir); err != nil { + return err + } return createConvenienceViews(ctx, db) } @@ -426,6 +431,37 @@ func createBaseViews(ctx context.Context, db *sql.DB, analyticsDir string, optCo return nil } +// The normalized relationship view is used by internal query paths even when +// legacy analytical views are disabled. +func createRelationshipActivityView(ctx context.Context, db *sql.DB, analyticsDir string) error { + activityFiles, err := filepath.Glob(filepath.Join(analyticsDir, identityindex.DatasetActivity, "*", "*.parquet")) + if err != nil { + return fmt.Errorf("find relationship activity: %w", err) + } + if len(activityFiles) > 0 { + parquet := func(dataset string, hive bool) string { + glob := filepath.Join(analyticsDir, dataset, "*.parquet") + options := "" + if hive { + glob = filepath.Join(analyticsDir, dataset, "**", "*.parquet") + options = ", hive_partitioning=true, union_by_name=true" + } + return "read_parquet('" + quoteIdentitySQLPath(glob) + "'" + options + ")" + } + relation := identityindex.ExpandedActivityRelation( + parquet(identityindex.DatasetActivity, true), + parquet(datasetConversationParticipants, false), + parquet(datasetParticipants, false), + parquet(datasetParticipantClusters, false), + parquet(datasetOwnerParticipants, false), + ) + if _, err := db.ExecContext(ctx, "CREATE OR REPLACE VIEW relationship_activity_expanded AS SELECT * FROM "+relation); err != nil { + return fmt.Errorf("create relationship activity view: %w", err) + } + } + return nil +} + // createConvenienceViews builds higher-level views on top of the // base Parquet views. Each view joins or aggregates the base views // to provide ready-to-query datasets. diff --git a/internal/store/attachments.go b/internal/store/attachments.go index d6e969b2c..cd36756e7 100644 --- a/internal/store/attachments.go +++ b/internal/store/attachments.go @@ -512,7 +512,7 @@ func (s *Store) SetDiscordAttachmentMetadata( changed += n } if changed > 0 { - if err := s.bumpDerivedDataRevision(tx); err != nil { + if err := s.bumpDerivedDataRevision(tx, true); err != nil { return fmt.Errorf("advance Discord attachment metadata revision: %w", err) } } diff --git a/internal/store/cache_related_change_journal_test.go b/internal/store/cache_related_change_journal_test.go new file mode 100644 index 000000000..63e21b7e4 --- /dev/null +++ b/internal/store/cache_related_change_journal_test.go @@ -0,0 +1,140 @@ +package store_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +func TestCacheRelatedChangeJournalTracksUnlinkedLabelRename(t *testing.T) { + require := require.New(t) + testutil.SkipIfPostgres(t, "analytics cache journal is SQLite-only") + f := storetest.New(t) + _, err := f.Store.EnsureLabelsBatch(f.Source.ID, map[string]store.LabelInfo{ + "remote-label": {Name: "Before", Type: "user"}, + }) + require.NoError(err) + var baseline int64 + require.NoError(f.Store.DB().QueryRow(`SELECT COALESCE(MAX(seq), 0) FROM cache_related_change_journal`).Scan(&baseline)) + _, err = f.Store.EnsureLabelsBatch(f.Source.ID, map[string]store.LabelInfo{ + "remote-label": {Name: "After", Type: "user"}, + }) + require.NoError(err) + var count int + require.NoError(f.Store.DB().QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal + WHERE seq > ? AND dataset = 'labels' AND message_id = 0`, baseline).Scan(&count)) + assert.Positive(t, count) +} + +func TestCacheRelatedChangeJournalTracksChildMutations(t *testing.T) { + require := require.New(t) + testutil.SkipIfPostgres(t, "analytics cache journal is SQLite-only") + f := storetest.New(t) + st := f.Store + first, err := st.UpsertMessage(f.NewMessage().WithSourceMessageID("first").Build()) + require.NoError(err) + second, err := st.UpsertMessage(f.NewMessage().WithSourceMessageID("second").Build()) + require.NoError(err) + participant := f.EnsureParticipant("recipient@example.com", "Recipient", "example.com") + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (1, 'synthetic')`) + require.NoError(err) + + var baseline int64 + require.NoError(st.DB().QueryRow(`SELECT COALESCE(MAX(seq), 0) FROM cache_related_change_journal`).Scan(&baseline)) + _, err = st.DB().Exec(`INSERT INTO message_recipients (message_id, participant_id, recipient_type) + VALUES (?, ?, 'to')`, first, participant) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE message_recipients SET message_id = ? WHERE message_id = ?`, second, first) + require.NoError(err) + _, err = st.DB().Exec(`DELETE FROM message_recipients WHERE message_id = ?`, second) + require.NoError(err) + + _, err = st.DB().Exec(`INSERT INTO message_labels (message_id, label_id) VALUES (?, 1)`, first) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE message_labels SET message_id = ? WHERE message_id = ?`, second, first) + require.NoError(err) + _, err = st.DB().Exec(`DELETE FROM message_labels WHERE message_id = ?`, second) + require.NoError(err) + + _, err = st.DB().Exec(`INSERT INTO attachments (id, message_id, storage_path) VALUES (1, ?, 'synthetic')`, first) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE attachments SET message_id = ? WHERE id = 1`, second) + require.NoError(err) + _, err = st.DB().Exec(`DELETE FROM attachments WHERE id = 1`) + require.NoError(err) + + rows, err := st.DB().Query(`SELECT dataset, message_id FROM cache_related_change_journal WHERE seq > ? ORDER BY seq`, baseline) + require.NoError(err) + defer func() { _ = rows.Close() }() + type change struct { + dataset string + messageID int64 + } + var got []change + for rows.Next() { + var entry change + require.NoError(rows.Scan(&entry.dataset, &entry.messageID)) + got = append(got, entry) + } + require.NoError(rows.Err()) + assert.Equal(t, []change{ + {"message_recipients", first}, {"message_recipients", first}, + {"message_recipients", second}, {"message_recipients", second}, + {"message_labels", first}, {"message_labels", first}, + {"message_labels", second}, {"message_labels", second}, + {"attachments", first}, {"attachments", first}, + {"attachments", second}, {"attachments", second}, + }, got) +} + +func TestCacheRelatedChangeJournalRollsBackWithMutation(t *testing.T) { + require := require.New(t) + testutil.SkipIfPostgres(t, "analytics cache journal is SQLite-only") + f := storetest.New(t) + st := f.Store + messageID, err := st.UpsertMessage(f.NewMessage().Build()) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (1, 'synthetic')`) + require.NoError(err) + var baseline int64 + require.NoError(st.DB().QueryRow(`SELECT COALESCE(MAX(seq), 0) FROM cache_related_change_journal`).Scan(&baseline)) + tx, err := st.DB().Begin() + require.NoError(err) + _, err = tx.Exec(`INSERT INTO message_labels (message_id, label_id) VALUES (?, 1)`, messageID) + require.NoError(err) + require.NoError(tx.Rollback()) + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal WHERE seq > ?`, baseline).Scan(&count)) + assert.Zero(t, count) +} + +func TestCacheRelatedChangeJournalInstallsOnExistingArchive(t *testing.T) { + require := require.New(t) + testutil.SkipIfPostgres(t, "analytics cache journal is SQLite-only") + f := storetest.New(t) + st := f.Store + for _, name := range []string{ + "trg_cache_recipients_insert", "trg_cache_recipients_update", "trg_cache_recipients_delete", + "trg_cache_labels_insert", "trg_cache_labels_update", "trg_cache_labels_delete", + "trg_cache_attachments_insert", "trg_cache_attachments_update", "trg_cache_attachments_delete", + } { + _, err := st.DB().Exec(`DROP TRIGGER IF EXISTS ` + name) + require.NoError(err) + } + _, err := st.DB().Exec(`DROP TABLE cache_related_change_journal`) + require.NoError(err) + require.NoError(st.InitSchema()) + messageID, err := st.UpsertMessage(f.NewMessage().Build()) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (1, 'synthetic')`) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO message_labels (message_id, label_id) VALUES (?, 1)`, messageID) + require.NoError(err) + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal WHERE dataset = 'message_labels' AND message_id = ?`, messageID).Scan(&count)) + assert.Equal(t, 1, count) +} diff --git a/internal/store/derived_data_revision.go b/internal/store/derived_data_revision.go index 1352ba2c1..90510aac8 100644 --- a/internal/store/derived_data_revision.go +++ b/internal/store/derived_data_revision.go @@ -20,7 +20,7 @@ func (s *Store) DerivedDataRevisionContext(ctx context.Context) (int64, error) { return readArchiveMetadataRevisionContext(ctx, s.db, derivedDataRevisionKey, "derived-data") } -func (s *Store) bumpDerivedDataRevision(tx *loggedTx) error { +func (s *Store) bumpDerivedDataRevision(tx *loggedTx, relatedOnly ...bool) error { if _, err := tx.Exec(s.dialect.InsertOrIgnore( `INSERT OR IGNORE INTO archive_metadata (key, value) VALUES (?, '0')`), derivedDataRevisionKey); err != nil { @@ -33,9 +33,30 @@ func (s *Store) bumpDerivedDataRevision(tx *loggedTx) error { `, derivedDataRevisionKey); err != nil { return fmt.Errorf("bump derived-data revision: %w", err) } + if len(relatedOnly) > 0 && relatedOnly[0] && !s.IsPostgreSQL() { + if _, err := tx.Exec(`INSERT INTO cache_related_revision_journal (revision) + SELECT CAST(value AS INTEGER) FROM archive_metadata WHERE key = ?`, + derivedDataRevisionKey); err != nil { + return fmt.Errorf("record related derived-data revision: %w", err) + } + } return nil } +// RelatedDerivedRevisionsOnly verifies that every derived revision after the +// committed cache marker was caused by a journaled child-row mutation. +func (s *Store) RelatedDerivedRevisionsOnly(ctx context.Context, previous, current int64) (bool, error) { + if current <= previous || s.IsPostgreSQL() { + return false, nil + } + var count int64 + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM cache_related_revision_journal + WHERE revision > ? AND revision <= ?`, previous, current).Scan(&count); err != nil { + return false, fmt.Errorf("inspect related derived-data revisions: %w", err) + } + return count == current-previous, nil +} + // AdvanceDerivedDataRevision records a repair attempt that may have committed // changes but was not complete enough to enter the migration ledger. The next // cache maintenance pass must still publish those partial, authoritative rows. diff --git a/internal/store/messages.go b/internal/store/messages.go index 9c4b1c452..42d559764 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -2869,7 +2869,7 @@ func (s *Store) AddMessageLabels(messageID int64, labelIDs []int64) error { if !changed { return nil } - return s.bumpDerivedDataRevision(tx) + return s.bumpDerivedDataRevision(tx, true) }) } diff --git a/internal/store/migrate_recipient_envelope.go b/internal/store/migrate_recipient_envelope.go index d094e4b2e..ccef2dadb 100644 --- a/internal/store/migrate_recipient_envelope.go +++ b/internal/store/migrate_recipient_envelope.go @@ -73,6 +73,9 @@ func (s *Store) ensureRecipientEnvelopeUniqueIndex(ctx context.Context) error { return fmt.Errorf("create idx_message_recipients_envelope: %w", err) } if rebuilt { + if err := restoreRecipientCacheJournalTriggers(tx); err != nil { + return err + } if err := s.dialect.EnsureTriggers(boundQuerier{ctx: ctx, q: tx}); err != nil { return fmt.Errorf("restore triggers after rebuilding message_recipients: %w", err) } @@ -100,6 +103,40 @@ func (s *Store) ensureRecipientEnvelopeUniqueIndex(ctx context.Context) error { ) } +// Rebuilding the legacy recipient table drops triggers attached to it. Restore +// the cache journal within the same migration transaction as the table swap. +func restoreRecipientCacheJournalTriggers(q querier) error { + for _, stmt := range []string{ + `CREATE TRIGGER trg_cache_recipients_insert + AFTER INSERT ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, NEW.message_id); + END`, + `CREATE TRIGGER trg_cache_recipients_update + AFTER UPDATE ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN OLD.recipient_type = 'from' OR NEW.recipient_type = 'from' + THEN 'message_facts' ELSE 'message_recipients' END, OLD.message_id); + INSERT INTO cache_related_change_journal (dataset, message_id) + SELECT CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, NEW.message_id + WHERE NEW.message_id <> OLD.message_id; + END`, + `CREATE TRIGGER trg_cache_recipients_delete + AFTER DELETE ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN OLD.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, OLD.message_id); + END`, + } { + if _, err := q.Exec(stmt); err != nil { + return fmt.Errorf("restore recipient cache journal trigger: %w", err) + } + } + return nil +} + // dropRecipientTableUniqueConstraintsPG drops every UNIQUE constraint on // message_recipients by its catalog name. Discovery instead of a hardcoded // name: the default constraint name is derived (and 63-byte truncated) by diff --git a/internal/store/migrate_recipient_envelope_test.go b/internal/store/migrate_recipient_envelope_test.go index 916b7a53c..dbb565fb1 100644 --- a/internal/store/migrate_recipient_envelope_test.go +++ b/internal/store/migrate_recipient_envelope_test.go @@ -128,11 +128,23 @@ func TestEnsureRecipientEnvelopeUniqueIndex_LegacyTableRebuild(t *testing.T) { // 3) An alias snapshot for the same participant now inserts, while a // case variant of an existing snapshot is still rejected. + var journalBaseline int + require.NoError(st.db.QueryRow(` + SELECT COUNT(*) FROM cache_related_change_journal + WHERE dataset = 'message_recipients' AND message_id = ? + `, msgID).Scan(&journalBaseline), "count recipient cache events before alias") _, err = st.db.Exec(` INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name, email_address) VALUES (?, ?, 'to', 'Primary', 'alias@example.test') `, msgID, participantID) require.NoError(err, "alias envelope row must insert after the migration") + var journalCount int + require.NoError(st.db.QueryRow(` + SELECT COUNT(*) FROM cache_related_change_journal + WHERE dataset = 'message_recipients' AND message_id = ? + `, msgID).Scan(&journalCount), "count recipient cache events after migration") + assert.Equal(journalBaseline+1, journalCount, + "recipient cache triggers must survive the legacy table swap") _, err = st.db.Exec(` INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name, email_address) VALUES (?, ?, 'to', 'Primary', 'ALIAS@example.test') diff --git a/internal/store/schema.sql b/internal/store/schema.sql index d1698cbac..81743d54a 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -1616,6 +1616,111 @@ CREATE TABLE IF NOT EXISTS message_labels ( PRIMARY KEY (message_id, label_id) ); +-- Durable cache repair boundary. Triggers write in the same transaction as +-- each child-row mutation, so a failed or interrupted cache publication can +-- replay every change after the last committed sequence. +CREATE TABLE IF NOT EXISTS cache_related_change_journal ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + dataset TEXT NOT NULL, + message_id INTEGER NOT NULL +); + +-- Revisions caused solely by related-row edits can be repaired from the +-- child-row journal. Every other derived revision still requires a full build. +CREATE TABLE IF NOT EXISTS cache_related_revision_journal ( + revision INTEGER PRIMARY KEY +); + +-- A label definition can change without touching any message_labels row. +-- Message ID zero denotes a dataset-wide change for cache invalidation. +CREATE TRIGGER IF NOT EXISTS trg_cache_label_definitions_insert +AFTER INSERT ON labels FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) VALUES ('labels', 0); +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_label_definitions_update +AFTER UPDATE ON labels FOR EACH ROW +WHEN OLD.name IS NOT NEW.name BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) VALUES ('labels', 0); +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_label_definitions_delete +AFTER DELETE ON labels FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) VALUES ('labels', 0); +END; + +-- Child edits that also change baked message facts require a message rebuild. +CREATE TRIGGER IF NOT EXISTS trg_cache_message_facts_update +AFTER UPDATE OF sender_id, is_from_me, + has_attachments, attachment_count ON messages FOR EACH ROW +WHEN OLD.sender_id IS NOT NEW.sender_id OR OLD.is_from_me IS NOT NEW.is_from_me + OR OLD.has_attachments IS NOT NEW.has_attachments + OR OLD.attachment_count IS NOT NEW.attachment_count BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_facts', NEW.id); +END; + +-- From rows determine owner_participant_id in message shards even when +-- replacing them leaves messages.is_from_me unchanged. +CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_insert +AFTER INSERT ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, NEW.message_id); +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_update +AFTER UPDATE ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN OLD.recipient_type = 'from' OR NEW.recipient_type = 'from' + THEN 'message_facts' ELSE 'message_recipients' END, OLD.message_id); + INSERT INTO cache_related_change_journal (dataset, message_id) + SELECT CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, NEW.message_id + WHERE NEW.message_id <> OLD.message_id; +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_delete +AFTER DELETE ON message_recipients FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES (CASE WHEN OLD.recipient_type = 'from' THEN 'message_facts' + ELSE 'message_recipients' END, OLD.message_id); +END; + +CREATE TRIGGER IF NOT EXISTS trg_cache_labels_insert +AFTER INSERT ON message_labels FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_labels', NEW.message_id); +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_labels_update +AFTER UPDATE ON message_labels FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_labels', OLD.message_id); + INSERT INTO cache_related_change_journal (dataset, message_id) + SELECT 'message_labels', NEW.message_id + WHERE NEW.message_id <> OLD.message_id; +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_labels_delete +AFTER DELETE ON message_labels FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_labels', OLD.message_id); +END; + +CREATE TRIGGER IF NOT EXISTS trg_cache_attachments_insert +AFTER INSERT ON attachments FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('attachments', NEW.message_id); +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_attachments_update +AFTER UPDATE ON attachments FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('attachments', OLD.message_id); + INSERT INTO cache_related_change_journal (dataset, message_id) + SELECT 'attachments', NEW.message_id + WHERE NEW.message_id <> OLD.message_id; +END; +CREATE TRIGGER IF NOT EXISTS trg_cache_attachments_delete +AFTER DELETE ON attachments FOR EACH ROW BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('attachments', OLD.message_id); +END; + -- ============================================================================ -- RAW DATA STORAGE -- ============================================================================ From 579be0d5e928b0217b7269eeac164c79e7453a65 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 28 Sep 2026 18:04:40 -0400 Subject: [PATCH 073/125] fix(vcard): derive and refresh full names for nameless contacts (#962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contacts without a full or structured name fail vCard 4.0 validation during publication. They now receive a full name from their organization, email, phone, or “Unnamed Contact”. Generated names preserve decoded text, remove matching email or phone URI prefixes regardless of case, and refresh when their source changes or is removed. Imported names remain intact, including names marked `DERIVED=true`. This PR covers vCard name generation and ownership. Repairing names saved by older versions is outside its scope. Co-authored-by: Steve Francia --- internal/vcard/resource.go | 64 ++++++++++- internal/vcard/resource_ownership.go | 25 ++++- internal/vcard/resource_render.go | 19 ++-- internal/vcard/resource_serialization.go | 3 + internal/vcard/resource_test.go | 56 ++++++++++ internal/vcard/semantic_property.go | 6 +- internal/vcardmap/profile.go | 4 +- internal/vcardmap/profile_merge.go | 2 +- internal/vcardmap/profile_merge_test.go | 130 +++++++++++++++++++++++ internal/vcardmap/profile_test.go | 10 +- 10 files changed, 295 insertions(+), 24 deletions(-) diff --git a/internal/vcard/resource.go b/internal/vcard/resource.go index 6e1197cc3..f2cf1161d 100644 --- a/internal/vcard/resource.go +++ b/internal/vcard/resource.go @@ -75,6 +75,8 @@ type PropertyOccurrence struct { Identity PropertyIdentity `json:"identity"` Property Property `json:"property"` Classification HandlingStrategy `json:"classification"` + // GeneratedFullName identifies a renderer-owned fallback, not an imported FN. + GeneratedFullName bool `json:"generated_full_name,omitzero"` } // PropertyEdit replaces or removes exactly one property occurrence. A zero @@ -271,6 +273,7 @@ func applyPropertyEdits( occurrence.Property = mergeEditedProperty( occurrence.Property, cloneProperty(edit.Property), edit.OwnedParameters, ) + occurrence.GeneratedFullName = false occurrence.Classification = classifyProperty(edit.Property) properties = append(properties, occurrence) } @@ -307,7 +310,7 @@ func indexPropertyEdits(edits []PropertyEdit) (map[string]PropertyEdit, error) { } // RenderView returns the requested wire-version view without mutating the -// envelope. A no-op view returns StoredBody byte-for-byte. +// envelope. A no-op view with a non-empty FN returns StoredBody byte-for-byte. func (e ResourceEnvelope) RenderView(version Version) ([]byte, error) { if version == "" { version = e.RenderMetadata.StoredVersion @@ -315,7 +318,11 @@ func (e ResourceEnvelope) RenderView(version Version) ([]byte, error) { if version == Version21 { return nil, errors.New("vCard 2.1 is read-only and cannot be emitted") } - if !e.RenderMetadata.RenderRequired && version == e.RenderMetadata.StoredVersion { + if !e.RenderMetadata.RenderRequired && version == e.RenderMetadata.StoredVersion && + slices.ContainsFunc(e.PropertyTree, func(occurrence PropertyOccurrence) bool { + return strings.EqualFold(occurrence.Property.Name, "FN") && + strings.TrimSpace(occurrence.Property.RawValue) != "" + }) { return append([]byte(nil), e.StoredBody...), nil } if version != Version30 && version != Version40 { @@ -423,6 +430,9 @@ func (e ResourceEnvelope) cardForRender(version Version) (Card, error) { sourceVersion := e.RenderMetadata.StoredVersion properties := make([]Property, 0, len(e.PropertyTree)+1) for _, occurrence := range e.PropertyTree { + if occurrence.GeneratedFullName { + continue + } property := cloneProperty(occurrence.Property) if strings.EqualFold(property.Name, "VERSION") { property.RawValue = string(version) @@ -489,6 +499,43 @@ func ensureRenderedFullName(properties []Property, version Version) ([]Property, } return slices.Insert(properties, index+1, fullName), nil } + for _, propName := range []string{"ORG", "EMAIL", "TEL"} { + for index, property := range properties { + if strings.EqualFold(property.Name, propName) && strings.TrimSpace(property.RawValue) != "" { + val, err := UnescapeText(property.RawValue) + if err != nil { + return nil, fmt.Errorf("derive FN from %s: %w", propName, err) + } + val = strings.TrimSpace(val) + if scheme, value, ok := strings.Cut(val, ":"); ok && + (propName == "EMAIL" && strings.EqualFold(scheme, "mailto") || + propName == "TEL" && semanticValueType(version, property) == valueTypeURI && + strings.EqualFold(scheme, "tel")) { + val = value + } + if val != "" { + fullName, err := NewProperty(property.Group, "FN", EscapeText(val)) + if err == nil { + if version == Version40 { + if derived, err := NewParameter("DERIVED", "true"); err == nil { + fullName.Parameters = append(fullName.Parameters, derived) + } + } + return slices.Insert(properties, index+1, fullName), nil + } + } + } + } + } + fullName, err := NewProperty("", "FN", "Unnamed Contact") + if err == nil { + if version == Version40 { + if derived, err := NewParameter("DERIVED", "true"); err == nil { + fullName.Parameters = append(fullName.Parameters, derived) + } + } + return append(properties, fullName), nil + } return properties, nil } @@ -612,6 +659,9 @@ func ResidueWithMappings( } residue := make([]PropertyOccurrence, 0) for _, property := range properties { + if property.GeneratedFullName { + continue + } if kind, ok := mapped[property.Identity.Key()]; ok { if kind == HandlingPreserve { residue = append(residue, property) @@ -884,7 +934,15 @@ func cloneResourceEnvelope(e ResourceEnvelope) ResourceEnvelope { // now carries; the rest receive ordinals after every identity in the stable // tree. func reconcilePropertyTree(stable, wire []PropertyOccurrence) []PropertyOccurrence { + generatedFullName := !slices.ContainsFunc(stable, func(occurrence PropertyOccurrence) bool { + return !occurrence.GeneratedFullName && strings.EqualFold(occurrence.Property.Name, "FN") && + strings.TrimSpace(occurrence.Property.RawValue) != "" + }) used := make([]bool, len(stable)) + for index, occurrence := range stable { + // An explicit FN replaces the generated fallback and keeps its own owner. + used[index] = occurrence.GeneratedFullName && !generatedFullName + } nextOrdinal := nextPropertyOrdinal(stable) reconciled := make([]PropertyOccurrence, 0, len(wire)) for _, wireOccurrence := range wire { @@ -896,6 +954,8 @@ func reconcilePropertyTree(stable, wire []PropertyOccurrence) []PropertyOccurren match = matchingStableOccurrence(stable, used, wireOccurrence, false) } candidate := clonePropertyOccurrence(wireOccurrence) + candidate.GeneratedFullName = generatedFullName && strings.EqualFold(candidate.Property.Name, "FN") && + strings.TrimSpace(candidate.Property.RawValue) != "" if match >= 0 { used[match] = true candidate.Identity.Ordinal = stable[match].Identity.Ordinal diff --git a/internal/vcard/resource_ownership.go b/internal/vcard/resource_ownership.go index e81edeb8e..9fb57c0da 100644 --- a/internal/vcard/resource_ownership.go +++ b/internal/vcard/resource_ownership.go @@ -12,10 +12,11 @@ var ErrResourceOwnershipMismatch = errors.New("vCard resource ownership cannot b // used for confirmation of an outgoing artifact; an inbound refresh may drop // ownership whose original value is no longer present or distinguishable. func RebindResourceOwnership(prepared, canonical ResourceEnvelope, requireAll bool) (ResourceEnvelope, error) { + canonical = cloneResourceEnvelope(canonical) preparedByIdentity := make(map[string]PropertyOccurrence) preparedPosition := make(map[string]int) preparedCount := make(map[string]int) - canonicalBySemantic := make(map[string][]PropertyOccurrence) + canonicalBySemantic := make(map[string][]int) key := func(version Version, occurrence PropertyOccurrence) string { encoded, _ := json.Marshal(NormalizeSemanticProperty(version, occurrence.Property), json.Deterministic(true)) // SemanticProperty holds only strings and string slices return string(encoded) @@ -26,9 +27,24 @@ func RebindResourceOwnership(prepared, canonical ResourceEnvelope, requireAll bo preparedPosition[occurrence.Identity.Key()] = preparedCount[k] preparedCount[k]++ } - for _, occurrence := range canonical.PropertyTree { + for index, occurrence := range canonical.PropertyTree { k := key(canonical.RenderMetadata.StoredVersion, occurrence) - canonicalBySemantic[k] = append(canonicalBySemantic[k], occurrence) + canonicalBySemantic[k] = append(canonicalBySemantic[k], index) + } + for _, occurrence := range prepared.PropertyTree { + if !occurrence.GeneratedFullName { + continue + } + k := key(prepared.RenderMetadata.StoredVersion, occurrence) + matches := canonicalBySemantic[k] + if len(matches) != preparedCount[k] { + if requireAll { + return ResourceEnvelope{}, ErrResourceOwnershipMismatch + } + continue + } + index := matches[preparedPosition[occurrence.Identity.Key()]] + canonical.PropertyTree[index].GeneratedFullName = true } canonical.NativeMappings = nil for _, mapping := range prepared.NativeMappings { @@ -47,7 +63,8 @@ func RebindResourceOwnership(prepared, canonical ResourceEnvelope, requireAll bo // The semantic key includes wire identity (group, PID, PROP-ID, ALTID). // Equal duplicates retain their occurrence order, independently of the // order in which native mappings are stored. - mapping.Identity = matches[preparedPosition[mapping.Identity.Key()]].Identity + index := matches[preparedPosition[mapping.Identity.Key()]] + mapping.Identity = canonical.PropertyTree[index].Identity canonical.NativeMappings = append(canonical.NativeMappings, mapping) } canonical.Residue = ResidueWithMappings(canonical.PropertyTree, canonical.NativeMappings) diff --git a/internal/vcard/resource_render.go b/internal/vcard/resource_render.go index adefc7278..f3b794f06 100644 --- a/internal/vcard/resource_render.go +++ b/internal/vcard/resource_render.go @@ -10,8 +10,11 @@ import ( "unicode" ) -// valueTypeText is the VALUE parameter value for free text (RFC 6350 §5.2). -const valueTypeText = "text" +// VALUE parameter types used by rendering (RFC 6350 §5.2). +const ( + valueTypeText = "text" + valueTypeURI = "uri" +) // propertyRender carries one property through its conversion to the target // wire version together with the facts lifted out of its source parameters. @@ -524,12 +527,12 @@ func (r *propertyRender) v3ValueType() string { case isInlineMediaName(name): return r.v3MediaValueType(name) case r.mediaURI: - return "uri" + return valueTypeURI case name == "TEL" && r.valueType != valueTypeText && - (r.valueType == "uri" || IsURIValue(r.property.RawValue)): + (r.valueType == valueTypeURI || IsURIValue(r.property.RawValue)): // A tel URI the v3 respelling could not unescape stays a URI. A // declared-text value is never one, however URI-shaped it looks. - return "uri" + return valueTypeURI default: return v3ValueTypeName(r.valueType) } @@ -542,7 +545,7 @@ func (r *propertyRender) v3MediaValueType(name string) string { // parameter 3.0 would read the value as inline binary. return valueTypeText case r.mediaURI || IsURIValue(r.property.RawValue): - return "uri" + return valueTypeURI default: return v3ValueTypeName(r.valueType) } @@ -570,13 +573,13 @@ func (r *propertyRender) v4ValueType() string { case r.mediaURI: // A decoded legacy binary on any other property is a URI, which is // not that property's default type. - return "uri" + return valueTypeURI case name == "UID" || name == "RELATED": if r.valueType == valueTypeText || !IsURIValue(r.property.RawValue) { return valueTypeText } return "" - case r.valueType == "uri" && v4URIValueByDefault(name): + case r.valueType == valueTypeURI && v4URIValueByDefault(name): return "" default: return r.valueType diff --git a/internal/vcard/resource_serialization.go b/internal/vcard/resource_serialization.go index 41ca01c0c..e7d388226 100644 --- a/internal/vcard/resource_serialization.go +++ b/internal/vcard/resource_serialization.go @@ -152,6 +152,9 @@ func validateResourceMetadata(envelope ResourceEnvelope) error { present := make(map[string]PropertyOccurrence, len(envelope.PropertyTree)) ordinals := make(map[int]struct{}, len(envelope.PropertyTree)) for _, occurrence := range envelope.PropertyTree { + if occurrence.GeneratedFullName && !strings.EqualFold(occurrence.Property.Name, "FN") { + return errors.New("generated full name marker belongs only on FN") + } ordinal := occurrence.Identity.Ordinal if ordinal < 0 { return fmt.Errorf("negative vCard occurrence ordinal %d", ordinal) diff --git a/internal/vcard/resource_test.go b/internal/vcard/resource_test.go index 5e2cdc43a..14960d5f8 100644 --- a/internal/vcard/resource_test.go +++ b/internal/vcard/resource_test.go @@ -484,6 +484,62 @@ func TestResourceEnvelopeDerivesFullNameForBothViewsWithoutV4OnlyParameter(t *te assert.Contains(string(v4), "FN;DERIVED=true:Jane Doe\r\n") } +func TestResourceEnvelopeDerivesFullNameFromContactPointsWhenNameMissing(t *testing.T) { + for _, tc := range []struct{ name, property, want string }{ + {"phone", "TEL:+15551234567", "+15551234567"}, + {"escaped organization", `ORG:Example\, Inc.`, "Example, Inc."}, + {"escaped email", `EMAIL:contact\,team@example.com`, "contact,team@example.com"}, + {"uppercase email scheme", "EMAIL:MAILTO:contact@example.com", "contact@example.com"}, + {"mixed case phone scheme", "TEL:TeL:+15551234567", "+15551234567"}, + {"literal organization prefix", "ORG:mailto:Example", "mailto:Example"}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + envelope, err := ParseResourceEnvelope( + []byte("BEGIN:VCARD\r\nVERSION:3.0\r\n" + tc.property + "\r\nEND:VCARD\r\n"), + ) + require.NoError(err) + v4, err := envelope.RenderView(Version40) + require.NoError(err) + rendered, err := ParseResourceEnvelope(v4) + require.NoError(err) + fullName := propertyByNameAndGroup(rendered.PropertyTree, "FN", "") + name, err := UnescapeText(fullName.Property.RawValue) + require.NoError(err) + assert.Equal(t, tc.want, name) + }) + } +} + +func TestResourceEnvelopeTelephoneFallbackRespectsValueType(t *testing.T) { + for _, tc := range []struct{ name, property, want string }{ + {"text", "TEL;VALUE=text:tel:+15551234567", "tel:+15551234567"}, + {"uri", "TEL;VALUE=uri:tel:+15551234567", "+15551234567"}, + } { + for _, version := range []Version{Version30, Version40} { + t.Run(tc.name+"/"+string(version), func(t *testing.T) { + require := require.New(t) + envelope, err := ParseResourceEnvelope( + []byte("BEGIN:VCARD\r\nVERSION:4.0\r\n" + tc.property + "\r\nEND:VCARD\r\n"), + ) + require.NoError(err) + note, err := NewProperty("", "NOTE", "edited") + require.NoError(err) + envelope, err = envelope.MergeProperties([]PropertyEdit{{Property: note}}) + require.NoError(err) + body, err := envelope.RenderView(version) + require.NoError(err) + rendered, err := ParseResourceEnvelope(body) + require.NoError(err) + fullName := propertyByNameAndGroup(rendered.PropertyTree, "FN", "") + name, err := UnescapeText(fullName.Property.RawValue) + require.NoError(err) + assert.Equal(t, tc.want, name) + }) + } + } +} + func TestResourceEnvelopeMovesLegacyReferencedMediaTypeToMediatype(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/internal/vcard/semantic_property.go b/internal/vcard/semantic_property.go index 6fcf34317..8746b36b2 100644 --- a/internal/vcard/semantic_property.go +++ b/internal/vcard/semantic_property.go @@ -63,10 +63,10 @@ func semanticValueType(version Version, property Property) string { } name := strings.ToUpper(property.Name) if version == Version40 && v4URIProperties[name] { - return "uri" + return valueTypeURI } if legacyURIProperties[name] { - return "uri" + return valueTypeURI } if textProperties[name] { return "text" @@ -79,7 +79,7 @@ func canonicalSemanticValue(version Version, property Property) string { switch semanticValueType(version, property) { case "text": return canonicalTextEscapes(value) - case "uri": + case valueTypeURI: return canonicalURI(value) case "boolean", "language-tag": return strings.ToLower(value) diff --git a/internal/vcardmap/profile.go b/internal/vcardmap/profile.go index 434832fd7..3ee415908 100644 --- a/internal/vcardmap/profile.go +++ b/internal/vcardmap/profile.go @@ -356,8 +356,8 @@ func appendNameParameters(property *vcard.Property, name store.PersonName) error // derivedFullName synthesizes FN when no name row projects one. It reports // false when the profile offers nothing to derive from. The result is only a // fallback: the merge drops it again when the card already carries an FN of -// its own, and a card that truly lacks one is rejected by rendering, where the -// mandatory-FN rule belongs. Components are ordered as the render layer +// its own. Rendering supplies a contact-point or placeholder fallback when +// needed to meet the mandatory-FN rule. Components are ordered as the render layer // orders the five vCard 3 components (prefix, given, additional, family, // suffix), with the RFC 9554 secondary surname and generation after family. func derivedFullName(profile store.PersonProfile) (projectedProperty, bool, error) { diff --git a/internal/vcardmap/profile_merge.go b/internal/vcardmap/profile_merge.go index 7fa05ebb6..8c940b41c 100644 --- a/internal/vcardmap/profile_merge.go +++ b/internal/vcardmap/profile_merge.go @@ -302,7 +302,7 @@ func hasUnclaimedFullName(tree []vcard.PropertyOccurrence, taken map[string]stru if _, used := taken[occurrence.Identity.Key()]; used { continue } - if strings.EqualFold(occurrence.Property.Name, "FN") && + if !occurrence.GeneratedFullName && strings.EqualFold(occurrence.Property.Name, "FN") && strings.TrimSpace(occurrence.Property.RawValue) != "" { return true } diff --git a/internal/vcardmap/profile_merge_test.go b/internal/vcardmap/profile_merge_test.go index 056a49eea..5ae7023f3 100644 --- a/internal/vcardmap/profile_merge_test.go +++ b/internal/vcardmap/profile_merge_test.go @@ -97,6 +97,136 @@ func TestProjectPersonEnvelopeDerivesFullNameOnlyWhenCardHasNone(t *testing.T) { assert.Equal(1, strings.Count(string(again.StoredBody), "\r\nFN")) } +func TestProjectPersonEnvelopeRefreshesGeneratedFullName(t *testing.T) { + for _, version := range []string{"3.0", "4.0"} { + for _, importedName := range []string{"", "FN:Imported Name\r\n", "FN;DERIVED=true:Imported Name\r\n"} { + t.Run(version+"/"+strings.TrimSpace(importedName), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + envelope := parseProjectEnvelope(t, []byte("BEGIN:VCARD\r\nVERSION:"+version+"\r\n"+ + importedName+"EMAIL:old@example.com\r\nNOTE:keep\r\nEND:VCARD\r\n")) + envelope.NativeMappings = []vcard.NativeMapping{{ + Identity: projectOccurrence(t, envelope, "EMAIL", 0).Identity, SourceRef: envelope.SourceRef, + Table: "person_contact_points", RowID: 5, Field: "original_value", Kind: vcard.HandlingNative, + }} + snapshot := store.PersonVCardSnapshot{Profile: store.PersonProfile{ + Person: store.Person{ID: 1}, + ContactPoints: []store.PersonContactPoint{{ + Envelope: store.ValueEnvelope{ID: 5, Source: store.ProvenanceUser}, + AddressKind: store.ContactAddressEmail, OriginalValue: "old@example.com", + }}, + }} + first, err := ProjectPersonEnvelope(snapshot, envelope) + require.NoError(err) + // Publication confirmation reparses server bytes and rebinds ownership. + first, err = vcard.RebindResourceOwnership(first, parseProjectEnvelope(t, first.StoredBody), true) + require.NoError(err) + metadata, err := vcard.MarshalResourceMetadata(first) + require.NoError(err) + restored, err := vcard.UnmarshalResourceMetadata(metadata) + require.NoError(err) + first.PropertyTree = restored.PropertyTree + unchanged, err := ProjectPersonEnvelope(snapshot, first) + require.NoError(err) + assert.Equal(first, unchanged, "unchanged projection must preserve the complete envelope") + for _, value := range []string{"new@example.com", ""} { + want := "FN;DERIVED=true:new@example.com\r\n" + if value == "" { + snapshot.Profile.ContactPoints = nil + want = "FN;DERIVED=true:Unnamed Contact\r\n" + } else { + snapshot.Profile.ContactPoints[0].OriginalValue = value + } + if importedName != "" { + want = importedName + } + first, err = ProjectPersonEnvelope(snapshot, first) + require.NoError(err) + assert.Contains(string(first.StoredBody), want) + assert.NotContains(string(first.StoredBody), "old@example.com") + } + snapshot.Profile.Person.DisplayName = new("Chosen Name") + named, err := ProjectPersonEnvelope(snapshot, first) + require.NoError(err) + want := "FN;DERIVED=true:Chosen Name\r\n" + if importedName != "" { + want = importedName + } + assert.Contains(string(named.StoredBody), want) + assert.Equal(1, strings.Count(string(named.StoredBody), "\r\nFN")) + }) + } + } +} + +func TestProjectPersonEnvelopeEditsNameAfterReplacingGeneratedFullName(t *testing.T) { + for _, name := range []string{"old@example.com", "Chosen Name"} { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + envelope := parseProjectEnvelope(t, []byte("BEGIN:VCARD\r\nVERSION:4.0\r\nEND:VCARD\r\n")) + snapshot := store.PersonVCardSnapshot{Profile: store.PersonProfile{ + Person: store.Person{ID: 1}, + ContactPoints: []store.PersonContactPoint{{ + Envelope: store.ValueEnvelope{ID: 5, Source: store.ProvenanceUser}, + AddressKind: store.ContactAddressEmail, OriginalValue: "old@example.com", + }}, + }} + generated, err := ProjectPersonEnvelope(snapshot, envelope) + require.NoError(err) + snapshot.Profile.Person.DisplayName = new(name) + named, err := ProjectPersonEnvelope(snapshot, generated) + require.NoError(err) + assert.Equal(name, projectOccurrence(t, named, "FN", 0).Property.RawValue) + + snapshot.Profile.Person.DisplayName = new("Updated Name") + renamed, err := ProjectPersonEnvelope(snapshot, named) + require.NoError(err) + assert.Equal("Updated Name", projectOccurrence(t, renamed, "FN", 0).Property.RawValue) + + snapshot.Profile.Person.DisplayName = nil + snapshot.Profile.ContactPoints[0].OriginalValue = "new@example.com" + removed, err := ProjectPersonEnvelope(snapshot, renamed) + require.NoError(err) + assert.Equal("new@example.com", projectOccurrence(t, removed, "FN", 0).Property.RawValue) + }) + } +} + +func TestProjectPersonEnvelopeGeneratesMissingFullNameWithoutEdits(t *testing.T) { + for _, tc := range []struct { + name string + properties string + want string + }{ + {"missing", "", "Unnamed Contact"}, + {"empty", "FN: \r\n", "Unnamed Contact"}, + {"email", "EMAIL:user@example.com\r\n", "user@example.com"}, + {"named", "FN:Known Name\r\n", "Known Name"}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + envelope := parseProjectEnvelope(t, []byte("BEGIN:VCARD\r\nVERSION:4.0\r\n"+ + "UID:resource\r\n"+tc.properties+"NOTE:keep\r\nEND:VCARD\r\n")) + snapshot := store.PersonVCardSnapshot{Profile: store.PersonProfile{Person: store.Person{ID: 1}}} + projected, err := ProjectPersonEnvelope(snapshot, envelope) + require.NoError(err) + doc, err := vcard.Decode(strings.NewReader(string(projected.StoredBody))) + require.NoError(err) + assert.NoError(vcard.Validate(doc)) + if tc.name == "named" { + assert.Equal(envelope.StoredBody, projected.StoredBody, "an existing name preserves the original bytes") + } else { + assert.Contains(string(projected.StoredBody), "FN;DERIVED=true:"+tc.want+"\r\n") + } + again, err := ProjectPersonEnvelope(snapshot, projected) + require.NoError(err) + assert.Equal(projected, again, "unchanged projection must preserve the complete envelope") + }) + } +} + func TestProjectPersonPropertiesPhoneticNameEmitsScriptOnce(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/internal/vcardmap/profile_test.go b/internal/vcardmap/profile_test.go index f451afe15..7211e3d45 100644 --- a/internal/vcardmap/profile_test.go +++ b/internal/vcardmap/profile_test.go @@ -831,16 +831,18 @@ func TestProjectPersonEnvelopeKeepsImportedFNWhenProfileHasNoName(t *testing.T) }} projected, err := ProjectPersonEnvelope(snapshot, envelope) - require.NoError(err, "a profile with nothing to derive FN from must not block other semantic writes") + require.NoError(err, "a profile without a name must not block other semantic writes") assert.Contains(string(projected.StoredBody), "FN:Imported Name\r\n") assert.Contains(string(projected.StoredBody), "EMAIL:alice@example.com\r\n") assert.NotContains(string(projected.StoredBody), "DERIVED") - // A card that has no FN anywhere is still refused, by rendering. + // Without an imported FN, rendering derives one from the email. bare := parseProjectEnvelope(t, []byte("BEGIN:VCARD\r\nVERSION:4.0\r\n"+ "NOTE:no name here\r\nEND:VCARD\r\n")) - _, err = ProjectPersonEnvelope(snapshot, bare) - require.Error(err) + projected, err = ProjectPersonEnvelope(snapshot, bare) + require.NoError(err) + assert.Contains(string(projected.StoredBody), "FN;DERIVED=true:alice@example.com\r\n") + assert.Contains(string(projected.StoredBody), "EMAIL:alice@example.com\r\n") } func TestProjectPersonPropertiesCarriesUnstructuredPostalAddresses(t *testing.T) { From 2a160c4906ea28109256359f469e8dee12e61802 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 28 Sep 2026 18:01:23 -0500 Subject: [PATCH 074/125] fix(people): reuse enrichment citations across lookups (#983) Repeat Exa lookups now reuse an existing citation when only its retrieval time changes, instead of failing with `person enrichment citation key has different immutable metadata`. The shared citation retains its first retrieval time; each attempt keeps its own timestamp in claim evidence. Other citation metadata must still match. Regression coverage exercises two completed attempts sharing one citation and rejects conflicting content. No schema change. Repairing existing failed runs and changing general commit-error cleanup are outside this fix. Fixes #972. Co-authored-by: Wes McKinney --- internal/store/person_enrichment_results.go | 6 +- .../store/person_enrichment_results_test.go | 97 ++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/internal/store/person_enrichment_results.go b/internal/store/person_enrichment_results.go index b68393396..eced0fb45 100644 --- a/internal/store/person_enrichment_results.go +++ b/internal/store/person_enrichment_results.go @@ -34,7 +34,7 @@ const ( ) // PersonEnrichmentCitation is the bounded public citation metadata linked to -// one provider attempt. It contains no credential or private identifier. +// provider attempts. It contains no credential or private identifier. type PersonEnrichmentCitation struct { ID int64 `json:"id"` PersonID int64 `json:"person_id"` @@ -1220,9 +1220,11 @@ func (s *Store) recheckPersonEnrichmentCatalogTx( func personEnrichmentCitationMatches( stored PersonEnrichmentCitation, citation personenrichment.Citation, ) bool { + // The citation keeps its first retrieval time; each attempt's evidence + // records that attempt's retrieval time. if stored.CitationKey != citation.Key || stored.CanonicalURL != citation.URL || stored.Title != citation.Title || stored.Publisher != citation.Publisher || - stored.Excerpt != citation.Excerpt || !stored.RetrievedAt.Equal(citation.RetrievedAt) { + stored.Excerpt != citation.Excerpt { return false } if citation.PublishedAt.IsZero() { diff --git a/internal/store/person_enrichment_results_test.go b/internal/store/person_enrichment_results_test.go index dc53b2714..49ad4d585 100644 --- a/internal/store/person_enrichment_results_test.go +++ b/internal/store/person_enrichment_results_test.go @@ -209,7 +209,7 @@ func (f *enrichmentResultFixture) reseal(t *testing.T) { f.commit, err = personenrichment.NewClaimCommit(personenrichment.ClaimCommitInput{ AttemptID: f.attempt.ID, RunID: f.attempt.RunID, PersonID: f.person.ID, LeaseFence: f.attempt.Token.Fence, ProfileFingerprint: f.profile.Fingerprint, - ProviderNamespace: f.profile.ProviderNamespace, RequestHash: strings.Repeat("2", 64), + ProviderNamespace: f.profile.ProviderNamespace, RequestHash: f.attempt.RequestHash, IdentityAssessment: f.commit.IdentityAssessment, }, f.result, hasher) require.NoError(t, err) @@ -616,6 +616,101 @@ func TestPersonEnrichmentResultDeduplicatesMetadataAndPreservesOpaqueIDs(t *test checks.Equal([]string{"Opaque ID/Not-A-URL:MiXeD?x=1#fragment"}, identities) } +func TestCommitEnrichmentClaimsReusesCitationAcrossAttempts(t *testing.T) { + checks := assert.New(t) + requirements := require.New(t) + f := newEnrichmentResultFixture(t) + firstAttemptID := f.attempt.ID + firstRetrievedAt := f.result.Citations[0].RetrievedAt + _, err := f.store.CommitEnrichmentClaims(t.Context(), f.commit) + requirements.NoError(err) + firstCitations, err := f.store.ListPersonEnrichmentAttemptCitationsContext(t.Context(), firstAttemptID) + requirements.NoError(err) + requirements.Len(firstCitations, 1) + + now := f.now.Add(f.profile.RefreshInterval) + SetPersonEnrichmentClockForTest(f.store, func() time.Time { return now }) + run, _, err := f.store.StartRun(t.Context(), personenrichment.RunStart{ + Kind: "scheduled", RequestedBy: "citation-refresh", RequestedAt: now, + }) + requirements.NoError(err) + lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ + RunID: run.ID, Owner: "result-worker", ProviderName: f.profile.Name, + Now: now, LeaseDuration: 5 * time.Minute, + }) + requirements.NoError(err) + requirements.NotNil(lease) + person, err := f.store.GetPersonContext(t.Context(), f.person.ID) + requirements.NoError(err) + f.attempt, _, err = f.store.BeginAttempt(t.Context(), lease.Token, personenrichment.AttemptStart{ + RunID: run.ID, PersonID: person.ID, ProfileFingerprint: f.profile.Fingerprint, + PayloadHash: strings.Repeat("1", 64), RequestHash: strings.Repeat("3", 64), + PersonRevision: person.Revision, Trigger: lease.Trigger, + }) + requirements.NoError(err) + requirements.NoError(f.store.AuthorizeAttemptDispatch(t.Context(), f.attempt.Token)) + f.result.RequestID = "refresh-request" + f.result.JobID = "refresh-job" + f.result.Citations[0].RetrievedAt = now + f.result.SourceAttempts[0].ObservedAt = now + f.reseal(t) + + outcome, err := f.store.CommitEnrichmentClaims(t.Context(), f.commit) + requirements.NoError(err) + checks.Equal(personenrichment.ClaimApplied, outcome.Status) + attempt, err := f.store.GetPersonEnrichmentAttemptContext(t.Context(), f.attempt.ID) + requirements.NoError(err) + checks.Equal("succeeded", attempt.State) + checks.Nil(attempt.LeaseUntil) + secondCitations, err := f.store.ListPersonEnrichmentAttemptCitationsContext(t.Context(), f.attempt.ID) + requirements.NoError(err) + checks.Equal(firstCitations, secondCitations, "reuse the citation and preserve its first retrieval time") + checks.Equal(int64(1), enrichmentTableCount(t, f.store, "person_enrichment_citations")) + checks.Equal(int64(2), enrichmentTableCount(t, f.store, "person_enrichment_attempt_citations")) + + evidence, err := f.store.ListPersonFactEvidenceContext(t.Context(), person.ID, personfacts.EvidenceFilter{}) + requirements.NoError(err) + recordedTimes := make([]time.Time, 0, len(evidence)) + for _, item := range evidence { + recordedTimes = append(recordedTimes, item.Input.RecordedTime) + } + checks.ElementsMatch([]time.Time{firstRetrievedAt, now}, recordedTimes) +} + +func TestCommitEnrichmentClaimsRejectsChangedCitationMetadata(t *testing.T) { + for _, tc := range []struct { + name string + change func(*personenrichment.Citation) + }{ + {"URL", func(c *personenrichment.Citation) { c.URL = "https://sources.example.test/other" }}, + {"title", func(c *personenrichment.Citation) { c.Title = "Different title" }}, + {"publisher", func(c *personenrichment.Citation) { c.Publisher = "Different publisher" }}, + {"excerpt", func(c *personenrichment.Citation) { c.Excerpt = "Different excerpt" }}, + {"published time", func(c *personenrichment.Citation) { c.PublishedAt = c.PublishedAt.Add(time.Hour) }}, + {"missing published time", func(c *personenrichment.Citation) { c.PublishedAt = time.Time{} }}, + } { + t.Run(tc.name, func(t *testing.T) { + checks := assert.New(t) + requirements := require.New(t) + f := newEnrichmentResultFixture(t) + citation := f.result.Citations[0] + _, err := f.store.db.ExecContext(t.Context(), `INSERT INTO person_enrichment_citations + (person_id, citation_key, canonical_url, title, publisher, excerpt, published_at, retrieved_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, f.person.ID, citation.Key, citation.URL, + citation.Title, citation.Publisher, citation.Excerpt, citation.PublishedAt, citation.RetrievedAt) + requirements.NoError(err) + tc.change(&f.result.Citations[0]) + f.reseal(t) + + _, err = f.store.CommitEnrichmentClaims(t.Context(), f.commit) + requirements.ErrorContains(err, "citation key has different immutable metadata") + citations, err := f.store.ListPersonEnrichmentAttemptCitationsContext(t.Context(), f.attempt.ID) + requirements.NoError(err) + checks.Empty(citations) + }) + } +} + func TestCommitEnrichmentClaimsRanksUnsupportedAggregatorEvidenceBelowThreshold(t *testing.T) { checks := assert.New(t) requirements := require.New(t) From f3add78b9ad6429096bfaeecd60254c522346234 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 28 Sep 2026 19:02:33 -0400 Subject: [PATCH 075/125] feat(beeper): route stored audio from captured sources (#939) Route stored WAV and MP3 attachments from captured sources, including email, through Msgvault's Docbank job. Keep each recording linked to its source message and preserve Beeper transcripts. Audio without a supplied transcript uses the configured Docbank speech-recognition profile when one is set. Uploads require the new `all_sources_upload_consent = true` setting. Existing users must opt in again, including for Beeper audio; the former `upload_consent` setting no longer permits uploads. Docbank's own consent still controls processing. Discovery reads attachment headers; upload preparation verifies the complete recording. Unsupported formats stay local. Supplied transcripts and speech-recognition profiles remain separate processing jobs, even for identical audio. Processing costs depend on the selected profile and provider. Builds on #876. Refs https://github.com/kenn-io/msgvault/issues/728#issuecomment-5629248605, slice 7 (audio from other sources). Co-authored-by: Rod Boev --- cmd/msgvault/cmd/beeper_media_scheduler.go | 9 +- .../cmd/beeper_media_scheduler_test.go | 95 ++- docs/changelog.md | 15 +- docs/configuration.md | 32 +- docs/usage/beeper.md | 69 +- internal/beeper/docbank_media.go | 367 +++++++--- internal/beeper/docbank_media_review_test.go | 251 +++++-- internal/beeper/docbank_media_test.go | 210 ++++-- internal/beeper/stored_media_test.go | 636 ++++++++++++++++++ internal/config/beeper_docbank_test.go | 23 +- internal/config/config.go | 21 +- internal/docbankmedia/client.go | 7 +- internal/docbankmedia/client_test.go | 16 +- internal/store/docbank_media.go | 175 ++++- internal/store/docbank_media_test.go | 73 +- 15 files changed, 1704 insertions(+), 295 deletions(-) create mode 100644 internal/beeper/stored_media_test.go diff --git a/cmd/msgvault/cmd/beeper_media_scheduler.go b/cmd/msgvault/cmd/beeper_media_scheduler.go index afa66994a..9a945599c 100644 --- a/cmd/msgvault/cmd/beeper_media_scheduler.go +++ b/cmd/msgvault/cmd/beeper_media_scheduler.go @@ -22,7 +22,7 @@ import ( const ( beeperMediaSubmitJob = "beeper-media-submit" beeperMediaSubmitCron = "* * * * *" - beeperMediaGateLabel = "Beeper media submission" + beeperMediaGateLabel = "Stored media submission" ) // beeperMediaGateWait bounds each wait for the operation gate; a busy gate ends @@ -119,7 +119,7 @@ func addBeeperMediaRoute( destination := beeperMediaDestinationKey(endpoint, archiveUID) // Without upload consent the job only records local discovery. var submitClient *docbankmedia.Client - if cfg.UploadConsent { + if cfg.AllSourcesUploadConsent { submitClient = client if err := withBeeperMediaGate(ctx, gate, func() error { return st.ReconsiderBlockedBeeperMediaOperations(ctx, destination) @@ -127,7 +127,8 @@ func addBeeperMediaRoute( return err } } - submitter := beeper.NewMediaSubmitter(st, blobs, submitClient, destination, spoolDir).WithOperationGate(beeperMediaGate(gate)) + submitter := beeper.NewMediaSubmitter(st, blobs, submitClient, destination, spoolDir). + WithASRProfile(cfg.ASRProfile).WithOperationGate(beeperMediaGate(gate)) return sched.AddJob(scheduler.Job{ Name: beeperMediaSubmitJob, Schedule: beeperMediaSubmitCron, @@ -137,7 +138,7 @@ func addBeeperMediaRoute( return err } if logger != nil && (result.Examined > 0 || result.Journaled > 0) { - logger.Debug("Beeper media submission pass", "examined", result.Examined, + logger.Debug("Stored media submission pass", "examined", result.Examined, "journaled", result.Journaled, "pending", result.Pending, "retained", result.Retained, "blocked", result.Blocked) } diff --git a/cmd/msgvault/cmd/beeper_media_scheduler_test.go b/cmd/msgvault/cmd/beeper_media_scheduler_test.go index 51c1db757..e835fbcf1 100644 --- a/cmd/msgvault/cmd/beeper_media_scheduler_test.go +++ b/cmd/msgvault/cmd/beeper_media_scheduler_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json/v2" + "fmt" "io" "log/slog" "net/http" @@ -177,7 +178,7 @@ func TestBeeperMediaConfig(t *testing.T) { // Remote plaintext is refused before any job exists. require.Error(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ - Enabled: true, URL: "http://docbank.example.com", APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true}, nil)) + Enabled: true, URL: "http://docbank.example.com", APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) assert.False(sched.IsJobScheduled(beeperMediaSubmitJob)) // Without upload consent the job records local discovery only. @@ -192,7 +193,7 @@ func TestBeeperMediaConfig(t *testing.T) { // A missing credential blocks the operation without scheduling a retry. t.Setenv(beeperMediaTestKeyEnv, "") require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ - Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true}, nil)) + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) var state, code, operationID string var scheduled bool @@ -209,7 +210,7 @@ func TestBeeperMediaConfig(t *testing.T) { // Startup reconsideration reopens the same operation. A failing peer still retries. t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ - Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true}, nil)) + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) require.NoError(st.DB().QueryRow(st.Rebind(` SELECT retention_state, error_code, retention_operation_id, next_action_at IS NOT NULL FROM beeper_media_occurrences WHERE destination_key = ?`), destination). @@ -249,7 +250,7 @@ func TestBeeperMediaInvalidConfigUnregisters(t *testing.T) { sched := scheduler.New(nil) defer func() { <-sched.Stop().Done() }() cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, - APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true} + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) require.True(consumerRegistered(t, st)) @@ -279,7 +280,7 @@ func TestBeeperMediaScheduledRoute(t *testing.T) { sched := scheduler.New(nil).WithWorkTracker(labelWorkTracker(gate, "media")) defer func() { <-sched.Stop().Done() }() cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, - APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true} + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) archiveUID, err := st.ArchiveUIDContext(t.Context()) require.NoError(err) @@ -340,6 +341,88 @@ func TestBeeperMediaScheduledRoute(t *testing.T) { require.Error(sched.TriggerJob(beeperMediaSubmitJob)) } +func TestStoredMediaSchedulerUsesOtherSourceProfile(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + _, err := st.DB().Exec(`UPDATE attachments SET attachment_role = 'preview' + WHERE message_id = (SELECT id FROM messages WHERE source_message_id = 'voice1')`) + require.NoError(err) + source, err := st.GetOrCreateSource("gmail", "rod@example.com") + require.NoError(err) + conversation, err := st.EnsureConversation(source.ID, "mail-thread-1", "Mail") + require.NoError(err) + messageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversation, SourceID: source.ID, SourceMessageID: "mail-audio-1", + MessageType: "gmail", SizeEstimate: int64(len(testWAV())), + }) + require.NoError(err) + wav := testWAV() + digest := sha256.Sum256(wav) + hash := hex.EncodeToString(digest[:]) + require.NoError(st.UpsertAttachmentRecord(t.Context(), messageID, store.AttachmentWrite{ + Filename: "meeting.wav", MIMEType: "application/octet-stream", StoragePath: hash[:2] + "/" + hash, + ContentHash: hash, Size: int64(len(wav)), SourceAttachmentID: "mail:attachment:1", + SourcePartKey: "mime:1.2", State: attachmentpolicy.StateStored, + Role: store.AttachmentRoleStandalone, RoleSource: store.AttachmentRoleSourceImporterSemantics, + })) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + _, httpServer := newRetentionServer(t) + sched := scheduler.New(nil) + cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true, ASRProfile: "asr"} + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + destination := beeperMediaDestinationKey(httpServer.URL, archiveUID) + var provider, profile string + require.NoError(st.DB().QueryRow(st.Rebind(`SELECT provider, profile FROM beeper_media_deliveries + WHERE destination_key = ?`), destination).Scan(&provider, &profile)) + assert.Equal("gmail", provider) + assert.Equal("asr", profile) +} + +func TestStoredMediaRequiresAllSourcesConsent(t *testing.T) { + for _, provider := range []string{"beeper", "gmail"} { + t.Run(provider, func(t *testing.T) { + require, assert := require.New(t), assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + _, err := st.DB().Exec(st.Rebind(`UPDATE sources SET source_type = ?`), provider) + require.NoError(err) + t.Setenv("MSGVAULT_HOME", t.TempDir()) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + server, httpServer := newRetentionServer(t) + sched := scheduler.New(nil) + t.Cleanup(func() { <-sched.Stop().Done() }) + path := filepath.Join(t.TempDir(), "config.toml") + content := fmt.Sprintf(`[integrations.docbank] +enabled = true +url = %q +api_key_env = %q +upload_consent = true +`, httpServer.URL, beeperMediaTestKeyEnv) + require.NoError(os.WriteFile(path, []byte(content), 0o600)) + cfg, err := config.Load(path, "") + require.NoError(err) + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg.Integrations.Docbank, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Zero(server.requestCount(), "existing consent must keep all audio local") + + content += "all_sources_upload_consent = true\n" + require.NoError(os.WriteFile(path, []byte(content), 0o600)) + cfg, err = config.Load(path, "") + require.NoError(err) + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg.Integrations.Docbank, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Equal(1, server.requestCount(), "new consent must allow the stored audio upload") + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + assert.Equal(map[string]string{beeperMediaDestinationKey(httpServer.URL, archiveUID): "retained::source"}, retentionRows(t, st)) + }) + } +} + // TestBeeperMediaGatedStoreWrites composes the daemon schedulers. A long // upload holds no operation gate, while every media Store write waits for it, // so a backup freeze sees no writer. @@ -358,7 +441,7 @@ func TestBeeperMediaGatedStoreWrites(t *testing.T) { require.NoError(sched.AddJob(scheduler.Job{Name: "test-gated-job", Schedule: "0 0 1 1 *", Run: func(context.Context) error { return nil }})) cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, - APIKeyEnv: beeperMediaTestKeyEnv, UploadConsent: true} + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} require.NoError(configureBeeperMediaJob(t.Context(), media, gate, st, blobs, t.TempDir(), cfg, logger)) archiveUID, err := st.ArchiveUIDContext(t.Context()) require.NoError(err) diff --git a/docs/changelog.md b/docs/changelog.md index f7c207541..4a1b1a5cb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -93,12 +93,15 @@ All notable changes to msgvault, grouped by release. source has `enabled = true` in `[[gmail.drafts]]`; `draft-send-as` lists owner-only Gmail aliases without sending mail. -- Send stored Beeper WAV and MP3 audio, with Beeper's own transcript, to a - separately running Docbank media service that has the media HTTP routes. The - daemon backfills in small scheduled batches after `upload_consent` is set, - and Docbank's own consent still controls processing. msgvault keeps each - live message mapped to its Docbank occurrence; search over processed audio is - not included yet. +- Send stored WAV and MP3 audio from any captured source, including email and + messaging imports, to a separately running Docbank media service with the + media HTTP routes. The daemon + backfills in small scheduled batches after `all_sources_upload_consent` is set and can + use an optional `asr_profile` when no usable source transcript exists. + Docbank's own consent still controls processing. msgvault keeps each live + message mapped to its Docbank occurrence; search over processed audio is not + included yet. Existing users must enable the new consent setting; the former + Beeper-only `upload_consent` setting no longer permits uploads. See [Beeper audio](usage/beeper.md#send-audio-to-docbank). diff --git a/docs/configuration.md b/docs/configuration.md index c73f63d7e..d5c5e5398 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -914,35 +914,43 @@ max_media_mb = 250 # per-attachment download cap (MiB) | `max_media_mb` | `250` | Per-attachment download cap in MiB (over-cap media is recorded as a `size_cap` skip and retried only after the cap changes) | | `accounts_config` | — | Per-accountID `media` and `max_media_mb` overrides | -#### Send Beeper audio to Docbank +#### Send stored audio to Docbank -The daemon can send stored Beeper WAV and MP3 audio, with Beeper's own -transcript, to a separately running Docbank media service. The service needs -Docbank's media HTTP routes. See -[Send audio to Docbank](/docs/usage/beeper/#send-audio-to-docbank) for what is -sent and how progress is tracked. +The daemon can send stored WAV and MP3 audio from any captured source, including +messaging and email imports, to a separately running Docbank media service. The +service needs Docbank's media HTTP routes. See +[Send audio to Docbank](/docs/usage/beeper/#send-audio-to-docbank) for the +capture and processing rules. ```toml [integrations.docbank] enabled = true url = "http://127.0.0.1:8080" # your Docbank daemon; the port is an example api_key_env = "DOCBANK_API_KEY" # daemon environment variable with the key -upload_consent = true # allow archive audio to leave msgvault +all_sources_upload_consent = true # allow audio from every captured source to leave msgvault +# asr_profile = "asr" # optional Docbank profile for audio without source text ``` | Key | Default | Description | |---|---|---| -| `enabled` | `false` | Schedule the Beeper media job in `msgvault serve` | +| `enabled` | `false` | Schedule the stored-media job in `msgvault serve` | | `url` | — | Docbank base URL: HTTPS, or HTTP on a loopback address. User info, query strings and fragments are rejected | | `api_key_env` | — | Name of the daemon environment variable that holds the Docbank API key. It is read for each request and sent as `X-Api-Key` | -| `upload_consent` | `false` | Allow audio and transcripts to be sent to `url`. Without it the job only records local state | +| `all_sources_upload_consent` | `false` | Allow stored audio and explicit source transcripts from every captured source, including future providers, to be sent to `url`. Without it the job only records local state | +| `asr_profile` | — | Optional Docbank processing profile for stored audio without usable source text. An empty value retains audio without requesting processing. Msgvault rejects `supplied-transcript`, which Docbank reserves for supplied transcript input. | + +The former Beeper-only `upload_consent` setting no longer enables uploads. +Existing users must explicitly set `all_sources_upload_consent = true` to resume +sending audio, including Beeper recordings. The daemon reads these settings at startup, so restart it after a change. A new `url` starts a separate delivery record; earlier rows stay. Disabling the route stops the job and keeps its rows. A failed setup, such as an invalid `url`, -does the same and logs a warning. `upload_consent` covers transport only; the -Docbank daemon's own processing consent still decides whether transcripts are -processed. +does the same and logs a warning. `all_sources_upload_consent` covers transport only; the +Docbank daemon's processing consent still decides whether a configured profile +may run. The route inspects stored CAS bytes, so MIME claims do not expand +Docbank's WAV and MP3 capability. Capture gaps and unsupported formats remain +typed local states. ### `[slack]` diff --git a/docs/usage/beeper.md b/docs/usage/beeper.md index de52772c1..2cdfba130 100644 --- a/docs/usage/beeper.md +++ b/docs/usage/beeper.md @@ -189,33 +189,52 @@ GROUP BY is_share; ## Send audio to Docbank -The daemon can copy stored Beeper audio to a separately running Docbank media -service. Docbank keeps the recording, imports Beeper's own transcript, and -processes it with its `supplied-transcript` profile. msgvault records which -Docbank source and occurrence belong to each live message. Configure the -destination in -[`[integrations.docbank]`](/docs/configuration/#send-beeper-audio-to-docbank). +The daemon can copy stored audio from any captured source, including messaging +and email importers, to a separately running Docbank media service. Docbank +keeps the recording and msgvault records which Docbank source and occurrence +belong to each live message. Beeper's complete attachment transcript is +imported as supplied evidence. Other sources use a configured ASR profile or +remain unprocessed. Configure the destination in +[`[integrations.docbank]`](/docs/configuration/#send-stored-audio-to-docbank). What you need: - A Docbank server with the media HTTP routes ([docbank#346](https://github.com/kenn-io/docbank/pull/346)). The Docbank library built into msgvault does not provide them. -- `upload_consent = true`. It allows transport to that URL only. Docbank's own - processing consent decides whether the transcript is processed. +- `all_sources_upload_consent = true`. It allows audio from every captured + source to be sent to that URL. See the [consent settings](/docs/configuration/#send-stored-audio-to-docbank) + when upgrading from the Beeper-only route. Docbank's own processing consent + decides whether a supplied transcript or configured ASR profile is processed. +- An optional `asr_profile = "asr"` requests that Docbank process stored audio + without usable source text. Leave it empty to retain the audio without a + processing request. - WAV or MP3 audio. msgvault checks the bytes and sends them as `audio/wav` or `audio/mpeg`, whatever type the provider reported. Docbank accepts no other - codec, so OGG/Opus, M4A and other formats stay local with the - `unsupported_media` code. msgvault never converts audio or runs speech - recognition. + codec, so OGG/Opus, M4A and other formats stay local. Attachments identified + as audio receive the `unsupported_media` code when full verification finds + an unsupported format. msgvault never converts audio or runs speech recognition. +- The route consumes WAV or MP3 bytes from every captured source, including + Beeper, messaging providers, email importers and future providers. An + importer must have captured the bytes and a stable part identity first; this + route never downloads missing media or invents a placeholder hash. What happens: -- Voice notes and ordinary audio both qualify when they are stored, standalone - Beeper attachments. Previews, stickers and other sources are skipped. -- msgvault reads the complete transcript for that attachment from the archived - raw message, not the 32 KiB metadata copy. Audio without a transcript is - still kept by Docbank and reported as `unprocessed`. +- Voice notes and ordinary audio qualify when they are stored, standalone + Beeper attachments. Other captured sources qualify when their stored row has + a stable source part, standalone or unknown role, and WAV or MP3 bytes. + Previews, stickers, other known inline roles and readable non-audio files + stay local. +- msgvault reads Beeper's complete transcript from the archived raw message, + not the 32 KiB metadata copy. Other sources have no generic transcript + metadata contract, so they use only the configured ASR profile or remain + unprocessed. Authored message text and caption URLs are never transcript + evidence. +- Audio without source text is retained and uses the configured ASR profile + when one is set. With an empty profile it remains `unprocessed`. +- For sources other than Beeper, the occurrence timestamp comes from the + archived message's `sent_at`. The route does not read the raw message. - The job backfills existing audio in pages of up to 100 attachments. After that first scan, it checks up to 100 attachment changes each minute. It starts another full scan a day after the previous scan finishes, to catch @@ -234,12 +253,18 @@ What happens: route accepts sources up to 1 GiB; memory use includes the recording plus inspection and allocation overhead. These file limits are not RAM limits. - The same recording in several messages gets one occurrence per message. - Docbank stores the bytes once, and each exact transcript is processed once. + Docbank stores the bytes once. Supplied transcripts share a processing job + when the provider, exact text, language, and audio match. Audio without a + transcript shares an ASR job when the bytes and configured profile match, + including across providers. These are separate jobs: a Beeper transcript + does not suppress ASR for an email attachment containing the same recording. - A hidden, source-deleted, removed or replaced message loses its mapping (`revoked`), including audio still waiting to be sent. Other messages sharing the audio keep theirs. Reaction changes leave the mapping live. - If no live or pending occurrence can supply the recording, an unstarted - transcript delivery stops waiting; restoring an occurrence reopens it. + If no live or pending occurrence can supply the recording, unprepared + processing stops waiting; restoring an occurrence reopens it. Prepared + requests keep their saved identity and retry even after revocation, since + Docbank may have accepted them before msgvault saved the receipt. msgvault decides which occurrences are live; Docbank keeps the shared evidence. - Network errors, HTTP 429 and 5xx responses retry after five minutes with the same operation ID. So does a request that runs out of time: each request @@ -250,6 +275,12 @@ What happens: changes, and so does their transcript delivery, with the same code. Missing or corrupt local bytes, or a temporary upload copy that can't be written, wait as `source_unavailable` and retry after five minutes. + For other sources, an unreadable attachment needs an audio type or WAV/MP3 + filename to enter that retry queue. Without those hints or a recognized + header, discovery leaves the attachment undecided and rechecks it in the + next daily full scan. It creates no media mapping or processing request. + Discovery reads only the header; upload preparation verifies the complete + audio before sending it. - A processed delivery reaches `done` only after Docbank reports coverage for its own processing request, not for another transcript of the same audio. A failed Docbank job or a failed processing request ends as `done` diff --git a/internal/beeper/docbank_media.go b/internal/beeper/docbank_media.go index 2a30686a9..27e1be852 100644 --- a/internal/beeper/docbank_media.go +++ b/internal/beeper/docbank_media.go @@ -19,6 +19,7 @@ import ( "go.kenn.io/docbank/document" "go.kenn.io/docbank/document/media" + "go.kenn.io/kit/pack" "go.kenn.io/msgvault/internal/attachmentstore" "go.kenn.io/msgvault/internal/docbankmedia" @@ -49,7 +50,7 @@ var ( errBeeperMediaGateBusy = errors.New("operation gate busy") ) -// MediaDescriptor contains immutable claims for one verified Beeper media +// MediaDescriptor contains immutable claims for one verified stored-media // version. It intentionally carries no transcript text or credentials. type MediaDescriptor struct { Occurrence docbankmedia.Occurrence @@ -67,6 +68,8 @@ type MediaDescriptor struct { Filename string MIMEType string ProcessingKey string + ProcessingProvider string + ProcessingProfile string } // MediaBatchResult counts one bounded pass. Retained counts a retention @@ -79,7 +82,7 @@ type MediaBatchResult struct { Journaled int } -// MediaSubmitter discovers current Beeper media, persists source mappings and +// MediaSubmitter discovers current stored media, persists source mappings and // performs at most one Docbank action per bounded pass. type MediaSubmitter struct { store *store.Store @@ -89,9 +92,19 @@ type MediaSubmitter struct { spoolDir string actionTimeout time.Duration uploadRate int64 + asrProfile string gate func(context.Context) (func(), bool) } +// WithASRProfile enables a configured Docbank processing profile for stored +// audio that has no usable source transcript. +func (w *MediaSubmitter) WithASRProfile(profile string) *MediaSubmitter { + if w != nil { + w.asrProfile = strings.TrimSpace(profile) + } + return w +} + // NewMediaSubmitter returns a worker for one destination. A nil client keeps // discovery local and sends nothing. func NewMediaSubmitter( @@ -222,7 +235,7 @@ func (w *MediaSubmitter) discover(ctx context.Context, archiveUID string) (Media result.Examined++ if mapping.RetentionState == store.BeeperMediaRetentionBlocked { result.Blocked++ - } else { + } else if mapping.OccurrenceRef != "" { result.Pending++ } scan.AfterAttachmentID = candidate.AttachmentID @@ -263,7 +276,7 @@ func (w *MediaSubmitter) reconcileCandidate( ctx context.Context, archiveUID string, candidate store.BeeperMediaCandidate, ) (store.BeeperMediaMapping, error) { mapping, err := w.mappingForCandidate(ctx, archiveUID, candidate) - if err != nil { + if err != nil || mapping.OccurrenceRef == "" { return store.BeeperMediaMapping{}, err } err = w.gated(ctx, func() error { return w.store.ReconcileBeeperMediaMapping(ctx, mapping) }) @@ -306,24 +319,169 @@ func (w *MediaSubmitter) replayJournal(ctx context.Context, archiveUID string) ( func (w *MediaSubmitter) mappingForCandidate( ctx context.Context, archiveUID string, candidate store.BeeperMediaCandidate, ) (store.BeeperMediaMapping, error) { - raw, err := w.store.GetMessageRawContext(ctx, candidate.MessageID) - if beeperMediaRawGap(err) { - return fallbackMediaMapping(w.destination, candidate, archiveUID, errBeeperMediaRawInvalid), nil - } - if err != nil { - return store.BeeperMediaMapping{}, err + if candidate.SourceType != "beeper" { + eligible, definitive, err := w.probeStoredMedia(ctx, candidate) + if err != nil { + return store.BeeperMediaMapping{}, err + } + if !eligible { + if !definitive { + // Without a recognized header or audio hint, recheck on the next full scan. + return store.BeeperMediaMapping{}, nil + } + return fallbackMediaMappingCode(w.destination, candidate, archiveUID, errBeeperMediaUnsupported.Error()), nil + } } - descriptor, _, err := describeMedia(raw, candidate, archiveUID) + descriptor, _, err := w.describeCandidate(ctx, archiveUID, candidate) if err != nil { + if beeperMediaGap(err) == nil { + return store.BeeperMediaMapping{}, err + } return fallbackMediaMapping(w.destination, candidate, archiveUID, err), nil } return descriptorMapping(w.destination, candidate, descriptor), nil } +// probeStoredMedia reads only a header. Upload preparation verifies the full +// source before sending it. Audio hints keep missing or corrupt audio retryable. +func (w *MediaSubmitter) probeStoredMedia( + ctx context.Context, candidate store.BeeperMediaCandidate, +) (eligible, definitive bool, err error) { + if candidate.ByteLength < 4 || candidate.ByteLength > beeperMediaSourceLimit { + return false, true, nil + } + _, mediaType := selectedMediaMetadata(candidate.Filename, candidate.MIMEType) + ext := strings.ToLower(filepath.Ext(candidate.Filename)) + audioHint := strings.HasPrefix(mediaType, "audio/") || + candidate.MediaType == "audio" || candidate.MediaType == "voice_note" || ext == ".wav" || ext == ".mp3" + if w.blobs == nil { + return audioHint, false, nil + } + reader, _, err := w.blobs.OpenStream(ctx, candidate.ContentHash) + if err != nil { + if ctx.Err() != nil { + return false, false, ctx.Err() + } + return audioHint, false, nil + } + var header [12]byte + n, readErr := io.ReadFull(reader, header[:]) + closeErr := reader.Close() + if ctx.Err() != nil { + return false, false, ctx.Err() + } + if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + return audioHint, false, nil + } + if closeErr != nil && !errors.Is(closeErr, pack.ErrVerificationIncomplete) { + return audioHint, false, nil + } + // An unrecognized header cannot distinguish non-audio from damaged audio. + return audioHint || supportedStoredMediaHeader(header[:n]), false, nil +} + +func supportedStoredMediaHeader(header []byte) bool { + if len(header) >= 12 && string(header[:4]) == "RIFF" && string(header[8:12]) == "WAVE" { + return true + } + if len(header) >= 10 && string(header[:3]) == "ID3" { + return true + } + return len(header) >= 4 && header[0] == 0xff && header[1]&0xe0 == 0xe0 && + header[1]&0x18 != 0x08 && header[1]&0x06 != 0 && header[2]&0xf0 != 0 && header[2]&0xf0 != 0xf0 +} + func beeperMediaRawGap(err error) bool { return errors.Is(err, sql.ErrNoRows) || errors.Is(err, store.ErrInvalidMessageRaw) } +// rawMediaEvidence reads raw messages only for providers whose media metadata +// comes from them. Other providers use the captured attachment and message rows. +func (w *MediaSubmitter) rawMediaEvidence(ctx context.Context, sourceType string, messageID int64) ([]byte, string, error) { + if sourceType != "beeper" { + return nil, "", nil + } + raw, err := w.store.GetMessageRawContext(ctx, messageID) + if beeperMediaRawGap(err) { + return nil, "", errBeeperMediaRawInvalid + } + if err != nil { + return nil, "", err + } + return raw, hashBytes(raw), nil +} + +func (w *MediaSubmitter) describeCandidate( + ctx context.Context, archiveUID string, candidate store.BeeperMediaCandidate, +) (MediaDescriptor, string, error) { + if candidate.SourceType == "beeper" { + raw, rawHash, err := w.rawMediaEvidence(ctx, candidate.SourceType, candidate.MessageID) + if err != nil { + return MediaDescriptor{}, "", err + } + descriptor, transcript, err := describeMedia(raw, candidate, archiveUID) + if err != nil { + return MediaDescriptor{RawHash: rawHash}, "", err + } + descriptor.RawHash = rawHash + return configureMediaProcessing(descriptor, transcript, w.asrProfile), transcript, nil + } + + descriptor, err := describeStoredMedia(candidate, archiveUID) + if err != nil { + return MediaDescriptor{}, "", err + } + return configureMediaProcessing(descriptor, "", w.asrProfile), "", nil +} + +func describeStoredMedia( + candidate store.BeeperMediaCandidate, archiveUID string, +) (MediaDescriptor, error) { + part := candidate.SourcePartKey + if part == "" { + part = candidate.SourceAttachmentID + } + if part == "" { + return MediaDescriptor{}, errBeeperMediaPartMissing + } + filename, requestMIME := selectedMediaMetadata(candidate.Filename, candidate.MIMEType) + var message docbankmedia.Timestamp + if candidate.SentAt.Valid { + message = mediaTimestamp(jsontext.Value(mustJSON(candidate.SentAt.Time.UTC().Format(time.RFC3339Nano)))) + } + descriptor := MediaDescriptor{ + Occurrence: docbankmedia.Occurrence{ + Ref: mediaOccurrenceRef(archiveUID, candidate.SourceType, candidate.SourceIdentifier, + candidate.SourceConversationID, candidate.SourceMessageID, part), + Filename: filename, Message: message, + }, + SourceType: candidate.SourceType, SourceIdentifier: candidate.SourceIdentifier, + SourceConversationID: candidate.SourceConversationID, SourceMessageID: candidate.SourceMessageID, + SourceAttachmentID: candidate.SourceAttachmentID, SourcePartKey: part, + SourceSHA256: candidate.ContentHash, ByteLength: candidate.ByteLength, + Filename: filename, MIMEType: requestMIME, + } + descriptor.Occurrence.Revision = mediaRevision(descriptor) + return descriptor, nil +} + +func configureMediaProcessing(descriptor MediaDescriptor, transcript, asrProfile string) MediaDescriptor { + descriptor.ProcessingKey = "" + descriptor.ProcessingProfile = "" + descriptor.ProcessingProvider = descriptor.SourceType + if descriptor.TranscriptSHA256 != "" && transcript != "" { + descriptor.ProcessingKey = mediaProcessingKey(descriptor) + descriptor.ProcessingProfile = "supplied-transcript" + return descriptor + } + asrProfile = strings.TrimSpace(asrProfile) + if asrProfile != "" { + descriptor.ProcessingKey = mediaASRProcessingKey(descriptor, asrProfile) + descriptor.ProcessingProfile = asrProfile + } + return descriptor +} + type beeperMediaEvidence struct { candidate store.BeeperMediaCandidate mapping store.BeeperMediaMapping @@ -345,17 +503,12 @@ func (w *MediaSubmitter) candidateEvidence( return beeperMediaEvidence{}, err } evidence := beeperMediaEvidence{candidate: candidate} - raw, err := w.store.GetMessageRawContext(ctx, candidate.MessageID) - if beeperMediaRawGap(err) { - evidence.gapCode = errBeeperMediaRawInvalid.Error() - return evidence, nil - } - if err != nil { - return beeperMediaEvidence{}, err - } - evidence.rawHash = hashBytes(raw) - descriptor, transcript, err := describeMedia(raw, candidate, archiveUID) + descriptor, transcript, err := w.describeCandidate(ctx, archiveUID, candidate) + evidence.rawHash = descriptor.RawHash if err != nil { + if beeperMediaGap(err) == nil { + return beeperMediaEvidence{}, err + } evidence.gapCode = mediaGapCode(err) return evidence, nil } @@ -366,18 +519,22 @@ func (w *MediaSubmitter) candidateEvidence( func (w *MediaSubmitter) mappingEvidence( ctx context.Context, archiveUID string, mapping store.BeeperMediaMapping, ) (beeperMediaEvidence, error) { - evidence := beeperMediaEvidence{candidate: mappingCandidate(mapping), mapping: mapping} - raw, err := w.store.GetMessageRawContext(ctx, mapping.MessageID) - if beeperMediaRawGap(err) { - evidence.gapCode = errBeeperMediaRawInvalid.Error() - return evidence, nil - } - if err != nil { - return beeperMediaEvidence{}, err + candidate := mappingCandidate(mapping) + if mapping.AttachmentID > 0 { + current, currentErr := w.store.GetBeeperMediaCandidate(ctx, mapping.AttachmentID) + if currentErr == nil { + candidate = current + } else if !errors.Is(currentErr, sql.ErrNoRows) { + return beeperMediaEvidence{}, currentErr + } } - evidence.rawHash = hashBytes(raw) - descriptor, transcript, err := describeMedia(raw, evidence.candidate, archiveUID) + evidence := beeperMediaEvidence{candidate: candidate, mapping: mapping} + descriptor, transcript, err := w.describeCandidate(ctx, archiveUID, candidate) + evidence.rawHash = descriptor.RawHash if err != nil { + if beeperMediaGap(err) == nil { + return beeperMediaEvidence{}, err + } evidence.gapCode = mediaGapCode(err) return evidence, nil } @@ -392,13 +549,14 @@ func hashBytes(value []byte) string { func sameBeeperMediaCandidate(a, b store.BeeperMediaCandidate) bool { return a.AttachmentID == b.AttachmentID && a.MessageID == b.MessageID && + a.SentAt.Valid == b.SentAt.Valid && a.SentAt.Time.Equal(b.SentAt.Time) && a.ConversationID == b.ConversationID && a.SourceID == b.SourceID && a.SourceType == b.SourceType && a.SourceIdentifier == b.SourceIdentifier && a.SourceConversationID == b.SourceConversationID && a.SourceMessageID == b.SourceMessageID && a.SourceAttachmentID == b.SourceAttachmentID && a.SourcePartKey == b.SourcePartKey && a.Filename == b.Filename && a.MIMEType == b.MIMEType && a.MediaType == b.MediaType && a.Role == b.Role && a.ContentHash == b.ContentHash && a.ByteLength == b.ByteLength && - a.AttachmentState == b.AttachmentState + a.AttachmentState == b.AttachmentState && a.AttachmentMetadata == b.AttachmentMetadata } func sameBeeperMediaMapping(a store.BeeperMediaMapping, b beeperMediaEvidence) bool { @@ -421,6 +579,7 @@ func descriptorMapping( Language: descriptor.Language, OccurrenceJSON: mustJSON(descriptor.Occurrence), Filename: descriptor.Filename, MIMEType: descriptor.MIMEType, RetentionState: store.BeeperMediaRetentionPending, ProcessingKey: descriptor.ProcessingKey, + ProcessingProvider: descriptor.ProcessingProvider, ProcessingProfile: descriptor.ProcessingProfile, } } @@ -440,7 +599,7 @@ func fallbackMediaMappingCode( part = candidate.SourceAttachmentID } if part == "" { - part = "beeper:unknown" + part = candidate.SourceType + ":unknown" } filename, mediaType := selectedMediaMetadata(candidate.Filename, candidate.MIMEType) descriptor := MediaDescriptor{ @@ -537,20 +696,21 @@ func (w *MediaSubmitter) retain( if err != nil { return err } - raw, err := w.store.GetMessageRawContext(ctx, candidate.MessageID) - if beeperMediaRawGap(err) { + _, currentRawHash, rawErr := w.rawMediaEvidence(ctx, candidate.SourceType, candidate.MessageID) + if errors.Is(rawErr, errBeeperMediaRawInvalid) { _, err = w.store.FinishBeeperMediaOperation(ctx, operation, store.BeeperMediaResult{ ErrorCode: errBeeperMediaRawInvalid.Error(), SourceUnavailable: true, }) return err } - if err != nil { - return err + if rawErr != nil { + return rawErr } - currentRawHash := hashBytes(raw) + rawMustMatch := candidate.SourceType == "beeper" && + (fresh.rawHash == "" || currentRawHash != fresh.rawHash) if fresh.missing || fresh.gapCode != "" || !sameBeeperMediaCandidate(candidate, fresh.candidate) || - fresh.rawHash == "" || currentRawHash != fresh.rawHash || + rawMustMatch || fresh.rawHash != evidence.rawHash || fresh.descriptor.Occurrence.Ref != operation.OccurrenceRef || fresh.descriptor.Occurrence.Revision != operation.Revision || @@ -644,16 +804,15 @@ func (w *MediaSubmitter) artifact( if snapshot == nil { continue } - raw, err := w.store.GetMessageRawContext(ctx, mapping.MessageID) - if beeperMediaRawGap(err) { + _, currentRawHash, rawErr := w.rawMediaEvidence(ctx, mapping.SourceType, mapping.MessageID) + if errors.Is(rawErr, errBeeperMediaRawInvalid) { gaps = append(gaps, fallbackMediaMapping(w.destination, mappingCandidate(mapping), archiveUID, errBeeperMediaRawInvalid)) continue } - if err != nil { - return err + if rawErr != nil { + return rawErr } - currentRawHash := hashBytes(raw) if snapshot.gapCode != "" { if snapshot.rawHash != "" && currentRawHash == snapshot.rawHash { gaps = append(gaps, fallbackMediaMappingCode(w.destination, @@ -661,7 +820,7 @@ func (w *MediaSubmitter) artifact( } continue } - if currentRawHash != snapshot.rawHash { + if snapshot.rawHash != "" && currentRawHash != snapshot.rawHash { if sourceChanged == nil { copyOf := *snapshot sourceChanged = ©Of @@ -686,7 +845,7 @@ func (w *MediaSubmitter) artifact( transcriptSHA := sourceChanged.descriptor.TranscriptSHA256 operation.FrozenRequestJSON = mustJSON(docbankmedia.ArtifactMetadata{ OccurrenceID: sourceChanged.mapping.DocbankOccurrenceID, Kind: "transcript", Origin: "provider", - Provider: "beeper", Language: operation.Language, Filename: "transcript.txt", + Provider: sourceChanged.mapping.SourceType, Language: operation.Language, Filename: "transcript.txt", MediaType: "text/plain", SHA256: transcriptSHA, ByteLength: int64(len(sourceChanged.transcript)), }) operation.DocbankSourceID, operation.SourceVersionID = sourceChanged.mapping.DocbankSourceID, sourceChanged.mapping.SourceVersionID @@ -713,7 +872,7 @@ func (w *MediaSubmitter) artifact( transcriptSHA := surviving.descriptor.TranscriptSHA256 operation.FrozenRequestJSON = mustJSON(docbankmedia.ArtifactMetadata{ OccurrenceID: donor.DocbankOccurrenceID, Kind: "transcript", Origin: "provider", - Provider: "beeper", Language: operation.Language, Filename: "transcript.txt", + Provider: donor.SourceType, Language: operation.Language, Filename: "transcript.txt", MediaType: "text/plain", SHA256: transcriptSHA, ByteLength: int64(len(transcript)), }) operation.DocbankSourceID, operation.SourceVersionID = donor.DocbankSourceID, donor.SourceVersionID @@ -748,10 +907,18 @@ func (w *MediaSubmitter) artifact( return w.finishOperation(ctx, prepared, store.BeeperMediaResult{SuppliedInputID: receipt.SuppliedInputID}) } -// process explicitly queues the supplied-transcript profile once per key. +// process explicitly queues the saved processing profile once per key. func (w *MediaSubmitter) process( ctx, actionCtx context.Context, archiveUID string, operation store.BeeperMediaOperation, ) error { + if operation.PreparedReplay { + // A saved request may have reached Docbank before its receipt was committed. + var processing docbankmedia.Processing + if err := json.Unmarshal([]byte(operation.FrozenRequestJSON), &processing); err != nil { + return fmt.Errorf("decode saved beeper processing request: %w", err) + } + return w.sendProcessing(ctx, actionCtx, operation, operation.VaultUID, processing) + } mappings, err := w.liveMappings(ctx, operation.ProcessingKey, 1) if err != nil { return err @@ -763,9 +930,14 @@ func (w *MediaSubmitter) process( if err != nil { return err } - operation.FrozenRequestJSON = mustJSON(docbankmedia.Processing{ - Profile: "supplied-transcript", SuppliedInputID: operation.SuppliedInputID, - }) + profile := strings.TrimSpace(operation.ProcessingProfile) + if profile == "" { + profile = "supplied-transcript" + } + processingRequest := docbankmedia.Processing{ + Profile: profile, SuppliedInputID: operation.SuppliedInputID, + } + operation.FrozenRequestJSON = mustJSON(processingRequest) var prepared store.BeeperMediaOperation var vaultUID string var retryScheduled bool @@ -785,17 +957,20 @@ func (w *MediaSubmitter) process( return nil } vaultUID = selected.VaultUID - raw, err := w.store.GetMessageRawContext(ctx, selected.MessageID) - if beeperMediaRawGap(err) { + operation.DocbankSourceID = selected.DocbankSourceID + operation.SourceVersionID = selected.SourceVersionID + operation.ContentVersionID = selected.ContentVersionID + operation.DocbankOccurrenceID = selected.DocbankOccurrenceID + _, currentRawHash, rawErr := w.rawMediaEvidence(ctx, selected.SourceType, selected.MessageID) + if errors.Is(rawErr, errBeeperMediaRawInvalid) { gap := fallbackMediaMapping(w.destination, mappingCandidate(*selected), archiveUID, errBeeperMediaRawInvalid) return w.store.ReconcileBeeperMediaMapping(ctx, gap) } - if err != nil { - return err + if rawErr != nil { + return rawErr } - currentRawHash := hashBytes(raw) - if currentRawHash != evidence.rawHash { + if evidence.rawHash != "" && currentRawHash != evidence.rawHash { prepared, err = w.store.PrepareBeeperMediaOperation(ctx, operation) if err != nil { return err @@ -830,7 +1005,14 @@ func (w *MediaSubmitter) process( if err := json.Unmarshal([]byte(prepared.FrozenRequestJSON), &processing); err != nil { return fmt.Errorf("decode saved beeper processing request: %w", err) } - receipt, err := w.client.Process(actionCtx, prepared.DocbankSourceID, prepared.OperationID, processing.SuppliedInputID) + return w.sendProcessing(ctx, actionCtx, prepared, vaultUID, processing) +} + +func (w *MediaSubmitter) sendProcessing( + ctx, actionCtx context.Context, prepared store.BeeperMediaOperation, + vaultUID string, processing docbankmedia.Processing, +) error { + receipt, err := w.client.Process(actionCtx, prepared.DocbankSourceID, prepared.OperationID, processing) if err != nil { return w.finishClientError(ctx, actionCtx, prepared, err) } @@ -878,6 +1060,8 @@ func (w *MediaSubmitter) status(ctx, actionCtx context.Context, operation store. return w.finishClientError(ctx, actionCtx, operation, err) } if own.VaultUID != operation.VaultUID || own.SourceID != operation.DocbankSourceID || + (own.SourceVersionID != "" && own.SourceVersionID != operation.SourceVersionID) || + (own.ContentVersionID != "" && own.ContentVersionID != operation.ContentVersionID) || own.OperationID != operation.OperationID { return w.finishOperation(ctx, operation, store.BeeperMediaResult{ErrorCode: "destination_mismatch"}) } @@ -896,12 +1080,20 @@ func (w *MediaSubmitter) status(ctx, actionCtx context.Context, operation store. // ownProcessingReceipt returns this operation's receipt. Docbank's source // status names only the newest operation and takes coverage from the newest -// succeeded one, so an older operation replays its saved retry receipt. +// succeeded one. Version fields describe the latest visible occurrence, so a +// different or omitted version also requires replaying the saved retry receipt. func (w *MediaSubmitter) ownProcessingReceipt( ctx context.Context, operation store.BeeperMediaOperation, source docbankmedia.Receipt, ) (docbankmedia.Receipt, error) { - if source.OperationID != operation.OperationID { - return w.client.Process(ctx, operation.DocbankSourceID, operation.OperationID, operation.SuppliedInputID) + if source.OperationID != operation.OperationID || + source.SourceVersionID != operation.SourceVersionID || source.ContentVersionID != operation.ContentVersionID { + profile := strings.TrimSpace(operation.ProcessingProfile) + if profile == "" { + return docbankmedia.Receipt{}, errors.New("stored media processing profile is missing") + } + return w.client.Process(ctx, operation.DocbankSourceID, operation.OperationID, docbankmedia.Processing{ + Profile: profile, SuppliedInputID: operation.SuppliedInputID, + }) } if source.OperationState != "succeeded" { // Until this operation succeeds, source coverage belongs to an earlier one. @@ -957,7 +1149,9 @@ func mappingCandidate(mapping store.BeeperMediaMapping) store.BeeperMediaCandida SourceType: mapping.SourceType, SourceIdentifier: mapping.SourceIdentifier, SourceConversationID: mapping.SourceConversationID, SourceMessageID: mapping.SourceMessageID, SourceAttachmentID: mapping.SourceAttachmentID, SourcePartKey: mapping.SourcePartKey, + Filename: mapping.Filename, MIMEType: mapping.MIMEType, ContentHash: mapping.SourceSHA256, ByteLength: mapping.ByteLength, + AttachmentState: "stored", Role: "standalone", } } @@ -975,9 +1169,6 @@ func describeMedia( if err := json.Unmarshal(raw, &envelope); err != nil { return MediaDescriptor{}, "", errBeeperMediaRawInvalid } - if candidate.SourceType != "beeper" { - return MediaDescriptor{}, "", errBeeperMediaUnsupported - } if candidate.SourceMessageID != "" && envelope.ID != candidate.SourceMessageID { return MediaDescriptor{}, "", errBeeperMediaSourceChanged } @@ -1015,7 +1206,6 @@ func describeMedia( return MediaDescriptor{}, "", errBeeperMediaTranscriptTooLarge } filename, requestMIME := selectedMediaMetadata(attachment.FileName, attachment.MimeType) - rawHash := sha256.Sum256(raw) transcriptHash := "" if transcript != "" { digest := sha256.Sum256([]byte(transcript)) @@ -1036,8 +1226,8 @@ func describeMedia( SourceIdentifier: candidate.SourceIdentifier, SourceConversationID: candidate.SourceConversationID, SourceMessageID: candidate.SourceMessageID, SourceAttachmentID: candidate.SourceAttachmentID, SourcePartKey: part, SourceSHA256: candidate.ContentHash, ByteLength: candidate.ByteLength, - RawHash: hex.EncodeToString(rawHash[:]), TranscriptSHA256: transcriptHash, - Language: language, Filename: filename, MIMEType: requestMIME, + TranscriptSHA256: transcriptHash, + Language: language, Filename: filename, MIMEType: requestMIME, } descriptor.Occurrence.Revision = mediaRevision(descriptor) descriptor.ProcessingKey = mediaProcessingKey(descriptor) @@ -1062,10 +1252,14 @@ func mediaProcessingKey(descriptor MediaDescriptor) string { if descriptor.TranscriptSHA256 == "" { return "" } - return hashDelimited("beeper", "supplied-transcript", descriptor.SourceSHA256, + return hashDelimited(descriptor.SourceType, "supplied-transcript", descriptor.SourceSHA256, descriptor.TranscriptSHA256, descriptor.Language) } +func mediaASRProcessingKey(descriptor MediaDescriptor, profile string) string { + return hashDelimited("msgvault-media/v1", "asr", descriptor.SourceSHA256, profile) +} + func mediaOccurrenceRef( archiveUID, sourceType, sourceIdentifier, conversationID, messageID, part string, ) string { @@ -1269,26 +1463,21 @@ func mustJSON(value any) string { } func mediaGapCode(err error) string { - switch { - case errors.Is(err, errBeeperMediaRawInvalid): - return errBeeperMediaRawInvalid.Error() - case errors.Is(err, errBeeperMediaPartMissing): - return errBeeperMediaPartMissing.Error() - case errors.Is(err, errBeeperMediaPartAmbiguous): - return errBeeperMediaPartAmbiguous.Error() - case errors.Is(err, errBeeperMediaTranscriptInvalid): - return errBeeperMediaTranscriptInvalid.Error() - case errors.Is(err, errBeeperMediaTranscriptTooLarge): - return errBeeperMediaTranscriptTooLarge.Error() - case errors.Is(err, errBeeperMediaSourceChanged): - return errBeeperMediaSourceChanged.Error() - case errors.Is(err, errBeeperMediaUnsupported): - return errBeeperMediaUnsupported.Error() - case errors.Is(err, errBeeperMediaNoLiveOccurrence): - return errBeeperMediaNoLiveOccurrence.Error() - case errors.Is(err, errBeeperMediaSourceUnavailable): - return errBeeperMediaSourceUnavailable.Error() - default: - return docbankmedia.ErrorCode(err) + if gap := beeperMediaGap(err); gap != nil { + return gap.Error() + } + return docbankmedia.ErrorCode(err) +} + +func beeperMediaGap(err error) error { + for _, candidate := range []error{ + errBeeperMediaRawInvalid, errBeeperMediaPartMissing, errBeeperMediaPartAmbiguous, + errBeeperMediaUnsupported, errBeeperMediaTranscriptInvalid, errBeeperMediaTranscriptTooLarge, + errBeeperMediaSourceChanged, errBeeperMediaSourceUnavailable, errBeeperMediaNoLiveOccurrence, + } { + if errors.Is(err, candidate) { + return candidate + } } + return nil } diff --git a/internal/beeper/docbank_media_review_test.go b/internal/beeper/docbank_media_review_test.go index 3cba90edc..a575cb710 100644 --- a/internal/beeper/docbank_media_review_test.go +++ b/internal/beeper/docbank_media_review_test.go @@ -42,21 +42,206 @@ func processDeliveryIdentities(t *testing.T, st *store.Store, destination string return result } +func nextPendingProcess(t *testing.T, worker *MediaSubmitter, destination string) store.BeeperMediaOperation { + t.Helper() + for range 8 { + operation, ok, err := worker.store.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(t, err) + if ok && operation.Kind == store.BeeperMediaOperationProcess { + return operation + } + _, err = worker.RunBatch(t.Context()) + require.NoError(t, err) + } + require.FailNow(t, "pending media process was not ready") + return store.BeeperMediaOperation{} +} + func preparePendingProcess(t *testing.T, worker *MediaSubmitter, destination string) store.BeeperMediaOperation { t.Helper() - runPasses(t, worker, 2) - operation, ok, err := worker.store.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + operation := nextPendingProcess(t, worker, destination) + mappings, err := worker.liveMappings(t.Context(), operation.ProcessingKey, 1) require.NoError(t, err) - require.True(t, ok) - require.Equal(t, store.BeeperMediaOperationProcess, operation.Kind) + require.Len(t, mappings, 1) + operation.VaultUID = mappings[0].VaultUID + operation.DocbankSourceID = mappings[0].DocbankSourceID + operation.SourceVersionID = mappings[0].SourceVersionID + operation.ContentVersionID = mappings[0].ContentVersionID + operation.DocbankOccurrenceID = mappings[0].DocbankOccurrenceID operation.FrozenRequestJSON = mustJSON(docbankmedia.Processing{ - Profile: "supplied-transcript", SuppliedInputID: operation.SuppliedInputID, + Profile: operation.ProcessingProfile, SuppliedInputID: operation.SuppliedInputID, }) prepared, err := worker.prepareOperation(t.Context(), operation) require.NoError(t, err) return prepared } +func TestStoredMediaProfileChangeRecoversPreparedProcessReceipt(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "profile-crash", asset: "mxc://beeper.local/profile-crash", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 72)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "stored-profile-crash" + oldWorker := world.submitter(t, server, destination).WithASRProfile("old-asr") + prepared := preparePendingProcess(t, oldWorker, destination) + oldKey := prepared.ProcessingKey + processing := docbankmedia.Processing{Profile: prepared.ProcessingProfile, + SuppliedInputID: prepared.SuppliedInputID} + firstReceipt, err := oldWorker.client.Process(t.Context(), prepared.DocbankSourceID, + prepared.OperationID, processing) + require.NoError(err) + require.NotEmpty(firstReceipt.JobID) + + archiveUID, err := world.st.ArchiveUIDContext(t.Context()) + require.NoError(err) + candidates, err := world.st.ListBeeperMediaCandidates(t.Context(), 0, 10) + require.NoError(err) + require.Len(candidates, 1) + updatedWorker := world.submitter(t, server, destination).WithASRProfile("new-asr") + _, err = updatedWorker.reconcileCandidate(t.Context(), archiveUID, candidates[0]) + require.NoError(err) + newKey := processingKeyForDestination(t, world.st, destination, "new-asr") + assert.NotEqual(oldKey, newKey) + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries + SET next_action_at = ? WHERE destination_key = ? AND processing_key = ?`), + time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), destination, oldKey) + require.NoError(err) + + replay, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + require.True(ok) + assert.Equal(store.BeeperMediaOperationProcess, replay.Kind) + assert.True(replay.PreparedReplay) + assert.Equal(oldKey, replay.ProcessingKey) + assert.Equal(prepared.OperationID, replay.OperationID) + assert.Equal(prepared.FrozenRequestJSON, replay.FrozenRequestJSON) + require.NoError(updatedWorker.process(t.Context(), t.Context(), archiveUID, replay)) + + deliveries := processDeliveryIdentities(t, world.st, destination) + assert.Equal("observing", findProcessDelivery(t, deliveries, oldKey).phase) + assert.Equal("pending-process", findProcessDelivery(t, deliveries, newKey).phase) + assert.Empty(findProcessDelivery(t, deliveries, newKey).operationID) + var processingOperationID, jobID string + require.NoError(world.st.DB().QueryRow(world.st.Rebind(`SELECT processing_operation_id, job_id + FROM beeper_media_deliveries WHERE destination_key = ? AND processing_key = ?`), destination, oldKey). + Scan(&processingOperationID, &jobID)) + assert.Equal(prepared.OperationID, processingOperationID) + assert.Equal(firstReceipt.JobID, jobID) + docbank.mu.Lock() + assert.Equal([]string{prepared.OperationID}, docbank.processOps) + assert.Equal(1, docbank.replays) + docbank.mu.Unlock() +} + +func TestStoredMediaPreparedProcessAfterRevocation(t *testing.T) { + for _, change := range []string{"deleted", "replaced"} { + t.Run(change, func(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "prepared-revocation", asset: "mxc://beeper.local/prepared-revocation", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 72)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "stored-prepared-revocation" + worker := world.submitter(t, server, destination).WithASRProfile("asr") + operation := nextPendingProcess(t, worker, destination) + archiveUID, err := world.st.ArchiveUIDContext(t.Context()) + require.NoError(err) + // A backup can prevent saving the receipt after Docbank accepts the request. + worker.WithOperationGate(func(context.Context) (func(), bool) { + docbank.mu.Lock() + defer docbank.mu.Unlock() + return func() {}, len(docbank.processOps) == 0 + }) + require.ErrorIs(worker.process(t.Context(), t.Context(), archiveUID, operation), errBeeperMediaGateBusy) + worker.WithOperationGate(nil) + identities := processDeliveryIdentities(t, world.st, destination) + require.Len(identities, 1) + prepared := identities[0] + require.NotEmpty(prepared.operationID) + require.NotEmpty(prepared.frozenRequest) + + if change == "deleted" { + var sourceID int64 + require.NoError(world.st.DB().QueryRow(`SELECT source_id FROM messages WHERE source_message_id = 'prepared-revocation'`).Scan(&sourceID)) + require.NoError(world.st.MarkMessageDeleted(sourceID, "prepared-revocation")) + } else { + _, err := world.st.DB().Exec(world.st.Rebind(`UPDATE attachments SET content_hash = ?`), strings.Repeat("f", 64)) + require.NoError(err) + } + require.NoError(world.st.UnregisterAttachmentChangeConsumer(t.Context(), store.BeeperMediaAttachmentConsumerKey)) + runPasses(t, NewMediaSubmitter(world.st, world.blobs, nil, destination, world.dir).WithASRProfile("asr"), 1) + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries + SET next_action_at = ? WHERE destination_key = ? AND processing_key = ?`), + time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), destination, operation.ProcessingKey) + require.NoError(err) + replay, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + require.True(ok) + require.True(replay.PreparedReplay) + assert.Equal(prepared.operationID, replay.OperationID) + assert.Equal(prepared.frozenRequest, replay.FrozenRequestJSON) + require.NoError(worker.process(t.Context(), t.Context(), archiveUID, replay)) + docbank.mu.Lock() + assert.Equal([]string{prepared.operationID}, docbank.processOps) + assert.Equal(1, docbank.replays) + docbank.coverage = "transcribed" + docbank.mu.Unlock() + rows := deliveryRows(t, world.st, destination) + for _, row := range rows { + if row.ProcessingOperationID != prepared.operationID { + continue + } + assert.Equal("observing", row.Phase) + status := replay + status.Kind, status.JobID = store.BeeperMediaOperationStatus, row.JobID + require.NoError(worker.status(t.Context(), t.Context(), status)) + } + assert.Equal("done", findProcessDelivery(t, processDeliveryIdentities(t, world.st, destination), operation.ProcessingKey).phase) + }) + } +} + +func TestStoredMediaProfileChangeRetiresUnpreparedProcess(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "profile-unprepared", asset: "mxc://beeper.local/profile-unprepared", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 73)}) + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "stored-profile-unprepared" + oldWorker := world.submitter(t, server, destination).WithASRProfile("old-asr") + operation := nextPendingProcess(t, oldWorker, destination) + require.Empty(operation.OperationID) + require.Empty(operation.FrozenRequestJSON) + archiveUID, err := world.st.ArchiveUIDContext(t.Context()) + require.NoError(err) + candidates, err := world.st.ListBeeperMediaCandidates(t.Context(), 0, 10) + require.NoError(err) + require.Len(candidates, 1) + updatedWorker := world.submitter(t, server, destination).WithASRProfile("new-asr") + _, err = updatedWorker.reconcileCandidate(t.Context(), archiveUID, candidates[0]) + require.NoError(err) + newKey := processingKeyForDestination(t, world.st, destination, "new-asr") + identities := processDeliveryIdentities(t, world.st, destination) + assert.Equal("blocked", findProcessDelivery(t, identities, operation.ProcessingKey).phase) + assert.Equal("processing_key_changed", findProcessDelivery(t, identities, operation.ProcessingKey).errorCode) + assert.Equal("pending-process", findProcessDelivery(t, identities, newKey).phase) + next, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + require.True(ok) + assert.Equal(store.BeeperMediaOperationProcess, next.Kind) + assert.Equal(newKey, next.ProcessingKey) + assert.Equal("new-asr", next.ProcessingProfile) + docbank.mu.Lock() + assert.Empty(docbank.processOps) + docbank.mu.Unlock() +} + func TestBeeperMediaDailyRescan(t *testing.T) { require, assert := require.New(t), assert.New(t) world := importVoiceChat(t, voiceSpec{id: "voice1", asset: "mxc://beeper.local/voice1", @@ -541,8 +726,8 @@ func TestBeeperMediaOperationRawReadFailure(t *testing.T) { assert.Empty(docbank.processOps) docbank.mu.Unlock() identity := findProcessDelivery(t, processDeliveryIdentities(t, world.st, destination), operation.ProcessingKey) - assert.Equal("blocked", identity.phase) - assert.Equal("source_raw_invalid", identity.errorCode) + assert.Equal("pending-process", identity.phase) + assert.Empty(identity.errorCode) assert.Equal(beforeOperationID, identity.operationID) assert.Equal(beforeInput, identity.suppliedInputID) assert.Equal(beforeFrozen, identity.frozenRequest) @@ -560,6 +745,12 @@ func TestBeeperMediaOperationRawReadFailure(t *testing.T) { require.NotNil(gap) assert.Equal("revoked", revoked.State) assert.Equal("blocked", gap.State) + replay, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + require.True(ok) + require.True(replay.PreparedReplay) + assert.Equal(beforeOperationID, replay.OperationID) + require.NoError(worker.process(t.Context(), t.Context(), archiveUID, replay)) }) } @@ -986,29 +1177,9 @@ func TestBeeperMediaProcessDescriptorRefresh(t *testing.T) { assert.Equal(beforeOperationID, oldIdentity.operationID) assert.Equal(beforeInput, oldIdentity.suppliedInputID) assert.Equal(beforeFrozen, oldIdentity.frozenRequest) - rows := occurrenceRows(t, world.st, destination) - if tc.keyChanges { - assert.Equal("blocked", oldIdentity.phase) - assert.Equal("no_live_occurrence", oldIdentity.errorCode) - var current *occurrenceRow - for i := range rows { - if rows[i].Revision != beforeMapping.Revision { - current = &rows[i] - } - } - require.NotNil(current) - assert.Equal("pending", current.State) - assert.NotEqual(beforeMapping.ProcessingKey, current.ProcessingKey) - for _, identity := range identities { - if identity.key != beforeMapping.ProcessingKey { - assert.NotEqual(beforeOperationID, identity.operationID) - } - } - return - } - assert.Equal("pending-process", oldIdentity.phase) assert.Empty(oldIdentity.errorCode) + rows := occurrenceRows(t, world.st, destination) var current *occurrenceRow for i := range rows { if rows[i].Revision != beforeMapping.Revision { @@ -1017,25 +1188,23 @@ func TestBeeperMediaProcessDescriptorRefresh(t *testing.T) { } require.NotNil(current) assert.Equal("pending", current.State) - assert.Equal(beforeMapping.ProcessingKey, current.ProcessingKey) - - _, err = worker.RunBatch(t.Context()) - require.NoError(err) - rows = occurrenceRows(t, world.st, destination) - for _, row := range rows { - if row.Revision == current.Revision { - assert.Equal("retained", row.State) + if tc.keyChanges { + assert.NotEqual(beforeMapping.ProcessingKey, current.ProcessingKey) + for _, identity := range identities { + if identity.key != beforeMapping.ProcessingKey { + assert.NotEqual(beforeOperationID, identity.operationID) + } } + } else { + assert.Equal(beforeMapping.ProcessingKey, current.ProcessingKey) } - identities = processDeliveryIdentities(t, world.st, destination) - oldIdentity = findProcessDelivery(t, identities, beforeMapping.ProcessingKey) - assert.Equal(beforeOperationID, oldIdentity.operationID) - assert.Equal(beforeInput, oldIdentity.suppliedInputID) - assert.Equal(beforeFrozen, oldIdentity.frozenRequest) + + // The saved request can finish before the replacement is retained. operation, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) require.NoError(err) require.True(ok) require.Equal(store.BeeperMediaOperationProcess, operation.Kind) + require.True(operation.PreparedReplay) assert.Equal(beforeOperationID, operation.OperationID) require.NoError(worker.process(t.Context(), t.Context(), archiveUID, operation)) docbank.mu.Lock() diff --git a/internal/beeper/docbank_media_test.go b/internal/beeper/docbank_media_test.go index e45949095..479033993 100644 --- a/internal/beeper/docbank_media_test.go +++ b/internal/beeper/docbank_media_test.go @@ -80,7 +80,9 @@ func importVoiceChat(t *testing.T, specs ...voiceSpec) *mediaWorld { }) f.setAsset(spec.asset, spec.data) } - ch.LastActivity = ch.Msgs[len(ch.Msgs)-1].Timestamp + if len(ch.Msgs) > 0 { + ch.LastActivity = ch.Msgs[len(ch.Msgs)-1].Timestamp + } f.addChat(ch) imp, st, done := newTestImporter(t, f) t.Cleanup(done) @@ -109,14 +111,14 @@ func runPasses(t *testing.T, submitter *MediaSubmitter, passes int) { } type occurrenceRow struct { - Ref, Revision, State, OperationID, ErrorCode, SourceID, SourceVersionID string - ContentVersionID, OccurrenceID, Coverage, ProcessingKey, MessageID string + Ref, Revision, State, OperationID, ErrorCode, SourceID, SourceVersionID, SourceType string + ContentVersionID, OccurrenceID, Coverage, ProcessingKey, MessageID string } func occurrenceRows(t *testing.T, st *store.Store, destination string) []occurrenceRow { t.Helper() rows, err := st.DB().Query(st.Rebind(` - SELECT occurrence_ref, revision, retention_state, retention_operation_id, error_code, + SELECT occurrence_ref, revision, retention_state, retention_operation_id, error_code, source_type, source_id, source_version_id, content_version_id, occurrence_id, coverage_state, processing_key, source_message_id FROM beeper_media_occurrences WHERE destination_key = ? @@ -126,7 +128,7 @@ func occurrenceRows(t *testing.T, st *store.Store, destination string) []occurre var result []occurrenceRow for rows.Next() { var row occurrenceRow - require.NoError(t, rows.Scan(&row.Ref, &row.Revision, &row.State, &row.OperationID, &row.ErrorCode, + require.NoError(t, rows.Scan(&row.Ref, &row.Revision, &row.State, &row.OperationID, &row.ErrorCode, &row.SourceType, &row.SourceID, &row.SourceVersionID, &row.ContentVersionID, &row.OccurrenceID, &row.Coverage, &row.ProcessingKey, &row.MessageID)) result = append(result, row) @@ -1126,13 +1128,16 @@ func TestBeeperMediaStartedJobAfterRevocation(t *testing.T) { func TestBeeperMediaRevokedObservingVaultMismatch(t *testing.T) { for _, tc := range []struct { - name string - sourceVault string - processVault string - sourceOp string + name string + sourceVault string + processVault string + processVersion string + processContent string + sourceOp string }{ {name: "source status", sourceVault: "foreign-vault"}, {name: "replayed processing receipt", processVault: "foreign-vault", sourceOp: "other-operation"}, + {name: "replayed processing identity", processVersion: "foreign-version", processContent: "foreign-content", sourceOp: "other-operation"}, } { t.Run(tc.name, func(t *testing.T) { require := require.New(t) @@ -1157,6 +1162,8 @@ func TestBeeperMediaRevokedObservingVaultMismatch(t *testing.T) { docbank.mu.Lock() docbank.sourceVaultUID = tc.sourceVault docbank.processVaultUID = tc.processVault + docbank.processVersionID = tc.processVersion + docbank.processContentVersionID = tc.processContent docbank.sourceOperationID = tc.sourceOp docbank.coverage = "transcribed" docbank.mu.Unlock() @@ -1314,38 +1321,81 @@ func TestBeeperMediaReceiptIdentity(t *testing.T) { assert.NotEqual(rows[0].ContentVersionID, docbank.artifactReceipts[0].ContentVersionID) } -func TestBeeperMediaSharedContent(t *testing.T) { +func TestStoredMediaSharedLifecycle(t *testing.T) { require := require.New(t) assert := assert.New(t) wav := syntheticWAV(800, 7) - world := importVoiceChat(t, - voiceSpec{id: "voice1", asset: "mxc://beeper.local/voice1", mime: "audio/wav", - fileName: "voice.wav", transcript: "shared words", data: wav}, - voiceSpec{id: "voice2", asset: "mxc://beeper.local/voice2", mime: "audio/wav", - fileName: "voice.wav", transcript: "shared words", data: wav}) + world := importVoiceChat(t) + addStoredMediaSource(t, world, "gmail", "rod@example.com", "mail-audio", wav, + "meeting.wav", "audio/wav", "", store.AttachmentRoleStandalone, "", nil, + "mail:attachment:1", "mime:1.2") + addStoredMediaSource(t, world, "whatsapp", "+15555550101", "whatsapp-audio", wav, + "voice.wav", "audio/wav", "", store.AttachmentRoleStandalone, "", nil, + "whatsapp:media", "whatsapp:media") docbank := newFakeDocbank(t) + docbank.coverage = "pending" server := httptest.NewServer(docbank) defer server.Close() - submitter := world.submitter(t, server, "destination-shared") - runPasses(t, submitter, 6) + submitter := world.submitter(t, server, "destination-shared").WithASRProfile("shared-asr") + runPasses(t, submitter, 3) rows := occurrenceRows(t, world.st, "destination-shared") require.Len(rows, 2) + assert.Equal("gmail", rows[0].SourceType) + assert.Equal("whatsapp", rows[1].SourceType) assert.NotEqual(rows[0].Ref, rows[1].Ref) assert.NotEqual(rows[0].OccurrenceID, rows[1].OccurrenceID) assert.Equal(rows[0].SourceID, rows[1].SourceID) assert.Equal(rows[0].ContentVersionID, rows[1].ContentVersionID) assert.Equal(rows[0].ProcessingKey, rows[1].ProcessingKey) - require.Len(deliveryRows(t, world.st, "destination-shared"), 1) + deliveries := deliveryRows(t, world.st, "destination-shared") + require.Len(deliveries, 1) + assert.Equal("observing", deliveries[0].Phase) docbank.mu.Lock() assert.Len(docbank.uploads, 2) - assert.Len(docbank.artifactOps, 1) assert.Len(docbank.processOps, 1) + docbank.mu.Unlock() + + var donorType, donorMessage string + require.NoError(world.st.DB().QueryRow(world.st.Rebind(` + SELECT s.source_type, m.source_message_id + FROM beeper_media_occurrences o + JOIN messages m ON m.id = o.message_id + JOIN sources s ON s.id = m.source_id + WHERE o.destination_key = ? AND o.occurrence_id = ?`), + "destination-shared", deliveries[0].Donor).Scan(&donorType, &donorMessage)) + _, err := world.st.DB().Exec(world.st.Rebind(` + UPDATE messages SET deleted_at = ? + WHERE source_message_id = ? AND source_id = (SELECT id FROM sources WHERE source_type = ? AND identifier = ?)`), + time.Now().UTC(), donorMessage, donorType, map[string]string{ + "gmail": "rod@example.com", "whatsapp": "+15555550101", + }[donorType]) + require.NoError(err) + require.NoError(world.st.RevokeStaleBeeperMediaMappings(t.Context(), "destination-shared")) + states := make(map[string]string, len(rows)) + for _, row := range occurrenceRows(t, world.st, "destination-shared") { + states[row.SourceType] = row.State + } + assert.Equal("revoked", states[donorType]) + otherType := "gmail" + if donorType == otherType { + otherType = "whatsapp" + } + assert.Equal("retained", states[otherType]) + docbank.mu.Lock() + docbank.coverage = "transcribed" + docbank.mu.Unlock() + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries + SET next_action_at = '2000-01-01 00:00:00.000' WHERE destination_key = ?`), "destination-shared") + require.NoError(err) + runPasses(t, submitter, 1) + assert.Equal("done", deliveryRows(t, world.st, "destination-shared")[0].Phase) + docbank.mu.Lock() requests := docbank.requests docbank.mu.Unlock() // A repeated complete backfill finds nothing to change or send. - _, err := world.st.DB().Exec(`UPDATE beeper_media_occurrences SET updated_at = '2001-02-03 04:05:06'`) + _, err = world.st.DB().Exec(`UPDATE beeper_media_occurrences SET updated_at = '2001-02-03 04:05:06'`) require.NoError(err) _, err = world.st.DB().Exec(`UPDATE beeper_media_deliveries SET updated_at = '2001-02-03 04:05:06'`) require.NoError(err) @@ -1395,7 +1445,7 @@ func TestBeeperMediaEligibility(t *testing.T) { runPasses(t, world.submitter(t, server, "destination-eligible"), 4) docbank.mu.Lock() - assert.ElementsMatch([][]byte{voice, ordinary}, docbank.uploads) + assert.ElementsMatch([][]byte{voice, ordinary, voice}, docbank.uploads) docbank.mu.Unlock() states := map[string]string{} for _, row := range occurrenceRows(t, world.st, "destination-eligible") { @@ -1403,13 +1453,14 @@ func TestBeeperMediaEligibility(t *testing.T) { } assert.Equal(map[string]string{ "voice1": "retained:", "audio1": "retained:", "fake1": "blocked:unsupported_media", + "beeper-message": "retained:", }, states) var messageType string require.NoError(world.st.DB().QueryRow(`SELECT message_type FROM messages WHERE source_message_id = 'audio1'`).Scan(&messageType)) assert.Equal("beeper", messageType) } -func TestBeeperMediaGaps(t *testing.T) { +func TestBeeperMediaAdmissionGaps(t *testing.T) { require := require.New(t) assert := assert.New(t) world := importVoiceChat(t, @@ -1612,43 +1663,48 @@ func deliveryNextActions(t *testing.T, st *store.Store, destination string) []st // metadata-then-file multipart, X-Api-Key, UUIDv4 replay and top-level // HTTP 200 receipts. It never returns an inline transcript field. type fakeDocbank struct { - t *testing.T - mu sync.Mutex - status int - malformed bool - hang bool - jobHTTPStatus int - sourceHTTPStatus int - jobState string - jobFailureCode string - sourceVaultUID string - processVaultUID string - sourceOperationID string - coverage string - failProcessing bool - dropRetention bool - submitDelay time.Duration - holdIndex map[int]bool - requests int - jobRequests int - sourceRequests int - replays int - next int - sources map[string]int - occurrenceIDs map[string]string - replies map[string]docbankmedia.Receipt - firstMetadata map[string]string - uploads [][]byte - occurrences []docbankmedia.Occurrence - retentionOps []string - retentionMetadata []string - artifactOps []string - artifactReceipts []docbankmedia.Receipt - transcripts []string - processOps []string - sourceOps map[string][]string - processReceipts map[string]docbankmedia.Receipt - rejected []string + t *testing.T + mu sync.Mutex + status int + malformed bool + hang bool + jobHTTPStatus int + sourceHTTPStatus int + jobState string + jobFailureCode string + sourceVaultUID string + sourceVersionID string + sourceContentVersionID string + processVaultUID string + processVersionID string + processContentVersionID string + sourceOperationID string + coverage string + failProcessing bool + dropRetention bool + submitDelay time.Duration + holdIndex map[int]bool + requests int + jobRequests int + sourceRequests int + replays int + next int + sources map[string]int + occurrenceIDs map[string]string + replies map[string]docbankmedia.Receipt + firstMetadata map[string]string + uploads [][]byte + occurrences []docbankmedia.Occurrence + retentionOps []string + retentionMetadata []string + artifactOps []string + artifactReceipts []docbankmedia.Receipt + transcripts []string + processOps []string + processRequests []docbankmedia.Processing + sourceOps map[string][]string + processReceipts map[string]docbankmedia.Receipt + rejected []string } func newFakeDocbank(t *testing.T) *fakeDocbank { @@ -1800,7 +1856,7 @@ func (f *fakeDocbank) artifact(w http.ResponseWriter, r *http.Request, source st } assert.Equal(f.t, "transcript", metadata.Kind) assert.Equal(f.t, "provider", metadata.Origin) - assert.Equal(f.t, "beeper", metadata.Provider) + assert.NotEmpty(f.t, metadata.Provider) assert.Equal(f.t, "text/plain", metadata.MediaType) f.mu.Lock() defer f.mu.Unlock() @@ -1832,21 +1888,33 @@ func (f *fakeDocbank) retry(w http.ResponseWriter, r *http.Request, source strin f.reject(w, err) return } - assert.Equal(f.t, "supplied-transcript", body.Processing.Profile) - assert.NotEmpty(f.t, body.Processing.SuppliedInputID) + if body.Processing.Profile == "supplied-transcript" { + assert.NotEmpty(f.t, body.Processing.SuppliedInputID) + } else { + assert.Empty(f.t, body.Processing.SuppliedInputID) + } f.mu.Lock() defer f.mu.Unlock() if _, replay := f.processReceipts[body.OperationID]; replay { // Docbank replays a known operation ID with its saved receipt. f.replays++ + f.processRequests = append(f.processRequests, body.Processing) receipt := f.processingReceiptLocked(body.OperationID) if f.processVaultUID != "" { receipt.VaultUID = f.processVaultUID } + if f.processVersionID != "" { + receipt.SourceVersionID = f.processVersionID + } + if f.processContentVersionID != "" { + receipt.ContentVersionID = f.processContentVersionID + } writeDocbankJSON(w, receipt) return } + sourceVersionID, sourceContentVersionID, occurrenceID := f.sourceIdentityLocked(source) receipt := docbankmedia.Receipt{VaultUID: "vault-1", SourceID: source, + SourceVersionID: sourceVersionID, ContentVersionID: sourceContentVersionID, OccurrenceID: occurrenceID, OperationID: body.OperationID, JobID: sha256Hex([]byte(body.OperationID)), OperationState: "queued", CoverageState: "pending", SuppliedInputID: body.Processing.SuppliedInputID} if f.failProcessing { @@ -1854,6 +1922,7 @@ func (f *fakeDocbank) retry(w http.ResponseWriter, r *http.Request, source strin receipt.JobID, receipt.OperationState, receipt.CoverageState = "", "failed", "unavailable" } f.processOps = append(f.processOps, body.OperationID) + f.processRequests = append(f.processRequests, body.Processing) f.sourceOps[source] = append(f.sourceOps[source], body.OperationID) f.processReceipts[body.OperationID] = receipt writeDocbankJSON(w, receipt) @@ -1870,6 +1939,15 @@ func (f *fakeDocbank) processingReceiptLocked(operationID string) docbankmedia.R return receipt } +func (f *fakeDocbank) sourceIdentityLocked(source string) (string, string, string) { + for _, receipt := range f.replies { + if receipt.SourceID == source && receipt.Outcome == "content_available" { + return receipt.SourceVersionID, receipt.ContentVersionID, receipt.OccurrenceID + } + } + return "", "", "" +} + // sourceStatus follows Docbank e33d77e4: the newest processing operation // supplies the operation fields, while coverage comes from the newest // succeeded operation on the same source. @@ -1880,7 +1958,15 @@ func (f *fakeDocbank) sourceStatus(w http.ResponseWriter, source string) { if vaultUID == "" { vaultUID = "vault-1" } + sourceVersionID, sourceContentVersionID, occurrenceID := f.sourceIdentityLocked(source) + if f.sourceVersionID != "" { + sourceVersionID = f.sourceVersionID + } + if f.sourceContentVersionID != "" { + sourceContentVersionID = f.sourceContentVersionID + } receipt := docbankmedia.Receipt{VaultUID: vaultUID, SourceID: source, + SourceVersionID: sourceVersionID, ContentVersionID: sourceContentVersionID, OccurrenceID: occurrenceID, OperationState: "succeeded", CoverageState: "unprocessed"} operations := f.sourceOps[source] if len(operations) > 0 { diff --git a/internal/beeper/stored_media_test.go b/internal/beeper/stored_media_test.go new file mode 100644 index 000000000..55f000c52 --- /dev/null +++ b/internal/beeper/stored_media_test.go @@ -0,0 +1,636 @@ +package beeper + +import ( + "database/sql" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/attachmentpolicy" + "go.kenn.io/msgvault/internal/export" + "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" + "go.kenn.io/msgvault/internal/whatsapp" +) + +func TestStoredMediaNonAudioProbe(t *testing.T) { + for _, mode := range []string{"missing", "no-blob-store", "unread-tail"} { + t.Run(mode, func(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t) + data := []byte("%PDF-1.7\n" + strings.Repeat("x", 1024)) + hash := addStoredMediaSource(t, world, "gmail", "test@example.com", "document", data, + "report.pdf", "application/pdf", "", store.AttachmentRoleStandalone, "", nil, "part", "part") + path := filepath.Join(world.dir, hash[:2], hash) + switch mode { + case "missing": + require.NoError(os.Remove(path)) + case "unread-tail": + // A changed tail would fail CAS verification if discovery drained the PDF. + data[len(data)-1] = 'y' + require.NoError(os.WriteFile(path, data, 0o600)) + } + worker := NewMediaSubmitter(world.st, world.blobs, nil, "non-audio", world.dir).WithASRProfile("asr") + if mode == "no-blob-store" { + worker.blobs = nil + } + for range 2 { + runPasses(t, worker, 1) + assert.Empty(occurrenceRows(t, world.st, "non-audio")) + assert.Empty(deliveryRows(t, world.st, "non-audio")) + _, ready, err := world.st.NextBeeperMediaOperation(t.Context(), "non-audio", time.Now().Add(time.Hour)) + require.NoError(err) + assert.False(ready) + require.NoError(world.st.UnregisterAttachmentChangeConsumer(t.Context(), store.BeeperMediaAttachmentConsumerKey)) + } + }) + } +} + +func TestStoredMediaUnhintedAudioHeaderRecovery(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t) + wav := syntheticWAV(800, 71) + hash := addStoredMediaSource(t, world, "gmail", "test@example.com", "unhinted-audio", wav, + "attachment.bin", "application/octet-stream", "", store.AttachmentRoleStandalone, "", nil, "part", "part") + path := filepath.Join(world.dir, hash[:2], hash) + corrupt := append([]byte(nil), wav...) + corrupt[0] ^= 1 + require.NoError(os.WriteFile(path, corrupt, 0o600)) + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "unhinted-header" + worker := world.submitter(t, server, destination).WithASRProfile("asr") + runPasses(t, worker, 1) + assert.Empty(occurrenceRows(t, world.st, destination)) + assert.Empty(deliveryRows(t, world.st, destination)) + + require.NoError(os.WriteFile(path, wav, 0o600)) + scan, err := world.st.LoadBeeperMediaScan(t.Context(), destination) + require.NoError(err) + due := scan + due.NextFullScanAt = time.Now().UTC().Add(-time.Hour) + changed, err := world.st.AdvanceBeeperMediaScan(t.Context(), destination, scan, due) + require.NoError(err) + require.True(changed) + runPasses(t, worker, 4) + docbank.mu.Lock() + assert.Equal([][]byte{wav}, docbank.uploads) + docbank.mu.Unlock() + deliveries := deliveryRows(t, world.st, destination) + require.Len(deliveries, 1) + assert.Equal("done", deliveries[0].Phase) +} + +func TestStoredMediaUsesMessageTimestampWithoutRawRead(t *testing.T) { + require, assert := require.New(t), assert.New(t) + testutil.SkipIfPostgres(t, "SQLite authorizer detects unnecessary MIME reads") + world := importVoiceChat(t) + for _, provider := range []string{"gmail", "imap", "mbox"} { + addStoredMediaSource(t, world, provider, "test@example.com", provider, syntheticWAV(800, 77), + "voice.wav", "audio/wav", "", store.AttachmentRoleStandalone, "", + []byte("Date: Tue, 10 Feb 2026 11:12:13 +0000\r\n\r\nmessage"), "part", "part") + } + _, err := world.st.DB().Exec(world.st.Rebind(`UPDATE messages SET sent_at = ?`), + time.Date(2026, 2, 10, 11, 12, 13, 0, time.UTC)) + require.NoError(err) + world.st.DB().SetMaxOpenConns(1) + conn, err := world.st.DB().Conn(t.Context()) + require.NoError(err) + require.NoError(conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok) + sqliteConn.RegisterAuthorizer(func(action int, table, _, _ string) int { + if action == sqlite3.SQLITE_READ && table == "message_raw" { + return sqlite3.SQLITE_DENY + } + return sqlite3.SQLITE_OK + }) + return nil + })) + require.NoError(conn.Close()) + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + runPasses(t, world.submitter(t, server, "timestamps").WithASRProfile("asr"), 10) + docbank.mu.Lock() + defer docbank.mu.Unlock() + require.Len(docbank.occurrences, 3) + for _, occurrence := range docbank.occurrences { + assert.Equal("2026-02-10T11:12:13Z", occurrence.Message.Normalized) + } + require.Len(deliveryRows(t, world.st, "timestamps"), 1) + assert.Equal("done", deliveryRows(t, world.st, "timestamps")[0].Phase) +} + +func TestBeeperMediaMissingPartStaysVisible(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "missing-part", asset: "mxc://beeper.local/part", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 78)}) + _, err := world.st.DB().Exec(`UPDATE attachments SET source_part_key = NULL, source_attachment_id = NULL`) + require.NoError(err) + runPasses(t, NewMediaSubmitter(world.st, world.blobs, nil, "missing-part", world.dir), 1) + rows := occurrenceRows(t, world.st, "missing-part") + require.Len(rows, 1) + assert.Equal("blocked", rows[0].State) + assert.Equal("source_part_missing", rows[0].ErrorCode) +} + +// addStoredMediaSource adds an importer-owned row around bytes already present +// in the test CAS. It intentionally leaves raw evidence optional. +func addStoredMediaSource( + t *testing.T, world *mediaWorld, sourceType, identifier, messageID string, + data []byte, filename, mimeType, mediaType string, role store.AttachmentRole, + state attachmentpolicy.DownloadState, raw []byte, sourceAttachmentID, sourcePartKey string, +) string { + t.Helper() + source, err := world.st.GetOrCreateSource(sourceType, identifier) + require.NoError(t, err) + conversation, err := world.st.EnsureConversation(source.ID, sourceType+"-thread", "Thread") + require.NoError(t, err) + message, err := world.st.UpsertMessage(&store.Message{ConversationID: conversation, SourceID: source.ID, + SourceMessageID: messageID, MessageType: sourceType, SizeEstimate: int64(len(data))}) + require.NoError(t, err) + if raw != nil { + require.NoError(t, world.st.UpsertMessageRawWithFormat(message, raw, sourceType+"_raw")) + } + stored := &mime.Attachment{Filename: filename, ContentType: mimeType, Content: data} + storagePath, err := export.StoreAttachmentFile(world.dir, stored) + require.NoError(t, err) + require.NoError(t, world.st.UpsertAttachmentRecord(t.Context(), message, store.AttachmentWrite{ + Filename: filename, MIMEType: mimeType, StoragePath: storagePath, + ContentHash: stored.ContentHash, Size: int64(len(data)), SourceAttachmentID: sourceAttachmentID, + SourcePartKey: sourcePartKey, MediaType: mediaType, + State: state, Role: role, RoleSource: store.AttachmentRoleSourceImporterSemantics, + })) + return stored.ContentHash +} + +func TestStoredMediaProviderMatrix(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + wav := syntheticWAV(800, 61) + mp3 := syntheticMP3(4) + world := importVoiceChat(t, + voiceSpec{id: "beeper-supplied", asset: "mxc://beeper.local/supplied", mime: "audio/wav", + fileName: "supplied.wav", transcript: "provider words", data: wav}, + voiceSpec{id: "beeper-asr", asset: "mxc://beeper.local/asr", mime: "audio/mp3", + fileName: "asr.mp3", data: mp3}) + addStoredMediaSource(t, world, "whatsapp", "+15555550101", "3EB0-message-1", wav, + "voice.bin", "application/octet-stream", "", store.AttachmentRoleStandalone, "", nil, "", "whatsapp:media") + addStoredMediaSource(t, world, "facebook_messenger", "test.user@facebook.messenger", "message:fb-1", wav, + "voice.wav", "audio/wav", "", store.AttachmentRoleUnknown, "", []byte("not json"), + "attachments/audio-1.mp3", "fbmessenger:attachment:a602fedf39561086320b2483a3dfac563bc7453ea3e224bd5aea06e7c4d3f79e") + addStoredMediaSource(t, world, "synctech_sms", "+15555550102", "mms:1", wav, + "voice.wav", "", "", store.AttachmentRoleStandalone, "", nil, "", "synctech:mms:1") + addStoredMediaSource(t, world, "slack", "T01:U01", "1712345678.000001", wav, + "voice.wav", "audio/wav", "", store.AttachmentRoleStandalone, attachmentpolicy.StateStored, nil, + "slack:F_FILE1", "slack:F_FILE1") + addStoredMediaSource(t, world, "discord", "200", "401", wav, + "voice.wav", "audio/wav", "", store.AttachmentRoleStandalone, attachmentpolicy.StateStored, nil, + "discord:401", "discord:401") + addStoredMediaSource(t, world, "google_voice", "+15555550103", "voicemail-1", mp3, + "voicemail.mp3", "application/octet-stream", "", store.AttachmentRoleStandalone, attachmentpolicy.StateStored, nil, + "gvoice:voicemail:audio", "gvoice:voicemail:audio") + addStoredMediaSource(t, world, "future_provider", "future-account", "future-message", wav, + "recording", "", "", store.AttachmentRoleStandalone, "", nil, "future:media:1", "future:media:1") + + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + runPasses(t, world.submitter(t, server, "stored-provider-matrix").WithASRProfile("configured-asr"), 40) + + rows := occurrenceRows(t, world.st, "stored-provider-matrix") + require.Len(rows, 9) + providers := make(map[string]string, len(rows)) + for _, row := range rows { + assert.Equal("retained", row.State, row.MessageID) + providers[row.MessageID] = row.SourceType + } + assert.Equal(map[string]string{ + "beeper-supplied": "beeper", "beeper-asr": "beeper", "3EB0-message-1": "whatsapp", + "message:fb-1": "facebook_messenger", "mms:1": "synctech_sms", "1712345678.000001": "slack", + "401": "discord", "voicemail-1": "google_voice", "future-message": "future_provider", + }, providers) + docbank.mu.Lock() + assert.Len(docbank.uploads, 9) + assert.Len(docbank.artifactOps, 1) + assert.Equal([]string{"provider words"}, docbank.transcripts) + profiles := make([]string, 0, len(docbank.processRequests)) + for _, request := range docbank.processRequests { + profiles = append(profiles, request.Profile) + } + docbank.mu.Unlock() + assert.GreaterOrEqual(len(profiles), 4) + assert.Contains(profiles, "supplied-transcript") + assert.Contains(profiles, "configured-asr") +} + +func TestStoredMediaEmptySlackExportDoesNotBlockLaterAudio(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t) + source, err := world.st.GetOrCreateSource("slack", "T01:EMPTY") + require.NoError(err) + conversation, err := world.st.EnsureConversation(source.ID, "slack-thread", "Thread") + require.NoError(err) + message, err := world.st.UpsertMessage(&store.Message{ + ConversationID: conversation, SourceID: source.ID, SourceMessageID: "empty-first", MessageType: "slack", + }) + require.NoError(err) + empty := &mime.Attachment{Filename: "empty.wav", ContentType: "audio/wav", Content: []byte{}} + storagePath, err := export.StoreAttachmentFileIncludingEmpty(world.dir, empty) + require.NoError(err) + require.NotEmpty(storagePath) + require.NoError(world.st.UpsertAttachmentRecord(t.Context(), message, store.AttachmentWrite{ + Filename: empty.Filename, MIMEType: empty.ContentType, StoragePath: storagePath, + ContentHash: empty.ContentHash, SourceAttachmentID: "slack:empty", SourcePartKey: "slack:empty", + MediaType: "audio", Role: store.AttachmentRoleStandalone, + RoleSource: store.AttachmentRoleSourceProviderExplicit, State: attachmentpolicy.StateStored, + })) + addStoredMediaSource(t, world, "slack", "T01:EMPTY", "valid-after-empty", syntheticWAV(800, 71), + "voice.wav", "audio/wav", "audio", store.AttachmentRoleStandalone, attachmentpolicy.StateStored, nil, + "slack:valid", "slack:valid") + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + worker := world.submitter(t, server, "stored-empty-slack").WithASRProfile("configured-asr") + result, err := worker.RunBatch(t.Context()) + require.NoError(err) + assert.Equal(1, result.Examined) + rows := occurrenceRows(t, world.st, "stored-empty-slack") + require.Len(rows, 1) + assert.Equal("valid-after-empty", rows[0].MessageID) +} + +func TestStoredMediaEmailFallback(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t) + wav := syntheticWAV(800, 70) + addStoredMediaSource(t, world, "gmail", "rod@example.com", "mail-audio-1", wav, + "meeting.wav", "application/octet-stream", "", store.AttachmentRoleStandalone, "", nil, + "mail:attachment:audio", "mime:1.2") + addStoredMediaSource(t, world, "gmail", "rod@example.com", "mail-document-1", []byte("%PDF-1.7\narchive"), + "report.pdf", "application/pdf", "", store.AttachmentRoleStandalone, "", nil, + "mail:attachment:document", "mime:1.3") + destination := "stored-email-fallback" + local := NewMediaSubmitter(world.st, world.blobs, nil, destination, world.dir).WithASRProfile("email-asr") + runPasses(t, local, 1) + rows := occurrenceRows(t, world.st, destination) + require.Len(rows, 1) + states := make(map[string]string, len(rows)) + for _, row := range rows { + states[row.MessageID] = row.State + ":" + row.ErrorCode + assert.Equal("gmail", row.SourceType) + } + assert.Equal(map[string]string{ + "mail-audio-1": "pending:", + }, states) + + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + worker := world.submitter(t, server, destination).WithASRProfile("email-asr") + runPasses(t, worker, 5) + rows = occurrenceRows(t, world.st, destination) + states = make(map[string]string, len(rows)) + for _, row := range rows { + states[row.MessageID] = row.State + ":" + row.ErrorCode + } + assert.Equal(map[string]string{ + "mail-audio-1": "retained:", + }, states) + docbank.mu.Lock() + assert.Equal([][]byte{wav}, docbank.uploads) + assert.Empty(docbank.artifactOps) + assert.Equal([]string{"email-asr"}, []string{docbank.processRequests[0].Profile}) + docbank.mu.Unlock() +} + +func TestStoredMediaRealWhatsAppImportWithoutRemoteRoute(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t) + mediaDir := t.TempDir() + wav := syntheticWAV(800, 72) + require.NoError(os.WriteFile(filepath.Join(mediaDir, "voice.wav"), wav, 0o600)) + waDBPath := filepath.Join(t.TempDir(), "msgstore.db") + waDB, err := sql.Open("sqlite3", waDBPath) + require.NoError(err) + _, err = waDB.Exec(` + PRAGMA journal_mode=WAL; + CREATE TABLE jid (_id INTEGER PRIMARY KEY, user TEXT, server TEXT, raw_string TEXT); + CREATE TABLE chat (_id INTEGER PRIMARY KEY, jid_row_id INTEGER UNIQUE, hidden INTEGER, + subject TEXT, sort_timestamp INTEGER); + CREATE TABLE message (_id INTEGER PRIMARY KEY, chat_row_id INTEGER, from_me INTEGER, + key_id TEXT, sender_jid_row_id INTEGER, timestamp INTEGER, message_type INTEGER, + text_data TEXT, status INTEGER, starred INTEGER); + CREATE TABLE message_media (message_row_id INTEGER PRIMARY KEY, mime_type TEXT, + file_size INTEGER, file_path TEXT, width INTEGER, height INTEGER, media_duration INTEGER); + INSERT INTO jid VALUES (1, '15555550101', 's.whatsapp.net', '15555550101@s.whatsapp.net'); + INSERT INTO chat VALUES (10, 1, 0, NULL, 2000); + INSERT INTO message VALUES (100, 10, 1, 'voice-message', NULL, 1000, 13, + 'caption text', 0, 0); + INSERT INTO message_media VALUES (100, 'audio/wav', 0, 'voice.wav', NULL, NULL, NULL); + `) + require.NoError(err) + require.NoError(waDB.Close()) + summary, err := whatsapp.NewImporter(world.st, nil).Import(t.Context(), waDBPath, whatsapp.ImportOptions{ + Phone: "+15555550100", MediaDir: mediaDir, AttachmentsDir: world.dir, + }) + require.NoError(err) + assert.Equal(int64(1), summary.MediaCopied) + candidates, err := world.st.ListBeeperMediaCandidates(t.Context(), 0, 10) + require.NoError(err) + require.Len(candidates, 1) + assert.Equal("whatsapp", candidates[0].SourceType) + assert.Empty(candidates[0].AttachmentState) + destination := "stored-real-whatsapp" + worker := NewMediaSubmitter(world.st, world.blobs, nil, destination, world.dir).WithASRProfile("import-asr") + result, err := worker.RunBatch(t.Context()) + require.NoError(err) + assert.Equal(1, result.Examined) + assert.Equal(1, result.Pending) + assert.Zero(result.Blocked) + rows := occurrenceRows(t, world.st, destination) + require.Len(rows, 1) + assert.Equal("whatsapp", rows[0].SourceType) + assert.Equal("pending", rows[0].State) + var partKey string + require.NoError(world.st.DB().QueryRow(world.st.Rebind(`SELECT source_part_key FROM attachments + WHERE content_hash = ?`), sha256Hex(wav)).Scan(&partKey)) + assert.Equal("whatsapp:media", partKey) + + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + runPasses(t, world.submitter(t, server, destination).WithASRProfile("import-asr"), 4) + rows = occurrenceRows(t, world.st, destination) + require.Len(rows, 1) + assert.Equal("retained", rows[0].State) + docbank.mu.Lock() + assert.Equal([][]byte{wav}, docbank.uploads) + assert.Empty(docbank.artifactOps) + assert.Equal("import-asr", docbank.processRequests[0].Profile) + docbank.mu.Unlock() +} + +func TestStoredMediaUnavailableCASTries(t *testing.T) { + for _, mode := range []string{"missing", "corrupt", "corrupt-header"} { + t.Run(mode, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t) + wav := syntheticWAV(800, 71) + hash := addStoredMediaSource(t, world, "gmail", "rod@example.com", "mail-"+mode, wav, + "voice.wav", "audio/wav", "", store.AttachmentRoleStandalone, "", nil, + "mail:attachment:"+mode, "mime:1.2") + path := filepath.Join(world.dir, hash[:2], hash) + if mode == "missing" { + require.NoError(os.Remove(path)) + } else { + corrupt := append([]byte(nil), wav...) + if mode == "corrupt-header" { + corrupt[0] ^= 1 + } else { + corrupt[len(corrupt)-1] ^= 1 + } + require.NoError(os.WriteFile(path, corrupt, 0o600)) + } + docbank := newFakeDocbank(t) + server := newTestDocbankServer(t, docbank) + defer server.Close() + runPasses(t, world.submitter(t, server, "stored-email-cas-"+mode), 1) + rows := occurrenceRows(t, world.st, "stored-email-cas-"+mode) + require.Len(rows, 1) + assert.Equal("source_unavailable", rows[0].State) + assert.Equal("source_unavailable", rows[0].ErrorCode) + assert.NotEmpty(rows[0].OperationID) + docbank.mu.Lock() + assert.Empty(docbank.uploads) + docbank.mu.Unlock() + }) + } +} + +func newTestDocbankServer(t *testing.T, docbank *fakeDocbank) *httptest.Server { + t.Helper() + return httptest.NewServer(docbank) +} + +func TestStoredMediaProcessingChoice(t *testing.T) { + assert := assert.New(t) + descriptor := MediaDescriptor{SourceType: "whatsapp", SourceSHA256: strings.Repeat("a", 64), + TranscriptSHA256: hashBytes([]byte("source words")), Language: "en"} + supplied := configureMediaProcessing(descriptor, "source words", "configured-asr") + assert.Equal("supplied-transcript", supplied.ProcessingProfile) + assert.Equal("whatsapp", supplied.ProcessingProvider) + assert.NotEmpty(supplied.ProcessingKey) + asr := configureMediaProcessing(MediaDescriptor{SourceType: "whatsapp", SourceSHA256: descriptor.SourceSHA256}, "", "configured-asr") + assert.Equal("configured-asr", asr.ProcessingProfile) + assert.Equal("whatsapp", asr.ProcessingProvider) + assert.Equal(mediaASRProcessingKey(asr, "configured-asr"), asr.ProcessingKey) + empty := configureMediaProcessing(asr, "", "") + assert.Empty(empty.ProcessingProfile) + assert.Empty(empty.ProcessingKey) + assert.NotEqual(supplied.ProcessingKey, asr.ProcessingKey) +} + +func TestStoredMediaProfileReplay(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "asr-replay", asset: "mxc://beeper.local/asr-replay", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 62)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + worker := world.submitter(t, server, "stored-profile-replay").WithASRProfile("configured-asr") + runPasses(t, worker, 6) + deliveries := deliveryRows(t, world.st, "stored-profile-replay") + require.Len(deliveries, 1) + assert.Equal("observing", deliveries[0].Phase) + assert.NotEmpty(deliveries[0].Donor) + rows := occurrenceRows(t, world.st, "stored-profile-replay") + require.Len(rows, 1) + assert.NotEqual(rows[0].Ref, deliveries[0].Donor) + var profile string + require.NoError(world.st.DB().QueryRow(world.st.Rebind(`SELECT profile FROM beeper_media_deliveries WHERE destination_key = ?`), + "stored-profile-replay").Scan(&profile)) + assert.Equal("configured-asr", profile) + + docbank.mu.Lock() + docbank.sourceOperationID = "other-operation" + docbank.coverage = "transcribed" + firstRequests := len(docbank.processRequests) + docbank.mu.Unlock() + _, err := world.st.DB().Exec(`UPDATE beeper_media_deliveries SET next_action_at = '2000-01-01 00:00:00.000'`) + require.NoError(err) + runPasses(t, worker, 1) + deliveries = deliveryRows(t, world.st, "stored-profile-replay") + require.Len(deliveries, 1) + assert.Equal("done", deliveries[0].Phase) + docbank.mu.Lock() + assert.Len(docbank.processRequests, firstRequests+1) + assert.Equal("configured-asr", docbank.processRequests[len(docbank.processRequests)-1].Profile) + docbank.mu.Unlock() +} + +func TestStoredMediaProfileChangeStartedJob(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "profile-change", asset: "mxc://beeper.local/profile-change", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 63)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "stored-profile-change" + worker := world.submitter(t, server, destination).WithASRProfile("old-asr") + runPasses(t, worker, 6) + oldDeliveries := deliveryRows(t, world.st, destination) + require.Len(oldDeliveries, 1) + require.Equal("observing", oldDeliveries[0].Phase) + oldKey := processingKeyForDestination(t, world.st, destination, "old-asr") + archiveUID, err := world.st.ArchiveUIDContext(t.Context()) + require.NoError(err) + candidates, err := world.st.ListBeeperMediaCandidates(t.Context(), 0, 10) + require.NoError(err) + var current store.BeeperMediaCandidate + for _, candidate := range candidates { + if candidate.SourceMessageID == "profile-change" { + current = candidate + break + } + } + require.NotZero(current.AttachmentID) + updated := world.submitter(t, server, destination).WithASRProfile("new-asr") + _, err = updated.reconcileCandidate(t.Context(), archiveUID, current) + require.NoError(err) + newKey := processingKeyForDestination(t, world.st, destination, "new-asr") + assert.NotEqual(oldKey, newKey) + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries SET next_action_at = '2000-01-01 00:00:00.000' + WHERE destination_key = ? AND processing_key = ?`), destination, oldKey) + require.NoError(err) + docbank.mu.Lock() + docbank.coverage = "transcribed" + docbank.mu.Unlock() + operation, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + require.True(ok) + assert.Equal(store.BeeperMediaOperationStatus, operation.Kind) + assert.Equal(oldKey, operation.ProcessingKey) + require.NoError(updated.status(t.Context(), t.Context(), operation)) + deliveries := processDeliveryIdentities(t, world.st, destination) + assert.Equal("done", findProcessDelivery(t, deliveries, oldKey).phase) + assert.Equal("pending-process", findProcessDelivery(t, deliveries, newKey).phase) + oldMappings, err := world.st.ListLiveBeeperMediaMappings(t.Context(), destination, oldKey, 10) + require.NoError(err) + assert.Empty(oldMappings) + newMappings, err := world.st.ListLiveBeeperMediaMappings(t.Context(), destination, newKey, 10) + require.NoError(err) + require.Len(newMappings, 1) + assert.Equal(newKey, newMappings[0].ProcessingKey) + assert.Equal("pending-process", newMappings[0].ProcessingPhase) + assert.Equal("new-asr", newMappings[0].ProcessingProfile) + assert.Empty(newMappings[0].ProcessingCoverage) +} + +func TestStoredMediaNewerSourceVersionUsesOwnReceipt(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "identity-mismatch", asset: "mxc://beeper.local/identity-mismatch", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 64)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + destination := "stored-identity-mismatch" + worker := world.submitter(t, server, destination).WithASRProfile("configured-asr") + runPasses(t, worker, 6) + started := deliveryRows(t, world.st, destination) + require.Len(started, 1) + require.Equal("observing", started[0].Phase) + for _, mismatch := range []struct { + column string + value string + }{ + {column: "source_id", value: "wrong-source"}, + {column: "source_version_id", value: "wrong-source-version"}, + {column: "content_version_id", value: "wrong-content-version"}, + } { + _, err := world.st.DB().Exec(world.st.Rebind("UPDATE beeper_media_deliveries SET "+mismatch.column+" = ?, next_action_at = '2000-01-01 00:00:00.000' WHERE destination_key = ?"), + mismatch.value, destination) + require.NoError(err) + _, ready, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) + require.NoError(err) + assert.False(ready, mismatch.column) + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries + SET source_id = ?, source_version_id = ?, content_version_id = ?, next_action_at = '2000-01-01 00:00:00.000' + WHERE destination_key = ?`), started[0].SourceID, started[0].SourceVersionID, + started[0].ContentVersionID, destination) + require.NoError(err) + } + docbank.mu.Lock() + docbank.sourceVersionID = "wrong-source-version" + docbank.sourceContentVersionID = "wrong-content-version" + docbank.coverage = "transcribed" + processes := len(docbank.processOps) + docbank.mu.Unlock() + _, err := world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries SET next_action_at = '2000-01-01 00:00:00.000' + WHERE destination_key = ?`), destination) + require.NoError(err) + runPasses(t, worker, 1) + ended := deliveryRows(t, world.st, destination) + require.Len(ended, 1) + assert.Equal("done", ended[0].Phase) + assert.Empty(ended[0].ErrorCode) + docbank.mu.Lock() + assert.Len(docbank.processOps, processes) + assert.Equal(1, docbank.replays) + docbank.mu.Unlock() +} + +func TestStoredMediaOmittedReceiptVersions(t *testing.T) { + require, assert := require.New(t), assert.New(t) + world := importVoiceChat(t, voiceSpec{id: "optional-versions", asset: "mxc://beeper.local/optional-versions", + mime: "audio/wav", fileName: "voice.wav", data: syntheticWAV(800, 79)}) + docbank := newFakeDocbank(t) + docbank.coverage = "pending" + server := newTestDocbankServer(t, docbank) + defer server.Close() + worker := world.submitter(t, server, "optional-versions").WithASRProfile("asr") + runPasses(t, worker, 4) + docbank.mu.Lock() + for id, receipt := range docbank.processReceipts { + receipt.SourceVersionID, receipt.ContentVersionID = "", "" + docbank.processReceipts[id] = receipt + } + docbank.sourceOperationID = "other-operation" + docbank.coverage = "transcribed" + docbank.mu.Unlock() + _, err := world.st.DB().Exec(`UPDATE beeper_media_deliveries SET next_action_at = '2000-01-01 00:00:00.000'`) + require.NoError(err) + runPasses(t, worker, 1) + deliveries := deliveryRows(t, world.st, "optional-versions") + require.Len(deliveries, 1) + assert.Equal("done", deliveries[0].Phase) + assert.Empty(deliveries[0].ErrorCode) +} + +func processingKeyForDestination(t *testing.T, st *store.Store, destination, profile string) string { + t.Helper() + var key string + require.NoError(t, st.DB().QueryRow(st.Rebind(`SELECT processing_key FROM beeper_media_deliveries + WHERE destination_key = ? AND profile = ?`), destination, profile).Scan(&key)) + return key +} diff --git a/internal/config/beeper_docbank_test.go b/internal/config/beeper_docbank_test.go index 2d96d2e28..e592860a1 100644 --- a/internal/config/beeper_docbank_test.go +++ b/internal/config/beeper_docbank_test.go @@ -16,7 +16,8 @@ func TestBeeperDocbankConfig(t *testing.T) { enabled = true url = "http://127.0.0.1:8080" api_key_env = "DOCBANK_TEST_KEY" -upload_consent = true +all_sources_upload_consent = true +asr_profile = "voice-asr" ` require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) @@ -24,10 +25,26 @@ upload_consent = true require.NoError(t, err) assert.Equal(t, DocbankIntegrationConfig{ Enabled: true, URL: "http://127.0.0.1:8080", - APIKeyEnv: "DOCBANK_TEST_KEY", UploadConsent: true, + APIKeyEnv: "DOCBANK_TEST_KEY", AllSourcesUploadConsent: true, ASRProfile: "voice-asr", }, cfg.Integrations.Docbank) } +func TestBeeperDocbankConfigRejectsReservedSuppliedTranscriptProfile(t *testing.T) { + t.Setenv("MSGVAULT_HOME", t.TempDir()) + path := filepath.Join(t.TempDir(), "config.toml") + content := `[integrations.docbank] +enabled = true +url = "http://127.0.0.1:8080" +api_key_env = "DOCBANK_TEST_KEY" +all_sources_upload_consent = true +asr_profile = " supplied-transcript " +` + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + + _, err := Load(path, "") + require.ErrorContains(t, err, "reserved for supplied transcript input") +} + func TestBeeperDocbankConfigDefaultsDisabled(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -38,5 +55,5 @@ func TestBeeperDocbankConfigDefaultsDisabled(t *testing.T) { cfg, err := Load(path, "") require.NoError(err) assert.False(cfg.Integrations.Docbank.Enabled) - assert.False(cfg.Integrations.Docbank.UploadConsent) + assert.False(cfg.Integrations.Docbank.AllSourcesUploadConsent) } diff --git a/internal/config/config.go b/internal/config/config.go index ea89af3b7..21df7fa2e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -117,13 +117,21 @@ type IntegrationsConfig struct { Docbank DocbankIntegrationConfig `toml:"docbank"` } -// DocbankIntegrationConfig configures the optional Beeper media destination. +// DocbankIntegrationConfig configures the optional stored-media destination. // The API key stays in the daemon environment and is read when a request runs. type DocbankIntegrationConfig struct { - Enabled bool `toml:"enabled"` - URL string `toml:"url"` - APIKeyEnv string `toml:"api_key_env"` - UploadConsent bool `toml:"upload_consent"` + Enabled bool `toml:"enabled"` + URL string `toml:"url"` + APIKeyEnv string `toml:"api_key_env"` + AllSourcesUploadConsent bool `toml:"all_sources_upload_consent"` + ASRProfile string `toml:"asr_profile"` +} + +func (d DocbankIntegrationConfig) validate() error { + if strings.TrimSpace(d.ASRProfile) == "supplied-transcript" { + return errors.New(`integrations.docbank.asr_profile: "supplied-transcript" is reserved for supplied transcript input`) + } + return nil } // TaskIntegrationConfig configures a task service connection. @@ -1019,6 +1027,9 @@ func decodeConfig(cfg *Config, path string, explicit, homeOverride bool, content if err := cfg.Integrations.Kata.Validate("kata"); err != nil { return nil, err } + if err := cfg.Integrations.Docbank.validate(); err != nil { + return nil, err + } cfg.Activity.ApplyDefaults() if err := cfg.Activity.Validate(); err != nil { return nil, err diff --git a/internal/docbankmedia/client.go b/internal/docbankmedia/client.go index 81279ba3a..e22d3568a 100644 --- a/internal/docbankmedia/client.go +++ b/internal/docbankmedia/client.go @@ -214,16 +214,13 @@ func (c *Client) ImportTranscript(ctx context.Context, sourceID string, metadata return receipt, nil } -func (c *Client) Process(ctx context.Context, sourceID, operationID, suppliedInputID string) (Receipt, error) { +func (c *Client) Process(ctx context.Context, sourceID, operationID string, processing Processing) (Receipt, error) { body := struct { OperationID string `json:"operation_id"` Processing Processing `json:"processing"` }{ OperationID: operationID, - Processing: Processing{ - Profile: "supplied-transcript", - SuppliedInputID: suppliedInputID, - }, + Processing: processing, } var receipt Receipt err := c.jsonRequest(ctx, http.MethodPost, diff --git a/internal/docbankmedia/client_test.go b/internal/docbankmedia/client_test.go index ad55683dc..ff1053862 100644 --- a/internal/docbankmedia/client_test.go +++ b/internal/docbankmedia/client_test.go @@ -183,7 +183,9 @@ func TestClientMediaWire(t *testing.T) { require.NoError(err) assert.Equal("input", artifact.SuppliedInputID) processID := uuid.NewString() - processed, err := client.Process(t.Context(), "source", processID, artifact.SuppliedInputID) + processed, err := client.Process(t.Context(), "source", processID, docbankmedia.Processing{ + Profile: "supplied-transcript", SuppliedInputID: artifact.SuppliedInputID, + }) require.NoError(err) status, err := client.JobStatus(t.Context(), processed.JobID) require.NoError(err) @@ -300,14 +302,18 @@ func TestClientMediaProcessReceipts(t *testing.T) { require.NoError(err) operationID := uuid.NewString() - receipt, err := client.Process(t.Context(), "failed", operationID, "input") + receipt, err := client.Process(t.Context(), "failed", operationID, docbankmedia.Processing{ + Profile: "supplied-transcript", SuppliedInputID: "input", + }) require.NoError(err) assert.Equal("failed", receipt.OperationState) assert.Equal("unavailable", receipt.CoverageState) assert.Empty(receipt.JobID) assert.Equal(operationID, receipt.OperationID) - _, err = client.Process(t.Context(), "queued", uuid.NewString(), "input") + _, err = client.Process(t.Context(), "queued", uuid.NewString(), docbankmedia.Processing{ + Profile: "supplied-transcript", SuppliedInputID: "input", + }) require.ErrorIs(err, docbankmedia.ErrInvalidReceipt) } @@ -354,7 +360,9 @@ func TestDocbankMediaLiveContract(t *testing.T) { require.NoError(err, sample.name) require.NotEmpty(artifact.SuppliedInputID) - processed, err := client.Process(t.Context(), first.SourceID, uuid.NewString(), artifact.SuppliedInputID) + processed, err := client.Process(t.Context(), first.SourceID, uuid.NewString(), docbankmedia.Processing{ + Profile: "supplied-transcript", SuppliedInputID: artifact.SuppliedInputID, + }) if err != nil { t.Logf("%s processing refused with %s", sample.name, docbankmedia.ErrorCode(err)) continue diff --git a/internal/store/docbank_media.go b/internal/store/docbank_media.go index b263d3ff5..b04abb2f8 100644 --- a/internal/store/docbank_media.go +++ b/internal/store/docbank_media.go @@ -20,7 +20,8 @@ var beeperMediaLocalGapCodes = []any{"source_raw_invalid", "source_part_missing" // beeperMediaResumedPhase is the step a reopened delivery resumes from. const beeperMediaResumedPhase = `CASE WHEN job_id <> '' THEN 'observing' - WHEN supplied_input_id = '' THEN 'pending-artifact' ELSE 'pending-process' END` + WHEN profile = 'supplied-transcript' AND supplied_input_id = '' THEN 'pending-artifact' + ELSE 'pending-process' END` // beeperMediaLocalGapFilter matches error_code against beeperMediaLocalGapCodes. func beeperMediaLocalGapFilter() string { @@ -53,13 +54,24 @@ const ( ) // beeperMediaEligible is the shared provider, capture and role predicate. It -// assumes the aliases a (attachments), m (messages) and src (sources). -const beeperMediaEligible = `src.source_type = 'beeper' - AND length(COALESCE(a.content_hash, '')) = 64 +// assumes a (attachments), m (messages), c (conversations) and src (sources). +const beeperMediaEligible = `length(COALESCE(a.content_hash, '')) = 64 + AND COALESCE(a.size, 0) > 0 AND COALESCE(a.storage_path, '') <> '' - AND COALESCE(a.attachment_state, '') = 'stored' - AND COALESCE(a.media_type, '') IN ('audio', 'voice_note') - AND COALESCE(a.attachment_role, 'unknown') = 'standalone'` + AND COALESCE(src.source_type, '') <> '' + AND COALESCE(src.identifier, '') <> '' + AND COALESCE(m.source_message_id, '') <> '' + AND ( + (src.source_type = 'beeper' + AND COALESCE(a.attachment_state, '') = 'stored' + AND COALESCE(a.media_type, '') IN ('audio', 'voice_note') + AND COALESCE(a.attachment_role, 'unknown') = 'standalone') + OR (src.source_type <> 'beeper' + AND COALESCE(NULLIF(a.source_part_key, ''), a.source_attachment_id, '') <> '' + AND COALESCE(c.source_conversation_id, '') <> '' + AND COALESCE(a.attachment_state, '') IN ('', 'stored') + AND COALESCE(a.attachment_role, 'unknown') IN ('standalone', 'unknown')) + )` // beeperMediaCurrentJoin follows occurrence row o to its live source // attachment by stable source tuple, never by attachment row ID. @@ -68,18 +80,19 @@ var beeperMediaCurrentJoin = ` JOIN messages m ON m.source_id = src.id AND m.source_message_id = o.source_message_id JOIN conversations c ON c.id = m.conversation_id JOIN attachments a ON a.message_id = m.id - WHERE c.source_conversation_id = o.source_conversation_id + WHERE COALESCE(c.source_conversation_id, '') = o.source_conversation_id AND COALESCE(a.source_attachment_id, '') = o.source_attachment_id - AND COALESCE(NULLIF(a.source_part_key, ''), a.source_attachment_id, '') = o.source_part_key + AND COALESCE(NULLIF(a.source_part_key, ''), NULLIF(a.source_attachment_id, ''), src.source_type || ':unknown') = o.source_part_key AND a.content_hash = o.source_sha256 AND ` + beeperMediaEligible + ` AND ` + LiveMessagesWhere("m", true) -// BeeperMediaCandidate is the current archive evidence for one stored Beeper -// audio occurrence. The source tuple, rather than AttachmentID, is stable. +// BeeperMediaCandidate is the current archive evidence for a stored attachment +// to inspect for audio. The source tuple, rather than AttachmentID, is stable. type BeeperMediaCandidate struct { AttachmentID int64 MessageID int64 + SentAt sql.NullTime ConversationID int64 SourceID int64 SourceType string @@ -95,6 +108,7 @@ type BeeperMediaCandidate struct { ContentHash string ByteLength int64 AttachmentState string + AttachmentMetadata string } // BeeperMediaMapping is one occurrence row together with the optional shared @@ -140,16 +154,21 @@ type BeeperMediaMapping struct { JobID string OperationState string ProcessingCoverage string + ProcessingProvider string + ProcessingProfile string } // BeeperMediaOperation is the one remote action selected by a bounded pass. // FrozenRequestJSON holds the saved nonsecret request for replay. type BeeperMediaOperation struct { Kind string + PreparedReplay bool DestinationKey string OccurrenceRef string Revision string ProcessingKey string + ProcessingProvider string + ProcessingProfile string OperationID string FrozenRequestJSON string SourceSHA256 string @@ -207,14 +226,15 @@ const beeperMediaCandidateColumns = ` COALESCE(a.filename, ''), COALESCE(a.mime_type, ''), COALESCE(a.media_type, ''), COALESCE(a.attachment_role, 'unknown'), COALESCE(a.content_hash, ''), COALESCE(a.size, 0), - COALESCE(a.attachment_state, '') + COALESCE(a.attachment_state, ''), + COALESCE(CAST(a.attachment_metadata AS TEXT), ''), m.sent_at FROM attachments a JOIN messages m ON m.id = a.message_id JOIN conversations c ON c.id = m.conversation_id JOIN sources src ON src.id = m.source_id` -// ListBeeperMediaCandidates returns a bounded page of stored, standalone -// Beeper audio on live messages. Provider ownership is an explicit filter. +// ListBeeperMediaCandidates returns a bounded page of captured attachments on +// live messages. Beeper candidates must be explicitly marked as standalone audio. func (s *Store) ListBeeperMediaCandidates( ctx context.Context, afterID int64, limit int, ) ([]BeeperMediaCandidate, error) { @@ -258,7 +278,7 @@ func (s *Store) queryBeeperMediaCandidates( &candidate.SourceAttachmentID, &candidate.SourcePartKey, &candidate.Filename, &candidate.MIMEType, &candidate.MediaType, &candidate.Role, &candidate.ContentHash, &candidate.ByteLength, - &candidate.AttachmentState, + &candidate.AttachmentState, &candidate.AttachmentMetadata, &candidate.SentAt, ); err != nil { return nil, fmt.Errorf("scan beeper media candidate: %w", err) } @@ -324,25 +344,54 @@ func (s *Store) ReconcileBeeperMediaMapping(ctx context.Context, mapping BeeperM } else if err != nil { return fmt.Errorf("read beeper media occurrence: %w", err) } else { + previousProcessingKey := existing.ProcessingKey next := reconciledBeeperMediaOccurrence(existing, mapping) if !sameBeeperMediaOccurrence(existing, next) { if err := s.updateBeeperMediaOccurrence(q, next); err != nil { return err } } + if previousProcessingKey != "" && previousProcessingKey != next.ProcessingKey { + if _, err := q.Exec(` + UPDATE beeper_media_deliveries + SET phase = 'blocked', next_action_at = NULL, error_code = 'processing_key_changed', + updated_at = `+s.dialect.Now()+` + WHERE destination_key = ? AND processing_key = ? + AND NOT EXISTS ( + SELECT 1 FROM beeper_media_occurrences o + WHERE o.destination_key = beeper_media_deliveries.destination_key + AND o.processing_key = beeper_media_deliveries.processing_key + AND o.retention_state IN ('pending', 'source_unavailable', 'retained')) + AND COALESCE(job_id, '') = '' + AND (phase = 'pending-artifact' OR (phase = 'pending-process' AND + (COALESCE(pending_operation_id, '') = '' OR COALESCE(frozen_request_json, '') = '' OR + COALESCE(source_id, '') = '' OR COALESCE(source_version_id, '') = '' OR + COALESCE(content_version_id, '') = '' OR COALESCE(donor_occurrence_id, '') = '')))`, + mapping.DestinationKey, previousProcessingKey); err != nil { + return fmt.Errorf("retire changed beeper media processing state: %w", err) + } + } reopened = existing.RetentionState != next.RetentionState mapping = next } if mapping.ProcessingKey != "" { + provider, profile := beeperMediaDeliveryProfile(mapping) + if provider == "" || profile == "" { + return errors.New("beeper media processing provider and profile are required") + } + phase := beeperMediaPhasePendingProcess + if profile == beeperMediaProcessingProfile { + phase = beeperMediaPhasePendingArtifact + } if _, err := q.Exec(` INSERT INTO beeper_media_deliveries (destination_key, processing_key, source_sha256, byte_length, transcript_sha256, language, provider, profile, phase, next_action_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, 'beeper', ?, 'pending-artifact', ?, `+s.dialect.Now()+`, `+s.dialect.Now()+`) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, `+s.dialect.Now()+`, `+s.dialect.Now()+`) ON CONFLICT (destination_key, processing_key) DO NOTHING`, mapping.DestinationKey, mapping.ProcessingKey, mapping.SourceSHA256, mapping.ByteLength, - mapping.TranscriptSHA256, mapping.Language, beeperMediaProcessingProfile, + mapping.TranscriptSHA256, mapping.Language, provider, profile, phase, s.timestampValue(time.Now().UTC())); err != nil { return fmt.Errorf("create beeper media processing state: %w", err) } @@ -371,8 +420,8 @@ func (s *Store) ReconcileBeeperMediaMapping(ctx context.Context, mapping BeeperM const errBeeperMediaNoLiveOccurrenceCode = "no_live_occurrence" -// retireBeeperMediaDeliveries blocks an unstarted transcript delivery of this -// occurrence once no pending or retained occurrence can supply its audio. +// retireBeeperMediaDeliveries blocks unprepared work once no occurrence can +// supply its audio. Prepared processing may have reached Docbank already. func (s *Store) retireBeeperMediaDeliveries(q boundQuerier, destination, occurrenceRef, code string) error { if !slices.Contains(beeperMediaLocalGapCodes, any(code)) { return nil @@ -381,7 +430,10 @@ func (s *Store) retireBeeperMediaDeliveries(q boundQuerier, destination, occurre UPDATE beeper_media_deliveries SET phase = 'blocked', next_action_at = NULL, error_code = ?, updated_at = `+s.dialect.Now()+` WHERE destination_key = ? - AND (phase = 'pending-artifact' OR (phase = 'pending-process' AND COALESCE(job_id, '') = '')) + AND (phase = 'pending-artifact' OR (phase = 'pending-process' AND COALESCE(job_id, '') = '' AND + (COALESCE(pending_operation_id, '') = '' OR COALESCE(frozen_request_json, '') = '' OR + COALESCE(source_id, '') = '' OR COALESCE(source_version_id, '') = '' OR + COALESCE(content_version_id, '') = '' OR COALESCE(donor_occurrence_id, '') = ''))) AND processing_key IN ( SELECT processing_key FROM beeper_media_occurrences WHERE destination_key = ? AND occurrence_ref = ? AND processing_key <> '') @@ -404,6 +456,8 @@ func reconciledBeeperMediaOccurrence(existing, current BeeperMediaMapping) Beepe next.SourceMessageID, next.SourceAttachmentID = current.SourceMessageID, current.SourceAttachmentID next.SourcePartKey, next.LocalSourceID = current.SourcePartKey, current.LocalSourceID next.MessageID, next.AttachmentID, next.RawHash = current.MessageID, current.AttachmentID, current.RawHash + next.ProcessingKey = current.ProcessingKey + next.ProcessingProvider, next.ProcessingProfile = current.ProcessingProvider, current.ProcessingProfile switch { case existing.DocbankSourceID != "": next.RetentionState = BeeperMediaRetentionRetained @@ -425,7 +479,7 @@ func sameBeeperMediaOccurrence(a, b BeeperMediaMapping) bool { a.SourcePartKey == b.SourcePartKey && a.LocalSourceID == b.LocalSourceID && a.MessageID == b.MessageID && a.AttachmentID == b.AttachmentID && a.RawHash == b.RawHash && a.RetentionState == b.RetentionState && a.RetentionOperationID == b.RetentionOperationID && - a.ErrorCode == b.ErrorCode && a.NextActionAt.Equal(b.NextActionAt) + a.ErrorCode == b.ErrorCode && a.ProcessingKey == b.ProcessingKey && a.NextActionAt.Equal(b.NextActionAt) } // ListLiveBeeperMediaMappings returns retained mappings whose current source, @@ -451,7 +505,8 @@ func (s *Store) ListLiveBeeperMediaMappings( SELECT `+beeperMediaOccurrenceColumns("o")+`, COALESCE(d.phase, ''), COALESCE(d.processing_operation_id, ''), COALESCE(d.donor_occurrence_id, ''), COALESCE(d.supplied_input_id, ''), - COALESCE(d.job_id, ''), COALESCE(d.operation_state, ''), COALESCE(d.coverage_state, '') + COALESCE(d.job_id, ''), COALESCE(d.operation_state, ''), COALESCE(d.coverage_state, ''), + COALESCE(d.provider, ''), COALESCE(d.profile, '') FROM beeper_media_occurrences o LEFT JOIN beeper_media_deliveries d ON d.destination_key = o.destination_key AND d.processing_key = o.processing_key @@ -467,7 +522,8 @@ func (s *Store) ListLiveBeeperMediaMappings( dest := beeperMediaOccurrenceScanTargets(&mapping) dest = append(dest, &mapping.ProcessingPhase, &mapping.ProcessingOperationID, &mapping.DonorOccurrenceID, &mapping.SuppliedInputID, &mapping.JobID, - &mapping.OperationState, &mapping.ProcessingCoverage) + &mapping.OperationState, &mapping.ProcessingCoverage, + &mapping.ProcessingProvider, &mapping.ProcessingProfile) var next sql.NullTime dest[beeperMediaNextActionIndex] = &next if err := rows.Scan(dest...); err != nil { @@ -617,17 +673,33 @@ func (s *Store) NextBeeperMediaOperation( retain.NextActionAt = nullTimeValue(retainNext) var delivery BeeperMediaOperation - var phase, pendingID, processingID, frozen string + var phase, pendingID, processingID, frozen, provider, profile string + var preparedReplay int var deliveryNext sql.NullTime deliveryErr := s.db.QueryRowContext(ctx, s.Rebind(` - SELECT d.phase, d.destination_key, d.processing_key, COALESCE(d.pending_operation_id, ''), + SELECT d.phase, + CASE WHEN d.phase = 'pending-process' AND COALESCE(d.pending_operation_id, '') <> '' + AND COALESCE(d.frozen_request_json, '') <> '' AND NOT EXISTS ( + SELECT 1 FROM beeper_media_occurrences o`+beeperMediaCurrentJoin+` + AND o.destination_key = d.destination_key AND o.processing_key = d.processing_key + AND o.retention_state = 'retained') THEN 1 ELSE 0 END, + d.destination_key, d.processing_key, COALESCE(d.pending_operation_id, ''), d.processing_operation_id, d.source_sha256, d.byte_length, d.transcript_sha256, d.language, d.source_id, d.source_version_id, d.content_version_id, d.donor_occurrence_id, d.supplied_input_id, d.job_id, COALESCE(d.frozen_request_json, ''), + COALESCE(d.provider, ''), COALESCE(d.profile, ''), COALESCE((SELECT o.vault_uid FROM beeper_media_occurrences o - WHERE o.destination_key = d.destination_key AND o.processing_key = d.processing_key - AND o.vault_uid <> '' LIMIT 1), ''), d.next_action_at + WHERE o.destination_key = d.destination_key + AND ((d.donor_occurrence_id <> '' AND o.occurrence_id = d.donor_occurrence_id) + OR (d.donor_occurrence_id = '' AND o.processing_key = d.processing_key)) + AND (d.donor_occurrence_id = '' OR (o.source_id = d.source_id + AND o.source_version_id = d.source_version_id + AND o.content_version_id = d.content_version_id)) + AND o.vault_uid <> '' + ORDER BY CASE WHEN o.occurrence_id = d.donor_occurrence_id THEN 0 ELSE 1 END + LIMIT 1), ''), + d.next_action_at FROM beeper_media_deliveries d WHERE d.destination_key = ? AND d.phase IN ('pending-artifact', 'pending-process', 'observing') @@ -637,22 +709,38 @@ func (s *Store) NextBeeperMediaOperation( SELECT 1 FROM beeper_media_occurrences o`+beeperMediaCurrentJoin+` AND o.destination_key = d.destination_key AND o.processing_key = d.processing_key AND o.retention_state = 'retained')) - OR (d.phase = 'observing' AND d.job_id <> '' AND EXISTS ( + OR (d.phase = 'pending-process' AND COALESCE(d.pending_operation_id, '') <> '' + AND COALESCE(d.frozen_request_json, '') <> '' AND COALESCE(d.source_id, '') <> '' + AND COALESCE(d.source_version_id, '') <> '' AND COALESCE(d.content_version_id, '') <> '' + AND d.donor_occurrence_id <> '' + AND EXISTS ( + SELECT 1 FROM beeper_media_occurrences o + WHERE o.destination_key = d.destination_key AND o.occurrence_id = d.donor_occurrence_id + AND o.source_id = d.source_id AND o.source_version_id = d.source_version_id + AND o.content_version_id = d.content_version_id AND o.source_sha256 = d.source_sha256 + AND o.byte_length = d.byte_length AND o.vault_uid <> '')) + OR (d.phase = 'observing' AND d.job_id <> '' AND d.donor_occurrence_id <> '' AND EXISTS ( SELECT 1 FROM beeper_media_occurrences o - WHERE o.destination_key = d.destination_key AND o.processing_key = d.processing_key + WHERE o.destination_key = d.destination_key AND o.occurrence_id = d.donor_occurrence_id + AND o.source_id = d.source_id + AND o.source_version_id = d.source_version_id + AND o.content_version_id = d.content_version_id AND o.vault_uid <> '')) ) ORDER BY d.next_action_at, d.processing_key LIMIT 1`), destination, s.dialect.TimestampParam(now)).Scan( - &phase, &delivery.DestinationKey, &delivery.ProcessingKey, &pendingID, &processingID, + &phase, &preparedReplay, &delivery.DestinationKey, &delivery.ProcessingKey, &pendingID, &processingID, &delivery.SourceSHA256, &delivery.ByteLength, &delivery.TranscriptSHA256, &delivery.Language, &delivery.DocbankSourceID, &delivery.SourceVersionID, &delivery.ContentVersionID, - &delivery.DocbankOccurrenceID, &delivery.SuppliedInputID, &delivery.JobID, &frozen, &delivery.VaultUID, &deliveryNext) + &delivery.DocbankOccurrenceID, &delivery.SuppliedInputID, &delivery.JobID, &frozen, + &provider, &profile, &delivery.VaultUID, &deliveryNext) if deliveryErr != nil && !errors.Is(deliveryErr, sql.ErrNoRows) { return BeeperMediaOperation{}, false, fmt.Errorf("select beeper media processing operation: %w", deliveryErr) } delivery.NextActionAt = nullTimeValue(deliveryNext) + delivery.PreparedReplay = preparedReplay != 0 delivery.FrozenRequestJSON = frozen + delivery.ProcessingProvider, delivery.ProcessingProfile = provider, profile switch phase { case beeperMediaPhasePendingArtifact: delivery.Kind, delivery.OperationID = BeeperMediaOperationArtifact, pendingID @@ -751,8 +839,20 @@ func (s *Store) PrepareBeeperMediaOperation( } operation.OperationID = storedID if operation.Kind == BeeperMediaOperationProcess { - operation.DocbankSourceID, operation.SourceVersionID = sourceID, sourceVersion - operation.ContentVersionID, operation.DocbankOccurrenceID = contentVersion, donor + // A first ASR request supplies the retained donor identity. Once + // saved, the delivery identity wins on every replay. + if sourceID != "" || operation.DocbankSourceID == "" { + operation.DocbankSourceID = sourceID + } + if sourceVersion != "" || operation.SourceVersionID == "" { + operation.SourceVersionID = sourceVersion + } + if contentVersion != "" || operation.ContentVersionID == "" { + operation.ContentVersionID = contentVersion + } + if donor != "" || operation.DocbankOccurrenceID == "" { + operation.DocbankOccurrenceID = donor + } operation.SuppliedInputID = supplied } if _, err := q.Exec(` @@ -1028,7 +1128,7 @@ func validateBeeperMediaMapping(mapping BeeperMediaMapping) error { return fmt.Errorf("beeper media %s is required", field.name) } } - if mapping.SourceType != "beeper" { + if strings.TrimSpace(mapping.SourceType) == "" { return errors.New("beeper media source type is invalid") } if mapping.ByteLength < 1 || !isBeeperMediaSHA256(mapping.SourceSHA256) { @@ -1084,6 +1184,10 @@ func beeperMediaOccurrenceColumns(alias string) string { return strings.Join(columns, ", ") } +func beeperMediaDeliveryProfile(mapping BeeperMediaMapping) (string, string) { + return strings.TrimSpace(mapping.ProcessingProvider), strings.TrimSpace(mapping.ProcessingProfile) +} + // beeperMediaNextActionIndex is next_action_at's position in the column list. const beeperMediaNextActionIndex = 22 @@ -1141,11 +1245,12 @@ func (s *Store) updateBeeperMediaOccurrence(q boundQuerier, m BeeperMediaMapping SET source_identifier = ?, source_conversation_id = ?, source_message_id = ?, source_attachment_id = ?, source_part_key = ?, source_row_id = ?, message_id = ?, attachment_id = ?, raw_hash = ?, retention_operation_id = ?, retention_state = ?, - next_action_at = ?, error_code = ?, updated_at = `+s.dialect.Now()+` + next_action_at = ?, error_code = ?, processing_key = ?, updated_at = `+s.dialect.Now()+` WHERE destination_key = ? AND occurrence_ref = ? AND revision = ?`, m.SourceIdentifier, m.SourceConversationID, m.SourceMessageID, m.SourceAttachmentID, m.SourcePartKey, m.LocalSourceID, m.MessageID, m.AttachmentID, m.RawHash, m.RetentionOperationID, m.RetentionState, s.timestampValue(m.NextActionAt), m.ErrorCode, + m.ProcessingKey, m.DestinationKey, m.OccurrenceRef, m.Revision) if err != nil { return fmt.Errorf("update beeper media occurrence: %w", err) diff --git a/internal/store/docbank_media_test.go b/internal/store/docbank_media_test.go index 8646c8fa0..73f63eeb1 100644 --- a/internal/store/docbank_media_test.go +++ b/internal/store/docbank_media_test.go @@ -70,6 +70,7 @@ func (a beeperAudio) mapping(destination, revision, processingKey string) store. RawHash: hex.EncodeToString(rawDigest[:]), TranscriptSHA256: transcript, OccurrenceJSON: `{"ref":"msgvault:` + a.sourceMessageID + `","revision":"` + revision + `"}`, Filename: "voice.wav", MIMEType: "audio/wav", ProcessingKey: processingKey, + ProcessingProvider: "beeper", ProcessingProfile: "supplied-transcript", } } @@ -92,6 +93,24 @@ func retainAudio(t *testing.T, st *store.Store, mapping store.BeeperMediaMapping require.True(t, applied) } +func TestBeeperMediaKeyChangeKeepsSharedDelivery(t *testing.T) { + require, assert := require.New(t), assert.New(t) + f := newBeeperMediaFixture(t) + first := addBeeperAudio(t, f.Store, f.Source.ID, f.ConvID, "first", strings.Repeat("a", 64)) + second := addBeeperAudio(t, f.Store, f.Source.ID, f.ConvID, "second", first.hash) + a := first.mapping("shared", "revision", "old-key") + b := second.mapping("shared", "revision", "old-key") + retainAudio(t, f.Store, a, "first-occurrence") + retainAudio(t, f.Store, b, "second-occurrence") + a.ProcessingKey = "new-key" + require.NoError(f.Store.ReconcileBeeperMediaMapping(t.Context(), a)) + operation, ready, err := f.Store.NextBeeperMediaOperation(t.Context(), "shared", time.Now().UTC()) + require.NoError(err) + require.True(ready) + assert.Equal("old-key", operation.ProcessingKey) + assert.Equal(store.BeeperMediaOperationArtifact, operation.Kind) +} + func liveMessageIDs(t *testing.T, st *store.Store) []string { t.Helper() mappings, err := st.ListLiveBeeperMediaMappings(t.Context(), "live", "", 100) @@ -166,6 +185,8 @@ func TestBeeperMediaOperationReplay(t *testing.T) { assert.Equal("input", process.SuppliedInputID) assert.NotEqual(replayed.OperationID, process.OperationID) process.FrozenRequestJSON = `{"profile":"supplied-transcript","supplied_input_id":"input"}` + process.SourceVersionID, process.ContentVersionID = "version", "content" + process.DocbankOccurrenceID = "occurrence" process, err = f.Store.PrepareBeeperMediaOperation(t.Context(), process) require.NoError(err) assert.Equal("source", process.DocbankSourceID) @@ -521,8 +542,8 @@ func TestBeeperMediaRevocationLifecycle(t *testing.T) { _, err = f.Store.DB().Exec(f.Store.Rebind(` UPDATE beeper_media_deliveries SET phase = 'observing', processing_operation_id = '11111111-1111-4111-8111-111111111111', - source_id = 'source-c', source_version_id = 'version-c', - content_version_id = 'content-c', donor_occurrence_id = 'observing-occurrence', + source_id = 'source-cccc', source_version_id = 'version', + content_version_id = 'content', donor_occurrence_id = 'observing-occurrence', job_id = 'job-c', operation_state = 'queued', coverage_state = 'pending', next_action_at = ? WHERE destination_key = ? AND processing_key = ?`), dueAt, observingMapping.DestinationKey, observingMapping.ProcessingKey) @@ -532,7 +553,7 @@ func TestBeeperMediaRevocationLifecycle(t *testing.T) { require.NoError(err) require.True(ok) assert.Equal(store.BeeperMediaOperationStatus, operation.Kind) - assert.Equal("source-c", operation.DocbankSourceID) + assert.Equal("source-cccc", operation.DocbankSourceID) assert.Equal("job-c", operation.JobID) } @@ -640,7 +661,9 @@ func TestBeeperMediaDiscovery(t *testing.T) { assert.Equal(audios[1].attachmentID, page[0].AttachmentID) rest, err := f.Store.ListBeeperMediaCandidates(t.Context(), page[99].AttachmentID, 100) require.NoError(err) - assert.Empty(rest) + require.Len(rest, 1) + assert.Equal("gmail", rest[0].SourceType) + assert.Equal("gmail-audio", rest[0].SourceMessageID) _, err = f.Store.ListBeeperMediaCandidates(t.Context(), 0, 1001) require.Error(err) @@ -697,6 +720,48 @@ func TestBeeperMediaDiscovery(t *testing.T) { assert.Empty(changes) } +func TestBeeperMediaCandidateAttachmentStates(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newBeeperMediaFixture(t) + gmail, err := f.Store.GetOrCreateSource("gmail", "rod@example.com") + require.NoError(err) + gmailConversation, err := f.Store.EnsureConversation(gmail.ID, "gmail-thread", "Thread") + require.NoError(err) + states := []string{"", "stored", "pending", "failed", "skipped"} + want := make(map[int64]string) + for i, state := range states { + audio := addBeeperAudio(t, f.Store, gmail.ID, gmailConversation, + fmt.Sprintf("gmail-%s-%d", state, i), fmt.Sprintf("%064x", i+1)) + if state == "" { + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE attachments SET attachment_state = NULL WHERE id = ?`), audio.attachmentID) + } else { + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE attachments SET attachment_state = ? WHERE id = ?`), + state, audio.attachmentID) + } + require.NoError(err) + if state == "" || state == "stored" { + want[audio.attachmentID] = state + } + } + zeroBytes := addBeeperAudio(t, f.Store, gmail.ID, gmailConversation, "gmail-zero-bytes", fmt.Sprintf("%064x", 9)) + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE attachments SET size = 0 WHERE id = ?`), zeroBytes.attachmentID) + require.NoError(err) + beeperAudio := addBeeperAudio(t, f.Store, f.Source.ID, f.ConvID, "beeper-empty-state", strings.Repeat("a", 64)) + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE attachments SET attachment_state = NULL WHERE id = ?`), beeperAudio.attachmentID) + require.NoError(err) + + candidates, err := f.Store.ListBeeperMediaCandidates(t.Context(), 0, 10) + require.NoError(err) + require.Len(candidates, len(want)) + for _, candidate := range candidates { + assert.Equal("gmail", candidate.SourceType) + assert.Equal(want[candidate.AttachmentID], candidate.AttachmentState) + delete(want, candidate.AttachmentID) + } + assert.Empty(want) +} + func TestBeeperMediaSchemaReopen(t *testing.T) { require := require.New(t) assert := assert.New(t) From b41344a914ce43d0b2d9c4471ea1ff8160ccc6c5 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Mon, 28 Sep 2026 21:00:14 -0400 Subject: [PATCH 076/125] test(web): wait for the directory drawer before keyboard disclosure (#984) The Directory keyboard test now waits for the mobile drawer's disclosure button to become enabled and receive focus before pressing Enter, fixing the race behind [this CI failure](https://github.com/kenn-io/msgvault/actions/runs/36495842794). Resizing remounts profile maintenance and reloads its catalog. The old assertion could match the desktop field before that transition; the CI trace then shows focus targeting a disabled button, so Enter never requested sensitive fields. The test now scopes readiness to the drawer and preserves its keyboard, privacy, request, layout, and accessibility checks. Co-authored-by: Rod Boev --- web/tests/e2e/directory.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/tests/e2e/directory.spec.ts b/web/tests/e2e/directory.spec.ts index 9cb252072..b4c4bd8a2 100644 --- a/web/tests/e2e/directory.spec.ts +++ b/web/tests/e2e/directory.spec.ts @@ -196,8 +196,12 @@ test('Directory profile maintenance uses exact safe requests and GET-only ambigu ]); await page.setViewportSize({ width: 390, height: 844 }); - await expect(maintenance.getByText('Time zone')).toBeVisible(); - await maintenance.getByRole('button', { name: 'Show sensitive eligible fields' }).focus(); + const drawerMaintenance = page.getByRole('dialog', { name: 'Person detail' }).getByRole('region', { name: 'Profile maintenance' }); + await expect(drawerMaintenance.getByText('Time zone')).toBeVisible(); + const drawerReveal = drawerMaintenance.getByRole('button', { name: 'Show sensitive eligible fields' }); + await expect(drawerReveal).toBeEnabled(); + await drawerReveal.focus(); + await expect(drawerReveal).toBeFocused(); await page.keyboard.press('Enter'); await expect(maintenance.getByText('Private note')).toBeVisible(); const targetCards = maintenance.locator('li'); From d2e82f7711a63fd2c9924296090f3823ddd54da6 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:07:44 -0700 Subject: [PATCH 077/125] fix(cache): keep queries responsive and label imports incremental (#915) Analytics queries keep serving published data while a refresh runs. Missing cache files trigger repair, including after manual sync during the normal rebuild interval. Fresh requests return a build job without advertising unverified cache metadata. Go clients can handle both query rows and accepted build jobs. Freshness checks skip journal entries for new messages instead of scanning the whole unpublished journal. Adding labels to those messages stays incremental and preserves existing message files. Source-attribution edits refresh cached message owners. The upgrade marks existing caches for one full rebuild to repair edits the old trigger missed. Builds on merged #914 and the cache refinements in #916. Raw SQL uses DuckDB for SQLite archives; PostgreSQL behavior is unchanged. Refs #912. Co-authored-by: Rusty Shackleford --- cmd/msgvault/cmd/build_cache.go | 7 +- cmd/msgvault/cmd/build_cache_identity_test.go | 92 +++++++++++++++++++ cmd/msgvault/cmd/cache_staleness.go | 47 ++++++---- cmd/msgvault/cmd/cache_staleness_test.go | 91 ++++++++++++++++++ cmd/msgvault/cmd/serve.go | 28 +++++- cmd/msgvault/cmd/serve_test.go | 89 +++++++++++++++++- cmd/msgvault/cmd/sync_cache_refresh.go | 27 ++---- cmd/msgvault/cmd/sync_cache_refresh_test.go | 65 +++++++++++++ docs/api-server.md | 3 + internal/query/cache_state.go | 29 ++++++ internal/query/cache_state_test.go | 21 +++++ .../store/migrate_cache_source_attribution.go | 40 ++++++++ internal/store/migrate_recipient_envelope.go | 17 ++-- .../store/migrate_recipient_envelope_test.go | 11 +++ internal/store/schema.sql | 38 +------- internal/store/store.go | 8 ++ internal/store/subset_test.go | 20 ++-- pkg/client/client.go | 38 ++++++++ pkg/client/client_test.go | 52 +++++++++++ 19 files changed, 623 insertions(+), 100 deletions(-) create mode 100644 internal/store/migrate_cache_source_attribution.go diff --git a/cmd/msgvault/cmd/build_cache.go b/cmd/msgvault/cmd/build_cache.go index a6a06643a..44687dabd 100644 --- a/cmd/msgvault/cmd/build_cache.go +++ b/cmd/msgvault/cmd/build_cache.go @@ -948,7 +948,12 @@ func derivedDriftOnly(staleness cacheStaleness) bool { func relatedDriftOnly(staleness cacheStaleness) bool { return staleness.HasUsablePublication && staleness.HasRelatedRowDrift && - staleness.Reason == "related rows changed" + !staleness.FullRebuild && !staleness.HasNew && !staleness.HasDeleted && + !staleness.HasUpdated && !staleness.HasDerivedDataDrift && + !staleness.HasIdentityDrift && !staleness.HasAccountIdentityDrift && + !staleness.HasConversationParticipantDrift && !staleness.HasConversationTypeDrift && + !staleness.HasParticipantIdentifierDrift && !staleness.HasParticipantDisplayNameDrift && + !staleness.HasPersonDisplayNameDrift } // refreshIdentityDatasetsOnly rebuilds every identity-derived dataset while diff --git a/cmd/msgvault/cmd/build_cache_identity_test.go b/cmd/msgvault/cmd/build_cache_identity_test.go index c5b7a25df..52e40c5bc 100644 --- a/cmd/msgvault/cmd/build_cache_identity_test.go +++ b/cmd/msgvault/cmd/build_cache_identity_test.go @@ -830,3 +830,95 @@ func TestBuildCache_OwnerParticipantsHonorPrimaryEmailGuard(t *testing.T) { assert.Equal(contactID, *counterpartsByTitle["Source-native sender before owner envelope"], "the matching envelope participant remains a counterpart for source-native attribution") } + +func TestSourceAttributionChangeRebuildsCachedOwner(t *testing.T) { + for _, upgrade := range []bool{false, true} { + name := "fresh" + if upgrade { + name = "upgrade" + } + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dbPath := filepath.Join(t.TempDir(), "msgvault.db") + analyticsDir := filepath.Join(t.TempDir(), "analytics") + st, err := store.OpenForTest(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("test", "synthetic@example.test") + require.NoError(err) + conv, err := st.EnsureConversationWithType(src.ID, "thread", "email_thread", "Synthetic") + require.NoError(err) + sender, err := st.EnsureParticipant("sender@example.test", "Sender", "example.test") + require.NoError(err) + envelope, err := st.EnsureParticipant("envelope@example.test", "Envelope", "example.test") + require.NoError(err) + require.NoError(st.AddAccountIdentity(src.ID, "sender@example.test", "manual")) + require.NoError(st.AddAccountIdentity(src.ID, "envelope@example.test", "manual")) + msg := &store.Message{ConversationID: conv, SourceID: src.ID, + SourceMessageID: "message", MessageType: "email", + SenderID: sql.NullInt64{Int64: sender, Valid: true}, + SentAt: sql.NullTime{Time: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC), Valid: true}} + id, err := st.PersistMessage(&store.MessagePersistData{Message: msg, + Recipients: []store.RecipientSet{{Type: "from", ParticipantIDs: []int64{envelope}, + DisplayNames: []string{"Envelope"}, EmailAddresses: []string{"envelope@example.test"}}}}) + require.NoError(err) + if upgrade { + _, err = st.DB().Exec(` + DROP TRIGGER trg_cache_message_facts_update; + CREATE TRIGGER trg_cache_message_facts_update + AFTER UPDATE OF sender_id, is_from_me, has_attachments, attachment_count + ON messages FOR EACH ROW + WHEN OLD.sender_id IS NOT NEW.sender_id OR OLD.is_from_me IS NOT NEW.is_from_me + OR OLD.has_attachments IS NOT NEW.has_attachments + OR OLD.attachment_count IS NOT NEW.attachment_count BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_facts', NEW.id); + END; + DELETE FROM applied_migrations WHERE name = 'cache_message_source_attribution'; + `) + require.NoError(err) + } + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + db, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + var cachedOwner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&cachedOwner)) + require.Equal(envelope, cachedOwner) + msg.IsFromMe = true + _, err = st.UpsertMessage(msg) + require.NoError(err) + var sourceNative, effective bool + require.NoError(st.DB().QueryRow(`SELECT source_is_from_me, is_from_me FROM messages WHERE id = ?`, id).Scan(&sourceNative, &effective)) + require.True(sourceNative) + require.True(effective) + if upgrade { + require.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild, + "legacy trigger misses source-only attribution changes") + require.NoError(st.InitSchema()) + } + stale := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(stale.NeedsBuild) + assert.True(stale.FullRebuild) + automatic, err := buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.False(automatic.Skipped) + var refreshedOwner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&refreshedOwner)) + assert.Equal(sender, refreshedOwner) + require.NoError(st.InitSchema()) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild, + "reopening an upgraded archive must not invalidate its cache again") + msg.IsFromMe = false + _, err = st.UpsertMessage(msg) + require.NoError(err) + assert.True(cacheNeedsBuild(dbPath, analyticsDir).FullRebuild, + "the installed trigger must also capture later source attribution edits") + }) + } +} diff --git a/cmd/msgvault/cmd/cache_staleness.go b/cmd/msgvault/cmd/cache_staleness.go index 02ce7bac5..bc8b06b27 100644 --- a/cmd/msgvault/cmd/cache_staleness.go +++ b/cmd/msgvault/cmd/cache_staleness.go @@ -91,6 +91,14 @@ func deletedSinceBuildCountSQL() string { AND ` + sentCacheExportMessageWhere("") } +// Bound freshness work by the published message IDs, not by the number of +// unpublished rows a sync has added. SQLite otherwise prefers the seq key. +func coveredRelatedChangesSQL() string { + return `SELECT COUNT(*) > 0, COALESCE(MAX(dataset = 'message_facts'), 0) + FROM cache_related_change_journal INDEXED BY idx_cache_related_change_message + WHERE seq > ? AND message_id <= ?` +} + // hiddenSinceBuildCountSQL counts exportable messages dedup-hidden since the // last cache build. Same cold-start constraint as deletedSinceBuildCountSQL: // it must be served by idx_messages_deleted_at. @@ -145,7 +153,7 @@ func cacheStalenessFailure(ctx context.Context, reason string) cacheStaleness { // for a builder that is staging the next generation. The shared lock excludes // only the brief publication step and destructive cache maintenance. func cacheNeedsBuildForQuery(ctx context.Context, dbPath, analyticsDir string) (cacheStaleness, error) { - return inspectCacheForQuery(ctx, dbPath, analyticsDir, true) + return inspectCacheForQuery(ctx, dbPath, analyticsDir, true, false) } // cacheNeedsBuildForServing omits the two archive-wide conversation hashes. @@ -153,10 +161,10 @@ func cacheNeedsBuildForQuery(ctx context.Context, dbPath, analyticsDir string) ( // rebuild interval expires; requests still see indexed sync and revision // signals immediately, without scanning millions of membership rows. func cacheNeedsBuildForServing(ctx context.Context, dbPath, analyticsDir string) (cacheStaleness, error) { - return inspectCacheForQuery(ctx, dbPath, analyticsDir, false) + return inspectCacheForQuery(ctx, dbPath, analyticsDir, false, true) } -func inspectCacheForQuery(ctx context.Context, dbPath, analyticsDir string, full bool) (cacheStaleness, error) { +func inspectCacheForQuery(ctx context.Context, dbPath, analyticsDir string, full, markerOnly bool) (cacheStaleness, error) { if store.IsPostgresURL(dbPath) { return cacheStaleness{}, nil } @@ -165,7 +173,7 @@ func inspectCacheForQuery(ctx context.Context, dbPath, analyticsDir string, full return cacheStaleness{}, err } defer release() - result := cacheNeedsBuildLockedWithConversationHashes(ctx, dbPath, analyticsDir, full) + result := cacheNeedsBuildLockedWithOptions(ctx, dbPath, analyticsDir, full, markerOnly) return result, ctx.Err() } @@ -179,10 +187,18 @@ func cacheNeedsBuildLocked(ctx context.Context, dbPath, analyticsDir string) cac } func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, analyticsDir string, full bool) cacheStaleness { + return cacheNeedsBuildLockedWithOptions(ctx, dbPath, analyticsDir, full, false) +} + +func cacheNeedsBuildLockedWithOptions(ctx context.Context, dbPath, analyticsDir string, full, markerOnly bool) cacheStaleness { if ctx.Err() != nil { return cacheStaleness{} } - readiness, err := query.InspectCacheReadiness(analyticsDir) + inspect := query.InspectCacheReadiness + if markerOnly { + inspect = query.InspectCacheMarkerReadiness + } + readiness, err := inspect(analyticsDir) if ctx.Err() != nil { return cacheStaleness{} } @@ -392,7 +408,7 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an return cacheStalenessFailure(ctx, "cannot inspect related-change journal") } if hasRelatedChangeJournal == 0 && state.LastRelatedChangeSeq != 0 { - return cacheStaleness{NeedsBuild: true, FullRebuild: true, Reason: "related-change journal is missing"} + return cacheStalenessFailure(ctx, "related-change journal is missing") } if hasRelatedChangeJournal > 0 { var latestSeq int64 @@ -407,11 +423,9 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an result.FullRebuild = true reasons = append(reasons, "related-change journal moved backwards") } else if latestSeq > state.LastRelatedChangeSeq { - var coveredChanges bool - err = db.DB().QueryRowContext(ctx, ` - SELECT EXISTS(SELECT 1 FROM cache_related_change_journal - WHERE seq > ? AND message_id <= ?) - `, state.LastRelatedChangeSeq, state.LastMessageID).Scan(&coveredChanges) + var coveredChanges, messageFactsChanged bool + err = db.DB().QueryRowContext(ctx, coveredRelatedChangesSQL(), + state.LastRelatedChangeSeq, state.LastMessageID).Scan(&coveredChanges, &messageFactsChanged) if err != nil { return cacheStalenessFailure(ctx, "cannot inspect related-row changes") } @@ -419,14 +433,6 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an result.HasRelatedRowDrift = true reasons = append(reasons, "related rows changed") } - var messageFactsChanged bool - err = db.DB().QueryRowContext(ctx, ` - SELECT EXISTS(SELECT 1 FROM cache_related_change_journal - WHERE seq > ? AND message_id <= ? AND dataset = 'message_facts') - `, state.LastRelatedChangeSeq, state.LastMessageID).Scan(&messageFactsChanged) - if err != nil { - return cacheStalenessFailure(ctx, "cannot inspect message fact changes") - } if messageFactsChanged { result.HasDerivedDataDrift = true result.FullRebuild = true @@ -448,7 +454,8 @@ func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, an if relatedErr != nil { return cacheStalenessFailure(ctx, "cannot classify derived-data revision") } - if !relatedOnly || !result.HasRelatedRowDrift { + // Appends already export related rows above the published message ID. + if !relatedOnly { result.HasDerivedDataDrift = true result.FullRebuild = true reasons = append(reasons, "derived message data changed") diff --git a/cmd/msgvault/cmd/cache_staleness_test.go b/cmd/msgvault/cmd/cache_staleness_test.go index dee672083..65cf4f77f 100644 --- a/cmd/msgvault/cmd/cache_staleness_test.go +++ b/cmd/msgvault/cmd/cache_staleness_test.go @@ -545,3 +545,94 @@ func TestCacheStalenessCounts_UseDeletionIndexes(t *testing.T) { assert.Equal(int64(1), deleted) assert.Equal(int64(1), hidden) } + +func TestCoveredRelatedChangesUseMessageBoundary(t *testing.T) { + s := testutil.NewSQLiteTestStore(t) + _, err := s.DB().Exec(`DELETE FROM cache_related_change_journal; + INSERT INTO cache_related_change_journal (seq, dataset, message_id) + VALUES (1, 'message_facts', 1), (2, 'labels', 0), + (3, 'message_labels', 100), (4, 'message_facts', 50), (5, 'message_facts', 101)`) + require.NoError(t, err) + plan := explainQueryPlan(t, s, coveredRelatedChangesSQL(), 1, 100) + assert.Contains(t, plan, "SEARCH cache_related_change_journal USING INDEX idx_cache_related_change_message (message_id OLD.message_id; END`, - `CREATE TRIGGER trg_cache_recipients_delete + `CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_delete AFTER DELETE ON message_recipients FOR EACH ROW BEGIN INSERT INTO cache_related_change_journal (dataset, message_id) VALUES (CASE WHEN OLD.recipient_type = 'from' THEN 'message_facts' @@ -131,7 +132,7 @@ func restoreRecipientCacheJournalTriggers(q querier) error { END`, } { if _, err := q.Exec(stmt); err != nil { - return fmt.Errorf("restore recipient cache journal trigger: %w", err) + return fmt.Errorf("ensure recipient cache journal trigger: %w", err) } } return nil diff --git a/internal/store/migrate_recipient_envelope_test.go b/internal/store/migrate_recipient_envelope_test.go index dbb565fb1..d733928bb 100644 --- a/internal/store/migrate_recipient_envelope_test.go +++ b/internal/store/migrate_recipient_envelope_test.go @@ -162,6 +162,17 @@ func TestEnsureRecipientEnvelopeUniqueIndex_LegacyTableRebuild(t *testing.T) { `SELECT COUNT(*) FROM message_recipients WHERE message_id = ?`, msgID, ).Scan(&rowCount), "count recipient rows") assert.Equal(2, rowCount, "no-op rerun must not change row count") + var factsBaseline int + require.NoError(st.db.QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal + WHERE dataset = 'message_facts' AND message_id = ?`, msgID).Scan(&factsBaseline)) + _, err = st.db.Exec(`INSERT INTO message_recipients + (message_id, participant_id, recipient_type, email_address) + VALUES (?, ?, 'from', 'primary@example.test')`, msgID, participantID) + require.NoError(err, "insert From row after legacy recipient table rebuild") + var factsCount int + require.NoError(st.db.QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal + WHERE dataset = 'message_facts' AND message_id = ?`, msgID).Scan(&factsCount)) + assert.Equal(factsBaseline+1, factsCount, "From-facts trigger must survive legacy table swap") } func TestInitSchema_RepairsDanglingLegacyRecipients(t *testing.T) { diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 81743d54a..208d65d3f 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -1624,6 +1624,8 @@ CREATE TABLE IF NOT EXISTS cache_related_change_journal ( dataset TEXT NOT NULL, message_id INTEGER NOT NULL ); +CREATE INDEX IF NOT EXISTS idx_cache_related_change_message + ON cache_related_change_journal(message_id); -- Revisions caused solely by related-row edits can be repaired from the -- child-row journal. Every other derived revision still requires a full build. @@ -1647,42 +1649,6 @@ AFTER DELETE ON labels FOR EACH ROW BEGIN INSERT INTO cache_related_change_journal (dataset, message_id) VALUES ('labels', 0); END; --- Child edits that also change baked message facts require a message rebuild. -CREATE TRIGGER IF NOT EXISTS trg_cache_message_facts_update -AFTER UPDATE OF sender_id, is_from_me, - has_attachments, attachment_count ON messages FOR EACH ROW -WHEN OLD.sender_id IS NOT NEW.sender_id OR OLD.is_from_me IS NOT NEW.is_from_me - OR OLD.has_attachments IS NOT NEW.has_attachments - OR OLD.attachment_count IS NOT NEW.attachment_count BEGIN - INSERT INTO cache_related_change_journal (dataset, message_id) - VALUES ('message_facts', NEW.id); -END; - --- From rows determine owner_participant_id in message shards even when --- replacing them leaves messages.is_from_me unchanged. -CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_insert -AFTER INSERT ON message_recipients FOR EACH ROW BEGIN - INSERT INTO cache_related_change_journal (dataset, message_id) - VALUES (CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' - ELSE 'message_recipients' END, NEW.message_id); -END; -CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_update -AFTER UPDATE ON message_recipients FOR EACH ROW BEGIN - INSERT INTO cache_related_change_journal (dataset, message_id) - VALUES (CASE WHEN OLD.recipient_type = 'from' OR NEW.recipient_type = 'from' - THEN 'message_facts' ELSE 'message_recipients' END, OLD.message_id); - INSERT INTO cache_related_change_journal (dataset, message_id) - SELECT CASE WHEN NEW.recipient_type = 'from' THEN 'message_facts' - ELSE 'message_recipients' END, NEW.message_id - WHERE NEW.message_id <> OLD.message_id; -END; -CREATE TRIGGER IF NOT EXISTS trg_cache_recipients_delete -AFTER DELETE ON message_recipients FOR EACH ROW BEGIN - INSERT INTO cache_related_change_journal (dataset, message_id) - VALUES (CASE WHEN OLD.recipient_type = 'from' THEN 'message_facts' - ELSE 'message_recipients' END, OLD.message_id); -END; - CREATE TRIGGER IF NOT EXISTS trg_cache_labels_insert AFTER INSERT ON message_labels FOR EACH ROW BEGIN INSERT INTO cache_related_change_journal (dataset, message_id) diff --git a/internal/store/store.go b/internal/store/store.go index a62b14a39..fbc1cc3cd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1380,6 +1380,11 @@ func (s *Store) InitSchemaContext(ctx context.Context) error { return fmt.Errorf("execute %s: %w", filename, err) } } + if !s.IsPostgreSQL() { + if err := ensureRecipientCacheJournalTriggers(boundQuerier{ctx: ctx, q: s.db}); err != nil { + return err + } + } if freshPostgreSQLSchema { if _, err := s.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS `+ rfc822CanonicalIndexName+` `+s.dialect.RFC822CanonicalIDIndexDefinition()); err != nil { @@ -1533,6 +1538,9 @@ func (s *Store) InitSchemaContext(ctx context.Context) error { lastModifiedColumnAdded = true } } + if err := s.ensureCacheSourceAttribution(ctx); err != nil { + return err + } // Older runs predate typed checkpoints. Restore types only when the source // or pinned Gmail handoff cursor identifies them unambiguously, then tag // unfinished Gmail recovery runs for the strict resume matcher. diff --git a/internal/store/subset_test.go b/internal/store/subset_test.go index 8ef3c36dd..6ad1ed4fc 100644 --- a/internal/store/subset_test.go +++ b/internal/store/subset_test.go @@ -1534,16 +1534,17 @@ func TestCopySubset_UpgradedMessageColumnOrder(t *testing.T) { st, err := Open(srcDB) require.NoError(err, "open source for upgrade") - // A pre-attribution archive also predates the activity queue trigger, which - // names source_is_from_me and would otherwise block the DROP COLUMN; the - // upgrade below reinstalls it through its own migration. + // A pre-attribution archive predates both triggers that name source_is_from_me. + // The upgrade below reinstalls them through their migrations. _, err = st.DB().Exec(` DROP TRIGGER trg_activity_queue_messages_update; + DROP TRIGGER trg_cache_message_facts_update; ALTER TABLE messages DROP COLUMN identity_is_from_me; ALTER TABLE messages DROP COLUMN source_is_from_me; DELETE FROM applied_migrations WHERE name IN ('message_attribution_provenance_v3', - 'activity_projection_triggers_v4'); + 'activity_projection_triggers_v4', + 'cache_message_source_attribution'); `) require.NoError(err, "simulate pre-attribution schema") require.NoError(st.InitSchema(), "upgrade source schema") @@ -3471,10 +3472,8 @@ func TestCopySubset_ControlCharInPath(t *testing.T) { // update messages, which resolves to main.messages, so the rename back — ALTER // TABLE ... RENAME TO messages — fails its schema reparse with "error in // trigger trg_message_bodies_last_modified_upd: no such table: main.messages". -// Neither attribution column is indexed, and the only trigger naming one -// (trg_activity_queue_messages_update, on source_is_from_me) postdates such -// archives, so it is dropped first and ALTER TABLE ... DROP COLUMN works -// directly. +// Neither attribution column is indexed. The activity and cache triggers that +// name source_is_from_me postdate such archives, so drop them before the column. func TestCopySubset_LegacySourceMissingAttributionColumns(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -3486,8 +3485,9 @@ func TestCopySubset_LegacySourceMissingAttributionColumns(t *testing.T) { db, err := sql.Open("sqlite3", srcDB+"?_foreign_keys=OFF") require.NoError(err, "open source db") - _, err = db.Exec(`DROP TRIGGER trg_activity_queue_messages_update`) - require.NoError(err, "drop the activity trigger that names source_is_from_me") + _, err = db.Exec(`DROP TRIGGER trg_activity_queue_messages_update; + DROP TRIGGER trg_cache_message_facts_update`) + require.NoError(err, "drop triggers that name source_is_from_me") for _, col := range []string{"source_is_from_me", "identity_is_from_me"} { _, err = db.Exec( diff --git a/pkg/client/client.go b/pkg/client/client.go index b01c94927..d02d7bd72 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -198,6 +198,44 @@ func (c *Client) APIClient() runtime.APIClient { return c.apiClient } +// RunQueryOutcome contains rows for an immediate result or a cache-build job +// when the query is accepted for asynchronous refresh. Exactly one field is set. +type RunQueryOutcome struct { + Result *generated.RunQueryResponse + Accepted *generated.CacheBuildAccepted +} + +// RunQueryWithAccepted handles both successful query statuses. The generated +// RunQuery convenience method returns only rows and treats a valid 202 as an +// error; callers that may need a cache build should use this method instead. +func (c *Client) RunQueryWithAccepted( + ctx context.Context, + options *generated.RunQueryRequestOptions, + reqEditors ...runtime.RequestEditorFn, +) (*RunQueryOutcome, error) { + resp, err := c.RunQueryWithResponse(ctx, options, reqEditors...) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errors.New("query: missing response") + } + switch resp.StatusCode { + case http.StatusOK: + if resp.JSON200 == nil { + return nil, errors.New("query: missing result") + } + return &RunQueryOutcome{Result: resp.JSON200}, nil + case http.StatusAccepted: + if resp.JSON202 == nil || resp.JSON202.JobID == "" { + return nil, errors.New("query accepted without job_id") + } + return &RunQueryOutcome{Accepted: resp.JSON202}, nil + default: + return nil, fmt.Errorf("query: unexpected status %d", resp.StatusCode) + } +} + // AddAccount accepts both documented success statuses. The generated // convenience method treats only 201 as success even though the daemon returns // 200 when the account already exists. diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 24b2bc7c2..863448505 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -1159,6 +1159,58 @@ func TestRunQueryDecodesScalarCells(t *testing.T) { assert.InDelta(1.0, numberCell, 0, "number cell") assert.Equal("x", got.Rows[0][1], "string cell") assert.Equal(true, got.Rows[0][2], "bool cell") + + outcome, err := c.RunQueryWithAccepted(t.Context(), &generated.RunQueryRequestOptions{ + Body: &generated.RunQueryBody{SQL: "SELECT 1"}, + }) + require.NoError(err, "RunQueryWithAccepted") + require.NotNil(outcome) + assert.Nil(outcome.Accepted) + require.NotNil(outcome.Result) + assert.Equal([]string{"n", "s", "b"}, outcome.Result.Columns) + assert.Equal(int64(1), outcome.Result.RowCount) +} + +func TestRunQueryWithAcceptedReturnsCacheBuildJob(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertions.Equal(http.MethodPost, r.Method) + assertions.Equal("/api/v1/query", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _, err := w.Write([]byte(`{"status":"queued","job_id":"job-123"}`)) + assertions.NoError(err) + })) + t.Cleanup(server.Close) + + c, err := New(server.URL) + requirements.NoError(err) + outcome, err := c.RunQueryWithAccepted(t.Context(), &generated.RunQueryRequestOptions{ + Body: &generated.RunQueryBody{SQL: "SELECT count(*) FROM messages"}, + }) + requirements.NoError(err) + requirements.NotNil(outcome) + assertions.Nil(outcome.Result) + requirements.NotNil(outcome.Accepted) + assertions.Equal("queued", outcome.Accepted.Status) + assertions.Equal("job-123", outcome.Accepted.JobID) +} + +func TestRunQueryWithAcceptedRejectsMissingJob(t *testing.T) { + requirements := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + })) + t.Cleanup(server.Close) + c, err := New(server.URL) + requirements.NoError(err) + outcome, err := c.RunQueryWithAccepted(t.Context(), &generated.RunQueryRequestOptions{ + Body: &generated.RunQueryBody{SQL: "SELECT 1"}, + }) + requirements.ErrorContains(err, "job_id") + requirements.Nil(outcome) } func TestGetMessageRendersLargeIDInPath(t *testing.T) { From 2bd64682946ba7f6452e6fbfacdd353ccee1bf09 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 28 Sep 2026 20:19:03 -0500 Subject: [PATCH 078/125] fix(store): sample SQLite planner maintenance on close (#986) Closing a short-lived SQLite store can start a full index analysis on a fresh connection and exhaust the one-second maintenance budget. Enable SQLite's built-in sampling limit for close-time optimization while retaining the deadline and single-connection shutdown behavior. The regression test uses fresh connections and verifies that close persists sampled planner statistics without waiting for the entire connection pool. Co-authored-by: Wes McKinney --- internal/store/sqlite_optimize_test.go | 18 +++++++++++++++++- internal/store/store.go | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/store/sqlite_optimize_test.go b/internal/store/sqlite_optimize_test.go index 1d6b92bef..1469f78c7 100644 --- a/internal/store/sqlite_optimize_test.go +++ b/internal/store/sqlite_optimize_test.go @@ -481,8 +481,12 @@ func TestCloseOptimizesWithoutDrainingSQLitePool(t *testing.T) { // Startup maintenance is best-effort and may leave no statistics table. _, err = s.db.ExecContext(t.Context(), "DROP TABLE IF EXISTS sqlite_stat1") require.NoError(err) - seedLiveMessages(t, s, 100) + const messageCount = 10_000 + seedLiveMessages(t, s, messageCount) assert.Zero(messagePlannerStatisticCount(t, s)) + // Short-lived stores can close without ever running InitSchema. Discard + // its connections so close cannot inherit their analysis_limit setting. + s.db.SetMaxIdleConns(0) s.db.SetMaxOpenConns(2) s.db.SetMaxIdleConns(2) blocker, err := s.db.Conn(t.Context()) @@ -505,6 +509,18 @@ func TestCloseOptimizesWithoutDrainingSQLitePool(t *testing.T) { defer func() { _ = reopened.Close() }() assert.Positive(messagePlannerStatisticCount(t, reopened), "store close must persist planner statistics") + var sourceStats string + require.NoError(reopened.db.QueryRow( + `SELECT stat FROM sqlite_stat1 WHERE idx = 'idx_messages_source'`, + ).Scan(&sourceStats)) + fields := strings.Fields(sourceStats) + require.Len(fields, 2) + rowsPerSource, err := strconv.Atoi(fields[1]) + require.NoError(err) + // Every message has the same source. Full analysis counts them all; + // sampled analysis stops counting this group at its sample limit. + assert.Less(rowsPerSource, messageCount, + "store close must sample indexes even on a fresh connection") } func TestSuccessfulSyncOptimizeThrottled(t *testing.T) { diff --git a/internal/store/store.go b/internal/store/store.go index fbc1cc3cd..a1d0dd91f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -566,7 +566,8 @@ func (s *Store) Close() error { // that may still have a checked-out connection during shutdown. ctx, cancel := context.WithTimeout(context.Background(), sqliteOptimizeTimeout) // Log maintenance errors here so expected deadlines do not also emit SQL warnings. - _, err := s.db.DB.ExecContext(ctx, "PRAGMA optimize=0x10002") + // Include 0x10 to bound ANALYZE even on a fresh connection without an analysis_limit. + _, err := s.db.DB.ExecContext(ctx, "PRAGMA optimize=0x10012") logSQLiteOptimizeError("store close", err) cancel() } From 5d1ce39233eba06c1d93bd75b736ee159d077228 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 28 Sep 2026 22:19:55 -0400 Subject: [PATCH 079/125] feat(msmail): sync Microsoft 365 mail through Microsoft Graph (#944) Closes #943 `add-o365 --graph` adds Microsoft 365 and Outlook.com mailboxes through the Microsoft Graph mail API when IMAP is unavailable. It requires the delegated `Mail.Read` permission. Each folder becomes a label; later syncs apply changed messages, moves, and deletions. The CLI and daemon scheduler both support these accounts. Failed message downloads are counted and saved for retry while other messages and folders continue syncing. MIME downloads allow ten minutes per attempt. Delta updates refresh full messages to preserve edits to received mail. Message downloads remain buffered in memory without a size cap. Graph mail uses the `msmail` account type and separate `msmail_.json` tokens. Teams and mail share the Graph transport, including retries for temporary connection failures and incomplete responses. Existing Teams tokens remain valid. A Graph account is separate from an IMAP account for the same mailbox. If both are synced, `msgvault dedup --collection` hides the extra copies. Co-authored-by: Mike Campbell --- cmd/msgvault/cmd/addo365.go | 115 ++- cmd/msgvault/cmd/backfill_teams_media.go | 2 +- cmd/msgvault/cmd/constants.go | 3 + cmd/msgvault/cmd/remove_account.go | 6 + cmd/msgvault/cmd/serve.go | 12 +- cmd/msgvault/cmd/sync.go | 33 +- cmd/msgvault/cmd/sync_msmail.go | 67 ++ cmd/msgvault/cmd/sync_teams.go | 2 +- cmd/msgvault/cmd/sync_test.go | 20 + cmd/msgvault/cmd/syncfull.go | 5 + docs/changelog.md | 4 + docs/cli-reference.md | 5 +- docs/guides/oauth-setup.md | 21 + internal/api/scheduler_jobs.go | 2 +- internal/dedup/dedup.go | 2 +- internal/microsoft/graph_oauth.go | 58 +- internal/microsoft/graph_oauth_test.go | 25 +- internal/msgraph/client.go | 281 +++++ internal/msgraph/client_test.go | 167 +++ internal/msmail/client.go | 120 +++ internal/msmail/importer.go | 610 +++++++++++ internal/msmail/importer_test.go | 968 ++++++++++++++++++ internal/store/attachment_roles.go | 48 + internal/store/docbank_media_test.go | 4 + internal/store/messages.go | 32 + internal/store/migrate_legacy_identity.go | 2 +- .../store/migrate_legacy_identity_test.go | 1 + internal/teams/client.go | 225 +--- internal/teams/client_test.go | 26 +- internal/teams/importer.go | 5 +- internal/teams/types.go | 8 - 31 files changed, 2587 insertions(+), 292 deletions(-) create mode 100644 cmd/msgvault/cmd/sync_msmail.go create mode 100644 internal/msgraph/client.go create mode 100644 internal/msgraph/client_test.go create mode 100644 internal/msmail/client.go create mode 100644 internal/msmail/importer.go create mode 100644 internal/msmail/importer_test.go diff --git a/cmd/msgvault/cmd/addo365.go b/cmd/msgvault/cmd/addo365.go index 02929dd10..d6295fd91 100644 --- a/cmd/msgvault/cmd/addo365.go +++ b/cmd/msgvault/cmd/addo365.go @@ -15,6 +15,7 @@ var ( o365Headless bool o365TenantID string noDefaultIdentityAddO365 bool + o365Graph bool ) func newAddO365Cmd() *cobra.Command { @@ -40,7 +41,6 @@ func preflightAddO365Authorize(cmd *cobra.Command, email string) error { return errors.New("configuration is unavailable") } cfg := state.cfg - logger := state.logger if IsRemoteMode(state) { // Tokens live on the remote host; authorization must happen there. return nil @@ -48,19 +48,8 @@ func preflightAddO365Authorize(cmd *cobra.Command, email string) error { if err := requireMicrosoftOAuthConfig(cfg); err != nil { return err } - msMgr := microsoft.NewManager( - cfg.Microsoft.ClientID, - microsoftTenantID(o365TenantID, cfg), - cfg.Microsoft.EffectiveRedirectURI(), - cfg.TokensDir(), - logger, - ) - if o365Headless { - msMgr.UseDeviceCode() - } - fmt.Printf("Authorizing %s with Microsoft...\n", email) - if err := msMgr.Authorize(cmd.Context(), email); err != nil { - return fmt.Errorf("authorization failed: %w", err) + if err := authorizeO365(cmd, email); err != nil { + return err } if err := cmd.Flags().Set(oauthPreflightedFlag, "true"); err != nil { return fmt.Errorf("set --%s after authorization: %w", oauthPreflightedFlag, err) @@ -81,10 +70,17 @@ to outlook.office365.com automatically using the XOAUTH2 SASL mechanism. Requires a [microsoft] section in config.toml with your Azure AD app's client_id. See the docs for Azure AD app registration setup. +With --graph, the account syncs through the Microsoft Graph mail API instead +of IMAP. Use it when IMAP is turned off for the mailbox. It needs the Mail.Read +permission on the app registration. A Graph account is a separate account: if +the mailbox is also synced over IMAP, the vault holds two copies, and +'msgvault dedup --collection' hides the extra ones. + Examples: msgvault add-o365 user@outlook.com msgvault add-o365 user@outlook.com --headless - msgvault add-o365 user@company.com --tenant my-tenant-id`, + msgvault add-o365 user@company.com --tenant my-tenant-id + msgvault add-o365 user@company.com --graph`, Args: cobra.ExactArgs(1), RunE: runAddO365Local, } @@ -93,6 +89,7 @@ Examples: cmd.Flags().BoolVar(&noDefaultIdentityAddO365, "no-default-identity", false, noDefaultIdentityHelp) cmd.Flags().BoolVar(&o365Headless, "headless", false, "Sign in with a device code instead of a local browser") + cmd.Flags().BoolVar(&o365Graph, "graph", false, "sync through the Microsoft Graph mail API instead of IMAP") registerOAuthPreflightedFlag(cmd) return cmd } @@ -109,6 +106,9 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { if err := requireMicrosoftOAuthConfig(cfg); err != nil { return err } + if o365Graph { + return runAddO365GraphLocal(cmd, email) + } msMgr := microsoft.NewManager( cfg.Microsoft.ClientID, @@ -117,18 +117,14 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { cfg.TokensDir(), logger, ) - if o365Headless { - msMgr.UseDeviceCode() - } preflighted, err := oauthPreflighted(cmd) if err != nil { return err } if !preflighted { - fmt.Printf("Authorizing %s with Microsoft...\n", email) - if err := msMgr.Authorize(cmd.Context(), email); err != nil { - return fmt.Errorf("authorization failed: %w", err) + if err := authorizeO365(cmd, email); err != nil { + return err } } @@ -215,6 +211,83 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { return nil } +// authorizeO365 runs the Microsoft sign-in for the account kind: Graph mail +// with --graph, IMAP otherwise. +func authorizeO365(cmd *cobra.Command, email string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + tenant := microsoftTenantID(o365TenantID, cfg) + redirect := cfg.Microsoft.EffectiveRedirectURI() + fmt.Printf("Authorizing %s with Microsoft...\n", email) + var err error + if o365Graph { + mgr := microsoft.NewGraphMailManager(cfg.Microsoft.ClientID, tenant, redirect, cfg.TokensDir(), logger) + if o365Headless { + mgr.UseDeviceCode() + } + err = mgr.Authorize(cmd.Context(), email) + } else { + mgr := microsoft.NewManager(cfg.Microsoft.ClientID, tenant, redirect, cfg.TokensDir(), logger) + if o365Headless { + mgr.UseDeviceCode() + } + err = mgr.Authorize(cmd.Context(), email) + } + if err != nil { + return fmt.Errorf("authorization failed: %w", err) + } + return nil +} + +// runAddO365GraphLocal creates an msmail source, the Graph mail counterpart of +// the IMAP source that add-o365 makes by default. +func runAddO365GraphLocal(cmd *cobra.Command, email string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + preflighted, err := oauthPreflighted(cmd) + if err != nil { + return err + } + if !preflighted { + if err := authorizeO365(cmd, email); err != nil { + return err + } + } + + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + + source, err := s.GetOrCreateSource(sourceTypeMSMail, email) + if err != nil { + return fmt.Errorf("create source: %w", err) + } + if err := s.UpdateSourceDisplayName(source.ID, email); err != nil { + return fmt.Errorf("set display name: %w", err) + } + if !noDefaultIdentityAddO365 { + confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) + } + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + + fmt.Printf("\nMicrosoft 365 account added for Graph mail sync!\n") + fmt.Printf(" Email: %s\n", email) + fmt.Println() + fmt.Println("You can now run:") + fmt.Printf(" msgvault sync %s\n", email) + return nil +} + // isMicrosoftIMAPSource returns true only if src is an IMAP source already // configured for Microsoft XOAUTH2 with the given username. This prevents // a non-Microsoft IMAP source (e.g. a password-auth source) that happens to diff --git a/cmd/msgvault/cmd/backfill_teams_media.go b/cmd/msgvault/cmd/backfill_teams_media.go index 4eae2d879..af735a93b 100644 --- a/cmd/msgvault/cmd/backfill_teams_media.go +++ b/cmd/msgvault/cmd/backfill_teams_media.go @@ -93,7 +93,7 @@ Examples: if qps <= 0 { qps = 5 } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) + client := teams.NewClient("https://graph.microsoft.com/v1.0", tokenFn, qps) imp := teams.NewImporter(s, client) _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Backfilling Teams inline media for %s\n\n", email) diff --git a/cmd/msgvault/cmd/constants.go b/cmd/msgvault/cmd/constants.go index 119a9fda7..b85a74b7b 100644 --- a/cmd/msgvault/cmd/constants.go +++ b/cmd/msgvault/cmd/constants.go @@ -1,5 +1,7 @@ package cmd +import "go.kenn.io/msgvault/internal/msmail" + // Source-type identifiers stored in sources.source_type and matched against // when dispatching sync/import logic per account kind. const ( @@ -7,6 +9,7 @@ const ( sourceTypeIMAP = "imap" sourceTypeMbox = "mbox" sourceTypeTeams = "teams" + sourceTypeMSMail = msmail.SourceType sourceTypeCalendar = "gcal" sourceTypeBeeper = "beeper" sourceTypeSlack = "slack" diff --git a/cmd/msgvault/cmd/remove_account.go b/cmd/msgvault/cmd/remove_account.go index f84f2060c..b52282b27 100644 --- a/cmd/msgvault/cmd/remove_account.go +++ b/cmd/msgvault/cmd/remove_account.go @@ -350,6 +350,12 @@ func runRemoveAccountLocal(cmd *cobra.Command, args []string) error { "Warning: could not remove Microsoft Graph token: %v\n", err, ) } + case sourceTypeMSMail: + if err := newGraphMailManager(state).DeleteToken(source.Identifier); err != nil { + fmt.Fprintf(os.Stderr, + "Warning: could not remove Microsoft Graph mail token: %v\n", err, + ) + } case sourceTypeDiscord: // Discord credential cleanup is part of the lifecycle-locked cascade // above so a concurrent guild registration cannot lose its bot token. diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index de5ec7ff0..1776f63db 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -3757,6 +3757,8 @@ func runScheduledSync(ctx context.Context, identifier string, s *store.Store, ge summary, err = runScheduledIMAPSync(ctx, src, s, state) case sourceTypeTeams: err = runScheduledTeamsSync(ctx, src, s, state) + case sourceTypeMSMail: + err = runScheduledMSMailSync(ctx, src, s, state) case sourceTypeDiscord: var discordSummary *discord.ImportSummary discordSummary, err = importDiscordSourceForScheduledRun( @@ -3841,7 +3843,7 @@ func logScheduledDiscordIssues(identifier string, summary *discord.ImportSummary // findScheduledSyncSources resolves ALL syncable source rows for a // scheduler identifier. Returns at most one row per syncable type -// (gmail, imap, teams, discord), in that stable order. Non-syncable types +// (gmail, imap, msmail, teams, discord), in that stable order. Non-syncable types // (mbox, apple-mail, etc.) are skipped. // // Returns an empty slice (not nil) when no syncable source matches — @@ -3858,7 +3860,7 @@ func findScheduledSyncSources(s *store.Store, identifier string) ([]*store.Sourc seen := make(map[string]*store.Source, 4) for _, src := range rows { switch src.SourceType { - case sourceTypeGmail, sourceTypeIMAP, sourceTypeTeams: + case sourceTypeGmail, sourceTypeIMAP, sourceTypeMSMail, sourceTypeTeams: if _, dup := seen[src.SourceType]; !dup { seen[src.SourceType] = src } @@ -3871,9 +3873,9 @@ func findScheduledSyncSources(s *store.Store, identifier string) ([]*store.Sourc } } - // Return in stable order: gmail, imap, teams, discord. + // Return in stable order: gmail, imap, msmail, teams, discord. var result []*store.Source - for _, t := range []string{sourceTypeGmail, sourceTypeIMAP, sourceTypeTeams, sourceTypeDiscord} { + for _, t := range []string{sourceTypeGmail, sourceTypeIMAP, sourceTypeMSMail, sourceTypeTeams, sourceTypeDiscord} { if src, ok := seen[t]; ok { result = append(result, src) } @@ -4088,7 +4090,7 @@ func runScheduledTeamsSync(ctx context.Context, src *store.Source, s *store.Stor if qps <= 0 { qps = 5 } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) + client := teams.NewClient("https://graph.microsoft.com/v1.0", tokenFn, qps) opts := scheduledTeamsImportOptions(email, cfg) _, err = teams.NewImporter(s, client).Import(ctx, opts) return err diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index ea6bb2b6d..d6d9b63ef 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -31,6 +31,10 @@ Requires a prior full sync to establish the history ID baseline. IMAP accounts use folder-based sync. Unchanged folders are skipped when UIDVALIDITY/UIDNEXT high water marks are available. +Microsoft Graph mail accounts (add-o365 --graph) use one delta cursor per +folder. The first sync downloads every folder; later syncs fetch only the +changes, including moves and deletes. + If no email is specified, syncs all accounts that have credentials configured. Accounts without tokens or history IDs are skipped. @@ -94,6 +98,7 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { } var gmailTargets []syncTarget var imapTargets []*store.Source + var msmailTargets []*store.Source var syncErrors []string if selectorSet { @@ -109,11 +114,13 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { gmailTargets = append(gmailTargets, syncTarget{source: src, email: src.Identifier}) case sourceTypeIMAP: imapTargets = append(imapTargets, src) + case sourceTypeMSMail: + msmailTargets = append(msmailTargets, src) } } - if len(gmailTargets) == 0 && len(imapTargets) == 0 { + if len(gmailTargets) == 0 && len(imapTargets) == 0 && len(msmailTargets) == 0 { if len(allMatches) > 0 { - return fmt.Errorf("%s exists but its source type cannot be synced (only gmail and imap are supported)", syncSelectorLabel(selector)) + return fmt.Errorf("%s exists but its source type cannot be synced (only gmail, imap and msmail are supported)", syncSelectorLabel(selector)) } if legacy { // Token not in DB — assume Gmail (legacy behaviour). @@ -165,11 +172,17 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { continue } imapTargets = append(imapTargets, src) + case sourceTypeMSMail: + if !newGraphMailManager(state).HasToken(src.Identifier) { + fmt.Printf("Skipping %s (no Microsoft Graph token - run 'add-o365 %s --graph' first)\n", src.Identifier, src.Identifier) + continue + } + msmailTargets = append(msmailTargets, src) default: continue } } - if len(gmailTargets) == 0 && len(imapTargets) == 0 { + if len(gmailTargets) == 0 && len(imapTargets) == 0 && len(msmailTargets) == 0 { if len(syncErrors) > 0 { // Surface the collected errors (e.g. broken OAuth config). return fmt.Errorf("%s", syncErrors[0]) @@ -189,6 +202,20 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { } } + // Sync Microsoft Graph mail sources. The first run walks every folder. + for _, src := range msmailTargets { + if ctx.Err() != nil { + break + } + fmt.Printf("Syncing Microsoft Graph mail for %s\n", src.Identifier) + sum, err := runMSMailSync(ctx, s, src.Identifier, func(line string) { fmt.Println(line) }, state) + if err != nil { + syncErrors = append(syncErrors, fmt.Sprintf("%s: %v", src.Identifier, err)) + continue + } + writeMSMailSyncSummary(os.Stdout, src.Identifier, sum) + } + // Sync Gmail sources via incremental sync. for _, target := range gmailTargets { if ctx.Err() != nil { diff --git a/cmd/msgvault/cmd/sync_msmail.go b/cmd/msgvault/cmd/sync_msmail.go new file mode 100644 index 000000000..8362f6528 --- /dev/null +++ b/cmd/msgvault/cmd/sync_msmail.go @@ -0,0 +1,67 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "time" + + "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/msmail" + "go.kenn.io/msgvault/internal/store" +) + +// msmailQPS is the Graph mail request rate. Microsoft documents 10,000 +// requests per 10 minutes per mailbox; 15 per second stays under it. +const msmailQPS = 15 + +func newGraphMailManager(state *invocation) *microsoft.GraphManager { + cfg := state.cfg + return microsoft.NewGraphMailManager( + cfg.Microsoft.ClientID, + cfg.Microsoft.EffectiveTenantID(), + cfg.Microsoft.EffectiveRedirectURI(), + cfg.TokensDir(), + state.logger, + ) +} + +// runMSMailSync syncs one Graph mail account. The first run downloads every +// folder; later runs fetch only the changes. +func runMSMailSync(ctx context.Context, s *store.Store, email string, progress func(string), state *invocation) (*msmail.Summary, error) { + cfg := state.cfg + tokenFn, err := newGraphMailManager(state).TokenSource(ctx, email) + if err != nil { + return nil, err + } + client := msmail.NewClient(msmail.GraphBaseURL, tokenFn, msmailQPS) + return msmail.Import(ctx, s, client, msmail.Options{ + Email: email, + AttachmentsDir: cfg.AttachmentsDir(), + Progress: progress, + }, state.logger) +} + +// runScheduledMSMailSync is the daemon path. Like runScheduledTeamsSync, it +// seeds the "me" identity and runs pending migrations before the sync. +func runScheduledMSMailSync(ctx context.Context, src *store.Source, s *store.Store, state *invocation) error { + confirmDefaultIdentity(io.Discard, s, src.ID, src.Identifier, src.Identifier, "account-identifier", state.logger) + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + _, err := runMSMailSync(ctx, s, src.Identifier, nil, state) + return err +} + +func writeMSMailSyncSummary(out io.Writer, email string, sum *msmail.Summary) { + _, _ = fmt.Fprintf(out, "\nMicrosoft Graph mail sync complete for %s\n", email) + _, _ = fmt.Fprintf(out, " Duration: %s\n", sum.Duration.Round(time.Second)) + _, _ = fmt.Fprintf(out, " Folders: %d\n", sum.Folders) + _, _ = fmt.Fprintf(out, " Messages added: %d\n", sum.Added) + _, _ = fmt.Fprintf(out, " Updated: %d\n", sum.Updated) + _, _ = fmt.Fprintf(out, " Moved: %d\n", sum.Moved) + _, _ = fmt.Fprintf(out, " Deleted: %d\n", sum.Deleted) + if sum.Errors > 0 { + _, _ = fmt.Fprintf(out, " Errors: %d\n", sum.Errors) + } +} diff --git a/cmd/msgvault/cmd/sync_teams.go b/cmd/msgvault/cmd/sync_teams.go index d4bd18b1f..c92790d9d 100644 --- a/cmd/msgvault/cmd/sync_teams.go +++ b/cmd/msgvault/cmd/sync_teams.go @@ -98,7 +98,7 @@ Examples: if qps <= 0 { qps = 5 } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) + client := teams.NewClient("https://graph.microsoft.com/v1.0", tokenFn, qps) imp := teams.NewImporter(s, client) _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Microsoft Teams for %s\n\n", email) diff --git a/cmd/msgvault/cmd/sync_test.go b/cmd/msgvault/cmd/sync_test.go index 288cdea00..f9201ecda 100644 --- a/cmd/msgvault/cmd/sync_test.go +++ b/cmd/msgvault/cmd/sync_test.go @@ -841,3 +841,23 @@ func TestTrimFolderFilter_DoesNotBlockOnErrorInSyncFull(t *testing.T) { require.NotPanics(func() { parseFolderFilter([]string{" ", " "}) }) require.NotPanics(func() { parseFolderFilter([]string{"", " ", ""}) }) } + +func TestSyncFullGraphAccountPointsToSync(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + st, err := store.Open(filepath.Join(dir, "msgvault.db")) + require.NoError(err) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource(sourceTypeMSMail, "mail@example.com") + require.NoError(err) + require.NoError(st.Close()) + cfg := &config.Config{HomeDir: dir, Data: config.DataConfig{DataDir: dir}} + cmd := &cobra.Command{} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.Flags().Int64("source-id", 0, "") + require.NoError(cmd.Flags().Set("source-id", strconv.FormatInt(src.ID, 10))) + err = runSyncFullLocal(cmd, nil) + require.Error(err) + assert.Contains(err.Error(), "msgvault sync --source-id "+strconv.FormatInt(src.ID, 10)) +} diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index fdbbc45fb..ce34acad6 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -122,6 +122,11 @@ func runSyncFullLocal(cmd *cobra.Command, args []string) error { } } if len(sources) == 0 { + for _, src := range allMatches { + if src.SourceType == sourceTypeMSMail { + return fmt.Errorf("use 'msgvault sync --source-id %d' for initial and later Microsoft Graph mail downloads", src.ID) + } + } if len(allMatches) > 0 { // Identifier exists but has no syncable source types. return fmt.Errorf("%s exists but its source type cannot be synced (only gmail and imap are supported)", syncSelectorLabel(selector)) diff --git a/docs/changelog.md b/docs/changelog.md index 4a1b1a5cb..44a287bfd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -50,6 +50,10 @@ All notable changes to msgvault, grouped by release. requests now log the upstream status, with body excerpts available at DEBUG. See [CardDAV contacts](usage/people-carddav.md#google-contacts) for the Google Contacts CardDAV API that your Google Cloud project must enable. +- `add-o365 --graph` syncs a Microsoft 365 or Outlook.com mailbox through the + Microsoft Graph mail API, for a mailbox that has IMAP turned off. Each folder + becomes a label, and later syncs fetch only the changes, including moves and + deletes. See [Microsoft Graph mail sync](guides/oauth-setup.md#microsoft-graph-mail-sync). - Saved View MCP tools publish canonical_state as a schema object, so MCP clients that validate tools/list strictly, such as those built on the official TypeScript SDK, load msgvault's tools. - `add-o365 --headless` and `add-teams --headless` sign in with a Microsoft device code, so no local browser is needed. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 852d3fd25..96f8a3086 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -396,8 +396,11 @@ Requires a `[microsoft]` section with `client_id` in `config.toml`. See the [OAu | `--tenant` | `common` | Azure AD tenant ID (restricts which accounts can authorize) | | `--headless` | `false` | Sign in with a device code instead of a local browser | | `--no-default-identity` | `false` | Do not auto-confirm the email address as this account's "me" identity | +| `--graph` | `false` | Sync through the Microsoft Graph mail API instead of IMAP. Creates an `msmail` account. Needs the `Mail.Read` permission | -After adding the account, sync it with `msgvault sync-full`. +After adding the account, sync it with `msgvault sync-full`. For a `--graph` +account, use `msgvault sync`. See +[Microsoft Graph mail sync](/docs/guides/oauth-setup/#microsoft-graph-mail-sync). --- diff --git a/docs/guides/oauth-setup.md b/docs/guides/oauth-setup.md index 577268d82..c24fca131 100644 --- a/docs/guides/oauth-setup.md +++ b/docs/guides/oauth-setup.md @@ -420,6 +420,27 @@ Some organizations require administrator consent before delegated channel message permissions can be used. See [Microsoft Teams](/docs/usage/teams/) for the full Teams workflow. +### Microsoft Graph Mail Sync + +If IMAP is turned off for a mailbox, `add-o365 --graph` syncs it through the +Microsoft Graph mail API. It uses the same `[microsoft] client_id` and redirect +URI. Add the **Microsoft Graph** delegated permission `Mail.Read` to the app +registration, then authorize and sync: + +```bash +msgvault add-o365 you@example.com --graph +msgvault sync you@example.com +``` + +The token is saved under `tokens/msmail_.json`, and the account has the +type `msmail`. Each mail folder becomes a label. The first sync downloads every +folder. Later syncs fetch only the changes, including moves between folders +and deletes. The daemon schedules the account like any other. + +A Graph account is a new account. If the same mailbox is also synced over +IMAP, the vault holds two copies. Run `msgvault dedup --collection` to hide the +extra copies, and `--undo` to reverse it. + ### Sync Your Email After adding the account, sync it the same way as any other account: diff --git a/internal/api/scheduler_jobs.go b/internal/api/scheduler_jobs.go index b3ae2fe2c..dd1140f04 100644 --- a/internal/api/scheduler_jobs.go +++ b/internal/api/scheduler_jobs.go @@ -51,7 +51,7 @@ const CardDAVJobName = "carddav" // types cannot borrow a scheduled account merely by sharing its identifier. func classifySourceScheduling(sourceType, identifier string) sourceScheduleClassification { switch sourceType { - case "", sourceTypeGmail, "imap", "teams", "discord": + case "", sourceTypeGmail, "imap", "msmail", "teams", "discord": return sourceScheduleClassification{kind: sourceScheduleAccount} case meetingimport.SourceType: return sourceScheduleClassification{kind: sourceScheduleNonSchedulable} diff --git a/internal/dedup/dedup.go b/internal/dedup/dedup.go index a63eb53c2..0d779066e 100644 --- a/internal/dedup/dedup.go +++ b/internal/dedup/dedup.go @@ -132,7 +132,7 @@ type Config struct { // DefaultSourcePreference is the default source-type authority order. var DefaultSourcePreference = []string{ - "gmail", "imap", "mbox", "emlx", "hey", + "gmail", "imap", "msmail", "mbox", "emlx", "hey", } // remoteSourceTypes lists source types whose messages can be deleted diff --git a/internal/microsoft/graph_oauth.go b/internal/microsoft/graph_oauth.go index 9e8c1a8d0..0f5a33043 100644 --- a/internal/microsoft/graph_oauth.go +++ b/internal/microsoft/graph_oauth.go @@ -30,6 +30,7 @@ const ( // Private and shared channels carry their own membership, read via // GET /teams/{id}/channels/{id}/members. scopeGraphChannelMemberRead = "https://graph.microsoft.com/ChannelMember.Read.All" + scopeGraphMailRead = "https://graph.microsoft.com/Mail.Read" ) // GraphScopes returns the OAuth scopes requested for Microsoft Teams ingestion @@ -44,9 +45,15 @@ func GraphScopes() []string { } } +// GraphMailScopes returns the OAuth scopes requested for mailbox ingestion via +// the Graph API. +func GraphMailScopes() []string { + return []string{scopeGraphMailRead, scopeGraphUserRead, scopeOfflineAccess, "openid", scopeEmail} +} + // GraphManager is a sibling of Manager that runs the same interactive browser // auth-code flow but requests Microsoft Graph scopes and persists tokens under -// a "teams_" filename prefix. It deliberately omits the IMAP scope-validation +// a "teams_" or "msmail_" filename prefix. It deliberately omits the IMAP scope-validation // and IMAP-host logic of Manager. // // The heavy browser-flow and ID-token verification machinery is reused via an @@ -60,15 +67,36 @@ type GraphManager struct { logger *slog.Logger deviceCode bool + // scopes, tokenPrefix and reauthCmd differ per capability: Teams or mail. + // reauthCmd is a format string that takes the account email. + scopes []string + tokenPrefix string + reauthCmd string + // Test hooks, mirrored onto the internal delegate. See Manager. authorityURL string browserFlowFn func(ctx context.Context, email string, scopes []string) (*oauth2.Token, string, error) verifyIDTokenFn func(ctx context.Context, rawIDToken string) (*idTokenClaims, error) } -// NewGraphManager constructs a GraphManager. An empty tenantID defaults to the -// multi-tenant "common" endpoint; a nil logger defaults to slog.Default(). +// NewGraphManager constructs a GraphManager for Teams. An empty tenantID +// defaults to the multi-tenant "common" endpoint; a nil logger defaults to +// slog.Default(). func NewGraphManager(clientID, tenantID, redirectURI, tokensDir string, logger *slog.Logger) *GraphManager { + m := newGraphManager(clientID, tenantID, redirectURI, tokensDir, logger) + m.scopes, m.tokenPrefix, m.reauthCmd = GraphScopes(), "teams_", "msgvault add-teams %s" + return m +} + +// NewGraphMailManager constructs a GraphManager for mailbox ingestion. Its +// tokens are saved under an "msmail_" prefix, apart from the Teams tokens. +func NewGraphMailManager(clientID, tenantID, redirectURI, tokensDir string, logger *slog.Logger) *GraphManager { + m := newGraphManager(clientID, tenantID, redirectURI, tokensDir, logger) + m.scopes, m.tokenPrefix, m.reauthCmd = GraphMailScopes(), "msmail_", "msgvault add-o365 %s --graph" + return m +} + +func newGraphManager(clientID, tenantID, redirectURI, tokensDir string, logger *slog.Logger) *GraphManager { if tenantID == "" { tenantID = DefaultTenant } @@ -92,7 +120,7 @@ func (m *GraphManager) UseDeviceCode() { // delegate builds an internal *Manager used only for its reusable browser-flow // and ID-token verification logic. Token storage is handled by GraphManager -// itself (with the teams_ prefix), so the delegate's tokensDir is irrelevant. +// itself (with its own prefix), so the delegate's tokensDir is irrelevant. func (m *GraphManager) delegate() *Manager { return &Manager{ clientID: m.clientID, @@ -108,10 +136,10 @@ func (m *GraphManager) delegate() *Manager { } // TokenPath returns the on-disk location of the persisted Graph token for an -// account, namespaced with a "teams_" prefix to keep it distinct from the IMAP -// Manager's "microsoft_" tokens. +// account, namespaced with a "teams_" or "msmail_" prefix to keep it distinct +// from the IMAP Manager's "microsoft_" tokens. func (m *GraphManager) TokenPath(email string) string { - return filepath.Join(m.tokensDir, "teams_"+sanitizeEmail(email)+".json") + return filepath.Join(m.tokensDir, m.tokenPrefix+sanitizeEmail(email)+".json") } // Authorize runs the interactive browser auth-code flow requesting Graph @@ -119,7 +147,7 @@ func (m *GraphManager) TokenPath(email string) string { // persists the token. Unlike Manager.Authorize there is no IMAP scope // correction step — Graph scopes are identical across account types. func (m *GraphManager) Authorize(ctx context.Context, email string) error { - scopes := GraphScopes() + scopes := m.scopes d := m.delegate() token, nonce, err := d.doBrowserFlow(ctx, email, scopes) if err != nil { @@ -151,11 +179,11 @@ func (m *GraphManager) TokenSource(ctx context.Context, email string) (func(cont scopes := tf.Scopes if len(scopes) == 0 { - scopes = GraphScopes() - } else if missing := missingGraphScopes(scopes); len(missing) > 0 { + scopes = m.scopes + } else if missing := missingScopes(scopes, m.scopes); len(missing) > 0 { return nil, fmt.Errorf( - "token for %s is missing Microsoft Graph scopes %s — run 'msgvault add-teams %s' to re-authorize", - email, strings.Join(missing, ", "), email, + "token for %s is missing Microsoft Graph scopes %s — run '%s' to re-authorize", + email, strings.Join(missing, ", "), fmt.Sprintf(m.reauthCmd, email), ) } @@ -239,7 +267,7 @@ func (m *GraphManager) DeleteToken(email string) error { } // saveToken atomically persists the token in the same on-disk JSON format as -// the IMAP Manager (tokenFile), under the teams_ filename. +// the IMAP Manager (tokenFile), under the capability's filename prefix. func (m *GraphManager) saveToken(email string, token *oauth2.Token, scopes []string, tenantID string) error { if err := fileutil.SecureMkdirAll(m.tokensDir, 0700); err != nil { return err @@ -271,13 +299,13 @@ func (m *GraphManager) loadTokenFile(email string) (*tokenFile, error) { return &tf, nil } -func missingGraphScopes(scopes []string) []string { +func missingScopes(scopes, want []string) []string { have := make(map[string]struct{}, len(scopes)) for _, scope := range scopes { have[scope] = struct{}{} } var missing []string - for _, scope := range GraphScopes() { + for _, scope := range want { if _, ok := have[scope]; !ok { missing = append(missing, scope) } diff --git a/internal/microsoft/graph_oauth_test.go b/internal/microsoft/graph_oauth_test.go index 4635cb043..9af6f07f3 100644 --- a/internal/microsoft/graph_oauth_test.go +++ b/internal/microsoft/graph_oauth_test.go @@ -14,7 +14,7 @@ import ( func TestGraphTokenPath(t *testing.T) { dir := filepath.Join("tmp", "tokens") - m := &GraphManager{tokensDir: dir} + m := NewGraphManager("", "", "", dir, nil) assert.Equal(t, filepath.Join(dir, "teams_user@example.com.json"), m.TokenPath("user@example.com")) } @@ -204,3 +204,26 @@ func TestGraphManager_TokenSource_Concurrent(t *testing.T) { } wg.Wait() } + +// Mail and Teams tokens live in separate files with separate scope sets, so a +// Teams token never satisfies the mail manager and the reverse. +func TestGraphMailManager_SeparateTokenAndScopes(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + teamsMgr := NewGraphManager("test-client", "common", "", dir, slog.Default()) + mailMgr := NewGraphMailManager("test-client", "common", "", dir, slog.Default()) + assert.Equal(filepath.Join(dir, "msmail_user@company.com.json"), mailMgr.TokenPath("user@company.com")) + + token := &oauth2.Token{AccessToken: "graph-access", RefreshToken: "graph-refresh", TokenType: "Bearer"} + require.NoError(teamsMgr.saveToken("user@company.com", token, GraphScopes(), "org-tid")) + _, err := teamsMgr.TokenSource(t.Context(), "user@company.com") + require.NoError(err) + assert.False(mailMgr.HasToken("user@company.com")) + + withoutMail := []string{"https://graph.microsoft.com/User.Read", scopeOfflineAccess, "openid", scopeEmail} + require.NoError(mailMgr.saveToken("user@company.com", token, withoutMail, "org-tid")) + _, err = mailMgr.TokenSource(t.Context(), "user@company.com") + require.ErrorContains(err, "https://graph.microsoft.com/Mail.Read") + require.ErrorContains(err, "msgvault add-o365 user@company.com --graph") +} diff --git a/internal/msgraph/client.go b/internal/msgraph/client.go new file mode 100644 index 000000000..b8a0bfd92 --- /dev/null +++ b/internal/msgraph/client.go @@ -0,0 +1,281 @@ +// Package msgraph is the Microsoft Graph REST transport shared by the Teams and +// mail connectors: bearer auth, a token-bucket rate limit, Retry-After back-off +// and @odata paging. +package msgraph + +import ( + "context" + "crypto/tls" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "go.kenn.io/msgvault/internal/httpretry" + "golang.org/x/time/rate" +) + +// ErrTooLarge classifies a response body that exceeds the caller's byte cap. +var ErrTooLarge = errors.New("graph response exceeds the configured size cap") + +// ErrNotFound classifies a 404 response. +var ErrNotFound = errors.New("graph resource not found") + +// ErrGone classifies an expired delta token: 410 Gone, or a syncStateNotFound +// error. The caller must restart the delta walk without a token. +var ErrGone = errors.New("graph delta token expired") + +const ( + maxRetries = 8 + maxRetryAfter = httpretry.ProviderMaxRetryAfter +) + +// TokenFunc returns a bearer token for a Graph API request. +type TokenFunc func(context.Context) (string, error) + +// Client is a minimal Microsoft Graph REST client supporting paging and +// Retry-After back-off. +type Client struct { + baseURL string + token TokenFunc + http *http.Client + limiter *rate.Limiter + + // Headers are extra request headers, for example the mail connector's + // Prefer header. They are set after the defaults and can replace them. + Headers map[string]string +} + +// NewClient creates a Client. baseURL is injected so tests can point at +// httptest servers. qps controls the token-bucket rate limit (default 5). +func NewClient(baseURL string, token TokenFunc, qps float64) *Client { + if qps <= 0 { + qps = 5 + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + token: token, + http: &http.Client{Timeout: 60 * time.Second}, + limiter: rate.NewLimiter(rate.Limit(qps), 1), + } +} + +// get fetches rawURL, respecting the rate limiter and retrying on 429/5xx with +// Retry-After or exponential back-off. +func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { + return c.getLimited(ctx, rawURL, 0) +} + +func (c *Client) getLimited(ctx context.Context, rawURL string, maxBytes int64) ([]byte, error) { + reqURL, err := c.resolveRequestURL(rawURL) + if err != nil { + return nil, err + } + var lastErr error + var retryAfter string + for attempt := range maxRetries { + if attempt > 0 { + if err := sleepCtx(ctx, httpretry.RetryAfter(retryAfter, attempt-1, maxRetryAfter)); err != nil { + return nil, err + } + } + retryAfter = "" + if err := c.limiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("wait for graph rate limit: %w", err) + } + tok, err := c.token(ctx) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+tok) + req.Header.Set("Accept", "application/json") + for k, v := range c.Headers { + req.Header.Set(k, v) + } + resp, err := c.http.Do(req) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if dnsErr, ok := errors.AsType[*net.DNSError](err); ok && dnsErr.IsNotFound { + return nil, err + } + if _, ok := errors.AsType[*tls.CertificateVerificationError](err); ok { + return nil, err + } + lastErr = err + continue + } + if resp.StatusCode == http.StatusOK && maxBytes > 0 && resp.ContentLength > maxBytes { + _ = resp.Body.Close() + return nil, ErrTooLarge + } + reader := io.Reader(resp.Body) + if maxBytes > 0 { + reader = io.LimitReader(resp.Body, maxBytes+1) + } + body, readErr := io.ReadAll(reader) + closeErr := resp.Body.Close() + if readErr != nil { + // A connection that breaks mid-body is transient, like a 5xx. + lastErr = fmt.Errorf("graph GET %s: read body: %w", reqURL, readErr) + continue + } + if closeErr != nil { + return nil, fmt.Errorf("graph GET %s: close body: %w", reqURL, closeErr) + } + expired := false + if resp.StatusCode >= http.StatusBadRequest { + var graphError struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + expired = json.Unmarshal(body, &graphError) == nil && graphError.Error.Code == "syncStateNotFound" + } + switch { + case resp.StatusCode == http.StatusOK: + if maxBytes > 0 && int64(len(body)) > maxBytes { + return nil, ErrTooLarge + } + return body, nil + case resp.StatusCode == http.StatusGone || expired: + return nil, fmt.Errorf("graph GET %s: status %d: %s: %w", reqURL, resp.StatusCode, string(body), ErrGone) + case resp.StatusCode == http.StatusNotFound: + return nil, fmt.Errorf("graph GET %s: status %d: %s: %w", reqURL, resp.StatusCode, string(body), ErrNotFound) + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500: + lastErr = fmt.Errorf("graph GET %s: status %d", reqURL, resp.StatusCode) + retryAfter = resp.Header.Get("Retry-After") + continue + default: + return nil, fmt.Errorf("graph GET %s: status %d: %s", reqURL, resp.StatusCode, string(body)) + } + } + return nil, fmt.Errorf("graph GET %s: exhausted %d retries: %w", reqURL, maxRetries, lastErr) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (c *Client) resolveRequestURL(rawURL string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("graph GET %q: parse URL: %w", rawURL, err) + } + if !u.IsAbs() { + return c.baseURL + rawURL, nil + } + base, err := url.Parse(c.baseURL) + if err != nil { + return "", fmt.Errorf("graph base URL %q: %w", c.baseURL, err) + } + if !strings.EqualFold(u.Scheme, base.Scheme) || !strings.EqualFold(u.Host, base.Host) { + return "", fmt.Errorf("graph GET %s: off-origin absolute URL", rawURL) + } + return u.String(), nil +} + +// GetRaw fetches url and returns the raw response bytes. url should be a +// path-relative string (e.g. "/me/messages/{id}/$value"); it is +// prefixed with the client's baseURL automatically by the underlying get method. +func (c *Client) GetRaw(ctx context.Context, url string) ([]byte, error) { + return c.get(ctx, url) +} + +// GetRawWithTimeout fetches raw bytes with a per-attempt deadline, including +// reading the body. Other requests on the client retain their usual timeout. +func (c *Client) GetRawWithTimeout(ctx context.Context, url string, timeout time.Duration) ([]byte, error) { + client := *c + httpClient := *c.http + httpClient.Timeout = timeout + client.http = &httpClient + return client.GetRaw(ctx, url) +} + +// GetRawLimited fetches raw bytes while enforcing a response-byte cap. +func (c *Client) GetRawLimited(ctx context.Context, url string, maxBytes int64) ([]byte, error) { + return c.getLimited(ctx, url, maxBytes) +} + +// BaseURL returns the client's configured base URL (scheme + host, no trailing slash). +// Importers use this to rewrite absolute graph.microsoft.com URLs to the configured +// host (supporting both production and httptest servers). +func (c *Client) BaseURL() string { + return c.baseURL +} + +// GetJSON fetches url and unmarshals the JSON body into out. +func (c *Client) GetJSON(ctx context.Context, url string, out any) error { + body, err := c.get(ctx, url) + if err != nil { + return err + } + return json.Unmarshal(body, out) +} + +// ListResponse is the Graph collection envelope. +type ListResponse[T any] struct { + Value []T `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` +} + +// PageThrough follows @odata.nextLink, decoding each page into []T, calling fn. +// Returns the terminal @odata.deltaLink (empty for non-delta endpoints). +func PageThrough[T any](ctx context.Context, c *Client, startURL string, fn func([]T)) (string, error) { + delta, _, err := PageThroughLimit(ctx, c, startURL, 0, fn) + return delta, err +} + +// PageThroughLimit is PageThrough with an optional item cap. When limit is +// positive, it stops before fetching a nextLink once enough items have been +// delivered and reports whether unread items/pages remain. +func PageThroughLimit[T any](ctx context.Context, c *Client, startURL string, limit int, fn func([]T)) (string, bool, error) { + url := startURL + delivered := 0 + for { + var page ListResponse[T] + if err := c.GetJSON(ctx, url, &page); err != nil { + return "", false, err + } + values := page.Value + if limit > 0 { + remaining := limit - delivered + if remaining <= 0 { + return "", true, nil + } + if len(values) > remaining { + fn(values[:remaining]) + return "", true, nil + } + if len(values) == remaining && page.NextLink != "" { + fn(values) + return "", true, nil + } + } + fn(values) + delivered += len(values) + if page.NextLink != "" { + url = page.NextLink + continue + } + return page.DeltaLink, false, nil + } +} diff --git a/internal/msgraph/client_test.go b/internal/msgraph/client_test.go new file mode 100644 index 000000000..8a7449403 --- /dev/null +++ b/internal/msgraph/client_test.go @@ -0,0 +1,167 @@ +package msgraph + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A connection that breaks in the middle of a body is retried, like a 5xx. +func TestGetRetriesTruncatedBody(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte("partial")) + return // the server closes the connection 93 bytes short + } + _, _ = w.Write([]byte("full body")) + })) + defer srv.Close() + + c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 1000) + body, err := c.GetRaw(context.Background(), "/me/messages/m1/$value") + require.NoError(t, err) + assert.Equal(t, "full body", string(body)) + assert.EqualValues(t, 2, calls.Load()) +} + +func TestClientContextCancelDuringRetry(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + server := httptest.NewTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "30") // long wait so cancellation wins + w.WriteHeader(http.StatusTooManyRequests) + })) + httpClient := server.Client() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := NewClient(server.URL, func(context.Context) (string, error) { return "t", nil }, 50) + c.http.Transport = httpClient.Transport + go func() { time.Sleep(50 * time.Millisecond); cancel() }() + _, err := c.GetRaw(ctx, "/x") + require.ErrorIs(t, err, context.Canceled) + }) +} + +func TestGetGraphErrorClassification(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + want error + }{ + {"expired_404", http.StatusNotFound, `{"error":{"code":"syncStateNotFound"}}`, ErrGone}, + {"expired_400", http.StatusBadRequest, `{"error":{"code":"syncStateNotFound"}}`, ErrGone}, + {"gone", http.StatusGone, "expired", ErrGone}, + {"missing", http.StatusNotFound, `{"error":{"code":"ErrorItemNotFound","message":"syncStateNotFound is not the error code"}}`, ErrNotFound}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + defer srv.Close() + c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 1000) + _, err := c.GetRaw(t.Context(), "/message") + require.ErrorIs(t, err, tc.want) + }) + } +} + +func TestGetStopsAfterLastAttempt(t *testing.T) { + for _, truncated := range []bool{false, true} { + t.Run(map[bool]string{false: "503", true: "truncated_body"}[truncated], func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + calls := 0 + srv := httptest.NewTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + if truncated { + w.Header().Set("Content-Length", "100") + _, _ = w.Write([]byte("partial")) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + httpClient := srv.Client() + c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 1000) + c.http.Transport = httpClient.Transport + start := time.Now() + _, err := c.GetRaw(t.Context(), "/message") + require.Error(t, err) + assert.Equal(t, 8, calls) + assert.Equal(t, 123*time.Second, time.Since(start)) + }) + }) + } +} + +// Dial errors are injected at the network boundary to exercise our retry +// classification without depending on a real DNS resolver or certificate store. +func TestGetConnectionErrorRetries(t *testing.T) { + for _, tc := range []struct { + name string + err error + attempts int + delay time.Duration + }{ + {"name_not_found", &net.DNSError{Err: "no such host", Name: "mail.example.com", IsNotFound: true}, 1, 0}, + {"certificate", &tls.CertificateVerificationError{Err: x509.UnknownAuthorityError{}}, 1, 0}, + {"temporary_dns", &net.DNSError{Err: "temporary lookup failure", Name: "mail.example.com", IsTemporary: true}, 8, 123 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + calls := 0 + c := NewClient("https://mail.example.com", func(context.Context) (string, error) { return "t", nil }, 1000) + c.http.Transport = &http.Transport{DialContext: func(context.Context, string, string) (net.Conn, error) { calls++; return nil, tc.err }} + start := time.Now() + _, err := c.GetRaw(t.Context(), "/message") + require.ErrorIs(t, err, tc.err) + assert.Equal(t, tc.attempts, calls) + assert.Equal(t, tc.delay, time.Since(start)) + }) + }) + } +} + +func TestGetRawWithTimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + srv := httptest.NewTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + assert.NoError(http.NewResponseController(w).Flush()) + select { + case <-time.After(2 * time.Minute): + _, _ = w.Write([]byte("complete MIME")) + case <-r.Context().Done(): + } + })) + httpClient := srv.Client() + c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 1000) + c.http.Transport = httpClient.Transport + body, err := c.GetRawWithTimeout(t.Context(), "/message", 10*time.Minute) + require.NoError(err) + assert.Equal("complete MIME", string(body)) + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _, err = c.GetRawWithTimeout(ctx, "/message", 10*time.Minute) + require.ErrorIs(err, context.DeadlineExceeded) + + // An ordinary request must still hit its original 60-second deadline. + start := time.Now() + _, err = c.GetRaw(t.Context(), "/metadata") + require.Error(err) + assert.Equal(603*time.Second, time.Since(start)) + }) +} diff --git a/internal/msmail/client.go b/internal/msmail/client.go new file mode 100644 index 000000000..903e186af --- /dev/null +++ b/internal/msmail/client.go @@ -0,0 +1,120 @@ +// Package msmail syncs a Microsoft 365 or Outlook.com mailbox through the +// Microsoft Graph mail API. It is the connector for accounts that cannot use +// IMAP, for example a tenant that turned IMAP off. +package msmail + +import ( + "context" + "net/url" + "time" + + "go.kenn.io/msgvault/internal/msgraph" +) + +// GraphBaseURL is the production Graph endpoint. +const GraphBaseURL = "https://graph.microsoft.com/v1.0" + +// Client adds the mail endpoints to the shared Graph transport. +type Client struct { + *msgraph.Client +} + +// NewClient creates a mail Client. Every request asks for immutable IDs, so a +// message keeps its ID when it moves between folders, and for 1,000-item pages. +func NewClient(baseURL string, token msgraph.TokenFunc, qps float64) *Client { + c := msgraph.NewClient(baseURL, token, qps) + c.Headers = map[string]string{"Prefer": `IdType="ImmutableId", odata.maxpagesize=1000`} + return &Client{c} +} + +// Folder is a mail folder. Path joins the display names from the top of the +// mailbox with "/". +type Folder struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + ChildFolderCount int `json:"childFolderCount"` + Path string `json:"-"` +} + +// DeltaMessage is one item of a folder's message delta. Removed is set when the +// message left the folder: it moved, or it was deleted. +type DeltaMessage struct { + ID string `json:"id"` + ReceivedDateTime time.Time `json:"receivedDateTime"` + Removed *struct { + Reason string `json:"reason"` + } `json:"@removed"` + + archiveID int64 // set when the message is already in the vault +} + +const folderSelect = "?$top=100&$select=id,displayName,childFolderCount" + +// ListFolders returns every folder in the mailbox, parents before children. +func (c *Client) ListFolders(ctx context.Context) ([]Folder, error) { + var out []Folder + var walk func(string, string) error + walk = func(listURL, parent string) error { + var level []Folder + if _, err := msgraph.PageThrough(ctx, c.Client, listURL, func(p []Folder) { level = append(level, p...) }); err != nil { + return err + } + for _, f := range level { + f.Path = f.DisplayName + if parent != "" { + f.Path = parent + "/" + f.DisplayName + } + out = append(out, f) + if f.ChildFolderCount > 0 { + if err := walk("/me/mailFolders/"+url.PathEscape(f.ID)+"/childFolders"+folderSelect, f.Path); err != nil { + return err + } + } + } + return nil + } + return out, walk("/me/mailFolders"+folderSelect, "") +} + +// WellKnownFolderID returns the ID of a well-known folder such as "sentitems". +// It returns msgraph.ErrNotFound when the mailbox does not have that folder. +func (c *Client) WellKnownFolderID(ctx context.Context, name string) (string, error) { + var f Folder + err := c.GetJSON(ctx, "/me/mailFolders/"+name+"?$select=id", &f) + return f.ID, err +} + +// DeltaStartURL is the first delta request for a folder with no saved cursor. +// It returns every message in the folder and ends with a deltaLink. +func DeltaStartURL(folderID string) string { + return "/me/mailFolders/" + url.PathEscape(folderID) + "/messages/delta?$select=receivedDateTime" +} + +// DeltaPage fetches one page of a delta walk. The page carries a NextLink +// while the walk continues, and a DeltaLink when it is complete. +func (c *Client) DeltaPage(ctx context.Context, pageURL string) (*msgraph.ListResponse[DeltaMessage], error) { + var page msgraph.ListResponse[DeltaMessage] + if err := c.GetJSON(ctx, pageURL, &page); err != nil { + return nil, err + } + return &page, nil +} + +// GetMIME returns the full RFC 5322 source of a message. +func (c *Client) GetMIME(ctx context.Context, id string) ([]byte, error) { + return c.GetRawWithTimeout(ctx, "/me/messages/"+url.PathEscape(id)+"/$value", 10*time.Minute) +} + +// MessageInfo is where a message is now and when it arrived. +type MessageInfo struct { + ParentFolderID string `json:"parentFolderId"` + ReceivedDateTime time.Time `json:"receivedDateTime"` +} + +// LookupMessage returns a message's folder and receipt time. It returns +// msgraph.ErrNotFound when the message no longer exists. +func (c *Client) LookupMessage(ctx context.Context, id string) (MessageInfo, error) { + var m MessageInfo + err := c.GetJSON(ctx, "/me/messages/"+url.PathEscape(id)+"?$select=parentFolderId,receivedDateTime", &m) + return m, err +} diff --git a/internal/msmail/importer.go b/internal/msmail/importer.go new file mode 100644 index 000000000..916cc0bec --- /dev/null +++ b/internal/msmail/importer.go @@ -0,0 +1,610 @@ +package msmail + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json/v2" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "go.kenn.io/msgvault/internal/importer" + "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/msgraph" + "go.kenn.io/msgvault/internal/store" + "golang.org/x/sync/errgroup" +) + +// SourceType is the sources.source_type value for a Graph mail account. +const SourceType = "msmail" + +// walkPrefix marks the saved nextLink of a walk. A walk must finish in one +// run, because its end looks up the archived messages it did not return, so +// an interrupted walk starts over. Known messages are not downloaded again. +const walkPrefix = "walk:" + +// retryPrefix marks a saved-state key that names a message to download again. +const retryPrefix = "retry:" + +// fetchWorkers is the number of parallel $value downloads. Microsoft documents +// four concurrent requests per mailbox as the limit. +const fetchWorkers = 4 + +// systemFolders maps Graph well-known folder names to the label system role +// they carry. Each one is labelled "system"; only Sent Items has a role. +var systemFolders = map[string]string{ + "inbox": "", + "sentitems": store.LabelSystemRoleSent, + "drafts": "", + "deleteditems": "", + "junkemail": "", + "archive": "", +} + +// Options configures one sync of one mailbox. +type Options struct { + Email string + AttachmentsDir string + Progress func(string) +} + +// Summary reports what one sync did. +type Summary struct { + SourceID int64 + Folders int + Added int + Updated int + Moved int + Deleted int + Errors int + Duration time.Duration +} + +// Import syncs every folder of the mailbox. A folder with no saved cursor is +// walked from the start. A folder with a cursor fetches only the changes since +// the last sync. Known messages are refreshed when delta reports an update. +func Import(ctx context.Context, st *store.Store, c *Client, opts Options, log *slog.Logger) (sum *Summary, err error) { + start := time.Now() + src, err := st.GetOrCreateSource(SourceType, opts.Email) + if err != nil { + return nil, err + } + sum = &Summary{SourceID: src.ID} + + // Cursors: the last completed run, then any checkpoint of an interrupted + // run after it. A delta link is opaque, so the newer checkpoint wins. + cursors := map[string]string{} + if prev, perr := st.GetLastSuccessfulSync(src.ID); perr == nil && prev != nil && prev.CursorAfter.Valid { + mergeCursors(cursors, prev.CursorAfter.String) + } + if cp, cerr := st.GetLatestCheckpointedSync(src.ID); cerr == nil && cp != nil && cp.CursorBefore.Valid { + mergeCursors(cursors, cp.CursorBefore.String) + } + + syncID, err := st.StartSync(src.ID, SourceType) + if err != nil { + return nil, err + } + st = st.ScopedToSync(src.ID, syncID) + checkpoint := func() *store.Checkpoint { + blob, _ := json.Marshal(cursors, json.Deterministic(true)) + return &store.Checkpoint{ + PageToken: string(blob), + MessagesProcessed: int64(sum.Added + sum.Updated + sum.Moved + sum.Deleted), + MessagesAdded: int64(sum.Added), + MessagesUpdated: int64(sum.Updated), + ErrorsCount: int64(sum.Errors), + } + } + defer func() { + if err != nil { + _ = st.FailSyncWithCheckpoint(syncID, err.Error(), checkpoint()) + } + }() + + s := &syncer{st: st, c: c, opts: opts, log: log, sourceID: src.ID, sum: sum, cursors: cursors} + folders, err := c.ListFolders(ctx) + if err != nil { + return sum, fmt.Errorf("list mail folders: %w", err) + } + if s.labels, err = s.ensureLabels(ctx, folders); err != nil { + return sum, err + } + if err = s.retryMessages(ctx); err != nil { + return sum, err + } + for id := range cursors { + if _, listed := s.labels[id]; listed || strings.HasPrefix(id, retryPrefix) { + continue + } + if err = s.retireFolder(ctx, id); err != nil { + return sum, fmt.Errorf("retire removed folder: %w", err) + } + delete(cursors, id) + } + + for _, f := range folders { + sum.Folders++ + s.progressf("Folder %s", f.Path) + link := cursors[f.ID] + // seen collects the IDs of a walk that starts in this run, so that + // archived messages it does not return can be looked up at its end. + var seen map[string]bool + restarted := false + switch { + case link == "": + link, seen = DeltaStartURL(f.ID), map[string]bool{} + case strings.HasPrefix(link, walkPrefix): + link, seen = DeltaStartURL(f.ID), map[string]bool{} + } + for { + page, perr := c.DeltaPage(ctx, link) + if errors.Is(perr, msgraph.ErrGone) && !restarted { + // The token expired. Walk the folder again; messages already + // in the vault are not downloaded again. + log.Info("delta token expired, walking folder again", "folder", f.Path) + link, seen, restarted = DeltaStartURL(f.ID), map[string]bool{}, true + continue + } + if perr != nil { + return sum, fmt.Errorf("folder %s: %w", f.Path, perr) + } + if err = s.applyPage(ctx, f.ID, page.Value, seen); err != nil { + return sum, fmt.Errorf("folder %s: %w", f.Path, err) + } + if page.NextLink == "" && seen != nil { + if err = s.reconcileWalk(ctx, s.labels[f.ID], seen); err != nil { + return sum, fmt.Errorf("folder %s: %w", f.Path, err) + } + } + if page.NextLink != "" { + link = page.NextLink + } else { + link = page.DeltaLink + } + cursors[f.ID] = link + if seen != nil && page.NextLink != "" { + cursors[f.ID] = walkPrefix + link + } + if err = st.UpdateSyncCheckpoint(syncID, checkpoint()); err != nil { + return sum, err + } + if page.NextLink == "" { + break + } + } + } + + if err = st.RecomputeConversationStats(src.ID); err != nil { + return sum, err + } + cp := checkpoint() + // Complete through the source so last_sync_at moves. The cursors live in + // the run's checkpoint, so the source cursor stays unchanged. + if err = st.CompleteSyncAndPreserveSourceCursorContext(ctx, syncID, src.ID, cp.PageToken); err != nil { + return sum, err + } + sum.Duration = time.Since(start) + return sum, nil +} + +func mergeCursors(dst map[string]string, blob string) { + var m map[string]string + if json.Unmarshal([]byte(blob), &m) != nil { + return + } + for k, v := range m { + if v != "" { + dst[k] = v + } + } +} + +type syncer struct { + // cursors is the saved state: folder ID -> delta link, and + // retryPrefix + message ID -> folder ID for messages to download again. + cursors map[string]string + + st *store.Store + c *Client + opts Options + log *slog.Logger + sourceID int64 + sum *Summary + labels map[string]int64 // Graph folder ID -> label ID + + // drafts is the Drafts folder. A draft keeps its ID while it is edited, + // so a known draft is downloaded again when delta reports it. + drafts string + + // deletions is the hidden Recoverable Items folder. A permanent delete + // (Shift+Delete, or emptying Deleted Items) moves a message there. + deletions string +} + +func (s *syncer) progressf(format string, args ...any) { + if s.opts.Progress != nil { + s.opts.Progress(fmt.Sprintf(format, args...)) + } +} + +// ensureLabels makes one label per folder. The label's source ID is the Graph +// folder ID, so a folder rename changes only the label name. +func (s *syncer) ensureLabels(ctx context.Context, folders []Folder) (map[string]int64, error) { + system := map[string]string{} // folder ID -> system role + for name, role := range systemFolders { + id, err := s.c.WellKnownFolderID(ctx, name) + if errors.Is(err, msgraph.ErrNotFound) { + continue + } + if err != nil { + return nil, fmt.Errorf("look up folder %s: %w", name, err) + } + system[id] = role + if name == "drafts" { + s.drafts = id + } + } + id, err := s.c.WellKnownFolderID(ctx, "recoverableitemsdeletions") + if err != nil && !errors.Is(err, msgraph.ErrNotFound) { + return nil, fmt.Errorf("look up folder recoverableitemsdeletions: %w", err) + } + s.deletions = id + infos := make(map[string]store.LabelInfo, len(folders)) + for _, f := range folders { + info := store.LabelInfo{Name: f.Path, Type: "user"} + if role, ok := system[f.ID]; ok { + info.Type, info.SystemRole = "system", role + } + infos[f.ID] = info + } + return s.st.EnsureLabelsBatch(s.sourceID, infos) +} + +// applyPage stores one delta page for a folder. New messages are downloaded. +// Known messages get the folder as their only label, because a mail item is +// in exactly one folder, and lose any deletion mark, because the mailbox has +// them again. In an incremental round (seen is nil), known messages are also +// downloaded again, because delta reports them only when they changed. A walk +// (seen is not nil) returns every message, so it downloads again only drafts, +// whose content can change under the same ID. Removed messages are looked up +// with relocate. A walk collects the IDs of live messages in seen. +func (s *syncer) applyPage(ctx context.Context, folderID string, items []DeltaMessage, seen map[string]bool) error { + folderLabel := s.labels[folderID] + var live []DeltaMessage + var liveIDs, removedIDs []string + for _, m := range items { + if m.Removed != nil { + removedIDs = append(removedIDs, m.ID) + continue + } + live = append(live, m) + liveIDs = append(liveIDs, m.ID) + if seen != nil { + seen[m.ID] = true + } + } + + known, err := s.st.MessageExistsBatch(s.sourceID, append(liveIDs, removedIDs...)) + if err != nil { + return err + } + var todo []DeltaMessage + for _, m := range live { + id, ok := known[m.ID] + if ok { + if err := s.st.ClearMessageDeletedFromSource(s.sourceID, m.ID); err != nil { + return err + } + } + if !ok { + todo = append(todo, m) + continue + } + if err := s.setFolder(id, folderLabel); err != nil { + return err + } + if seen == nil || (s.drafts != "" && folderID == s.drafts) { + m.archiveID = id + todo = append(todo, m) + } + } + if err := s.download(ctx, folderID, todo); err != nil { + return err + } + + removed := map[string]int64{} + for _, id := range removedIDs { + if msgID, ok := known[id]; ok { + removed[id] = msgID + } + } + return s.relocate(ctx, removed) +} + +// afterStore makes sure that every attachment of a stored MIME has a row. For +// a refreshed message, it then drops the rows of parts that the new MIME no +// longer has. If a row is missing, the old rows stay and it returns an error. +func (s *syncer) afterStore(ctx context.Context, m DeltaMessage, raw []byte) error { + if s.opts.AttachmentsDir == "" { + return nil // no attachment rows are written + } + msgID := m.archiveID + if msgID == 0 { + ids, err := s.st.MessageExistsBatch(s.sourceID, []string{m.ID}) + if err != nil { + return err + } + msgID = ids[m.ID] + } + parsed, err := mime.ParseWithRecovery(raw, "") + if err != nil { + // The MIME did not parse, so its attachments are unknown. Keep the + // rows that are there. + s.log.Warn("MIME did not parse, keeping attachment rows", "id", m.ID, "error", err) + return nil + } + complete, err := s.attachmentsStored(ctx, msgID, parsed.Attachments) + if err != nil { + return err + } + if !complete { + return errors.New("an attachment was not stored") + } + if m.archiveID == 0 { + return nil + } + keep := make([]string, 0, len(parsed.Attachments)) + for _, a := range parsed.Attachments { + if len(a.Content) > 0 { // storage writes no row for an empty file + keep = append(keep, a.PartKey) + } + } + if err := s.st.DeleteMIMEAttachmentsExceptContext(ctx, msgID, keep); err != nil { + return err + } + return s.st.RecomputeMessageAttachmentStats(msgID) +} + +// attachmentsStored checks only nonempty parts, since storage skips empty files. +func (s *syncer) attachmentsStored(ctx context.Context, messageID int64, atts []mime.Attachment) (bool, error) { + parts := make([]store.AttachmentRef, 0, len(atts)) + for _, a := range atts { + if len(a.Content) > 0 { + parts = append(parts, store.AttachmentRef{SourcePartKey: a.PartKey, ContentHash: a.ContentHash}) + } + } + return s.st.AttachmentPartsStoredContext(ctx, messageID, parts) +} + +// retryMessages retries failed message downloads and incomplete attachments. A message that is gone is marked deleted. +func (s *syncer) retryMessages(ctx context.Context) error { + var ids []string + for key := range s.cursors { + if id, ok := strings.CutPrefix(key, retryPrefix); ok { + ids = append(ids, id) + } + } + // A marker is removed only when its retry is done, so every error path + // leaves it in the checkpoint for the next sync. + for _, id := range ids { + key := retryPrefix + id + known, err := s.st.MessageExistsBatch(s.sourceID, []string{id}) + if err != nil { + return err + } + info, err := s.c.LookupMessage(ctx, id) + parent := info.ParentFolderID + if errors.Is(err, msgraph.ErrNotFound) || (err == nil && s.deletions != "" && parent == s.deletions) { + if known[id] != 0 { + if err := s.st.MarkMessagesDeletedBatch(s.sourceID, []string{id}); err != nil { + return err + } + s.sum.Deleted++ + } + delete(s.cursors, key) + continue + } + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + s.log.Warn("message lookup failed, retrying on the next sync", "id", id, "error", err) + s.sum.Errors++ + continue + } + if _, ok := s.labels[parent]; !ok { + s.cursors[key] = parent // a folder this run did not list + continue + } + // download clears the marker only after the message is stored. + if err := s.download(ctx, parent, []DeltaMessage{{ID: id, ReceivedDateTime: info.ReceivedDateTime, archiveID: known[id]}}); err != nil { + s.cursors[key] = parent + return err + } + } + return nil +} + +// retireFolder handles a folder that has a saved cursor but is no longer in +// the mailbox. Each archived message still labeled with it is looked up: it +// moved to another folder, or it is gone. +func (s *syncer) retireFolder(ctx context.Context, folderID string) error { + labelID, err := s.st.LabelIDContext(ctx, s.sourceID, folderID) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + return s.reconcileWalk(ctx, labelID, nil) +} + +// reconcileWalk looks up the archived messages of a folder that a complete +// walk did not return. They left the folder while no delta cursor covered it, +// for example after the cursor expired. +func (s *syncer) reconcileWalk(ctx context.Context, folderLabel int64, seen map[string]bool) error { + missing, err := s.st.MessageIDsWithLabelContext(ctx, s.sourceID, folderLabel) + if err != nil { + return err + } + for sourceMsgID := range missing { + if seen[sourceMsgID] { + delete(missing, sourceMsgID) + } + } + return s.relocate(ctx, missing) +} + +// relocate finds where known messages went: source message ID -> message ID. +// A message that Graph still finds in a mail folder moved, and one it cannot +// find, or finds in Recoverable Items, is marked deleted. +func (s *syncer) relocate(ctx context.Context, msgs map[string]int64) error { + var gone []string + for id, msgID := range msgs { + info, err := s.c.LookupMessage(ctx, id) + parent := info.ParentFolderID + if errors.Is(err, msgraph.ErrNotFound) || (err == nil && s.deletions != "" && parent == s.deletions) { + gone = append(gone, id) + continue + } + if err != nil { + return fmt.Errorf("look up removed message: %w", err) + } + // A folder this run did not list is picked up on the next sync. + if label, ok := s.labels[parent]; ok { + if err := s.setFolder(msgID, label); err != nil { + return err + } + } + } + if len(gone) > 0 { + if err := s.st.MarkMessagesDeletedBatch(s.sourceID, gone); err != nil { + return err + } + s.sum.Deleted += len(gone) + } + return nil +} + +func (s *syncer) setFolder(messageID, label int64) error { + changed, err := s.st.ReconcileMessageLabels(messageID, []int64{label}, true) + if changed { + s.sum.Moved++ + } + return err +} + +type fetched struct { + msg DeltaMessage + raw []byte + err error +} + +// download fetches messages with fetchWorkers parallel requests and stores +// them one at a time. A message that disappears before its download is +// skipped. Other download failures are saved for retry while the page advances. +func (s *syncer) download(ctx context.Context, folderID string, msgs []DeltaMessage) error { + folderLabel := s.labels[folderID] + if len(msgs) == 0 { + return nil + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + g, gctx := errgroup.WithContext(ctx) + jobs := make(chan DeltaMessage) + results := make(chan fetched, fetchWorkers) + g.Go(func() error { + defer close(jobs) + for _, m := range msgs { + select { + case jobs <- m: + case <-gctx.Done(): + return gctx.Err() + } + } + return nil + }) + for range fetchWorkers { + g.Go(func() error { + for m := range jobs { + raw, err := s.c.GetMIME(gctx, m.ID) + if gctx.Err() != nil { + return gctx.Err() + } + select { + case results <- fetched{m, raw, err}: + case <-gctx.Done(): + return gctx.Err() + } + } + return nil + }) + } + var fetchErr error + go func() { + fetchErr = g.Wait() + close(results) + }() + + // A message that fails to store stops the page, so the cursor does not + // move past it; the next sync retries the page. Results are drained so + // the workers can exit. + var storeErr error + vanished := map[string]int64{} + for r := range results { + if storeErr != nil { + continue + } + if errors.Is(r.err, msgraph.ErrNotFound) { + delete(s.cursors, retryPrefix+r.msg.ID) + if r.msg.archiveID != 0 { + vanished[r.msg.ID] = r.msg.archiveID + } + continue + } + if r.err != nil { + s.log.Warn("message download failed, retrying on the next sync", "id", r.msg.ID, "error", r.err) + s.sum.Errors++ + s.cursors[retryPrefix+r.msg.ID] = folderID + continue + } + sum := sha256.Sum256(r.raw) + if err := importer.IngestRawMessage(ctx, s.st, s.sourceID, s.opts.Email, s.opts.AttachmentsDir, + []int64{folderLabel}, r.msg.ID, hex.EncodeToString(sum[:]), r.raw, r.msg.ReceivedDateTime, s.log); err != nil { + storeErr = fmt.Errorf("store message %s: %w", r.msg.ID, err) + s.sum.Errors++ + cancel() + continue + } + delete(s.cursors, retryPrefix+r.msg.ID) + if err := s.afterStore(ctx, r.msg, r.raw); err != nil { + // The message is stored, but an attachment is not. A later walk + // would skip the known message, so it goes on the retry list. + s.log.Warn("attachment not stored, retrying on the next sync", "id", r.msg.ID, "error", err) + s.sum.Errors++ + s.cursors[retryPrefix+r.msg.ID] = folderID + } + if r.msg.archiveID == 0 { + s.sum.Added++ + continue + } + s.sum.Updated++ + } + if storeErr != nil { + return storeErr + } + if ctx.Err() != nil { + return ctx.Err() + } + if fetchErr != nil { + return fmt.Errorf("download messages: %w", fetchErr) + } + // A known message that delta reported but $value no longer finds moved or + // is gone since the page was read. + return s.relocate(ctx, vanished) +} diff --git a/internal/msmail/importer_test.go b/internal/msmail/importer_test.go new file mode 100644 index 000000000..f73ce9125 --- /dev/null +++ b/internal/msmail/importer_test.go @@ -0,0 +1,968 @@ +package msmail + +import ( + "context" + "encoding/json/v2" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/msgraph" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +// fakeGraph is a Graph mail server that applies what it is given. It holds +// folders and messages, and a change log. A deltaLink encodes a position in +// that log, so the next delta returns only the changes made after it. +type fakeGraph struct { + t *testing.T + srv *httptest.Server + + mu sync.Mutex + folders []string // folder IDs, in list order + names map[string]string // folder ID -> display name, when not derived from the ID + folder map[string]string // message ID -> folder ID; absent when deleted + log []change // one entry per change + expired map[string]bool // folder IDs whose next delta reports an expired token + gone map[string]bool // folder IDs whose every delta answers 410 + version map[string]int // message ID -> content version + + withAttachment map[string]bool // message IDs whose MIME carries a file + shifted map[string]bool // message IDs whose file moves to another part + attachmentBody map[string]string // message ID -> base64 file content + broken map[string]bool // message IDs whose MIME does not parse + goneOnValue map[string]bool // message IDs deleted just before their $value + badLookup map[string]bool // message IDs whose metadata lookup answers 400 + badValue map[string]bool // message IDs whose $value answers 400 + attachDir string // attachments directory; a fresh one when empty + throttle bool // answer the next $value with 429 once + pageSize int + stopAt int // fail the delta page at this skip offset, when non-zero + + mimeCalls atomic.Int32 + walkStarts atomic.Int32 // delta requests with no token and no nextLink + expiredStatus int +} + +// change records a message and the folder it was in before the change. +type change struct{ id, from string } + +func newFakeGraph(t *testing.T) *fakeGraph { + t.Helper() + f := &fakeGraph{t: t, folder: map[string]string{}, expired: map[string]bool{}, gone: map[string]bool{}, version: map[string]int{}, withAttachment: map[string]bool{}, shifted: map[string]bool{}, attachmentBody: map[string]string{}, broken: map[string]bool{}, goneOnValue: map[string]bool{}, badValue: map[string]bool{}, pageSize: 2} + f.folders = []string{"inbox", "archive"} + f.expiredStatus = http.StatusGone + f.srv = httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeGraph) put(id, folder string) { + f.mu.Lock() + defer f.mu.Unlock() + f.log = append(f.log, change{id, f.folder[id]}) + f.folder[id] = folder +} + +func (f *fakeGraph) remove(id string) { + f.mu.Lock() + defer f.mu.Unlock() + f.log = append(f.log, change{id, f.folder[id]}) + delete(f.folder, id) +} + +func raw(id string, version int) string { + return "From: a@example.com\r\nTo: me@example.com\r\nSubject: " + id + + "\r\nMessage-ID: <" + id + "@example.com>\r\nDate: Mon, 1 Jan 2024 10:00:00 +0000\r\n\r\nbody " + id + + " v" + strconv.Itoa(version) + "\r\n" +} + +// rawWithAttachment carries one file. When shifted, a second text part comes +// first, so the file gets another MIME part key. +func rawWithAttachment(id string, shifted bool, content string) string { + switch content { + case "": + content = "aGVsbG8=" + case "-": + content = "" // an empty file + } + extra := "" + if shifted { + extra = "--b\r\nContent-Type: text/plain\r\n\r\nnote\r\n" + } + return "From: a@example.com\r\nTo: me@example.com\r\nSubject: " + id + + "\r\nMessage-ID: <" + id + "@example.com>\r\nDate: Mon, 1 Jan 2024 10:00:00 +0000\r\n" + + "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=b\r\n\r\n" + + "--b\r\nContent-Type: text/plain\r\n\r\nbody " + id + "\r\n" + extra + + "--b\r\nContent-Type: application/octet-stream\r\nContent-Disposition: attachment; filename=a.bin\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n" + content + "\r\n--b--\r\n" +} + +func (f *fakeGraph) writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + assert.NoError(f.t, json.MarshalWrite(w, v)) +} + +func (f *fakeGraph) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + p := r.URL.Path + q := r.URL.Query() + switch { + case p == "/me/mailFolders": + var out []map[string]any + for _, id := range f.folders { + name := f.names[id] + if name == "" { + name = strings.ToUpper(id[:1]) + id[1:] + } + out = append(out, map[string]any{"id": id, "displayName": name}) + } + f.writeJSON(w, map[string]any{"value": out}) + case strings.HasPrefix(p, "/me/mailFolders/") && strings.HasSuffix(p, "/messages/delta"): + f.delta(w, strings.Split(p, "/")[3], q) + case strings.HasPrefix(p, "/me/mailFolders/"): + id := strings.TrimPrefix(p, "/me/mailFolders/") + if slices.Contains(f.folders, id) { + f.writeJSON(w, map[string]any{"id": id}) + return + } + if id == "recoverableitemsdeletions" { + f.writeJSON(w, map[string]any{"id": "deletions"}) + return + } + http.Error(w, `{"error":{"code":"ErrorFolderNotFound"}}`, http.StatusNotFound) + case strings.HasPrefix(p, "/me/messages/") && strings.HasSuffix(p, "/$value"): + f.mimeCalls.Add(1) + if f.throttle { + f.throttle = false + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + return + } + id := strings.Split(p, "/")[3] + if f.badValue[id] { + http.Error(w, "bad", http.StatusBadRequest) + return + } + if f.goneOnValue[id] { + delete(f.folder, id) + } + if _, ok := f.folder[id]; !ok { + http.Error(w, "gone", http.StatusNotFound) + return + } + body := raw(id, f.version[id]) + if f.withAttachment[id] { + body = rawWithAttachment(id, f.shifted[id], f.attachmentBody[id]) + } + if f.broken[id] { + body = "From: a@example.com\r\nSubject: " + id + "\r\nMIME-Version: 1.0\r\n" + + "Content-Type: multipart mixed; boundary=b\r\n\r\n--b\r\n\r\nbody\r\n--b--\r\n" + } + _, _ = w.Write([]byte(body)) //nolint:gosec // local test server returns fixture MIME + case strings.HasPrefix(p, "/me/messages/"): + id := strings.TrimPrefix(p, "/me/messages/") + if f.badLookup[id] { + http.Error(w, "bad lookup", http.StatusBadRequest) + return + } + folder, ok := f.folder[id] + if !ok { + http.Error(w, `{"error":{"code":"ErrorItemNotFound"}}`, http.StatusNotFound) + return + } + f.writeJSON(w, map[string]any{"parentFolderId": folder, "receivedDateTime": "2024-01-01T10:00:00Z"}) + default: + http.Error(w, "unexpected "+p, http.StatusBadRequest) + } +} + +// delta answers a walk (no token) or a round (token = log position). +func (f *fakeGraph) delta(w http.ResponseWriter, folder string, q map[string][]string) { + get := func(k string) string { + if v := q[k]; len(v) > 0 { + return v[0] + } + return "" + } + link := func(kind string, vals ...string) string { + return f.srv.URL + "/me/mailFolders/" + folder + "/messages/delta?" + kind + "&" + strings.Join(vals, "&") + } + if f.gone[folder] { + http.Error(w, `{"error":{"code":"syncStateNotFound"}}`, http.StatusGone) + return + } + if tok := get("token"); tok != "" { + if f.expired[folder] { + delete(f.expired, folder) + http.Error(w, `{"error":{"code":"syncStateNotFound"}}`, f.expiredStatus) + return + } + pos, _ := strconv.Atoi(tok) + seen := map[string]bool{} + var out []map[string]any + for _, c := range f.log[pos:] { + id := c.id + if seen[id] || (c.from != folder && f.folder[id] != folder) { + continue + } + seen[id] = true + if f.folder[id] == folder { + out = append(out, map[string]any{"id": id}) + } else { + out = append(out, map[string]any{"id": id, "@removed": map[string]any{"reason": "deleted"}}) + } + } + f.writeJSON(w, map[string]any{"value": out, "@odata.deltaLink": link("t", "token="+strconv.Itoa(len(f.log)))}) + return + } + // A walk lists the folder in log order. pos pins the log position that + // the final deltaLink carries, taken when the walk started. + pos := get("pos") + if pos == "" { + pos = strconv.Itoa(len(f.log)) + } + skip, _ := strconv.Atoi(get("skip")) + if get("skip") == "" { + f.walkStarts.Add(1) + } + if f.stopAt != 0 && skip == f.stopAt { + f.stopAt = 0 + http.Error(w, "boom", http.StatusBadRequest) + return + } + var ids []string + for _, c := range f.log { + if id := c.id; f.folder[id] == folder && !slices.Contains(ids, id) { + ids = append(ids, c.id) + } + } + end := min(skip+f.pageSize, len(ids)) + var out []map[string]any + for _, id := range ids[skip:end] { + out = append(out, map[string]any{"id": id, "receivedDateTime": "2024-01-01T10:00:00Z"}) + } + resp := map[string]any{"value": out} + if end < len(ids) { + resp["@odata.nextLink"] = link("w", "skip="+strconv.Itoa(end), "pos="+pos) + } else { + resp["@odata.deltaLink"] = link("t", "token="+pos) + } + f.writeJSON(w, resp) +} + +func (f *fakeGraph) sync(t *testing.T, st *store.Store) (*Summary, error) { + t.Helper() + c := NewClient(f.srv.URL, func(context.Context) (string, error) { return "tok", nil }, 1000) + dir := f.attachDir + if dir == "" { + dir = f.t.TempDir() + } + return Import(context.Background(), st, c, Options{Email: "me@example.com", AttachmentsDir: dir}, slog.Default()) +} + +// state returns message ID -> "folder label name" or "deleted". +func state(t *testing.T, st *store.Store) map[string]string { + t.Helper() + rows, err := st.DB().Query(` + SELECT m.source_message_id, COALESCE(l.name, ''), m.deleted_from_source_at IS NOT NULL + FROM messages m + LEFT JOIN message_labels ml ON ml.message_id = m.id + LEFT JOIN labels l ON l.id = ml.label_id`) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + out := map[string]string{} + for rows.Next() { + var id, label string + var deleted bool + require.NoError(t, rows.Scan(&id, &label, &deleted)) + if deleted { + label = "deleted" + } + _, dup := out[id] + assert.False(t, dup, "message %s has more than one row or label", id) + out[id] = label + } + require.NoError(t, rows.Err()) + return out +} + +func TestImportFirstSyncThenNoChange(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + f.put("m2", "inbox") + f.put("m3", "inbox") + f.put("m4", "archive") + f.throttle = true + + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(4, sum.Added) + assert.Equal(map[string]string{"m1": "Inbox", "m2": "Inbox", "m3": "Inbox", "m4": "Archive"}, state(t, st)) + assert.EqualValues(5, f.mimeCalls.Load(), "four downloads plus one 429") + + var labelType string + require.NoError(st.DB().QueryRow(`SELECT label_type FROM labels WHERE source_label_id = 'inbox'`).Scan(&labelType)) + assert.Equal("system", labelType) + + var synced bool + require.NoError(st.DB().QueryRow(`SELECT last_sync_at IS NOT NULL FROM sources`).Scan(&synced)) + assert.True(synced, "a completed sync sets last_sync_at") + + f.mimeCalls.Store(0) + sum, err = f.sync(t, st) + require.NoError(err) + assert.Equal(0, sum.Added) + assert.EqualValues(0, f.mimeCalls.Load()) +} + +func TestImportMoveAndDelete(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + f.put("m2", "inbox") + f.put("m3", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.put("m1", "archive") // move + f.remove("m2") // purged + f.put("m3", "deletions") // Shift+Delete: hidden Recoverable Items + f.put("m5", "inbox") // new + f.mimeCalls.Store(0) + + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "Archive", "m2": "deleted", "m3": "deleted", "m5": "Inbox"}, state(t, st)) + assert.Equal(1, sum.Added) + assert.Equal(1, sum.Moved) + assert.Equal(2, sum.Deleted) + assert.EqualValues(2, f.mimeCalls.Load(), "the new message, and the moved one again") +} + +func TestImportResumesFromCheckpoint(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + for i := range 5 { + f.put(fmt.Sprintf("m%d", i), "inbox") + } + f.stopAt = 4 // the third page fails + + _, err := f.sync(t, st) + require.Error(err) + assert.Len(state(t, st), 4) + + f.walkStarts.Store(0) + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Added) + assert.EqualValues(2, f.walkStarts.Load(), "inbox starts its interrupted walk over; archive starts its first") + assert.Len(state(t, st), 5) +} + +func TestImportExpiredTokenWalksAgain(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + f.put("m2", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.put("m3", "inbox") + f.expired["inbox"] = true + f.mimeCalls.Store(0) + f.walkStarts.Store(0) + + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Added) + assert.EqualValues(1, f.mimeCalls.Load()) + assert.EqualValues(1, f.walkStarts.Load(), "inbox walks again") + assert.Equal(map[string]string{"m1": "Inbox", "m2": "Inbox", "m3": "Inbox"}, state(t, st)) +} + +// snippets returns message ID -> snippet, the stored body start. +func snippets(t *testing.T, st *store.Store) map[string]string { + t.Helper() + rows, err := st.DB().Query(`SELECT source_message_id, COALESCE(snippet, '') FROM messages`) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + out := map[string]string{} + for rows.Next() { + var id, snippet string + require.NoError(t, rows.Scan(&id, &snippet)) + out[id] = snippet + } + require.NoError(t, rows.Err()) + return out +} + +// An incremental round stores every changed message again. A walk after an +// expired cursor returns every message, so it downloads again only drafts, +// which keep their ID while they are edited. +func TestImportRefreshesChangedMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.folders = append(f.folders, "drafts") + f.put("d1", "drafts") + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.version["d1"], f.version["m1"] = 1, 1 + f.put("d1", "drafts") + f.put("m1", "inbox") + f.mimeCalls.Store(0) + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(2, sum.Updated) + var runUpdated int64 + require.NoError(st.DB().QueryRow(`SELECT messages_updated FROM sync_runs ORDER BY id DESC LIMIT 1`).Scan(&runUpdated)) + assert.EqualValues(2, runUpdated) + assert.EqualValues(2, f.mimeCalls.Load()) + assert.Equal(map[string]string{"d1": "body d1 v1", "m1": "body m1 v1"}, snippets(t, st)) + + f.version["d1"], f.version["m1"] = 2, 2 + f.expired["inbox"], f.expired["drafts"] = true, true + f.mimeCalls.Store(0) + _, err = f.sync(t, st) + require.NoError(err) + assert.EqualValues(1, f.mimeCalls.Load(), "the walk downloads only the draft again") + assert.Equal(map[string]string{"d1": "body d1 v2", "m1": "body m1 v1"}, snippets(t, st)) +} + +// A message stored again replaces the attachments of its old MIME. +func TestImportRefreshReplacesAttachments(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{1, 1}, attachments(t, st)) + + f.withAttachment["m1"] = false + f.put("m1", "inbox") + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{0, 0}, attachments(t, st)) +} + +// attachments returns the attachment row count and attachment_count of m1. +func attachments(t *testing.T, st *store.Store) [2]int { + t.Helper() + var rows, count int + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT (SELECT COUNT(*) FROM attachments a WHERE a.message_id = m.id), m.attachment_count + FROM messages m WHERE m.source_message_id = 'm1'`)).Scan(&rows, &count)) + return [2]int{rows, count} +} + +// A message deleted while no cursor covered its folder is marked deleted when +// the walk after an expired cursor does not return it. +func TestImportExpiredTokenReconcilesMissedDelete(t *testing.T) { + for _, status := range []int{http.StatusNotFound, http.StatusGone} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.expiredStatus = status + f.put("m1", "inbox") + f.put("m2", "inbox") + f.put("m3", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.remove("m2") // purged during the gap + f.put("m3", "archive") // moved during the gap + f.expired["inbox"] = true + f.expired["archive"] = true + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "Inbox", "m2": "deleted", "m3": "Archive"}, state(t, st)) + assert.Equal(1, sum.Deleted) + }) + } +} + +// A message restored from Recoverable Items loses its deletion mark. +func TestImportRestoredMessageClearsDeletion(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.put("m1", "deletions") + _, err = f.sync(t, st) + require.NoError(err) + require.Equal(map[string]string{"m1": "deleted"}, state(t, st)) + + f.put("m1", "inbox") + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "Inbox"}, state(t, st)) +} + +// A folder that answers 410 even to a fresh walk fails the sync instead of +// walking again forever. +func TestImportRepeatedGoneFails(t *testing.T) { + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.gone["inbox"] = true + _, err := f.sync(t, st) + require.ErrorIs(t, err, msgraph.ErrGone) +} + +// A message that fails to store stops the sync before its page cursor is +// saved, so the next sync stores it. +func TestImportStoreFailureDoesNotAdvanceCursor(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger to fail one insert") + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + f.put("m2", "inbox") + _, err := st.DB().Exec(`CREATE TRIGGER fail_m2 BEFORE INSERT ON messages + WHEN NEW.source_message_id = 'm2' BEGIN SELECT RAISE(ABORT, 'boom'); END`) + require.NoError(err) + + _, err = f.sync(t, st) + require.Error(err) + + _, err = st.DB().Exec(`DROP TRIGGER fail_m2`) + require.NoError(err) + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "Inbox", "m2": "Inbox"}, state(t, st)) +} + +// A walk after an expired cursor that is interrupted starts over on the next +// sync, so its end still finds the messages that left during the gap. +func TestImportInterruptedRewalkStartsOver(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + for i := range 5 { + f.put(fmt.Sprintf("m%d", i), "inbox") + } + _, err := f.sync(t, st) + require.NoError(err) + + f.remove("m0") // purged during the gap + f.expired["inbox"] = true + f.stopAt = 2 // the second page of the walk fails + _, err = f.sync(t, st) + require.Error(err) + + f.walkStarts.Store(0) + _, err = f.sync(t, st) + require.NoError(err) + assert.EqualValues(1, f.walkStarts.Load(), "inbox walks again from the start") + assert.Equal("deleted", state(t, st)["m0"]) +} + +// When the attachment of a refreshed message cannot be written, the row of +// the old MIME stays, and the next sync downloads the message again. +func TestImportRefreshKeepsAttachmentWhenWriteFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(os.WriteFile(blocker, nil, 0o600)) + f.attachDir = filepath.Join(blocker, "attachments") // cannot be created + f.shifted["m1"] = true // the file gets a new part key + f.put("m1", "inbox") + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + assert.Equal([2]int{1, 1}, attachments(t, st)) + + f.attachDir = "" + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{1, 1}, attachments(t, st)) + assert.Equal("mime:3", attachmentKey(t, st), "the old row is replaced by the new part") +} + +// A refreshed part with the same key but new bytes that cannot be written is +// retried on the next sync, so the old content is not kept as if current. +func TestImportRefreshRetriesChangedPartNotWritten(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + before := attachmentHash(t, st) + + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(os.WriteFile(blocker, nil, 0o600)) + f.attachDir = filepath.Join(blocker, "attachments") // cannot be created + f.attachmentBody["m1"] = "d29ybGQ=" // same part, new bytes + f.put("m1", "inbox") + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + + f.attachDir = "" + _, err = f.sync(t, st) + require.NoError(err) + assert.NotEqual(before, attachmentHash(t, st)) + assert.Equal([2]int{1, 1}, attachments(t, st)) +} + +func attachmentKey(t *testing.T, st *store.Store) string { + t.Helper() + var key string + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT a.source_part_key FROM attachments a JOIN messages m ON m.id = a.message_id + WHERE m.source_message_id = 'm1'`)).Scan(&key)) + return key +} + +func attachmentHash(t *testing.T, st *store.Store) string { + t.Helper() + var hash string + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT a.content_hash FROM attachments a JOIN messages m ON m.id = a.message_id + WHERE m.source_message_id = 'm1'`)).Scan(&hash)) + return hash +} + +// A new message whose attachment cannot be written is downloaded again on the +// next sync, even though a walk skips known messages. +func TestImportNewMessageAttachmentWriteFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(os.WriteFile(blocker, nil, 0o600)) + f.attachDir = filepath.Join(blocker, "attachments") // cannot be created + f.withAttachment["m1"] = true + f.put("m1", "inbox") + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + assert.Equal([2]int{0, 0}, attachments(t, st)) + + f.attachDir = "" + f.expired["inbox"] = true // the next sync walks, and a walk skips known messages + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{1, 1}, attachments(t, st)) + var hasDate bool + require.NoError(st.DB().QueryRow(`SELECT internal_date IS NOT NULL FROM messages WHERE source_message_id = 'm1'`).Scan(&hasDate)) + assert.True(hasDate, "the retry keeps the receipt time") +} + +// Storage skips an empty attachment on purpose, so it does not fail the sync. +func TestImportEmptyAttachmentDoesNotFail(t *testing.T) { + require := require.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.attachmentBody["m1"] = "-" + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.put("m1", "inbox") // refresh + _, err = f.sync(t, st) + require.NoError(err) +} + +// A refreshed message whose file became empty loses the old attachment row, +// because storage writes no row for an empty file. +func TestImportRefreshDropsAttachmentThatBecameEmpty(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{1, 1}, attachments(t, st)) + + f.attachmentBody["m1"] = "-" + f.put("m1", "inbox") // refresh + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{0, 0}, attachments(t, st)) +} + +// A folder removed from the mailbox is retired: its messages that moved get +// their new folder, and the ones that are gone are marked deleted. +func TestImportRemovedFolderIsRetired(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.folders = append(f.folders, "old") + f.put("m1", "old") + f.put("m2", "old") + _, err := f.sync(t, st) + require.NoError(err) + + f.folders = []string{"inbox", "archive"} // "old" is removed + f.remove("m1") + f.folder["m2"] = "inbox" // moved before the removal, with no change log entry + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "deleted", "m2": "Inbox"}, state(t, st)) +} + +// A folder deleted and created again under the same name has a new ID. The +// new folder takes over the old label by name, so the old ID is not retired. +// The walk of the new folder still reconciles the old folder's messages. +func TestImportRecreatedFolderReconcilesOldMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.names = map[string]string{"old": "Projects", "new": "Projects"} + f.folders = append(f.folders, "old") + f.put("m1", "old") + f.put("m2", "old") + f.put("m3", "old") + _, err := f.sync(t, st) + require.NoError(err) + + f.folders = []string{"inbox", "archive", "new"} // "old" is deleted, "new" created + f.remove("m1") + f.folder["m2"] = "inbox" // moved before the deletion, with no change log entry + f.folder["m3"] = "new" + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "deleted", "m2": "Inbox", "m3": "Projects"}, state(t, st)) + var sourceLabelID string + require.NoError(st.DB().QueryRow(`SELECT source_label_id FROM labels WHERE name = 'Projects'`).Scan(&sourceLabelID)) + assert.Equal("new", sourceLabelID, "the new folder took over the old label") +} + +// A known message that delta reports but that is deleted before its $value is +// looked up and marked deleted. +func TestImportKnownMessageGoneBeforeDownload(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.put("m1", "inbox") // changed, so the round downloads it again + f.goneOnValue["m1"] = true + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(map[string]string{"m1": "deleted"}, state(t, st)) +} + +// A refreshed MIME that does not parse keeps the attachment rows there. +func TestImportRefreshWithBrokenMIMEKeepsAttachments(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.withAttachment["m1"] = true + f.put("m1", "inbox") + _, err := f.sync(t, st) + require.NoError(err) + + f.broken["m1"] = true + f.put("m1", "inbox") + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal(1, attachments(t, st)[0]) +} + +// A retry whose download fails keeps its marker. The marker here comes from a +// sync that failed later, so no completed sync holds it, and the checkpoint of +// the failed run is the only place left for it. The next run can complete +// while retaining the marker for a failed download. +func TestImportFailedRetryKeepsMarker(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(os.WriteFile(blocker, nil, 0o600)) + f.attachDir = filepath.Join(blocker, "attachments") // cannot be created + f.withAttachment["m1"] = true + f.put("m1", "inbox") + f.put("m2", "inbox") + f.put("m3", "inbox") + f.stopAt = 2 // the second page fails after m1 is stored + _, err := f.sync(t, st) + require.Error(err) + + f.attachDir = "" + f.badValue["m1"] = true + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + + f.badValue["m1"] = false + _, err = f.sync(t, st) + require.NoError(err) + assert.Equal([2]int{1, 1}, attachments(t, st)) +} + +// Failed new and refreshed messages remain retryable after a completed sync, +// while other messages and later folders continue to advance. +func TestImportDownloadFailureDoesNotBlockAccount(t *testing.T) { + for _, archived := range []bool{false, true} { + t.Run(fmt.Sprint("archived=", archived), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("bad", "inbox") + if archived { + _, err := f.sync(t, st) + require.NoError(err) + f.version["bad"] = 1 + f.put("bad", "inbox") + } + f.badValue["bad"] = true + f.put("good1", "inbox") + f.put("good2", "archive") + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + assert.Equal("Archive", state(t, st)["good2"]) + assert.Equal("Inbox", state(t, st)["good1"]) + + f.put("good3", "archive") + sum, err = f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + assert.Equal("Archive", state(t, st)["good3"]) + + f.badLookup = map[string]bool{"bad": true} + f.put("good4", "archive") + sum, err = f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Errors) + assert.Equal("Archive", state(t, st)["good4"]) + + f.badLookup["bad"] = false + f.badValue["bad"] = false + f.folder["bad"] = "archive" // Retry must find its current folder, without a delta entry. + sum, err = f.sync(t, st) + require.NoError(err) + assert.Zero(sum.Errors) + assert.Equal("Archive", state(t, st)["bad"]) + want := "body bad v0" + if archived { + want = "body bad v1" + } + assert.Equal(want, snippets(t, st)["bad"]) + var hasDate bool + require.NoError(st.DB().QueryRow(`SELECT internal_date IS NOT NULL FROM messages WHERE source_message_id = 'bad'`).Scan(&hasDate)) + assert.True(hasDate) + + f.mimeCalls.Store(0) + _, err = f.sync(t, st) + require.NoError(err) + assert.Zero(f.mimeCalls.Load(), "successful retries clear their markers") + }) + } +} + +func TestImportCanceledDownloadDoesNotAdvance(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + f.put("m1", "inbox") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/$value") { + cancel() + <-r.Context().Done() + return + } + f.serve(w, r) + })) + defer srv.Close() + c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 1000) + _, err := Import(ctx, st, c, Options{Email: "me@example.com", AttachmentsDir: t.TempDir()}, slog.Default()) + require.ErrorIs(err, context.Canceled) + assert.Empty(state(t, st)) + sum, err := f.sync(t, st) + require.NoError(err) + assert.Equal(1, sum.Added) +} + +func TestImportRetryDeletionErrorKeepsMarker(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger to fail a deletion write") + require := require.New(t) + st := testutil.NewTestStore(t) + f := newFakeGraph(t) + blocker := filepath.Join(t.TempDir(), "file") + require.NoError(os.WriteFile(blocker, nil, 0o600)) + f.attachDir = filepath.Join(blocker, "attachments") + f.withAttachment["m1"] = true + f.put("m1", "inbox") + f.put("m2", "inbox") + f.put("m3", "inbox") + f.stopAt = 2 + _, err := f.sync(t, st) + require.Error(err) + + f.goneOnValue["m1"] = true + _, err = st.DB().Exec(`CREATE TRIGGER reject_delete BEFORE UPDATE OF deleted_from_source_at ON messages + WHEN NEW.source_message_id = 'm1' BEGIN SELECT RAISE(ABORT, 'deletion failed'); END`) + require.NoError(err) + sum, err := f.sync(t, st) + require.ErrorContains(err, "deletion failed") + run, err := st.GetLatestCheckpointedSync(sum.SourceID) + require.NoError(err) + require.NotNil(run) + var cursors map[string]string + require.NoError(json.Unmarshal([]byte(run.CursorBefore.String), &cursors)) + assert.Equal(t, "inbox", cursors["retry:m1"]) +} diff --git a/internal/store/attachment_roles.go b/internal/store/attachment_roles.go index c063322cc..385db2880 100644 --- a/internal/store/attachment_roles.go +++ b/internal/store/attachment_roles.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strings" "go.kenn.io/msgvault/internal/attachmentpolicy" ) @@ -274,6 +275,27 @@ func (s *Store) DeleteKeyedAttachmentsExceptContext( }) } +// DeleteMIMEAttachmentsExceptContext removes the MIME-owned attachment rows of +// a message whose source part key is not in keep. A connector that stores a +// newer MIME for the same message calls it after the new rows are written, so +// a failed write never loses the old rows. +func (s *Store) DeleteMIMEAttachmentsExceptContext( + ctx context.Context, messageID int64, keep []string, +) error { + query := `DELETE FROM attachments WHERE message_id = ? AND source_attachment_id IS NULL` + args := []any{messageID} + if len(keep) > 0 { + query += ` AND COALESCE(source_part_key, '') NOT IN (?` + strings.Repeat(`, ?`, len(keep)-1) + `)` + for _, k := range keep { + args = append(args, k) + } + } + return s.withSyncMessageWriteContext(ctx, messageID, func(q querier) error { + _, err := q.Exec(query, args...) + return err + }) +} + // DeleteUnstoredAttachmentByHashContext removes an obsolete placeholder for // one exact content or synthetic hash. func (s *Store) DeleteUnstoredAttachmentByHashContext( @@ -514,3 +536,29 @@ func (s *Store) replaceMIMEAttachmentsWith( } return nil } + +// AttachmentPartsStoredContext reports whether every part key and content hash +// pair has a stored row for the message. An empty key matches NULL or empty keys. +func (s *Store) AttachmentPartsStoredContext(ctx context.Context, messageID int64, parts []AttachmentRef) (bool, error) { + want := make(map[[2]string]struct{}, len(parts)) + for _, p := range parts { + want[[2]string{p.SourcePartKey, p.ContentHash}] = struct{}{} + } + if len(want) == 0 { + return true, nil + } + args := []any{messageID} + match := make([]string, 0, len(want)) + for kh := range want { + match = append(match, "(COALESCE(source_part_key, '') = ? AND content_hash = ?)") + args = append(args, kh[0], kh[1]) + } + var n int + err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(DISTINCT COALESCE(source_part_key, '') || ':' || content_hash) FROM attachments + WHERE message_id = ? AND (`+strings.Join(match, " OR ")+`)`, args...).Scan(&n) + if err != nil { + return false, fmt.Errorf("check stored attachment parts: %w", err) + } + return n == len(want), nil +} diff --git a/internal/store/docbank_media_test.go b/internal/store/docbank_media_test.go index 73f63eeb1..f24152350 100644 --- a/internal/store/docbank_media_test.go +++ b/internal/store/docbank_media_test.go @@ -102,6 +102,10 @@ func TestBeeperMediaKeyChangeKeepsSharedDelivery(t *testing.T) { b := second.mapping("shared", "revision", "old-key") retainAudio(t, f.Store, a, "first-occurrence") retainAudio(t, f.Store, b, "second-occurrence") + // Make the old delivery due first without depending on clock precision. + _, err := f.Store.DB().Exec(`UPDATE beeper_media_deliveries + SET next_action_at = '2000-01-01 00:00:00' WHERE processing_key = 'old-key'`) + require.NoError(err) a.ProcessingKey = "new-key" require.NoError(f.Store.ReconcileBeeperMediaMapping(t.Context(), a)) operation, ready, err := f.Store.NextBeeperMediaOperation(t.Context(), "shared", time.Now().UTC()) diff --git a/internal/store/messages.go b/internal/store/messages.go index 42d559764..522a7087a 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -6076,3 +6076,35 @@ func (s *Store) ReplaceMessageLinkAttachments(messageID int64, refs []Attachment `(storage_path LIKE 'http://%' OR storage_path LIKE 'https://%') AND COALESCE(source_attachment_id, '') NOT LIKE 'teams:inline:%'`, false, refs) } + +// LabelIDContext finds a label by its provider ID within a source. +func (s *Store) LabelIDContext(ctx context.Context, sourceID int64, sourceLabelID string) (int64, error) { + var id int64 + if err := s.db.QueryRowContext(ctx, `SELECT id FROM labels WHERE source_id = ? AND source_label_id = ?`, sourceID, sourceLabelID).Scan(&id); err != nil { + return 0, fmt.Errorf("look up source label: %w", err) + } + return id, nil +} + +// MessageIDsWithLabelContext returns source message IDs and archive IDs for +// messages in a label that have not been marked deleted at the source. +func (s *Store) MessageIDsWithLabelContext(ctx context.Context, sourceID, labelID int64) (map[string]int64, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT m.source_message_id, m.id FROM messages m + JOIN message_labels ml ON ml.message_id = m.id + WHERE m.source_id = ? AND ml.label_id = ? AND m.deleted_from_source_at IS NULL`, sourceID, labelID) + if err != nil { + return nil, fmt.Errorf("list messages in label: %w", err) + } + defer func() { _ = rows.Close() }() + ids := map[string]int64{} + for rows.Next() { + var sourceMsgID string + var id int64 + if err := rows.Scan(&sourceMsgID, &id); err != nil { + return nil, fmt.Errorf("scan message in label: %w", err) + } + ids[sourceMsgID] = id + } + return ids, rows.Err() +} diff --git a/internal/store/migrate_legacy_identity.go b/internal/store/migrate_legacy_identity.go index 7d6e48e0b..a50ba26bd 100644 --- a/internal/store/migrate_legacy_identity.go +++ b/internal/store/migrate_legacy_identity.go @@ -307,7 +307,7 @@ func (s *Store) legacyIdentityMigrationAppliedTx(ctx context.Context, tx *logged func SourceTypeUsesEmailIdentity(sourceType string) bool { switch sourceType { - case "gmail", "imap", "o365", "mbox", "hey", "apple-mail", "pst", "eml", "maildir": + case "gmail", "imap", "o365", "msmail", "mbox", "hey", "apple-mail", "pst", "eml", "maildir": return true } return false diff --git a/internal/store/migrate_legacy_identity_test.go b/internal/store/migrate_legacy_identity_test.go index ad933edbf..c85a73056 100644 --- a/internal/store/migrate_legacy_identity_test.go +++ b/internal/store/migrate_legacy_identity_test.go @@ -20,6 +20,7 @@ func TestSourceTypeUsesEmailIdentity(t *testing.T) { {"gmail", true}, {"imap", true}, {"o365", true}, + {"msmail", true}, {"mbox", true}, {"hey", true}, {"apple-mail", true}, diff --git a/internal/teams/client.go b/internal/teams/client.go index b07c43ff2..ac33061b0 100644 --- a/internal/teams/client.go +++ b/internal/teams/client.go @@ -2,212 +2,19 @@ package teams import ( "context" - "encoding/json/v2" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - "go.kenn.io/msgvault/internal/httpretry" - "golang.org/x/time/rate" + "go.kenn.io/msgvault/internal/msgraph" ) -// ErrMediaTooLarge classifies hosted media that exceeds its configured cap. -var ErrMediaTooLarge = errors.New("teams hosted media exceeds the configured size cap") - -var errGraphNotFound = errors.New("graph resource not found") - -const ( - maxRetries = 8 - maxRetryAfter = httpretry.ProviderMaxRetryAfter -) - -// TokenFunc returns a bearer token for a Graph API request. -type TokenFunc func(context.Context) (string, error) - -// Client is a minimal Microsoft Graph REST client supporting paging and -// Retry-After back-off. +// Client adds the Teams endpoints to the shared Graph transport. type Client struct { - baseURL string - token TokenFunc - http *http.Client - limiter *rate.Limiter + *msgraph.Client } // NewClient creates a Client. baseURL is injected so tests can point at // httptest servers. qps controls the token-bucket rate limit (default 5). -func NewClient(baseURL string, token TokenFunc, qps float64) *Client { - if qps <= 0 { - qps = 5 - } - return &Client{ - baseURL: strings.TrimRight(baseURL, "/"), - token: token, - http: &http.Client{Timeout: 60 * time.Second}, - limiter: rate.NewLimiter(rate.Limit(qps), 1), - } -} - -// get fetches rawURL, respecting the rate limiter and retrying on 429/5xx with -// Retry-After or exponential back-off. -func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { - return c.getLimited(ctx, rawURL, 0) -} - -func (c *Client) getLimited(ctx context.Context, rawURL string, maxBytes int64) ([]byte, error) { - reqURL, err := c.resolveRequestURL(rawURL) - if err != nil { - return nil, err - } - for attempt := range maxRetries { - if err := c.limiter.Wait(ctx); err != nil { - return nil, fmt.Errorf("wait for graph rate limit: %w", err) - } - tok, err := c.token(ctx) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Bearer "+tok) - req.Header.Set("Accept", "application/json") - resp, err := c.http.Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode == http.StatusOK && maxBytes > 0 && resp.ContentLength > maxBytes { - _ = resp.Body.Close() - return nil, ErrMediaTooLarge - } - reader := io.Reader(resp.Body) - if maxBytes > 0 { - reader = io.LimitReader(resp.Body, maxBytes+1) - } - body, readErr := io.ReadAll(reader) - closeErr := resp.Body.Close() - if readErr != nil { - return nil, fmt.Errorf("graph GET %s: read body: %w", reqURL, readErr) - } - if closeErr != nil { - return nil, fmt.Errorf("graph GET %s: close body: %w", reqURL, closeErr) - } - switch { - case resp.StatusCode == http.StatusOK: - if maxBytes > 0 && int64(len(body)) > maxBytes { - return nil, ErrMediaTooLarge - } - return body, nil - case resp.StatusCode == http.StatusNotFound: - return nil, fmt.Errorf("graph GET %s: status %d: %s: %w", reqURL, resp.StatusCode, string(body), errGraphNotFound) - case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500: - wait := httpretry.RetryAfter(resp.Header.Get("Retry-After"), attempt, maxRetryAfter) - timer := time.NewTimer(wait) - select { - case <-ctx.Done(): - timer.Stop() - return nil, ctx.Err() - case <-timer.C: - } - continue - default: - return nil, fmt.Errorf("graph GET %s: status %d: %s", reqURL, resp.StatusCode, string(body)) - } - } - return nil, fmt.Errorf("graph GET %s: exhausted %d retries", reqURL, maxRetries) -} - -func (c *Client) resolveRequestURL(rawURL string) (string, error) { - u, err := url.Parse(rawURL) - if err != nil { - return "", fmt.Errorf("graph GET %q: parse URL: %w", rawURL, err) - } - if !u.IsAbs() { - return c.baseURL + rawURL, nil - } - base, err := url.Parse(c.baseURL) - if err != nil { - return "", fmt.Errorf("graph base URL %q: %w", c.baseURL, err) - } - if !strings.EqualFold(u.Scheme, base.Scheme) || !strings.EqualFold(u.Host, base.Host) { - return "", fmt.Errorf("graph GET %s: off-origin absolute URL", rawURL) - } - return u.String(), nil -} - -// GetRaw fetches url and returns the raw response bytes. url should be a -// path-relative string (e.g. "/me/chats/.../hostedContents/1/$value"); it is -// prefixed with the client's baseURL automatically by the underlying get method. -func (c *Client) GetRaw(ctx context.Context, url string) ([]byte, error) { - return c.get(ctx, url) -} - -// GetRawLimited fetches raw hosted media while enforcing a response-byte cap. -func (c *Client) GetRawLimited(ctx context.Context, url string, maxBytes int64) ([]byte, error) { - return c.getLimited(ctx, url, maxBytes) -} - -// BaseURL returns the client's configured base URL (scheme + host, no trailing slash). -// Importers use this to rewrite absolute graph.microsoft.com URLs to the configured -// host (supporting both production and httptest servers). -func (c *Client) BaseURL() string { - return c.baseURL -} - -// getJSON fetches url and unmarshals the JSON body into out. -func (c *Client) getJSON(ctx context.Context, url string, out any) error { - body, err := c.get(ctx, url) - if err != nil { - return err - } - return json.Unmarshal(body, out) -} - -// pageThrough follows @odata.nextLink, decoding each page into []T, calling fn. -// Returns the terminal @odata.deltaLink (empty for non-delta endpoints). -func pageThrough[T any](ctx context.Context, c *Client, startURL string, fn func([]T)) (string, error) { - delta, _, err := pageThroughLimit(ctx, c, startURL, 0, fn) - return delta, err -} - -// pageThroughLimit is pageThrough with an optional item cap. When limit is -// positive, it stops before fetching a nextLink once enough items have been -// delivered and reports whether unread items/pages remain. -func pageThroughLimit[T any](ctx context.Context, c *Client, startURL string, limit int, fn func([]T)) (string, bool, error) { - url := startURL - delivered := 0 - for { - var page listResponse[T] - if err := c.getJSON(ctx, url, &page); err != nil { - return "", false, err - } - values := page.Value - if limit > 0 { - remaining := limit - delivered - if remaining <= 0 { - return "", true, nil - } - if len(values) > remaining { - fn(values[:remaining]) - return "", true, nil - } - if len(values) == remaining && page.NextLink != "" { - fn(values) - return "", true, nil - } - } - fn(values) - delivered += len(values) - if page.NextLink != "" { - url = page.NextLink - continue - } - return page.DeltaLink, false, nil - } +func NewClient(baseURL string, token msgraph.TokenFunc, qps float64) *Client { + return &Client{msgraph.NewClient(baseURL, token, qps)} } // SelfChatID is the Teams chat a user holds with themselves. Graph never @@ -220,19 +27,19 @@ const SelfChatID = "48:notes" func (c *Client) ListChats(ctx context.Context) ([]Chat, error) { var out []Chat - _, err := pageThrough[Chat](ctx, c, "/me/chats?$top=50", func(p []Chat) { out = append(out, p...) }) + _, err := msgraph.PageThrough[Chat](ctx, c.Client, "/me/chats?$top=50", func(p []Chat) { out = append(out, p...) }) return out, err } func (c *Client) ListJoinedTeams(ctx context.Context) ([]JoinedTeam, error) { var out []JoinedTeam - _, err := pageThrough[JoinedTeam](ctx, c, "/me/joinedTeams", func(p []JoinedTeam) { out = append(out, p...) }) + _, err := msgraph.PageThrough[JoinedTeam](ctx, c.Client, "/me/joinedTeams", func(p []JoinedTeam) { out = append(out, p...) }) return out, err } func (c *Client) ListChannels(ctx context.Context, teamID string) ([]Channel, error) { var out []Channel - _, err := pageThrough[Channel](ctx, c, "/teams/"+teamID+"/channels", func(p []Channel) { out = append(out, p...) }) + _, err := msgraph.PageThrough[Channel](ctx, c.Client, "/teams/"+teamID+"/channels", func(p []Channel) { out = append(out, p...) }) return out, err } @@ -248,7 +55,7 @@ func (c *Client) ListChatMessages(ctx context.Context, chatID, sinceISO string, url += "&$filter=lastModifiedDateTime%20gt%20" + sinceISO + "&$orderby=lastModifiedDateTime%20desc" } var out []ChatMessage - _, truncated, err := pageThroughLimit[ChatMessage](ctx, c, url, limit, func(p []ChatMessage) { out = append(out, p...) }) + _, truncated, err := msgraph.PageThroughLimit[ChatMessage](ctx, c.Client, url, limit, func(p []ChatMessage) { out = append(out, p...) }) return out, truncated, err } @@ -260,19 +67,19 @@ func (c *Client) ChannelMessagesDelta(ctx context.Context, teamID, channelID, de start = "/teams/" + teamID + "/channels/" + channelID + "/messages/delta" } var out []ChatMessage - newDelta, truncated, err := pageThroughLimit[ChatMessage](ctx, c, start, limit, func(p []ChatMessage) { out = append(out, p...) }) + newDelta, truncated, err := msgraph.PageThroughLimit[ChatMessage](ctx, c.Client, start, limit, func(p []ChatMessage) { out = append(out, p...) }) return out, newDelta, truncated, err } func (c *Client) ListChannelMessages(ctx context.Context, teamID, channelID string, limit int) ([]ChatMessage, bool, error) { var out []ChatMessage - _, truncated, err := pageThroughLimit[ChatMessage](ctx, c, "/teams/"+teamID+"/channels/"+channelID+"/messages?$top=50", limit, func(p []ChatMessage) { out = append(out, p...) }) + _, truncated, err := msgraph.PageThroughLimit[ChatMessage](ctx, c.Client, "/teams/"+teamID+"/channels/"+channelID+"/messages?$top=50", limit, func(p []ChatMessage) { out = append(out, p...) }) return out, truncated, err } func (c *Client) ListReplies(ctx context.Context, teamID, channelID, messageID string, limit int) ([]ChatMessage, bool, error) { var out []ChatMessage - _, truncated, err := pageThroughLimit[ChatMessage](ctx, c, "/teams/"+teamID+"/channels/"+channelID+"/messages/"+messageID+"/replies", limit, func(p []ChatMessage) { out = append(out, p...) }) + _, truncated, err := msgraph.PageThroughLimit[ChatMessage](ctx, c.Client, "/teams/"+teamID+"/channels/"+channelID+"/messages/"+messageID+"/replies", limit, func(p []ChatMessage) { out = append(out, p...) }) return out, truncated, err } @@ -280,7 +87,7 @@ func (c *Client) ListReplies(ctx context.Context, teamID, channelID, messageID s // selecting the fields needed for participant email resolution. func (c *Client) GetUser(ctx context.Context, id string) (*GraphUser, error) { var u GraphUser - if err := c.getJSON(ctx, "/users/"+id+"?$select=id,mail,userPrincipalName,displayName", &u); err != nil { + if err := c.GetJSON(ctx, "/users/"+id+"?$select=id,mail,userPrincipalName,displayName", &u); err != nil { return nil, err } return &u, nil @@ -289,7 +96,7 @@ func (c *Client) GetUser(ctx context.Context, id string) (*GraphUser, error) { // ListChatMembers returns all members of the given chat. func (c *Client) ListChatMembers(ctx context.Context, chatID string) ([]ChatMember, error) { var out []ChatMember - _, err := pageThrough[ChatMember](ctx, c, "/chats/"+chatID+"/members", func(p []ChatMember) { out = append(out, p...) }) + _, err := msgraph.PageThrough[ChatMember](ctx, c.Client, "/chats/"+chatID+"/members", func(p []ChatMember) { out = append(out, p...) }) return out, err } @@ -298,7 +105,7 @@ func (c *Client) ListChatMembers(ctx context.Context, chatID string) ([]ChatMemb // against. func (c *Client) ListTeamMembers(ctx context.Context, teamID string) ([]ChatMember, error) { var out []ChatMember - _, err := pageThrough[ChatMember](ctx, c, "/teams/"+teamID+"/members", func(p []ChatMember) { out = append(out, p...) }) + _, err := msgraph.PageThrough[ChatMember](ctx, c.Client, "/teams/"+teamID+"/members", func(p []ChatMember) { out = append(out, p...) }) return out, err } @@ -306,7 +113,7 @@ func (c *Client) ListTeamMembers(ctx context.Context, teamID string) ([]ChatMemb // shared channels are governed by this roster rather than the team's. func (c *Client) ListChannelMembers(ctx context.Context, teamID, channelID string) ([]ChatMember, error) { var out []ChatMember - _, err := pageThrough[ChatMember](ctx, c, "/teams/"+teamID+"/channels/"+channelID+"/members", + _, err := msgraph.PageThrough[ChatMember](ctx, c.Client, "/teams/"+teamID+"/channels/"+channelID+"/members", func(p []ChatMember) { out = append(out, p...) }) return out, err } diff --git a/internal/teams/client_test.go b/internal/teams/client_test.go index 371ff89ae..9832e8081 100644 --- a/internal/teams/client_test.go +++ b/internal/teams/client_test.go @@ -2,13 +2,12 @@ package teams import ( "context" + "go.kenn.io/msgvault/internal/msgraph" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" - "testing/synctest" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -32,7 +31,7 @@ func TestClientGetJSONPaging(t *testing.T) { c := NewClient(srv.URL, func(context.Context) (string, error) { return "test-token", nil }, 50) var got []Chat - delta, err := pageThrough[Chat](context.Background(), c, "/me/chats", func(page []Chat) { got = append(got, page...) }) + delta, err := msgraph.PageThrough[Chat](context.Background(), c.Client, "/me/chats", func(page []Chat) { got = append(got, page...) }) require.NoError(t, err) assert.Equal(t, "DELTA", delta) assert.Len(t, got, 2) @@ -87,7 +86,7 @@ func TestClientGetRawLimitedRejectsDeclaredAndStreamedOversizeBodies(t *testing. defer srv.Close() client := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 50) _, err := client.GetRawLimited(context.Background(), "/hostedContents/1/$value", 10) - assert.ErrorIs(t, err, ErrMediaTooLarge) + assert.ErrorIs(t, err, msgraph.ErrTooLarge) }) } } @@ -125,28 +124,11 @@ func TestClientRetryAfter(t *testing.T) { defer srv.Close() c := NewClient(srv.URL, func(context.Context) (string, error) { return "t", nil }, 50) - _, err := pageThrough[Chat](context.Background(), c, "/x", func([]Chat) {}) + _, err := msgraph.PageThrough[Chat](context.Background(), c.Client, "/x", func([]Chat) {}) require.NoError(t, err) assert.EqualValues(t, 2, calls.Load()) } -func TestClientContextCancelDuringRetry(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - server := httptest.NewTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Retry-After", "30") // long wait so cancellation wins - w.WriteHeader(http.StatusTooManyRequests) - })) - httpClient := server.Client() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - c := NewClient(server.URL, func(context.Context) (string, error) { return "t", nil }, 50) - c.http.Transport = httpClient.Transport - go func() { time.Sleep(50 * time.Millisecond); cancel() }() - _, err := pageThrough[Chat](ctx, c, "/x", func([]Chat) {}) - require.ErrorIs(t, err, context.Canceled) - }) -} - func TestListChatsAndMessages(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/teams/importer.go b/internal/teams/importer.go index ca89d406f..37cf610c6 100644 --- a/internal/teams/importer.go +++ b/internal/teams/importer.go @@ -6,6 +6,7 @@ import ( "encoding/json/v2" "errors" "fmt" + "go.kenn.io/msgvault/internal/msgraph" "net/url" "regexp" "strings" @@ -353,7 +354,7 @@ func (imp *Importer) syncChats(ctx context.Context, sourceID, syncID int64, opts // user, chatMembers leaves the roster unresolved so a later sync can retry it. func (imp *Importer) selfChat(ctx context.Context, email string) ([]Chat, []ChatMember, error) { msgs, _, err := imp.client.ListChatMessages(ctx, SelfChatID, "", 50) - if errors.Is(err, errGraphNotFound) { + if errors.Is(err, msgraph.ErrNotFound) { return nil, nil, nil } if err != nil { @@ -1210,7 +1211,7 @@ func (imp *Importer) downloadInlineImages(ctx context.Context, messageID int64, data, derr := imp.client.GetRawLimited(ctx, fetchPath, maxBytes) if derr != nil || len(data) == 0 { replacementComplete = false - if errors.Is(derr, ErrMediaTooLarge) { + if errors.Is(derr, msgraph.ErrTooLarge) { marker.Size = attachmentpolicy.OversizeMarkerSize(maxBytes, int64(marker.Size)) marker.State = attachmentpolicy.StateSkipped marker.SkipReason = attachmentpolicy.SkipSizeCap diff --git a/internal/teams/types.go b/internal/teams/types.go index deefd85e7..61cda50d5 100644 --- a/internal/teams/types.go +++ b/internal/teams/types.go @@ -9,14 +9,6 @@ import ( "go.kenn.io/msgvault/internal/attachmentpolicy" ) -// ---- Graph response envelopes ---- - -type listResponse[T any] struct { - Value []T `json:"value"` - NextLink string `json:"@odata.nextLink"` - DeltaLink string `json:"@odata.deltaLink"` -} - // ---- Chats & channels ---- type Chat struct { From 61a8d1f1a07b357e8b53fa4926d2c435746a07cd Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 29 Sep 2026 07:27:03 -0500 Subject: [PATCH 080/125] fix(identity): preserve account opt-out during scheduled sync (#987) Accounts added with `--no-default-identity` keep that choice during scheduled sync and re-authorization. The add commands save the opt-out in `sync_config`, and the shared identity helper checks it before confirming an address. Omitting the flag preserves the saved choice; explicit `--no-default-identity=false` re-enables defaults. This covers Gmail (including service accounts), IMAP, Microsoft 365 (IMAP and Graph mail), and Teams without a schema migration. Provider config updates preserve the choice. Existing identity rows and the one-time legacy identity migration retain their behavior. The CLI reference describes the flag contract, and the account guide explains how to save an older opt-out and remove an unwanted identity. If the shared helper cannot read or parse `sync_config`, it logs a warning and skips automatic identity confirmation. This also applies to import commands that use the helper. A Beeper regression test sets its intended operation order explicitly so equal clock timestamps do not fail Windows CI. The shared test transport moved to an untagged test file so identity coverage compiles without optional build tags. Fixes #981. Co-authored-by: Wes McKinney --- cmd/msgvault/cmd/account_identity.go | 75 +++++- .../cmd/account_identity_sync_test.go | 224 ++++++++++++++++++ cmd/msgvault/cmd/add_teams.go | 5 +- cmd/msgvault/cmd/addaccount.go | 12 +- cmd/msgvault/cmd/addaccount_test.go | 27 ++- cmd/msgvault/cmd/addimap.go | 5 +- cmd/msgvault/cmd/addo365.go | 8 +- cmd/msgvault/cmd/eval_rerank_test.go | 4 - docs/cli-reference.md | 21 +- docs/usage/multi-account.md | 5 + internal/beeper/docbank_media_review_test.go | 5 + 11 files changed, 375 insertions(+), 16 deletions(-) create mode 100644 cmd/msgvault/cmd/account_identity_sync_test.go diff --git a/cmd/msgvault/cmd/account_identity.go b/cmd/msgvault/cmd/account_identity.go index 7d22afbb0..4882f45f5 100644 --- a/cmd/msgvault/cmd/account_identity.go +++ b/cmd/msgvault/cmd/account_identity.go @@ -1,26 +1,82 @@ package cmd import ( + "encoding/json/jsontext" + "encoding/json/v2" "fmt" "io" "log/slog" "strings" + "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/store" ) // noDefaultIdentityHelp is the flag help text for --no-default-identity. // Each ingest command registers its own bool variable and reuses this constant. -const noDefaultIdentityHelp = "Suppress auto-default-identity at account creation. " + +const noDefaultIdentityHelp = "Suppress automatic default identity confirmation. " + "Note: a one-time legacy [identity] config migration may still write confirmed " + "identifiers to the account on first post-upgrade startup." +const savedDefaultIdentityHelp = noDefaultIdentityHelp + + " Saved for later syncs and re-authorization; omit the flag to keep the choice, " + + "or use --no-default-identity=false to re-enable defaults." + +// setDefaultIdentityOptOut changes the saved choice only for an explicit flag. +// source must contain the config from before registration: IMAP add commands +// replace provider settings, so an omitted flag must carry the old choice forward. +func setDefaultIdentityOptOut(cmd *cobra.Command, s *store.Store, source *store.Source, optOut bool) error { + src, err := s.GetSourceByID(source.ID) + if err != nil { + return fmt.Errorf("read identity preference: %w", err) + } + if !cmd.Flags().Changed("no-default-identity") { + if src.SyncConfig == source.SyncConfig { + return nil + } + var previous struct { + NoDefaultIdentity bool `json:"no_default_identity"` + } + if source.SyncConfig.Valid { + if err := json.Unmarshal([]byte(source.SyncConfig.String), &previous); err != nil { + return fmt.Errorf("parse saved identity preference: %w", err) + } + } + optOut = previous.NoDefaultIdentity + } + cfg := make(map[string]jsontext.Value) + if src.SyncConfig.Valid { + if err := json.Unmarshal([]byte(src.SyncConfig.String), &cfg); err != nil { + return fmt.Errorf("parse identity preference: %w", err) + } + } + if optOut { + if cfg == nil { + cfg = make(map[string]jsontext.Value) + } + cfg["no_default_identity"] = jsontext.Value("true") + } else { + if _, exists := cfg["no_default_identity"]; !exists { + return nil + } + delete(cfg, "no_default_identity") + } + encoded, err := json.Marshal(cfg, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("encode identity preference: %w", err) + } + if err := s.UpdateSourceSyncConfig(source.ID, string(encoded)); err != nil { + return fmt.Errorf("save identity preference: %w", err) + } + return nil +} + // confirmDefaultIdentity writes one confirmed identifier to a freshly // created source's identity. Best-effort: any error is logged and swallowed // so a partially failed identity write never breaks ingest. Empty identifiers // are a silent no-op. // -// Skips the write when the source already has at least one identity row. +// Skips the write when the source opted out or already has an identity row. // add-account / add-imap / add-o365 / import-* commands all call this on // every invocation (including reruns and rebinds), so without this guard // an identity the user explicitly removed via `identity remove` would be @@ -46,6 +102,21 @@ func confirmDefaultIdentity(out io.Writer, s *store.Store, sourceID int64, accou if id == "" { return } + src, err := s.GetSourceByID(sourceID) + var cfg struct { + NoDefaultIdentity bool `json:"no_default_identity"` + } + if err == nil && src.SyncConfig.Valid { + err = json.Unmarshal([]byte(src.SyncConfig.String), &cfg) + } + if err != nil { + logger.Warn("auto-default-identity preference check failed", + "source_id", sourceID, "account", account, "error", err.Error()) + return + } + if cfg.NoDefaultIdentity { + return + } existing, err := s.ListAccountIdentities(sourceID) if err != nil { logger.Warn("auto-default-identity precheck failed", diff --git a/cmd/msgvault/cmd/account_identity_sync_test.go b/cmd/msgvault/cmd/account_identity_sync_test.go new file mode 100644 index 000000000..a2913bcc9 --- /dev/null +++ b/cmd/msgvault/cmd/account_identity_sync_test.go @@ -0,0 +1,224 @@ +package cmd + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json/v2" + "encoding/pem" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type testTransport func(*http.Request) (*http.Response, error) + +func (f testTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestAddServiceAccountDefaultIdentityScheduledSync(t *testing.T) { + assert, require := assert.New(t), require.New(t) + saveAddAccountFlags(t) + // Only Google's token and profile responses are simulated. Registration, + // service-account token creation, scheduled sync, and database writes are real. + // Scheduled Gmail sync constructs its HTTP client with context.Background(), + // so oauth2.HTTPClient in the invocation context cannot intercept its requests. + // Keep this test nonparallel and restore the transport after it runs. + savedTransport := http.DefaultTransport + t.Cleanup(func() { http.DefaultTransport = savedTransport }) + http.DefaultTransport = testTransport(func(req *http.Request) (*http.Response, error) { + var body string + switch req.Method + " " + req.URL.String() { + case "POST https://token.example.com/oauth2": + body = `{"access_token":"synthetic-token","token_type":"Bearer","expires_in":3600}` + case "GET https://gmail.googleapis.com/gmail/v1/users/me/profile": + body = `{"emailAddress":"user@example.com","historyId":"100"}` + default: + return nil, fmt.Errorf("unexpected request: %s %s", req.Method, req.URL) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil + }) + + home := t.TempDir() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(err) + keyJSON, err := json.Marshal(map[string]string{ + "type": "service_account", + "client_email": "service@example.com", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})), + "token_uri": "https://token.example.com/oauth2", + }) + require.NoError(err) + keyPath := filepath.Join(home, "service-account.json") + require.NoError(os.WriteFile(keyPath, keyJSON, 0600)) + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, + OAuth: config.OAuthConfig{ServiceAccountKey: keyPath}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + optOut := flag != "--no-default-identity=false" + cmd := &cobra.Command{Use: addAccountUse, RunE: runAddAccountLocal} + registerAddAccountFlags(cmd) + args := []string{"user@example.com"} + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + src, err := findGmailSource(st, "user@example.com") + require.NoError(err) + // Seed the cursor of an already-synced mailbox so this scheduled run + // completes with no new messages. + require.NoError(st.UpdateSourceSyncCursor(src.ID, "100")) + summary, err := runScheduledGmailSync(ctx, "user@example.com", src, st, nil, invocationFromContext(ctx)) + require.NoError(err) + assert.Zero(summary.Errors) + ids, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + if optOut { + assert.Empty(ids, "scheduled sync must preserve the service-account opt-out") + } else { + require.Len(ids, 1) + assert.Equal("user@example.com", ids[0].Address) + } + } +} + +func TestAddIMAPDefaultIdentityScheduledSync(t *testing.T) { + assert, require := assert.New(t), require.New(t) + savedHost, savedPort, savedUsername := imapHost, imapPort, imapUsername + savedNoTLS, savedSTARTTLS, savedNoDefault := imapNoTLS, imapSTARTTLS, noDefaultIdentityAddImap + t.Cleanup(func() { + imapHost, imapPort, imapUsername = savedHost, savedPort, savedUsername + imapNoTLS, imapSTARTTLS, noDefaultIdentityAddImap = savedNoTLS, savedSTARTTLS, savedNoDefault + }) + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + t.Setenv("MSGVAULT_IMAP_PASSWORD", testutil.IMAPTestPassword) + addr, _ := testutil.StartIMAPMemServerWithSpecialUse(t, map[string]int{"INBOX": 1}, nil) + host, port, err := net.SplitHostPort(addr) + require.NoError(err) + home := t.TempDir() + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + + // Re-registering preserves the choice unless the flag is explicit. + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + optOut := flag != "--no-default-identity=false" + cmd := newAddIMAPCmd() + args := []string{"--host", host, "--port", port, "--username", testutil.IMAPTestUsername, "--no-tls"} + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + sources, err := st.ListSources(sourceTypeIMAP) + require.NoError(err) + require.Len(sources, 1) + src := sources[0] + // Removing the last identity on an account that did not opt out must + // still allow the next scheduled sync to restore it. + if !optOut { + removed, err := st.RemoveAccountIdentity(src.ID, testutil.IMAPTestUsername) + require.NoError(err) + require.EqualValues(1, removed) + } + summary, err := runScheduledIMAPSync(ctx, src, st, invocationFromContext(ctx)) + require.NoError(err) + assert.Zero(summary.Errors) + ids, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + if optOut { + assert.Empty(ids, "scheduled sync must preserve the opt-out") + } else { + require.Len(ids, 1) + assert.Equal(testutil.IMAPTestUsername, ids[0].Address) + } + } +} + +func TestAddMicrosoftDefaultIdentityOptOut(t *testing.T) { + savedGraph := o365Graph + savedO365, savedTeams := noDefaultIdentityAddO365, noDefaultIdentityAddTeams + savedO365Headless, savedTeamsHeadless := o365Headless, teamsHeadless + savedO365Tenant, savedTeamsTenant := o365TenantID, teamsTenantID + t.Cleanup(func() { + o365Graph = savedGraph + noDefaultIdentityAddO365, noDefaultIdentityAddTeams = savedO365, savedTeams + o365Headless, teamsHeadless = savedO365Headless, savedTeamsHeadless + o365TenantID, teamsTenantID = savedO365Tenant, savedTeamsTenant + }) + for _, tc := range []struct { + name string + newCommand func() *cobra.Command + args []string + }{ + {"o365", newAddO365LocalCmd, nil}, + {"graph", newAddO365LocalCmd, []string{"--graph"}}, + {"teams", newAddTeamsLocalCmd, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + home := t.TempDir() + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, + Microsoft: config.MicrosoftConfig{ClientID: "synthetic-client"}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + const email = "user@example.com" + mgr := microsoft.NewManager(cfg.Microsoft.ClientID, "common", cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), testDiscardLogger()) + require.NoError(os.MkdirAll(cfg.TokensDir(), 0700)) + require.NoError(os.WriteFile(mgr.TokenPath(email), []byte(`{"access_token":"synthetic-token"}`), 0600)) + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + cmd := tc.newCommand() + args := append([]string{email, "--" + oauthPreflightedFlag}, tc.args...) + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + sources, err := st.ListSources("") + require.NoError(err) + require.Len(sources, 1) + if tc.name == "graph" { + // Identity setup precedes token loading. With no Graph token, + // the real scheduled path stops before making network requests. + err := runScheduledMSMailSync(ctx, sources[0], st, invocationFromContext(ctx)) + require.ErrorContains(err, "no valid token") + } else { + confirmDefaultIdentity(io.Discard, st, sources[0].ID, email, email, "account-identifier", testDiscardLogger()) + } + ids, err := st.ListAccountIdentities(sources[0].ID) + require.NoError(err) + if flag == "--no-default-identity=false" { + require.Len(ids, 1, "explicit false restores the default") + assert.Equal(email, ids[0].Address) + } else { + assert.Empty(ids, "default identity confirmation must honor the saved opt-out (flag %q)", flag) + } + } + }) + } +} diff --git a/cmd/msgvault/cmd/add_teams.go b/cmd/msgvault/cmd/add_teams.go index ae0b2ba2a..a9d277c55 100644 --- a/cmd/msgvault/cmd/add_teams.go +++ b/cmd/msgvault/cmd/add_teams.go @@ -87,7 +87,7 @@ Examples: } cmd.Flags().StringVar(&teamsTenantID, "tenant", "", "Azure AD tenant ID (default: \"common\" for multi-tenant)") - cmd.Flags().BoolVar(&noDefaultIdentityAddTeams, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddTeams, "no-default-identity", false, savedDefaultIdentityHelp) cmd.Flags().BoolVar(&teamsHeadless, "headless", false, "Sign in with a device code instead of a local browser") registerOAuthPreflightedFlag(cmd) @@ -142,6 +142,9 @@ func runAddTeamsLocal(cmd *cobra.Command, args []string) error { return fmt.Errorf("set display name: %w", err) } + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddTeams); err != nil { + return err + } if !noDefaultIdentityAddTeams { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } diff --git a/cmd/msgvault/cmd/addaccount.go b/cmd/msgvault/cmd/addaccount.go index 9478517a8..aef34f841 100644 --- a/cmd/msgvault/cmd/addaccount.go +++ b/cmd/msgvault/cmd/addaccount.go @@ -512,6 +512,10 @@ func runAddAccountLocal(cmd *cobra.Command, args []string) error { } } + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddAccount); err != nil { + return err + } + fmt.Printf("Account %s authorized via service account.\n", email) fmt.Println("Next step: msgvault sync-full", email) return nil @@ -601,6 +605,9 @@ func runAddAccountLocal(cmd *cobra.Command, args []string) error { // [identity] block contains the same address. Reverse order // would leave the source without its own account identifier // because confirmDefaultIdentity skips on any existing rows. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddAccount); err != nil { + return err + } if !noDefaultIdentityAddAccount { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } @@ -657,6 +664,9 @@ func runAddAccountLocal(cmd *cobra.Command, args []string) error { } // Auto-default-identity must run BEFORE the legacy migration // retry — see comment on the token-reusable path above. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddAccount); err != nil { + return err + } if !noDefaultIdentityAddAccount { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } @@ -1043,7 +1053,7 @@ func registerAddAccountFlags(cmd *cobra.Command) { cmd.Flags().BoolVar(&forceReauth, "force", false, "Delete existing token and re-authorize") cmd.Flags().StringVar(&accountDisplayName, "display-name", "", "Display name for the account (e.g., \"Work\", \"Personal\")") cmd.Flags().StringVar(&oauthAppName, "oauth-app", "", "Named OAuth app from config (for Google Workspace orgs)") - cmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, savedDefaultIdentityHelp) cmd.Flags().BoolVar(&readonlyGrant, "readonly", false, "Request Gmail read-only access instead of read+write (refused if the account already holds write access)") cmd.Flags().Bool(addAccountGrantDecidedFlag, false, "Internal: the grant decision was already applied by the frontend CLI") if err := cmd.Flags().MarkHidden(addAccountGrantDecidedFlag); err != nil { diff --git a/cmd/msgvault/cmd/addaccount_test.go b/cmd/msgvault/cmd/addaccount_test.go index 2cf2509eb..ea54a15a9 100644 --- a/cmd/msgvault/cmd/addaccount_test.go +++ b/cmd/msgvault/cmd/addaccount_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "io" "log/slog" "os" "path/filepath" @@ -864,7 +865,7 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { cfg := testConfigValue() logger := testLoggerValue() - require := require.New(t) + assert, require := assert.New(t), require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -930,7 +931,29 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { ids, err := s.ListAccountIdentities(src.ID) require.NoError(err, "ListAccountIdentities") - assert.Empty(t, ids, "expected 0 identity rows with --no-default-identity") + assert.Empty(ids, "expected 0 identity rows with --no-default-identity") + + // The daemon's shared helper must honor the choice after the store reopens. + confirmDefaultIdentity(io.Discard, s, src.ID, "user@example.com", "user@example.com", "account-identifier", logger) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "the shared identity helper must preserve the opt-out") + + // Use a new command, as a separate re-authorization invocation would. + rerun := &cobra.Command{Use: addAccountUse, RunE: runAddAccountLocal} + registerAddAccountFlags(rerun) + rerun.SetArgs([]string{"user@example.com"}) + require.NoError(rerun.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "omitting the flag must preserve the saved choice") + + root.SetArgs([]string{"add-account", "user@example.com", "--no-default-identity=false"}) + require.NoError(root.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + require.Len(ids, 1, "explicit false restores the default") + assert.Equal("user@example.com", ids[0].Address) } // TestAddAccount_DeferredLegacyIdentityMigrationFires verifies that legacy diff --git a/cmd/msgvault/cmd/addimap.go b/cmd/msgvault/cmd/addimap.go index 917094b62..72dafb5cf 100644 --- a/cmd/msgvault/cmd/addimap.go +++ b/cmd/msgvault/cmd/addimap.go @@ -167,6 +167,9 @@ Examples: // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddImap); err != nil { + return err + } if !noDefaultIdentityAddImap { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, imapUsername, imapUsername, "account-identifier", state.logger) } @@ -189,7 +192,7 @@ Examples: cmd.Flags().StringVar(&imapUsername, "username", "", "IMAP username / email address (required)") cmd.Flags().BoolVar(&imapNoTLS, "no-tls", false, "Disable TLS (plain connection, not recommended)") cmd.Flags().BoolVar(&imapSTARTTLS, "starttls", false, "Use STARTTLS instead of implicit TLS") - cmd.Flags().BoolVar(&noDefaultIdentityAddImap, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddImap, "no-default-identity", false, savedDefaultIdentityHelp) return cmd } diff --git a/cmd/msgvault/cmd/addo365.go b/cmd/msgvault/cmd/addo365.go index d6295fd91..6a71c89a2 100644 --- a/cmd/msgvault/cmd/addo365.go +++ b/cmd/msgvault/cmd/addo365.go @@ -86,7 +86,7 @@ Examples: } cmd.Flags().StringVar(&o365TenantID, "tenant", "", "Azure AD tenant ID (default: \"common\" for multi-tenant)") - cmd.Flags().BoolVar(&noDefaultIdentityAddO365, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddO365, "no-default-identity", false, savedDefaultIdentityHelp) cmd.Flags().BoolVar(&o365Headless, "headless", false, "Sign in with a device code instead of a local browser") cmd.Flags().BoolVar(&o365Graph, "graph", false, "sync through the Microsoft Graph mail API instead of IMAP") @@ -194,6 +194,9 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddO365); err != nil { + return err + } if !noDefaultIdentityAddO365 { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } @@ -273,6 +276,9 @@ func runAddO365GraphLocal(cmd *cobra.Command, email string) error { if err := s.UpdateSourceDisplayName(source.ID, email); err != nil { return fmt.Errorf("set display name: %w", err) } + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddO365); err != nil { + return err + } if !noDefaultIdentityAddO365 { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } diff --git a/cmd/msgvault/cmd/eval_rerank_test.go b/cmd/msgvault/cmd/eval_rerank_test.go index 69ac96d07..afe2c75fb 100644 --- a/cmd/msgvault/cmd/eval_rerank_test.go +++ b/cmd/msgvault/cmd/eval_rerank_test.go @@ -242,10 +242,6 @@ func TestRunEvalJevSlowRequestWaves(t *testing.T) { assert.GreaterOrEqual(t, arm.Latency.P95MS, 12000.0) } -type testTransport func(*http.Request) (*http.Response, error) - -func (f testTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } - type fakeReranker struct{} func (fakeReranker) Rerank(_ context.Context, _ rerank.Request) (rerank.Result, error) { diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 96f8a3086..effe4165c 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -148,10 +148,23 @@ msgvault add-account --oauth-app | `--force` | Delete existing token and re-authorize | | `--readonly` | Request Gmail read-only access instead of read + write. Refused if the account already holds write access — see [OAuth Setup](/docs/guides/oauth-setup/#read-only-access) | | `--display-name` | Set a display name for the account | -| `--no-default-identity` | Do not auto-confirm the email address as this account's "me" identity | +| `--no-default-identity` | Do not auto-confirm the email address as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | If `[oauth].service_account_key` or `[oauth.apps.].service_account_key` is configured, `add-account` authorizes via Google service account domain-wide delegation instead of browser OAuth. Service-account accounts do not use `--headless`, `--force`, or `--readonly`; their scope comes from the domain-wide delegation grant in the Admin Console. +### Saved default identity choice + +`add-account` (including service accounts), `add-imap`, `add-o365` (IMAP and +`--graph`), and `add-teams` save `--no-default-identity` for later syncs. +Omitting the flag when adding or re-authorizing an existing account keeps its +saved choice. Use `--no-default-identity=false` to re-enable automatic defaults. +New accounts confirm their default identity unless the flag is set. + +The flag does not remove existing identities or prevent the one-time legacy +`[identity]` migration from adding configured addresses. If the saved sync +configuration cannot be read, automatic identity confirmation logs a warning +and skips the write. + --- ## add-imap @@ -179,7 +192,7 @@ It tests the connection before saving credentials. | `--port` | `993` | IMAP server port (993 for TLS, 143 for STARTTLS/plain) | | `--starttls` | `false` | Use STARTTLS instead of implicit TLS | | `--no-tls` | `false` | Disable TLS entirely (plaintext, not recommended) | -| `--no-default-identity` | `false` | Do not auto-confirm the username as this account's "me" identity | +| `--no-default-identity` | `false` | Do not auto-confirm the username as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | Credentials are stored in `tokens/imap_.json` with restricted file permissions (0600). Use app-specific passwords when your provider supports them. @@ -395,7 +408,7 @@ Requires a `[microsoft]` section with `client_id` in `config.toml`. See the [OAu |---|---|---| | `--tenant` | `common` | Azure AD tenant ID (restricts which accounts can authorize) | | `--headless` | `false` | Sign in with a device code instead of a local browser | -| `--no-default-identity` | `false` | Do not auto-confirm the email address as this account's "me" identity | +| `--no-default-identity` | `false` | Do not auto-confirm the email address as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | | `--graph` | `false` | Sync through the Microsoft Graph mail API instead of IMAP. Creates an `msmail` account. Needs the `Mail.Read` permission | After adding the account, sync it with `msgvault sync-full`. For a `--graph` @@ -423,7 +436,7 @@ the Microsoft IMAP token used by `add-o365`. Requires `[microsoft].client_id` in |---|---|---| | `--tenant` | `common` | Azure AD tenant ID to use for authorization | | `--headless` | `false` | Sign in with a device code instead of a local browser | -| `--no-default-identity` | `false` | Do not auto-confirm the email address as this source's "me" identity | +| `--no-default-identity` | `false` | Do not auto-confirm the email address as this source's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | After adding the account, sync it with `msgvault sync-teams`. diff --git a/docs/usage/multi-account.md b/docs/usage/multi-account.md index a125a65d4..72bd2e8a7 100644 --- a/docs/usage/multi-account.md +++ b/docs/usage/multi-account.md @@ -161,6 +161,11 @@ Fastmail alias inventory, person promotion, and typed attributes. New Gmail, IMAP, Microsoft 365, MBOX, EML, EMLX, WhatsApp, and Google Voice sources auto-confirm the source identifier by default. Use `--no-default-identity` on supported add/import commands when that is not correct. (iMessage imports are exempt, because iMessage contacts are not self-identifying.) +For accounts added with older versions, run the add command again with +`--no-default-identity` to save the choice, then remove any unwanted identity. +See the CLI reference for [saved identity choices](../cli-reference.md#saved-default-identity-choice), +including re-authorization and re-enabling defaults. + ```bash # List confirmed identifiers across all accounts msgvault identity list diff --git a/internal/beeper/docbank_media_review_test.go b/internal/beeper/docbank_media_review_test.go index a575cb710..ab7ced5d7 100644 --- a/internal/beeper/docbank_media_review_test.go +++ b/internal/beeper/docbank_media_review_test.go @@ -1200,6 +1200,11 @@ func TestBeeperMediaProcessDescriptorRefresh(t *testing.T) { } // The saved request can finish before the replacement is retained. + // Set the order explicitly; equal clock timestamps favor retention. + _, err = world.st.DB().Exec(world.st.Rebind(`UPDATE beeper_media_deliveries + SET next_action_at = ? WHERE destination_key = ? AND processing_key = ?`), + time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), destination, beforeMapping.ProcessingKey) + require.NoError(err) operation, ok, err := world.st.NextBeeperMediaOperation(t.Context(), destination, time.Now().UTC()) require.NoError(err) require.True(ok) From e54b14f2a8b0b813517c2d85949e92647f0a5818 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:46:06 -0700 Subject: [PATCH 081/125] feat(meetings): add Muesli and link meeting attendees to people (#946) Archive Muesli meeting notes and transcripts alongside mail and chat, and connect meeting attendees to existing people through email or phone. Muesli previously required a post-meeting hook, which missed later edits and Contacts-only attendees. - Add `[[muesli]]`, `add-muesli`, `sync-muesli`, and scheduled syncs. Read the local Muesli and Contacts databases in SQLite read-only mode, including committed WAL data. Do not invoke `muesli-cli`, which can migrate the source database. - Archive completed meetings with AI notes, typed notes, transcripts, and participants. Scan the full source to catch edits that do not advance meeting timestamps. Meetings still recording wait; deleted meetings stay archived. Archive keys combine local row ID and creation time. - Allow phone attendees in the shared meeting archive and import API. Stable Notion user IDs, Contacts cards, and import attendee IDs can link addresses to the same person. Owner addresses are excluded; conflicting people and prior source claims go to review. Names never establish identity. - Create new automatic Contacts links only when every Contacts store is readable, and exclude addresses shared by separate Contacts groups. Retain those addresses as meeting evidence. If a Contacts card disappears or becomes unreadable, preserve the known addresses for an attendee still present in Muesli. Refresh archived Contacts status when access recovers. - Keep email-only API payloads compatible and require an email or phone in Go client validation. Pass cancellation through phone-participant database writes. Existing person bindings remain durable. Moving a phone between Contacts cards does not automatically move its person ownership; a correction that preserves existing profiles is deferred. Cross-device or restored-database deduplication is not added. msgvault must run on the Mac containing Muesli. Grant the daemon Full Disk Access to read Contacts; meetings still sync when Contacts is unavailable. ```toml [[muesli]] identifier = "mac" account_email = "you@example.com" phone_country_code = "1" # optional: normalize national-format Contacts phones schedule = "*/30 * * * *" enabled = true ``` ```bash msgvault add-muesli mac msgvault sync-muesli mac ``` Import API attendees can include a phone and a source-specific stable ID: ```json {"name":"Alex Example","email":"alex@example.com","phone":"+1 604 555 0100","id":"crm-42"} ``` Closes #945 Closes #951 Co-authored-by: Rusty Shackleford --- api/openapi.yaml | 25 +- cmd/msgvault/cmd/constants.go | 1 + cmd/msgvault/cmd/meeting_source_test.go | 4 + cmd/msgvault/cmd/muesli.go | 309 ++++++++++++++ cmd/msgvault/cmd/muesli_test.go | 203 +++++++++ cmd/msgvault/cmd/serve.go | 27 ++ docs/api-server.md | 7 + docs/architecture/storage.md | 4 +- docs/changelog.md | 11 + docs/cli-reference.md | 44 ++ docs/configuration.md | 33 ++ docs/guides/sources.md | 4 +- docs/introduction.md | 4 +- docs/usage/meetings.md | 162 ++++++- docs/usage/multi-account.md | 4 +- docs/usage/people.md | 7 +- docs/usage/tui.md | 6 +- docs/usage/vector-search.md | 6 +- internal/api/cli_allowlist_slack_test.go | 14 + internal/api/cli_handlers.go | 2 + internal/api/handlers_test.go | 2 + internal/api/meeting_import.go | 31 +- internal/api/meeting_import_test.go | 23 + internal/api/openapi.go | 18 + internal/api/openapi_test.go | 8 + internal/api/scheduler_jobs.go | 5 + internal/circleback/person_link_test.go | 42 ++ internal/config/config.go | 109 +++++ internal/config/config_meeting_test.go | 108 +++++ internal/config/edit.go | 3 + internal/config/edit_test.go | 21 + internal/granola/person_link_test.go | 39 ++ internal/meetingarchive/archive.go | 120 ++++-- internal/meetingarchive/identity.go | 106 +++++ internal/meetingarchive/identity_test.go | 147 +++++++ internal/meetingarchive/links.go | 298 +++++++++++++ internal/meetingarchive/links_test.go | 260 ++++++++++++ internal/meetingcontent/decode.go | 21 +- internal/meetingcontent/decode_test.go | 90 ++++ internal/meetingcontent/muesli.go | 74 ++++ internal/meetingcontent/render.go | 29 +- internal/meetingcontent/render_test.go | 34 ++ internal/meetingcontent/types.go | 1 + internal/meetingimport/identities_test.go | 142 +++++++ internal/meetingimport/importer.go | 21 +- internal/meetingimport/models.go | 78 +++- internal/muesli/contacts.go | 363 ++++++++++++++++ internal/muesli/contacts_fixture_test.go | 65 +++ internal/muesli/contacts_test.go | 167 ++++++++ internal/muesli/fixture_test.go | 163 +++++++ internal/muesli/format.go | 365 ++++++++++++++++ internal/muesli/format_test.go | 214 ++++++++++ internal/muesli/importer.go | 233 ++++++++++ internal/muesli/importer_test.go | 298 +++++++++++++ internal/muesli/metadata_test.go | 29 ++ internal/muesli/models.go | 60 +++ internal/muesli/person_link_test.go | 53 +++ internal/muesli/phone.go | 58 +++ internal/muesli/reader.go | 333 +++++++++++++++ internal/muesli/reader_test.go | 263 ++++++++++++ internal/muesli/resolve.go | 174 ++++++++ internal/muesli/resolve_test.go | 397 ++++++++++++++++++ internal/notionmeetings/format.go | 10 + internal/notionmeetings/hydrate.go | 5 +- internal/notionmeetings/importer.go | 11 +- internal/notionmeetings/person_link_test.go | 73 ++++ internal/store/docbank_media_test.go | 8 +- .../store/meeting_phone_participants_test.go | 213 ++++++++++ internal/store/meetings.go | 11 +- internal/store/messages.go | 184 ++++++-- internal/store/messages_test.go | 53 +++ .../store/participant_identifier_revision.go | 7 +- internal/store/participant_observations.go | 4 +- internal/store/stable_anchor.go | 128 ++++++ internal/tui/meeting_mode_test.go | 6 +- internal/tui/meeting_state.go | 3 +- internal/tui/meeting_view.go | 2 + internal/tui/meeting_view_test.go | 15 + pkg/client/generated/types.go | 10 +- pkg/client/meeting_person_test.go | 45 ++ pkg/client/openapi.yaml | 14 +- .../lib/api/generated/models/meetingPerson.ts | 38 +- website/index.html | 2 +- website/index.md | 2 +- 84 files changed, 6631 insertions(+), 155 deletions(-) create mode 100644 cmd/msgvault/cmd/muesli.go create mode 100644 cmd/msgvault/cmd/muesli_test.go create mode 100644 internal/circleback/person_link_test.go create mode 100644 internal/granola/person_link_test.go create mode 100644 internal/meetingarchive/identity.go create mode 100644 internal/meetingarchive/identity_test.go create mode 100644 internal/meetingarchive/links.go create mode 100644 internal/meetingarchive/links_test.go create mode 100644 internal/meetingcontent/muesli.go create mode 100644 internal/meetingimport/identities_test.go create mode 100644 internal/muesli/contacts.go create mode 100644 internal/muesli/contacts_fixture_test.go create mode 100644 internal/muesli/contacts_test.go create mode 100644 internal/muesli/fixture_test.go create mode 100644 internal/muesli/format.go create mode 100644 internal/muesli/format_test.go create mode 100644 internal/muesli/importer.go create mode 100644 internal/muesli/importer_test.go create mode 100644 internal/muesli/metadata_test.go create mode 100644 internal/muesli/models.go create mode 100644 internal/muesli/person_link_test.go create mode 100644 internal/muesli/phone.go create mode 100644 internal/muesli/reader.go create mode 100644 internal/muesli/reader_test.go create mode 100644 internal/muesli/resolve.go create mode 100644 internal/muesli/resolve_test.go create mode 100644 internal/notionmeetings/person_link_test.go create mode 100644 internal/store/meeting_phone_participants_test.go create mode 100644 internal/store/stable_anchor.go create mode 100644 pkg/client/meeting_person_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index 58e6d8e04..4411da732 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -5911,14 +5911,35 @@ components: type: object MeetingPerson: additionalProperties: false + anyOf: + - properties: + email: + format: email + type: string + required: + - email + type: object + - properties: + phone: + maxLength: 64 + type: string + required: + - phone + type: object properties: email: format: email type: string + id: + description: Stable identifier for this person in the import source + maxLength: 200 + type: string name: type: string - required: - - email + phone: + description: International phone number starting with + or 00; normalized to E.164 + maxLength: 64 + type: string type: object MeetingRef: additionalProperties: true diff --git a/cmd/msgvault/cmd/constants.go b/cmd/msgvault/cmd/constants.go index b85a74b7b..2f4ec145d 100644 --- a/cmd/msgvault/cmd/constants.go +++ b/cmd/msgvault/cmd/constants.go @@ -17,6 +17,7 @@ const ( sourceTypeGranola = "granola" sourceTypeCircleback = "circleback" sourceTypeNotionMeetings = "notion_meetings" + sourceTypeMuesli = "muesli" ) // Analytics dataset / SQLite table names: the Parquet subdirectory under diff --git a/cmd/msgvault/cmd/meeting_source_test.go b/cmd/msgvault/cmd/meeting_source_test.go index 0915e0b37..355a17c42 100644 --- a/cmd/msgvault/cmd/meeting_source_test.go +++ b/cmd/msgvault/cmd/meeting_source_test.go @@ -37,6 +37,7 @@ func TestMeetingConfigurationHintsLoad(t *testing.T) { {name: "Granola", hint: granolaConfigHint}, {name: "Circleback", hint: circlebackConfigHint}, {name: "Notion", hint: notionMeetingsConfigHint}, + {name: "Muesli", hint: muesliConfigHint}, } { t.Run(tt.name, func(t *testing.T) { assert := assert.New(t) @@ -57,6 +58,9 @@ func TestMeetingConfigurationHintsLoad(t *testing.T) { case "Notion": require.Len(cfg.NotionMeetings, 1) assert.Equal("you@example.com", cfg.NotionMeetings[0].AccountEmail) + case "Muesli": + require.Len(cfg.Muesli, 1) + assert.Equal("you@example.com", cfg.Muesli[0].AccountEmail) } }) } diff --git a/cmd/msgvault/cmd/muesli.go b/cmd/msgvault/cmd/muesli.go new file mode 100644 index 000000000..65f2ff3a3 --- /dev/null +++ b/cmd/msgvault/cmd/muesli.go @@ -0,0 +1,309 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/muesli" + "go.kenn.io/msgvault/internal/store" +) + +var ( + syncMuesliLimit int + syncMuesliAfter string + syncMuesliFull bool +) + +var ( + rebuildMuesliCacheAfterWrite = rebuildCacheAfterWrite + rebuildMuesliCacheAfterScheduledSync = rebuildCacheAfterScheduledSync +) + +const muesliConfigHint = `Add to your config.toml: + + [[muesli]] + identifier = "mac" # stable label for this Mac's Muesli database + account_email = "you@example.com" # you, the person who records the meetings + enabled = true + # db_path = "~/Library/Application Support/Muesli/muesli.db" # default shown + # schedule = "*/30 * * * *" # optional daemon schedule + # phone_country_code = "1" # convert national-format Contacts phones + # contacts = false # skip Apple Contacts attendee lookup` + +// resolveMuesliSources picks [[muesli]] entries: an explicit identifier must +// match one entry; with no argument every configured entry is returned. +func resolveMuesliSources(args []string, cfg *config.Config) ([]config.MuesliSource, error) { + if len(cfg.Muesli) == 0 { + return nil, errors.New("no [[muesli]] sources configured\n\n" + muesliConfigHint) + } + if len(args) == 0 { + return cfg.Muesli, nil + } + source := cfg.GetMuesliSource(args[0]) + if source == nil { + identifiers := make([]string, 0, len(cfg.Muesli)) + for _, candidate := range cfg.Muesli { + identifiers = append(identifiers, candidate.Identifier) + } + return nil, fmt.Errorf("no [[muesli]] entry with identifier %q (configured: %s)", + args[0], strings.Join(identifiers, ", ")) + } + return []config.MuesliSource{*source}, nil +} + +// probeMuesliDatabase proves the configured database opens read-only and +// has Muesli's meetings table. +func probeMuesliDatabase(ctx context.Context, path string) error { + reader, err := muesli.Open(ctx, path) + if err != nil { + return fmt.Errorf("%w\n\nCheck db_path in the [[muesli]] entry. msgvault reads the database on the "+ + "daemon's host; if macOS blocks the read, grant the daemon Full Disk Access", err) + } + return reader.Close() +} + +var addMuesliCmd = &cobra.Command{ + Use: "add-muesli [identifier]", + Short: "Register a local Muesli meeting database", + Long: `Register a configured Muesli database as a msgvault meeting source. + +Reads db_path from the matching [[muesli]] entry in config.toml (default: +~/Library/Application Support/Muesli/muesli.db) and checks that it opens +read-only as a Muesli database. The daemon reads the file on its own host, +so msgvault must run on the Mac where Muesli records. + +Examples: + msgvault add-muesli + msgvault add-muesli mac`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + sources, err := resolveMuesliSources(args, cfg) + if err != nil { + return err + } + if len(sources) > 1 { + return errors.New("multiple [[muesli]] sources configured; pass an identifier") + } + source := sources[0] + accountEmail, err := source.EffectiveAccountEmail() + if err != nil { + return err + } + if err := probeMuesliDatabase(cmd.Context(), source.EffectiveDBPath()); err != nil { + return err + } + if source.ContactsEnabled() { + contacts, err := muesli.OpenContacts(cmd.Context(), source.EffectiveContactsPath()) + if err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Apple Contacts: %s\n", contacts.State()) + if contacts.State() != muesli.ContactsComplete { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), + "Grant the msgvault daemon Full Disk Access so attendees picked from Contacts link to people.") + } + } + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + if _, err := registerMeetingSource(cmd.OutOrStdout(), st, sourceTypeMuesli, + source.Identifier, accountEmail); err != nil { + return err + } + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nMuesli source %s registered.\n", source.Identifier) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Run: msgvault sync-muesli %s\n", source.Identifier) + return nil + }, +} + +var syncMuesliCmd = &cobra.Command{ + Use: "sync-muesli [identifier]", + Short: "Sync meetings from a local Muesli database", + Long: `Archive completed Muesli meetings: AI notes, typed notes, transcript, and +participant emails. With no identifier, every configured [[muesli]] source is +synced. + +Every run reads the whole database read-only and updates meetings that +changed in place. Meetings still recording or processing wait for a later run. +Meetings deleted in Muesli stay archived. + +Examples: + msgvault sync-muesli + msgvault sync-muesli mac --limit 5 + msgvault sync-muesli --after 2026-01-01 # UTC date + msgvault sync-muesli --full`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + sources, err := resolveMuesliSources(args, cfg) + if err != nil { + return err + } + var after time.Time + if syncMuesliAfter != "" { + parsed, err := time.Parse(time.DateOnly, syncMuesliAfter) + if err != nil { + return usageErr(cmd, fmt.Errorf("invalid --after %q (expected YYYY-MM-DD): %w", syncMuesliAfter, err)) + } + after = parsed.UTC() + } + for _, source := range sources { + if _, err := source.EffectiveAccountEmail(); err != nil { + return err + } + } + + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + dbPath := cfg.DatabaseDSN() + + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sigChan) + go func() { + select { + case <-sigChan: + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Finishing current meeting...") + cancel() + case <-ctx.Done(): + } + }() + + pendingWrites := &muesli.ImportSummary{} + for _, source := range sources { + accountEmail, _ := source.EffectiveAccountEmail() + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Muesli for %s\n\n", source.Identifier) + options := muesliImportOptions(source) + options.AccountEmail = accountEmail + options.Full, options.Limit, options.StartedAfter = syncMuesliFull, syncMuesliLimit, after + options.Progress = func(line string) { _, _ = fmt.Fprintln(cmd.OutOrStdout(), " "+line) } + summary, importErr := muesli.NewImporter(st).Import(ctx, options) + if summary != nil { + pendingWrites.MeetingsAdded += summary.MeetingsAdded + pendingWrites.MeetingsUpdated += summary.MeetingsUpdated + } + if err := finishMuesliImport(source.Identifier, pendingWrites, importErr, + func() error { return rebuildMuesliCacheAfterWrite(dbPath, state) }); err != nil { + return err + } + writeMuesliSummary(cmd.OutOrStdout(), summary) + } + return rebuildMuesliCacheAfterWrite(dbPath, state) + }, +} + +// finishMuesliImport reports a failed sync, first refreshing the cache when +// the run still archived meetings. +func finishMuesliImport(identifier string, summary *muesli.ImportSummary, importErr error, refresh func() error) error { + if importErr == nil { + return nil + } + var refreshErr error + if summary != nil && summary.MeetingsAdded+summary.MeetingsUpdated > 0 && refresh != nil { + refreshErr = refresh() + } + return errors.Join(fmt.Errorf("muesli sync %s failed: %w", identifier, importErr), refreshErr) +} + +func writeMuesliSummary(out io.Writer, summary *muesli.ImportSummary) { + _, _ = fmt.Fprintln(out, "\nMuesli sync complete!") + _, _ = fmt.Fprintf(out, " Duration: %s\n", summary.Duration.Round(time.Second)) + _, _ = fmt.Fprintf(out, " Meetings processed: %d\n", summary.MeetingsProcessed) + _, _ = fmt.Fprintf(out, " Meetings added: %d\n", summary.MeetingsAdded) + _, _ = fmt.Fprintf(out, " Meetings updated: %d\n", summary.MeetingsUpdated) + if summary.SkippedInProgress > 0 { + _, _ = fmt.Fprintf(out, " Still in progress: %d (archived by a later sync)\n", summary.SkippedInProgress) + } + if summary.SkippedDeleted > 0 { + _, _ = fmt.Fprintf(out, " Deleted in Muesli: %d (kept archived)\n", summary.SkippedDeleted) + } + if summary.SkippedEmpty > 0 { + _, _ = fmt.Fprintf(out, " Empty: %d (no notes or transcript)\n", summary.SkippedEmpty) + } + if summary.ContactsState != "" { + _, _ = fmt.Fprintf(out, " Contacts: %s\n", summary.ContactsState) + } + if summary.ContactsState == muesli.ContactsUnavailable || summary.ContactsState == muesli.ContactsPartial { + _, _ = fmt.Fprintln(out, " Grant the msgvault daemon Full Disk Access so attendees picked from Contacts link to people.") + } +} + +// muesliImportOptions carries a configured source's paths and Contacts +// settings into an import. +func muesliImportOptions(source config.MuesliSource) muesli.ImportOptions { + return muesli.ImportOptions{ + Identifier: source.Identifier, AccountEmail: source.AccountEmail, + DBPath: source.EffectiveDBPath(), + ContactsEnabled: source.ContactsEnabled(), ContactsPath: source.EffectiveContactsPath(), + PhoneCountryCode: source.PhoneCountryCode, + } +} + +// runConfiguredMuesliSync is the daemon-scheduler entry point for one +// [[muesli]] source. +func runConfiguredMuesliSync(ctx context.Context, st *store.Store, source config.MuesliSource) error { + if _, err := st.GetSourceByTypeAndIdentifier(muesli.SourceType, source.Identifier); err != nil { + if errors.Is(err, store.ErrSourceNotFound) { + return fmt.Errorf("muesli source %q is not registered; run msgvault add-muesli %s first", + source.Identifier, source.Identifier) + } + return err + } + accountEmail, err := source.EffectiveAccountEmail() + if err != nil { + return err + } + options := muesliImportOptions(source) + options.AccountEmail = accountEmail + summary, importErr := muesli.NewImporter(st).Import(ctx, options) + refreshCtx := context.WithoutCancel(ctx) + refresh := func() error { + return rebuildMuesliCacheAfterScheduledSync(refreshCtx, "muesli:"+source.Identifier) + } + if err := finishMuesliImport(source.Identifier, summary, importErr, refresh); err != nil { + return err + } + return refresh() +} + +func init() { + syncMuesliCmd.Flags().IntVar(&syncMuesliLimit, "limit", 0, "max meetings processed per run (0 = no limit)") + syncMuesliCmd.Flags().StringVar(&syncMuesliAfter, "after", "", "only meetings that start on or after this UTC date (YYYY-MM-DD)") + syncMuesliCmd.Flags().BoolVar(&syncMuesliFull, "full", false, "rewrite every archived meeting, even unchanged ones (refreshes attribution)") + rootCmd.AddCommand(addMuesliCmd) + rootCmd.AddCommand(syncMuesliCmd) +} diff --git a/cmd/msgvault/cmd/muesli_test.go b/cmd/msgvault/cmd/muesli_test.go new file mode 100644 index 000000000..aa55839a7 --- /dev/null +++ b/cmd/msgvault/cmd/muesli_test.go @@ -0,0 +1,203 @@ +package cmd + +import ( + "bytes" + "context" + "database/sql" + "encoding/json/v2" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/muesli" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestResolveMuesliSources(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + cfg := &config.Config{} + _, err := resolveMuesliSources(nil, cfg) + require.Error(err) + assert.Contains(err.Error(), "[[muesli]]") + + cfg = &config.Config{Muesli: []config.MuesliSource{ + {Identifier: "mac", AccountEmail: "you@example.com"}, + {Identifier: "studio", AccountEmail: "you@example.com"}, + }} + all, err := resolveMuesliSources(nil, cfg) + require.NoError(err) + assert.Len(all, 2) + + one, err := resolveMuesliSources([]string{"STUDIO"}, cfg) + require.NoError(err) + require.Len(one, 1) + assert.Equal("studio", one[0].Identifier) + + _, err = resolveMuesliSources([]string{"laptop"}, cfg) + require.Error(err) + assert.Contains(err.Error(), "configured: mac, studio") +} + +func TestProbeMuesliDatabaseRejectsForeignFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "notes.txt") + require.NoError(t, os.WriteFile(path, []byte("not sqlite"), 0o600)) + + err := probeMuesliDatabase(context.Background(), path) + + require.Error(t, err) + assert.Contains(t, err.Error(), "db_path") +} + +func TestRunConfiguredMuesliSyncRefusesUnregisteredSource(t *testing.T) { + st := testutil.NewTestStore(t) + + err := runConfiguredMuesliSync(context.Background(), st, config.MuesliSource{ + Identifier: "removed", AccountEmail: "you@example.com", + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "add-muesli removed") +} + +func TestServeScheduledMuesliSyncCompletes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + cfg := lifecycleTestConfig(t.TempDir()) + cfg.Server.APIPort = freeTCPPort(t) + cfg.Analytics.Engine = config.AnalyticsEngineSQL + cfg.Analytics.AutoBuildCache = false + cfg.Vector.Enabled = false + path := filepath.Join(t.TempDir(), "muesli.db") + db, err := sql.Open("sqlite3", path) + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec(`CREATE TABLE meetings (id INTEGER PRIMARY KEY, title TEXT, start_time TEXT, created_at TEXT, raw_transcript TEXT); + INSERT INTO meetings VALUES (1, 'Planning', '2026-09-01T14:00:00Z', '2026-09-01 14:00:03', 'Synthetic meeting notes')`) + require.NoError(err) + require.NoError(db.Close()) + contacts := false + cfg.Muesli = []config.MuesliSource{{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: path, + Contacts: &contacts, Enabled: true, Schedule: "0 0 1 1 *", + }} + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + _, err = st.GetOrCreateSource(muesli.SourceType, "mac") + require.NoError(err) + + ctx, cancel := context.WithCancel(t.Context()) + cmd := &cobra.Command{Use: serveCmd.Use} + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) + errCh := make(chan error, 1) + go func() { errCh <- runServe(cmd, nil) }() + t.Cleanup(func() { + cancel() + select { + case err := <-errCh: + require.NoError(err) + case <-time.After(serveLifecycleTestTimeout): + require.FailNow("daemon did not stop") + } + }) + waitForServeHealth(t, cfg.Server.APIPort, errCh) + baseURL := fmt.Sprintf("http://127.0.0.1:%d", cfg.Server.APIPort) + client := &http.Client{Timeout: time.Second} + response, err := client.Post(baseURL+"/api/v1/sync/mac?source_type=muesli", "application/json", nil) + require.NoError(err) + require.NoError(response.Body.Close()) + require.Equal(http.StatusAccepted, response.StatusCode) + var status api.SourceStatusResponse + require.Eventually(func() bool { + response, err := client.Get(baseURL + "/api/v1/sources/status?source_type=muesli") + if err != nil { + return false + } + defer func() { _ = response.Body.Close() }() + return json.UnmarshalRead(response.Body, &status) == nil && len(status.Sources) == 1 && + status.Sources[0].LastSuccessfulSync != nil && status.Sources[0].CanSync + }, serveLifecycleTestTimeout, 20*time.Millisecond, "scheduled import did not finish") + assert.Equal(int64(1), status.Sources[0].LastSuccessfulSync.MessagesAdded) + assert.Empty(status.Sources[0].SchedulerLastError, "post-import cache refresh must receive the daemon configuration") +} + +func TestFinishMuesliImportRefreshesCacheAfterPartialWrites(t *testing.T) { + refreshed := 0 + err := finishMuesliImport("mac", &muesli.ImportSummary{MeetingsAdded: 1}, + errors.New("meeting 3 failed"), func() error { refreshed++; return nil }) + + require.Error(t, err) + assert.Equal(t, 1, refreshed) + assert.Contains(t, err.Error(), "muesli sync mac failed") +} + +func TestWriteMuesliSummaryReportsSkippedMeetings(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{ + MeetingsProcessed: 3, MeetingsAdded: 2, SkippedDeleted: 4, SkippedEmpty: 1, SkippedInProgress: 1, + }) + + assert.Contains(t, out.String(), "Deleted in Muesli: 4 (kept archived)") + assert.Contains(t, out.String(), "Empty: 1 (no notes or transcript)") + assert.Contains(t, out.String(), "Still in progress: 1") +} + +func TestWriteMuesliSummaryOmitsZeroSkipCounts(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{MeetingsProcessed: 1, MeetingsAdded: 1}) + + assert.NotContains(t, out.String(), "Deleted in Muesli") + assert.NotContains(t, out.String(), "Empty:") +} + +func TestWriteMuesliSummaryReportsContactsState(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{ContactsState: muesli.ContactsUnavailable}) + + assert.Contains(t, out.String(), "Contacts: unavailable") + assert.Contains(t, out.String(), "Full Disk Access") +} + +func TestMuesliImportOptionsCarryContactsSettings(t *testing.T) { + enabled := false + opts := muesliImportOptions(config.MuesliSource{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: "/tmp/muesli.db", + Contacts: &enabled, ContactsPath: "/tmp/AddressBook", PhoneCountryCode: "44", + }) + + assert.Equal(t, muesli.ImportOptions{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: "/tmp/muesli.db", + ContactsEnabled: false, ContactsPath: "/tmp/AddressBook", PhoneCountryCode: "44", + }, opts) +} + +func TestMuesliCommandsUseInvocationConfiguration(t *testing.T) { + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + cfg := &config.Config{Muesli: []config.MuesliSource{{Identifier: "mac"}}} + for _, command := range []*cobra.Command{addMuesliCmd, syncMuesliCmd} { + t.Run(command.Name(), func(t *testing.T) { + cmd := &cobra.Command{} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + + err := command.RunE(cmd, []string{"missing"}) + + assert.ErrorContains(t, err, `no [[muesli]] entry with identifier "missing" (configured: mac)`) + }) + } +} diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index 1776f63db..1fedbd655 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -31,6 +31,7 @@ import ( "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/muesli" "go.kenn.io/msgvault/internal/notionmeetings" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/operations" @@ -649,6 +650,32 @@ func runServe(cmd *cobra.Command, args []string) error { logger.Info("scheduled notion meeting source", "source", source.Identifier, "schedule", source.Schedule) } } + for _, src := range cfg.Muesli { + if src.Enabled && src.Schedule == "" { + logger.Warn("muesli source is enabled but has no schedule — the daemon will not sync it; its freshness will eventually go stale", + "source", src.Identifier, + "hint", `set a cron schedule (e.g. "*/30 * * * *") on the [[muesli]] entry`) + } + } + for _, src := range cfg.ScheduledMuesliSources() { + source := src + jobName, ok := api.SchedulerJobNameForSource(muesli.SourceType, source.Identifier) + if !ok { + logger.Error("no scheduler job mapping for muesli source", "source", source.Identifier) + continue + } + if err := sched.AddJob(scheduler.Job{ + Name: jobName, + Schedule: source.Schedule, + Run: invocationBoundJobRun(state, func(ctx context.Context) error { + return runConfiguredMuesliSync(ctx, s, source) + }), + }); err != nil { + logger.Error("failed to schedule muesli source", "source", source.Identifier, "error", err) + } else { + logger.Info("scheduled muesli source", "source", source.Identifier, "schedule", source.Schedule) + } + } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) diff --git a/docs/api-server.md b/docs/api-server.md index f6c4d51da..8b5641fd3 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -1735,6 +1735,13 @@ omission means unsupported; `null` is rejected. See the [complete import example](usage/meetings.md#import-from-any-meeting-source). Unknown fields are rejected except within `meeting.metadata`. +Each organizer or attendee needs an `email`, a `phone`, or both. `phone` must be +international (a leading `+` or `00`) and is normalized to E.164. An optional +`id` (up to 200 characters) identifies the person within the import source; +msgvault links the person's email and phone through it, including across +meetings. See [how meetings connect to people](usage/meetings.md#how-meetings-connect-to-people). +Validation errors name the failing field without echoing its value. + --- ### OAuth token exchange {#post-apiv1authtokenemail} diff --git a/docs/architecture/storage.md b/docs/architecture/storage.md index 6cb29e9ca..426e2ea58 100644 --- a/docs/architecture/storage.md +++ b/docs/architecture/storage.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-25" title: Data Storage description: Database schema, Parquet analytics cache, content-addressed attachments, and token storage. --- @@ -33,7 +33,7 @@ separately. SQLite is the default and stores the archive at `~/.msgvault/msgvaul | Column | Type | Description | |---|---|---| | `id` | INTEGER PK | Auto-increment | -| `source_type` | TEXT | Provider/import type, for example `gmail`, `imap`, `gcal`, `teams`, `discord`, `beeper`, `mbox`, `whatsapp`, `granola`, `circleback`, or `notion_meetings` | +| `source_type` | TEXT | Provider/import type, for example `gmail`, `imap`, `gcal`, `teams`, `discord`, `beeper`, `mbox`, `whatsapp`, `granola`, `circleback`, `notion_meetings`, or `muesli` | | `identifier` | TEXT | Provider-stable identifier such as an email address, phone number, or Discord guild ID | | `display_name` | TEXT | Account display name | | `sync_cursor` | TEXT | Sync cursor (Gmail history ID for Gmail accounts) | diff --git a/docs/changelog.md b/docs/changelog.md index 44a287bfd..0eafe2448 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -54,6 +54,17 @@ All notable changes to msgvault, grouped by release. Microsoft Graph mail API, for a mailbox that has IMAP turned off. Each folder becomes a label, and later syncs fetch only the changes, including moves and deletes. See [Microsoft Graph mail sync](guides/oauth-setup.md#microsoft-graph-mail-sync). +- **Muesli meetings.** Archive meetings recorded by Muesli on the same Mac: + AI notes, typed notes, transcripts, and participant emails. msgvault reads + Muesli's local database read-only with `add-muesli` and `sync-muesli`, or on + a daemon schedule. See [Muesli](usage/meetings.md#muesli). +- **Meetings reach the right people.** Meeting attendees can be identified by + phone as well as email, and sources that know a stable identity link an + attendee's emails and phones: Notion user IDs, Apple Contacts cards for + Muesli, and a new attendee `id` (with `phone`) in `POST /api/v1/import/meeting`. + A meeting then appears on the person you already know from mail or chat. + Links follow the existing rules: no name matching, and conflicts between two + people go to review. See [how meetings connect to people](usage/meetings.md#how-meetings-connect-to-people). - Saved View MCP tools publish canonical_state as a schema object, so MCP clients that validate tools/list strictly, such as those built on the official TypeScript SDK, load msgvault's tools. - `add-o365 --headless` and `add-teams --headless` sign in with a Microsoft device code, so no local browser is needed. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index effe4165c..ca051f4c8 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -770,6 +770,50 @@ See [Meeting Transcripts](/docs/usage/meetings/) for setup and what gets stored. --- +## add-muesli + +Check a configured local Muesli database and register it as a meeting source. + +```bash +msgvault add-muesli [identifier] +``` + +The matching `[[muesli]]` entry requires `account_email`; `db_path` defaults +to `~/Library/Application Support/Muesli/muesli.db`. With one entry, the +identifier may be omitted. The command fails unless the file opens read-only +as a Muesli database on the daemon's host. + +--- + +## sync-muesli + +Archive completed meetings from a local Muesli database. + +```bash +msgvault sync-muesli [identifier] +msgvault sync-muesli mac --limit 5 +msgvault sync-muesli --after 2026-01-01 +msgvault sync-muesli --full +``` + +Every run reads the whole database read-only and updates changed meetings in +place; unchanged meetings are skipped. Attendees are resolved through Apple +Contacts unless `contacts = false`; the summary reports whether Contacts was +`complete`, `partial`, `unavailable`, or `off`. Meetings still recording or processing +wait for a later run, and meetings deleted in Muesli stay archived. With no +identifier, every configured `[[muesli]]` source is synced. + +| Flag | Default | Description | +|---|---|---| +| `--limit` | `0` | Maximum meetings processed per run (`0` = unlimited) | +| `--after` | — | Only meetings that start on or after this UTC date (`YYYY-MM-DD`) | +| `--full` | `false` | Rewrite every archived meeting, even unchanged ones, to refresh attribution | + +See [Meeting Transcripts](/docs/usage/meetings/#muesli) for setup and what gets +stored. + +--- + ## archive-remote-images Download remote `` images from existing email for offline viewing. diff --git a/docs/configuration.md b/docs/configuration.md index d5c5e5398..8661ef9af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1108,6 +1108,39 @@ scheduler from recreating it. See [Meeting Transcripts](/docs/usage/meetings/) f the 50-result discovery limit, attendee visibility, transcript retries, and stored data. +### Muesli Sources + +Muesli meeting sync uses one top-level `[[muesli]]` entry per Muesli database. +The daemon reads the database read-only on its own host, so msgvault must run +on the Mac where Muesli records. No credential is needed. + +```toml +[[muesli]] +identifier = "mac" # stable source label; defaults to "default" for one entry +account_email = "you@example.com" # required; you, the person who records +db_path = "~/Library/Application Support/Muesli/muesli.db" # optional; this is the default +phone_country_code = "1" # optional; convert national-format Contacts phones +schedule = "*/30 * * * *" # optional 5-field cron, no seconds +enabled = true +``` + +| Key | Default | Description | +|---|---|---| +| `identifier` | `default` (single entry) | Source name used by `sync-muesli ` and scheduler logs | +| `account_email` | (required) | Normalized primary identity; attributed as the organizer of every meeting | +| `db_path` | `~/Library/Application Support/Muesli/muesli.db` | Muesli database path; `~` expands, and a relative path resolves against the config directory when `--config` is used | +| `contacts` | `true` | Resolve attendees through Apple Contacts; needs Full Disk Access for the daemon | +| `contacts_path` | `~/Library/Application Support/AddressBook` | Apple Contacts data folder; expands like `db_path` | +| `phone_country_code` | — | Country calling code (1–3 digits, such as `"1"` or `"44"`) used for Contacts phone numbers typed without one; unset means only international numbers are used | +| `schedule` | — | Cron expression used by `msgvault serve` | +| `enabled` | `false` | Whether the source is daemon-scheduled | + +Run `msgvault add-muesli ` to check the database and register the +source before enabling a schedule. Run `msgvault sync-muesli --full` +after identity changes to repair existing meeting attribution. Removing the +source prevents the scheduler from recreating it. See +[Meeting Transcripts](/docs/usage/meetings/#muesli) for what gets stored. + ### `[vector]` Top-level toggle and backend marker for semantic/hybrid search. SQLite vector search requires a build with `sqlite_vec` support (default via `make build`). PostgreSQL vector search requires a build with the `pgvector` tag and a PostgreSQL `[data].database_url`. See [Vector Search](/docs/usage/vector-search/) for prerequisites, initial embedding, and the full workflow. diff --git a/docs/guides/sources.md b/docs/guides/sources.md index 25ff57915..be901f504 100644 --- a/docs/guides/sources.md +++ b/docs/guides/sources.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-22" +last_edited: "2026-09-25" title: Choose a Source description: Find the right sync or import path for mail, chat, meetings, calendars, and contacts. --- @@ -50,7 +50,7 @@ missing downloads. | Your source | Start here | What it adds | |---|---|---| -| Granola, Circleback, or Notion AI Meeting Notes | [Meeting notes and transcripts](../usage/meetings.md) | Searchable notes, context exports, recorded actions, and meeting-time coverage | +| Granola, Circleback, Notion AI Meeting Notes, or Muesli | [Meeting notes and transcripts](../usage/meetings.md) | Searchable notes, context exports, recorded actions, and meeting-time coverage | | Another meeting capture tool | [Meeting import workflow](../usage/meetings.md#import-from-any-meeting-source) | Provider-neutral ingestion keyed by source and external meeting ID | | Google Calendar | [Calendar sync](../usage/calendar.md) | Events, organizers, attendees, recurrence, and cancellation state | | CardDAV address book | [CardDAV contacts](../usage/people-carddav.md) | Imported contacts and explicit publication of curated profiles | diff --git a/docs/introduction.md b/docs/introduction.md index f747a91d0..da6095233 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-08-30" +last_edited: "2026-09-25" title: Introduction description: Why msgvault was created and what problem it solves. --- @@ -36,7 +36,7 @@ This means: I started with Gmail but I want all my life's messages in this system, including Google Calendar, Microsoft Teams, Discord guilds, Slack workspaces, -Beeper Desktop chats, Granola, Circleback, and Notion meeting notes, WhatsApp, iMessage, +Beeper Desktop chats, Granola, Circleback, Notion, and Muesli meeting notes, WhatsApp, iMessage, Google Voice, Facebook Messenger, SMS Backup & Restore archives, and old local email archives. diff --git a/docs/usage/meetings.md b/docs/usage/meetings.md index a993b47f4..8ccaa4af7 100644 --- a/docs/usage/meetings.md +++ b/docs/usage/meetings.md @@ -1,7 +1,7 @@ --- -last_edited: "2026-09-17" +last_edited: "2026-09-28" title: Meeting Transcripts -description: Archive AI meeting notes and transcripts from Granola, Circleback, and Notion into your searchable local archive. +description: Archive AI meeting notes and transcripts from Granola, Circleback, Notion, and Muesli into your searchable local archive. --- Find meeting decisions and transcripts in the same archive as your email and @@ -16,10 +16,11 @@ emails connect meetings to the people you already know in msgvault. | [Granola](#granola) | API key | Requires access to Granola's public API | | [Notion AI Meeting Notes](#notion-ai-meeting-notes) | Notion integration token | At most 50 attendee-visible meetings per discovery query | | [Circleback](#circleback) | Browser authorization to its MCP server | Older note edits require a full refresh | +| [Muesli](#muesli) | Local database on the same Mac | msgvault must run on the Mac where Muesli records | | [Another meeting source](#import-from-any-meeting-source) | Authenticated JSON import | Your integration supplies each meeting and its updates | Provider sync reads meeting data without changing the source service. Recording -media is not downloaded by the Notion or Circleback integrations. +media is not downloaded by the Notion, Circleback, or Muesli integrations. ## Browse and search @@ -131,7 +132,7 @@ Each meeting source has two distinct values: rejects a missing or invalid value with guidance to preserve the source label and add the account email separately. -`add-granola`, `add-circleback`, and `add-notion-meetings` always confirm the primary email for their +`add-granola`, `add-circleback`, `add-notion-meetings`, and `add-muesli` always confirm the primary email for their source. Add other confirmed aliases with the identity command: ```bash @@ -141,6 +142,43 @@ msgvault identity add work you+meetings@example.com Adding a new confirmed identity immediately repairs `is_from_me` on matching messages already stored for that source. No provider resync is required. +## How meetings connect to people + +A meeting shows up on a person when one of its attendees is an email or phone +number that already belongs to that person, for example from mail, chat, or a +promoted profile. The attendee becomes a participant, and the next activity +update (hourly, or `msgvault activity build`) adds the meeting to the person's +timeline, relationships, and last-contact information. + +Some sources also know that several emails and phone numbers are the same +human. msgvault links those identities through the source's stable identifier, +so a meeting reaches the person even when it only names an address the person +has never used with you: + +| Source | Identities per attendee | Linked through | +|---|---|---| +| Granola, Circleback | One email | Not needed | +| Notion AI Meeting Notes | The user's verified email | The Notion user ID, so a user whose email changes keeps one person | +| Muesli | Email, or the emails and phones on the attendee's Apple Contacts card | The Contacts card, excluding addresses shared by unlinked cards | +| Import API | `email` and `phone` | The person's `id` within the import source | + +These links follow the same rules as other automatic identity links: + +- Only a shared stable identifier links identities. Matching names never do. +- Muesli excludes addresses shared by unlinked Contacts cards from automatic + linking, even when only one card appears in a meeting. The addresses remain + in the meeting evidence. +- If the identities already belong to two different people, msgvault leaves + them apart and records a conflict. So does an address that another card or + `id` from the same source already claims in archived observations. + Review conflicts in the Web [Directory review queues](/docs/web-ui/#directory-and-reviews). +- Rejecting a proposed link keeps that pair apart. The same identities can + still connect through another identity of the same person. + +Attendees known only by name appear in the meeting body but do not link to a +person. A profile imported from CardDAV links meetings once its email or phone +participant is promoted or linked to it; see [people](/docs/usage/people/). + ## Import from any meeting source The provider-neutral import API archives one meeting at a time and requires no @@ -173,6 +211,15 @@ curl http://localhost:8080/api/v1/import/meeting \ }' ``` +Each organizer or attendee needs an `email`, a `phone`, or both. A phone must be +international (a leading `+` or `00`) and is stored in E.164 form; national +numbers are rejected rather than guessed. Add a stable `id` for the person in +your source to link their email and phone, including across meetings: + +```json +{"name": "Alex Example", "email": "alex@example.com", "phone": "+1 604 555 0100", "id": "crm-42"} +``` + Choose a stable `source.identifier` for the upstream dataset and preserve the upstream meeting ID as `meeting.external_id`. That pair is the idempotency key: the first import returns `201` with status `created`; unchanged retries and @@ -455,3 +502,110 @@ status), insights, and tags land in the message metadata and body; the meeting recording URL and `recording_url_fetched_at` remain in the archived provider metadata. msgvault does not expose recording URLs as durable attachments, and downloading or archiving recording media is not supported. + +## Muesli + +[Muesli](https://github.com/Muesli-HQ/muesli) records and transcribes meetings +on a Mac and keeps them in a local SQLite database. msgvault reads that +database directly and read-only. It never changes Muesli's meetings or schema, +and it does not call `muesli-cli`, which updates the database whenever it runs. + +### Prerequisites + +The msgvault daemon reads the database on its own host, so run msgvault on the +Mac where Muesli records. If your archive lives on another machine, send each +meeting to that daemon with the [import API](#import-from-any-meeting-source) +from a Muesli post-meeting hook instead. + +If macOS blocks the read, grant the process that runs `msgvault serve` Full +Disk Access in System Settings. + +### Configure and register + +```toml +[[muesli]] +identifier = "mac" +account_email = "you@example.com" # you, the person who records +# db_path = "~/Library/Application Support/Muesli/muesli.db" # default +schedule = "*/30 * * * *" # optional daemon schedule +enabled = true +``` + +`db_path` defaults to the stable app's database. Development builds of Muesli +use a different support folder, such as `MuesliDev`; set `db_path` for those. + +### Attendees from Apple Contacts + +People you tag in Muesli usually come from Apple Contacts. msgvault reads the +Mac's Contacts stores read-only, finds each attendee's card by its Contacts ID +or exact email, and links its unshared emails and phones. An address on multiple +unlinked Contacts cards stays in the meeting evidence but does not create an +identity link. A contact tagged +with only a phone number therefore reaches the person you already chat with at +that number. + +- Reading Contacts needs Full Disk Access for the process that runs + `msgvault serve`. Without it, meetings still sync, and `sync-muesli` reports + `Contacts: unavailable`. +- Phone numbers typed with `+` or `00` always work. Set `phone_country_code` + (for example `"1"` or `"44"`) to also use numbers typed without a country + code. +- When only some Contacts accounts can be read, msgvault still uses Contacts + IDs to retain meeting evidence, but stops matching by email and creating + automatic identity links. An unreadable account could hold another card + with the same address. Linking resumes when every account is readable. +- When Contacts is unreadable or a card disappears, an attendee still present + in Muesli keeps the identities archived for that meeting. Those retained + addresses do not assert current Contacts ownership. Removing the attendee + in Muesli removes its meeting association. +- Set `contacts = false` to turn the lookup off. + +```bash +msgvault add-muesli mac +``` + +`add-muesli` checks that the file opens read-only as a Muesli database, then +registers the source. + +### Sync + +```bash +msgvault sync-muesli # all configured databases +msgvault sync-muesli mac --limit 5 +msgvault sync-muesli --after 2026-01-01 +msgvault sync-muesli --full +``` + +Every run reads the whole database and updates meetings that changed in place, +including title, notes, transcript, participant, and folder edits. Unchanged +meetings are skipped. + +- Meetings still recording or processing wait for a later run. +- Meetings deleted in Muesli stay in the archive unchanged. +- `--after` keeps meetings that start on or after the date, read as UTC. +- `--limit` caps the meetings processed in one run. +- `--full` rewrites every archived meeting, which refreshes attribution after + you add an identity. + +### What gets stored (Muesli) + +Each meeting becomes one `meeting_transcript` message in a `meeting` +conversation. The body holds the title, time, participant names, Muesli's AI +notes, the notes you typed, and the transcript. When Muesli skipped or failed +the summary, the body omits that notice instead of showing it as a summary. + +Muesli does not record an organizer. msgvault attributes each meeting to +`account_email` as its organizer. Participants with an email address become +recipients and connect to your existing people. Participants without one, +such as a contact picked by name, appear only by name. + +The raw archive (`muesli_json`) keeps the meeting's text, times, status, +template name, calendar event ID, folder path, and participant names, emails, +phones, and sources. It never stores audio or audio file paths, screen text, +template prompts, or Apple Contacts identifiers. + +Remove the archive source with: + +```bash +msgvault remove-account mac --type muesli --yes +``` diff --git a/docs/usage/multi-account.md b/docs/usage/multi-account.md index 72bd2e8a7..b1883eaa2 100644 --- a/docs/usage/multi-account.md +++ b/docs/usage/multi-account.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-25" title: Accounts, Identities, and Collections description: How msgvault organizes every source into accounts, tracks which identifiers are "you," and groups accounts into collections for scoped search, stats, and deduplication. --- @@ -283,7 +283,7 @@ collection; selections spanning sources are rejected. Deduplication remains available through the collection-scoped CLI commands, not through the TUI. Meetings mode uses the same key for a separate source selector. It lists -Granola, Circleback, and Notion meeting sources. Changing it does not replace +Granola, Circleback, Notion, and Muesli meeting sources. Changing it does not replace the Email account filter. ## Command Reference diff --git a/docs/usage/people.md b/docs/usage/people.md index 3864e5be8..6b54cf6d9 100644 --- a/docs/usage/people.md +++ b/docs/usage/people.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-09-25" title: People and Profiles description: Find people across your archive, keep their details together, and understand your contact history. --- @@ -58,7 +58,10 @@ msgvault person set-display-name 7 "Alex Example" ``` Repeating promotion returns the same profile. Archive observation alone does -not promote people. Linking another cluster into a promoted one expands that +not promote people. Sources that assert a stable identity link addresses +automatically: Beeper user IDs, Notion user IDs, Apple Contacts cards for +Muesli attendees, and person `id` values in meeting imports. See +[how meetings connect to people](/docs/usage/meetings/#how-meetings-connect-to-people). Linking another cluster into a promoted one expands that profile's participant bindings. Linking two clusters that already belong to different profiles reports a conflict instead of silently merging curated data. Unlinking evidence does not move or delete profile bindings. diff --git a/docs/usage/tui.md b/docs/usage/tui.md index 8320d469f..162947c36 100644 --- a/docs/usage/tui.md +++ b/docs/usage/tui.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-25" title: Interactive TUI description: Browse messages and people, search your archive, save attachments, and stage email deletion from the terminal. --- @@ -160,8 +160,8 @@ Navigation and interaction in Text mode work the same as Email mode. Press `Ente ## Meetings Meetings mode is a read-only browser for archived transcripts and notes from -[meeting sources](/docs/usage/meetings/), including Granola, Circleback, and -Notion. It shows a flat, newest-first list +[meeting sources](/docs/usage/meetings/), including Granola, Circleback, +Notion, and Muesli. It shows a flat, newest-first list with each meeting's date, title, organizer, and source. Press `Enter` to open the transcript and notes, `Esc` or `Backspace` to return to the list, and the left/right arrow keys to move between meeting details. diff --git a/docs/usage/vector-search.md b/docs/usage/vector-search.md index 1536182bd..ed48469f5 100644 --- a/docs/usage/vector-search.md +++ b/docs/usage/vector-search.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-23" +last_edited: "2026-09-25" title: Vector Search description: Find messages by meaning and set up separate people, visual, and document search indexes. --- @@ -429,9 +429,9 @@ trigger). |---|---| | Manual `sync-full` / `sync` (Gmail, IMAP) | No. Run `msgvault embeddings build` afterward | | Manual `sync-calendar` / `sync-teams` / `sync-discord` | No. Run `msgvault embeddings build` afterward | -| Manual `sync-slack` / `sync-beeper` / `sync-granola` / `sync-circleback` / `sync-notion-meetings` | No. Run `msgvault embeddings build` afterward | +| Manual `sync-slack` / `sync-beeper` / `sync-granola` / `sync-circleback` / `sync-notion-meetings` / `sync-muesli` | No. Run `msgvault embeddings build` afterward | | Scheduled account syncs in `msgvault serve` (Gmail, IMAP, Teams, Discord) | Yes, when `[vector.embed.schedule].run_after_sync = true` | -| Scheduled calendar, Slack, Beeper, Granola, Circleback, and Notion syncs in `msgvault serve` | No immediate post-sync run. Picked up by the embed worker's `[vector.embed.schedule].cron` schedule | +| Scheduled calendar, Slack, Beeper, Granola, Circleback, Notion, and Muesli syncs in `msgvault serve` | No immediate post-sync run. Picked up by the embed worker's `[vector.embed.schedule].cron` schedule | | Local email imports (`import-pst`, `import-emlx`, `import-eml`, `import-mbox`) | No. Run `msgvault embeddings build` afterward | | Local chat/text imports | No. Run `msgvault embeddings build` afterward | diff --git a/internal/api/cli_allowlist_slack_test.go b/internal/api/cli_allowlist_slack_test.go index 2f7d8a756..31b9d20f9 100644 --- a/internal/api/cli_allowlist_slack_test.go +++ b/internal/api/cli_allowlist_slack_test.go @@ -38,3 +38,17 @@ func TestCLIRunCommandAllowedNotionMeetingsCommands(t *testing.T) { }) } } + +func TestCLIRunCommandAllowedMuesliCommands(t *testing.T) { + t.Parallel() + for _, args := range [][]string{ + {"add-muesli"}, + {"add-muesli", "mac"}, + {"sync-muesli"}, + {"sync-muesli", "mac", "--full"}, + } { + t.Run(args[0], func(t *testing.T) { + assert.True(t, cliRunCommandAllowed(args), "%v must be runnable via the daemon CLI", args) + }) + } +} diff --git a/internal/api/cli_handlers.go b/internal/api/cli_handlers.go index f3d9b172f..682cb772f 100644 --- a/internal/api/cli_handlers.go +++ b/internal/api/cli_handlers.go @@ -1694,6 +1694,7 @@ func cliRunCommandAllowed(args []string) bool { "add-discord", "add-granola", "add-imap", + "add-muesli", "add-notion-meetings", "add-o365", "add-slack", @@ -1744,6 +1745,7 @@ func cliRunCommandAllowed(args []string) bool { "sync-circleback", "sync-discord", "sync-granola", + "sync-muesli", "sync-notion-meetings", "sync-slack", "sync-synctech-sms", diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go index da56af325..e60145f56 100644 --- a/internal/api/handlers_test.go +++ b/internal/api/handlers_test.go @@ -40,6 +40,7 @@ import ( "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" "go.kenn.io/msgvault/internal/meetingimport" + "go.kenn.io/msgvault/internal/muesli" "go.kenn.io/msgvault/internal/notionmeetings" "go.kenn.io/msgvault/internal/opserr" "go.kenn.io/msgvault/internal/personenrichment" @@ -4304,6 +4305,7 @@ func TestSchedulerJobNameForSource(t *testing.T) { {"granola", granola.SourceType, "acct-1", "granola:acct-1", true}, {"circleback", circleback.SourceType, "acct-2", "circleback:acct-2", true}, {"notion meetings", notionmeetings.SourceType, "acct-3", "notion-meetings:acct-3", true}, + {"muesli", muesli.SourceType, "mac", "muesli:mac", true}, {"beeper", "beeper", "beeper-account-1", "beeper", true}, {"slack", "slack", "T01:U01", "slack", true}, {"account scheduler type", "gmail", "alice@example.com", "", false}, diff --git a/internal/api/meeting_import.go b/internal/api/meeting_import.go index 96d4c2135..171047c06 100644 --- a/internal/api/meeting_import.go +++ b/internal/api/meeting_import.go @@ -5,6 +5,8 @@ import ( "errors" "mime" "net/http" + "slices" + "strings" "github.com/danielgtaylor/huma/v2" "go.kenn.io/msgvault/internal/meetingimport" @@ -83,6 +85,26 @@ func hardenMeetingImportSchemas(doc *huma.OpenAPI) { email.Format = "email" } } + if person := doc.Components.Schemas.Map()["MeetingPerson"]; person != nil { + idLimit, phoneLimit := 200, 64 + if id := person.Properties["id"]; id != nil { + id.MaxLength = &idLimit + } + if phone := person.Properties["phone"]; phone != nil { + phone.MaxLength = &phoneLimit + } + person.Required = slices.DeleteFunc(person.Required, func(field string) bool { return field == "email" }) + person.AnyOf = []*huma.Schema{ + { + Type: huma.TypeObject, Required: []string{"email"}, + Properties: map[string]*huma.Schema{"email": {Type: huma.TypeString, Format: "email"}}, + }, + { + Type: huma.TypeObject, Required: []string{"phone"}, + Properties: map[string]*huma.Schema{"phone": {Type: huma.TypeString, MaxLength: &phoneLimit}}, + }, + } + } one := 1 contentRequired := []*huma.Schema{ { @@ -150,8 +172,13 @@ func (s *Server) handleMeetingImport(w http.ResponseWriter, r *http.Request) { return } if _, err := req.Normalize(); err != nil { - writeError(w, http.StatusUnprocessableEntity, "validation_failed", - "Meeting import request failed validation") + // Validation messages name the field and the rule, never the value. + message := "Meeting import request failed validation" + if detail, ok := strings.CutPrefix(err.Error(), meetingimport.ErrValidation.Error()+": "); ok && + errors.Is(err, meetingimport.ErrValidation) { + message += ": " + detail + } + writeError(w, http.StatusUnprocessableEntity, "validation_failed", message) return } diff --git a/internal/api/meeting_import_test.go b/internal/api/meeting_import_test.go index 19497bdd4..4f92c32c8 100644 --- a/internal/api/meeting_import_test.go +++ b/internal/api/meeting_import_test.go @@ -210,6 +210,29 @@ func TestMeetingImportRejectsInvalidRequests(t *testing.T) { assert.Equal(t, 0, store.calls) } +func TestMeetingImportValidationNamesFieldWithoutEchoingValue(t *testing.T) { + t.Parallel() + assert := assert.New(t) + require := require.New(t) + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + body := strings.Replace(validMeetingImportBody, `"email": "attendee@example.com"`, + `"phone": "(604) 555-0100"`, 1) + require.NotEqual(validMeetingImportBody, body, "fixture must contain the attendee email") + + req := meetingImportRequest(body) + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, req) + + require.Equal(http.StatusUnprocessableEntity, resp.Code, "body: %s", resp.Body.String()) + var decoded ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&decoded)) + assert.Equal("validation_failed", decoded.Error) + assert.Contains(decoded.Message, "meeting.attendees[0].phone") + assert.NotContains(decoded.Message, "555") + assert.Equal(0, store.calls) +} + func TestMeetingImportRejectsOversizedBody(t *testing.T) { t.Parallel() assert := assert.New(t) diff --git a/internal/api/openapi.go b/internal/api/openapi.go index f479557fc..266e75d28 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -859,6 +859,24 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { } } } + if person := schemas["MeetingPerson"]; person != nil { + // As with Meeting, keep the email-or-phone rule in the public schema but + // give the Go client a plain struct instead of a union wrapper. + person.AnyOf = nil + if email := person.Properties[emailProperty]; email != nil { + // Email used to be required. Keep the Go client field a plain string + // so existing callers still compile, and omit it when empty so a + // phone-only attendee never sends "" against format: email. + if email.Extensions == nil { + email.Extensions = map[string]any{} + } + email.Extensions["x-go-type-skip-optional-pointer"] = true + email.Extensions["x-omitempty"] = true + email.Extensions["x-oapi-codegen-extra-tags"] = map[string]any{ + "validate": "required_without=Phone", + } + } + } if totals := schemas["DurationTotals"]; totals != nil { if average := totals.Properties["average_known_seconds"]; average != nil { setCodegenGoType(average, "*float64") diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go index 1e2d764eb..c1a342788 100644 --- a/internal/api/openapi_test.go +++ b/internal/api/openapi_test.go @@ -1095,6 +1095,14 @@ func TestOpenAPIMeetingImportContract(t *testing.T) { ) assertions.Equal("email", schemas["MeetingPerson"].Properties["email"].Format) + person := schemas["MeetingPerson"] + assertions.NotContains(person.Required, "email", "an attendee may be identified by phone alone") + requirements.Len(person.AnyOf, 2, "attendee requires an email or a phone") + assertions.ElementsMatch([]string{"email", "phone"}, + []string{person.AnyOf[0].Required[0], person.AnyOf[1].Required[0]}) + requirements.NotNil(person.Properties["id"].MaxLength) + assertions.Equal(200, *person.Properties["id"].MaxLength) + requirements.NotNil(person.Properties["phone"].MaxLength) offset := schemas["TranscriptSegment"].Properties["offset_seconds"] requirements.NotNil(offset.Minimum) assertions.Zero(*offset.Minimum) diff --git a/internal/api/scheduler_jobs.go b/internal/api/scheduler_jobs.go index dd1140f04..905434c08 100644 --- a/internal/api/scheduler_jobs.go +++ b/internal/api/scheduler_jobs.go @@ -7,6 +7,7 @@ import ( "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" "go.kenn.io/msgvault/internal/meetingimport" + "go.kenn.io/msgvault/internal/muesli" "go.kenn.io/msgvault/internal/notionmeetings" "go.kenn.io/msgvault/internal/synctechsms" ) @@ -102,6 +103,10 @@ func SchedulerJobNameForSource(sourceType, identifier string) (string, bool) { return "circleback:" + identifier, true case notionmeetings.SourceType: return "notion-meetings:" + identifier, true + case muesli.SourceType: + // Store identifier == config Identifier (see + // internal/muesli/importer.go GetSourceByTypeAndIdentifier call). + return "muesli:" + identifier, true case sourceTypeBeeper: // One scheduler job syncs every beeper source (see // internal/beeper/importer.go GetOrCreateSource, one store source diff --git a/internal/circleback/person_link_test.go b/internal/circleback/person_link_test.go new file mode 100644 index 000000000..46102a38b --- /dev/null +++ b/internal/circleback/person_link_test.go @@ -0,0 +1,42 @@ +package circleback + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/activity" + "go.kenn.io/msgvault/internal/store" +) + +// TestImportedMeetingReachesExistingPerson pins the contract that matters to +// a user: an attendee whose email already belongs to a person gets the +// meeting on that person after activity projection, with no name matching. +func TestImportedMeetingReachesExistingPerson(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + imp, st := newTestImporter(t, &fakeSource{ + meetings: map[string]json.RawMessage{"42": json.RawMessage(meeting42)}, + transcripts: map[string]json.RawMessage{"42": json.RawMessage(transcript42)}, + }) + participantID, err := st.EnsureParticipant("bob@example.com", "Bob", "example.com") + require.NoError(err) + person, _, err := st.CreatePersonFromParticipantContext(t.Context(), participantID) + require.NoError(err) + + _, err = imp.Import(t.Context(), ImportOptions{Identifier: "alice@example.com", AccountEmail: "alice@example.com"}) + require.NoError(err) + projector, err := activity.NewProjector(st, activity.Options{Timezone: "UTC", BatchSize: 10, MaxDirectCounterparts: 25}) + require.NoError(err) + _, err = projector.RunOnce(t.Context()) + require.NoError(err) + + var role string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT aep.role FROM activity_event_persons aep + JOIN messages m ON m.id = aep.message_id + WHERE m.source_message_id = ? AND aep.person_id = ?`), + "meeting:42", person.ID).Scan(&role)) + assert.Equal(string(store.RoleAttendee), role) +} diff --git a/internal/config/config.go b/internal/config/config.go index 21df7fa2e..6cc06caa8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -508,6 +508,7 @@ type Config struct { Granola []GranolaSource `toml:"granola"` Circleback []CirclebackSource `toml:"circleback"` NotionMeetings []NotionMeetingsSource `toml:"notion_meetings"` + Muesli []MuesliSource `toml:"muesli"` Backup BackupConfig `toml:"backup"` Discord DiscordConfig `toml:"discord"` Attachments documentindex.AttachmentsConfig `toml:"attachments"` @@ -959,6 +960,10 @@ func decodeConfig(cfg *Config, path string, explicit, homeOverride bool, content cfg.Vector.DBPath = expandPath(cfg.Vector.DBPath) cfg.Vector.Multimodal.CapabilitiesFile = expandPath(cfg.Vector.Multimodal.CapabilitiesFile) cfg.Backup.Repo = expandPath(cfg.Backup.Repo) + for i := range cfg.Muesli { + cfg.Muesli[i].DBPath = expandPath(cfg.Muesli[i].DBPath) + cfg.Muesli[i].ContactsPath = expandPath(cfg.Muesli[i].ContactsPath) + } for name, app := range cfg.OAuth.Apps { app.ClientSecrets = expandPath(app.ClientSecrets) app.ServiceAccountKey = expandPath(app.ServiceAccountKey) @@ -976,6 +981,10 @@ func decodeConfig(cfg *Config, path string, explicit, homeOverride bool, content cfg.Vector.DBPath = resolveRelative(cfg.Vector.DBPath, cfg.HomeDir) cfg.Vector.Multimodal.CapabilitiesFile = resolveRelative(cfg.Vector.Multimodal.CapabilitiesFile, cfg.HomeDir) cfg.Backup.Repo = resolveRelative(cfg.Backup.Repo, cfg.HomeDir) + for i := range cfg.Muesli { + cfg.Muesli[i].DBPath = resolveRelative(cfg.Muesli[i].DBPath, cfg.HomeDir) + cfg.Muesli[i].ContactsPath = resolveRelative(cfg.Muesli[i].ContactsPath, cfg.HomeDir) + } for name, app := range cfg.OAuth.Apps { app.ClientSecrets = resolveRelative(app.ClientSecrets, cfg.HomeDir) app.ServiceAccountKey = resolveRelative(app.ServiceAccountKey, cfg.HomeDir) @@ -1763,6 +1772,60 @@ func (s NotionMeetingsSource) EffectiveAccountEmail() (string, error) { return effectiveMeetingAccountEmail("notion_meetings", s.Identifier, s.AccountEmail) } +// MuesliSource is one local Muesli meeting database. Each entry is a +// top-level [[muesli]] table. The daemon reads the database on its own host. +type MuesliSource struct { + Identifier string `toml:"identifier"` // stable source label for add-/sync-muesli; defaults to "default" for a single entry + AccountEmail string `toml:"account_email"` // the person who records; attributed as each meeting's organizer + DBPath string `toml:"db_path"` // muesli.db path; empty = the stable app's default location + Schedule string `toml:"schedule"` // 5-field cron; empty = not daemon-scheduled + Enabled bool `toml:"enabled"` + // Contacts resolves attendees through the Mac's Contacts app; nil = on. + Contacts *bool `toml:"contacts"` + // ContactsPath is the Contacts data folder; empty = macOS's default. + ContactsPath string `toml:"contacts_path"` + // PhoneCountryCode lets national-format Contacts phone numbers convert to + // E.164, for example "1" or "44". Empty = only international numbers. + PhoneCountryCode string `toml:"phone_country_code"` +} + +// EffectiveAccountEmail returns the normalized primary identity configured +// for this source. +func (s MuesliSource) EffectiveAccountEmail() (string, error) { + return effectiveMeetingAccountEmail("muesli", s.Identifier, s.AccountEmail) +} + +// DefaultMuesliDBPath is where the stable Muesli app keeps its database. +func DefaultMuesliDBPath() string { + return expandPath(filepath.Join("~", "Library", "Application Support", "Muesli", "muesli.db")) +} + +// ContactsEnabled reports whether attendees are resolved through Contacts. +func (s MuesliSource) ContactsEnabled() bool { + return s.Contacts == nil || *s.Contacts +} + +// DefaultContactsPath is where macOS keeps the Contacts stores. +func DefaultContactsPath() string { + return expandPath(filepath.Join("~", "Library", "Application Support", "AddressBook")) +} + +// EffectiveContactsPath returns the configured Contacts folder, or macOS's. +func (s MuesliSource) EffectiveContactsPath() string { + if strings.TrimSpace(s.ContactsPath) == "" { + return DefaultContactsPath() + } + return s.ContactsPath +} + +// EffectiveDBPath returns the configured database path, or Muesli's default. +func (s MuesliSource) EffectiveDBPath() string { + if strings.TrimSpace(s.DBPath) == "" { + return DefaultMuesliDBPath() + } + return s.DBPath +} + // EffectiveAccountEmail returns the normalized primary identity configured // for this source. func (s CirclebackSource) EffectiveAccountEmail() (string, error) { @@ -1807,6 +1870,9 @@ func (c *Config) applyMeetingSourceDefaults() { if len(c.NotionMeetings) == 1 && c.NotionMeetings[0].Identifier == "" { c.NotionMeetings[0].Identifier = "default" } + if len(c.Muesli) == 1 && c.Muesli[0].Identifier == "" { + c.Muesli[0].Identifier = "default" + } } // validateMeetingSources rejects native meeting-source lists with empty @@ -1875,6 +1941,26 @@ func (c *Config) validateMeetingSources() error { c.NotionMeetings[i].AccountEmail = email } } + muesliIDs := make([]string, len(c.Muesli)) + for i, s := range c.Muesli { + muesliIDs[i] = s.Identifier + } + if err := check("muesli", muesliIDs); err != nil { + return err + } + for i := range c.Muesli { + email, err := c.Muesli[i].EffectiveAccountEmail() + if err != nil { + return err + } + c.Muesli[i].AccountEmail = email + code := strings.TrimPrefix(strings.TrimSpace(c.Muesli[i].PhoneCountryCode), "+") + if code != "" && (len(code) > 3 || strings.Trim(code, "0123456789") != "" || code[0] == '0') { + return fmt.Errorf("[[muesli]] identifier %q has invalid phone_country_code %q; use 1 to 3 digits such as \"1\" or \"44\"", + c.Muesli[i].Identifier, c.Muesli[i].PhoneCountryCode) + } + c.Muesli[i].PhoneCountryCode = code + } return nil } @@ -1948,6 +2034,29 @@ func (c *Config) ScheduledNotionMeetingsSources() []NotionMeetingsSource { return out } +// GetMuesliSource returns the configured Muesli source matching identifier +// (case-insensitive), or nil. +func (c *Config) GetMuesliSource(identifier string) *MuesliSource { + for _, src := range c.Muesli { + if strings.EqualFold(src.Identifier, identifier) { + cp := src + return &cp + } + } + return nil +} + +// ScheduledMuesliSources returns enabled Muesli sources with a cron schedule. +func (c *Config) ScheduledMuesliSources() []MuesliSource { + var out []MuesliSource + for _, src := range c.Muesli { + if src.Enabled && src.Schedule != "" { + out = append(out, src) + } + } + return out +} + func (c *Config) GetSynctechSMSSource(name string) *SynctechSMSSource { for _, src := range c.SynctechSMS.Sources { if strings.EqualFold(src.Name, name) { diff --git a/internal/config/config_meeting_test.go b/internal/config/config_meeting_test.go index b8349afb1..7b28baa89 100644 --- a/internal/config/config_meeting_test.go +++ b/internal/config/config_meeting_test.go @@ -236,3 +236,111 @@ identifier = "second" require.Error(err) require.Contains(err.Error(), "identifier") } + +func TestLoadMuesliSourceDefaultsAndPaths(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + configPath := writeMeetingConfig(t, ` +[[muesli]] +account_email = " You@Example.com " +db_path = "~/muesli/muesli.db" +schedule = "*/30 * * * *" +enabled = true +`) + + cfg, err := Load(configPath, "") + require.NoError(err, "Load()") + + require.Len(cfg.Muesli, 1) + src := cfg.Muesli[0] + assert.Equal("default", src.Identifier) + assert.Equal("you@example.com", src.AccountEmail) + home, err := os.UserHomeDir() + require.NoError(err) + assert.Equal(filepath.Join(home, "muesli", "muesli.db"), src.EffectiveDBPath()) + assert.Len(cfg.ScheduledMuesliSources(), 1) + found := cfg.GetMuesliSource("DEFAULT") + require.NotNil(found, "lookup is case-insensitive") + assert.Equal("you@example.com", found.AccountEmail) + assert.Nil(cfg.GetMuesliSource("nope")) +} + +func TestMuesliSourceEffectiveDBPathDefaultsToMuesliSupportFolder(t *testing.T) { + home, err := os.UserHomeDir() + require.NoError(t, err) + + assert.Equal(t, + filepath.Join(home, "Library", "Application Support", "Muesli", "muesli.db"), + MuesliSource{}.EffectiveDBPath()) +} + +func TestLoadMuesliSourceRejectsInvalidEntries(t *testing.T) { + for _, tt := range []struct { + name string + content string + want string + }{ + { + name: "duplicate identifiers", + content: ` +[[muesli]] +identifier = "mac" +account_email = "a@example.com" +[[muesli]] +identifier = "MAC" +account_email = "b@example.com" +`, + want: `[[muesli]]: duplicate identifier "MAC"`, + }, + { + name: "missing account email", + content: ` +[[muesli]] +identifier = "mac" +`, + want: `[[muesli]] identifier "mac" requires account_email`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := Load(writeMeetingConfig(t, tt.content), "") + require.Error(t, err) + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestLoadMuesliContactsSettings(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + home, err := os.UserHomeDir() + require.NoError(err) + + defaults, err := Load(writeMeetingConfig(t, ` +[[muesli]] +account_email = "you@example.com" +`), "") + require.NoError(err) + assert.True(defaults.Muesli[0].ContactsEnabled(), "Contacts enrichment is on by default") + assert.Equal(filepath.Join(home, "Library", "Application Support", "AddressBook"), + defaults.Muesli[0].EffectiveContactsPath()) + + custom, err := Load(writeMeetingConfig(t, ` +[[muesli]] +account_email = "you@example.com" +contacts = false +contacts_path = "~/contacts-copy" +phone_country_code = "+44" +`), "") + require.NoError(err) + assert.False(custom.Muesli[0].ContactsEnabled()) + assert.Equal(filepath.Join(home, "contacts-copy"), custom.Muesli[0].EffectiveContactsPath()) + assert.Equal("44", custom.Muesli[0].PhoneCountryCode, "a leading + is accepted and removed") + + _, err = Load(writeMeetingConfig(t, ` +[[muesli]] +account_email = "you@example.com" +phone_country_code = "UK" +`), "") + require.Error(err) + assert.Contains(err.Error(), "phone_country_code") +} diff --git a/internal/config/edit.go b/internal/config/edit.go index 52e0fc411..d02075432 100644 --- a/internal/config/edit.go +++ b/internal/config/edit.go @@ -916,6 +916,9 @@ func validateEditableCandidate(cfg *Config) error { for index, source := range cfg.NotionMeetings { schedules[fmt.Sprintf("notion_meetings[%d].schedule", index)] = source.Schedule } + for index, source := range cfg.Muesli { + schedules[fmt.Sprintf("muesli[%d].schedule", index)] = source.Schedule + } for key, expression := range schedules { if expression == "" { continue diff --git a/internal/config/edit_test.go b/internal/config/edit_test.go index c7b28dac0..e523c9a60 100644 --- a/internal/config/edit_test.go +++ b/internal/config/edit_test.go @@ -2122,3 +2122,24 @@ func TestLoadConfigFileUsesLogicalSymlinkPathForRelativeDefaults(t *testing.T) { assert.Equal(fromDaemon.HomeDir, fromSnapshot.HomeDir) assert.Equal(logicalDir, fromSnapshot.HomeDir) } + +func TestEditConfigRejectsInvalidMuesliSchedule(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + path := filepath.Join(t.TempDir(), "config.toml") + before := "[[muesli]]\n" + + "identifier = \"mac\"\n" + + "account_email = \"you@example.com\"\n" + + "schedule = \"*/30 * * * *\"\n" + + "enabled = true\n" + require.NoError(os.WriteFile(path, []byte(before), 0o600)) + snapshot, err := ReadConfigFile(path) + require.NoError(err) + + _, err = EditConfigFile(path, snapshot.ETag, []Edit{{Key: "muesli.schedule", Value: "not a cron"}}) + require.ErrorIs(err, ErrInvalidConfigCandidate) + assert.Contains(err.Error(), "invalid muesli[0].schedule") + got, readErr := os.ReadFile(path) + require.NoError(readErr) + assert.Equal(before, string(got)) +} diff --git a/internal/granola/person_link_test.go b/internal/granola/person_link_test.go new file mode 100644 index 000000000..4bb276595 --- /dev/null +++ b/internal/granola/person_link_test.go @@ -0,0 +1,39 @@ +package granola + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/activity" + "go.kenn.io/msgvault/internal/store" +) + +// TestImportedNoteReachesExistingPerson pins the contract that matters to a +// user: an attendee whose email already belongs to a person gets the meeting +// on that person after activity projection, with no name matching. +func TestImportedNoteReachesExistingPerson(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + api := &fakeAPI{notes: map[string][]byte{"not_Ab12Cd34Ef56Gh": loadFixture(t, "note_full.json")}} + imp, st := newTestImporter(t, api) + participantID, err := st.EnsureParticipant("carol@example.com", "Carol", "example.com") + require.NoError(err) + person, _, err := st.CreatePersonFromParticipantContext(t.Context(), participantID) + require.NoError(err) + + _, err = imp.Import(t.Context(), ImportOptions{Identifier: "alice@example.com", AccountEmail: "alice@example.com"}) + require.NoError(err) + projector, err := activity.NewProjector(st, activity.Options{Timezone: "UTC", BatchSize: 10, MaxDirectCounterparts: 25}) + require.NoError(err) + _, err = projector.RunOnce(t.Context()) + require.NoError(err) + + var role string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT aep.role FROM activity_event_persons aep + JOIN messages m ON m.id = aep.message_id + WHERE m.source_message_id = ? AND aep.person_id = ?`), + "not_Ab12Cd34Ef56Gh", person.ID).Scan(&role)) + assert.Equal(string(store.RoleAttendee), role) +} diff --git a/internal/meetingarchive/archive.go b/internal/meetingarchive/archive.go index 50157743e..43445ec7e 100644 --- a/internal/meetingarchive/archive.go +++ b/internal/meetingarchive/archive.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json/jsontext" "errors" "fmt" "strings" @@ -23,9 +24,22 @@ const ( var ErrUnavailable = errors.New("meeting archiver is unavailable") +// Person is one meeting organizer or attendee. Email, then Phone, is the +// identity recorded as the meeting recipient. OtherEmails and OtherPhones are +// further identities of the same human; with an Anchor they are linked to the +// recipient identity through LinkIdentities. Names never match anything. type Person struct { - Name string - Email string + Name string + Email string + Phone string // E.164 + OtherEmails []string + OtherPhones []string + // Anchor is a stable, provider-scoped identifier for this human, built + // with Anchor(). Empty means the provider asserts no stable identity. + Anchor string + // LinkExcludedAddresses are normalized addresses kept as meeting evidence + // but excluded from this anchor's automatic ownership assertions. + LinkExcludedAddresses []string } type Snapshot struct { @@ -48,8 +62,11 @@ type Result struct { MessageID int64 Created bool // Changed means the archive write committed, including when Upsert returns - // a later conversation-stat maintenance error. + // a later conversation-stat maintenance or identity-linking error. Changed bool + // Links reports anchored attendee identity linking, which runs on every + // Upsert so an interrupted link is repaired by the next sync. + Links LinkResult } // UpsertOptions controls archive repair behavior. @@ -79,62 +96,73 @@ func (a *Archiver) Upsert( return Result{}, err } - existing, err := a.store.MessageExistsBatch(snapshot.SourceID, []string{snapshot.SourceMessageID}) + existing, err := a.store.MessageMetadataBatch(snapshot.SourceID, []string{snapshot.SourceMessageID}) if err != nil { return Result{}, fmt.Errorf("lookup existing meeting: %w", err) } - existingMessageID, existed := existing[snapshot.SourceMessageID] + existingMessage, existed := existing[snapshot.SourceMessageID] + existingMessageID := existingMessage.ID identities, err := meetingidentity.ForSource(a.store, snapshot.SourceID, snapshot.AccountEmail) if err != nil { return Result{}, err } - organizerEmail, organizerName := "", "" + var organizer Person if snapshot.Organizer != nil { - organizerEmail = normalizeEmail(snapshot.Organizer.Email) - organizerName = strings.TrimSpace(snapshot.Organizer.Name) + organizer = snapshot.Organizer.Normalized() } - expectedIsFromMe := organizerEmail != "" && identities.Contains(organizerEmail) + organizerEmail, organizerName := organizer.Email, organizer.Name + organizerAddress := organizerEmail + if organizerAddress == "" { + organizerAddress = organizer.Phone + } + expectedIsFromMe := organizerAddress != "" && identities.Contains(organizerAddress) if existed && !opts.Force { storedRaw, rawErr := a.store.GetMessageRaw(existingMessageID) storedIsFromMe, attributionErr := a.store.GetMessageIsFromMe(existingMessageID) - if rawErr == nil && attributionErr == nil && bytes.Equal(storedRaw, snapshot.Raw) && storedIsFromMe == expectedIsFromMe { + if rawErr == nil && attributionErr == nil && bytes.Equal(storedRaw, snapshot.Raw) && + storedIsFromMe == expectedIsFromMe && equalMetadata([]byte(existingMessage.Metadata.String), snapshot.Metadata) { if err := a.store.RecomputeConversationStatsForMessageContext(ctx, existingMessageID); err != nil { return Result{}, fmt.Errorf("recompute meeting conversation stats: %w", err) } - return Result{MessageID: existingMessageID}, nil + result := Result{MessageID: existingMessageID} + result.Links, err = a.LinkIdentities(ctx, snapshot.SourceID, snapshotPeople(snapshot)) + if err != nil { + return result, fmt.Errorf("link meeting attendee identities: %w", err) + } + return result, nil } } participants := make([]store.ParticipantPersistData, 0, len(snapshot.Attendees)+1) - hasOrganizer := organizerEmail != "" + hasOrganizer := organizer.PrimaryKey() != "" if hasOrganizer { - participants = append(participants, store.ParticipantPersistData{ - EmailAddress: organizerEmail, - DisplayName: organizerName, - Domain: emailDomain(organizerEmail), - }) + participants = append(participants, persistData(organizer)) } attendeeNames := make([]string, 0, len(snapshot.Attendees)) attendeeEmails := make([]string, 0, len(snapshot.Attendees)) - for _, attendee := range snapshot.Attendees { + attendeeAddresses := make([]string, 0, len(snapshot.Attendees)) + seenAttendees := make(map[string]bool, len(snapshot.Attendees)) + for _, raw := range snapshot.Attendees { if err := ctx.Err(); err != nil { return Result{}, err } - email := normalizeEmail(attendee.Email) - if email == "" { + attendee := raw.Normalized() + key := attendee.PrimaryKey() + if key == "" || seenAttendees[key] { continue } - name := strings.TrimSpace(attendee.Name) - participants = append(participants, store.ParticipantPersistData{ - EmailAddress: email, - DisplayName: name, - Domain: emailDomain(email), - }) - attendeeNames = append(attendeeNames, name) - attendeeEmails = append(attendeeEmails, email) + seenAttendees[key] = true + participants = append(participants, persistData(attendee)) + attendeeNames = append(attendeeNames, attendee.Name) + attendeeEmails = append(attendeeEmails, attendee.Email) + if attendee.Email != "" { + attendeeAddresses = append(attendeeAddresses, attendee.Email) + } else { + attendeeAddresses = append(attendeeAddresses, attendee.Phone) + } } conversationID := strings.TrimSpace(snapshot.SourceConversationID) @@ -208,8 +236,8 @@ func (a *Archiver) Upsert( FTS: &store.FTSDoc{ Subject: snapshot.Title, Body: snapshot.Body, - FromAddr: organizerEmail, - ToAddrs: strings.Join(attendeeEmails, " "), + FromAddr: organizerAddress, + ToAddrs: strings.Join(attendeeAddresses, " "), }, } }, @@ -221,11 +249,20 @@ func (a *Archiver) Upsert( if err := a.store.RecomputeConversationStatsForMessageContext(ctx, messageID); err != nil { return result, fmt.Errorf("recompute meeting conversation stats: %w", err) } + result.Links, err = a.LinkIdentities(ctx, snapshot.SourceID, snapshotPeople(snapshot)) + if err != nil { + return result, fmt.Errorf("link meeting attendee identities: %w", err) + } return result, nil } -func normalizeEmail(email string) string { - return strings.ToLower(strings.TrimSpace(email)) +func equalMetadata(stored, incoming []byte) bool { + if bytes.Equal(stored, incoming) { + return true + } + // PostgreSQL JSONB changes object key order and whitespace on storage. + left, right := jsontext.Value(stored).Clone(), jsontext.Value(incoming).Clone() + return left.Canonicalize() == nil && right.Canonicalize() == nil && bytes.Equal(left, right) } func emailDomain(email string) string { @@ -235,3 +272,22 @@ func emailDomain(email string) string { } return strings.ToLower(email[at+1:]) } + +func persistData(person Person) store.ParticipantPersistData { + if person.Email != "" { + return store.ParticipantPersistData{ + EmailAddress: person.Email, + DisplayName: person.Name, + Domain: emailDomain(person.Email), + } + } + return store.ParticipantPersistData{PhoneNumber: person.Phone, DisplayName: person.Name} +} + +func snapshotPeople(snapshot Snapshot) []Person { + people := make([]Person, 0, len(snapshot.Attendees)+1) + if snapshot.Organizer != nil { + people = append(people, *snapshot.Organizer) + } + return append(people, snapshot.Attendees...) +} diff --git a/internal/meetingarchive/identity.go b/internal/meetingarchive/identity.go new file mode 100644 index 000000000..522481f7a --- /dev/null +++ b/internal/meetingarchive/identity.go @@ -0,0 +1,106 @@ +package meetingarchive + +import ( + "strings" + "unicode" +) + +// Normalized returns the person with trimmed, lowercased emails, E.164-only +// phones, invalid values dropped, and extra identities de-duplicated against +// the primary ones. The name is presentation data only. +func (p Person) Normalized() Person { + out := Person{ + Name: strings.TrimSpace(p.Name), + Email: normalizePersonEmail(p.Email), + Phone: normalizePersonPhone(p.Phone), + Anchor: strings.TrimSpace(p.Anchor), + LinkExcludedAddresses: p.LinkExcludedAddresses, + } + seenEmails := map[string]bool{out.Email: true} + for _, email := range p.OtherEmails { + if email = normalizePersonEmail(email); email != "" && !seenEmails[email] { + seenEmails[email] = true + out.OtherEmails = append(out.OtherEmails, email) + } + } + seenPhones := map[string]bool{out.Phone: true} + for _, phone := range p.OtherPhones { + if phone = normalizePersonPhone(phone); phone != "" && !seenPhones[phone] { + seenPhones[phone] = true + out.OtherPhones = append(out.OtherPhones, phone) + } + } + return out +} + +// PrimaryKey names the identity that makes this person a recipient: the +// email when present, otherwise the phone. Empty means the person has no +// usable identity. +func (p Person) PrimaryKey() string { + switch { + case p.Email != "": + return "email:" + p.Email + case p.Phone != "": + return "phone:" + p.Phone + default: + return "" + } +} + +// identities lists every email and phone of a normalized person, primary +// identities first. +func (p Person) identities() []identity { + var out []identity + if p.Email != "" { + out = append(out, identity{kind: identityEmail, value: p.Email}) + } + if p.Phone != "" { + out = append(out, identity{kind: identityPhone, value: p.Phone}) + } + for _, email := range p.OtherEmails { + out = append(out, identity{kind: identityEmail, value: email}) + } + for _, phone := range p.OtherPhones { + out = append(out, identity{kind: identityPhone, value: phone}) + } + return out +} + +type identityKind string + +const ( + identityEmail identityKind = "email" + identityPhone identityKind = "phone" +) + +type identity struct { + kind identityKind + value string +} + +// normalizePersonEmail keeps the archiver's historical leniency (providers +// validate their own addresses) while refusing values that cannot be an +// address at all. +func normalizePersonEmail(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + if !strings.Contains(value, "@") || strings.IndexFunc(value, unicode.IsSpace) >= 0 { + return "" + } + return value +} + +// normalizePersonPhone accepts only E.164 values. Providers normalize their +// own formats before building a snapshot; guessing a country here could +// attach a meeting to the wrong person. +func normalizePersonPhone(value string) string { + value = strings.TrimSpace(value) + if len(value) < 8 || len(value) > 16 || value[0] != '+' || value[1] == '0' { + return "" + } + for _, r := range value[1:] { + if r < '0' || r > '9' { + return "" + } + } + return value +} diff --git a/internal/meetingarchive/identity_test.go b/internal/meetingarchive/identity_test.go new file mode 100644 index 000000000..1163e09fd --- /dev/null +++ b/internal/meetingarchive/identity_test.go @@ -0,0 +1,147 @@ +package meetingarchive + +import ( + "encoding/json/v2" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/meetingcontent" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestPersonNormalized(t *testing.T) { + for _, tt := range []struct { + name string + in Person + want Person + }{ + { + name: "lowercases and drops duplicates of the primary", + in: Person{ + Name: " Alex Example ", Email: " Alex@Example.com ", Phone: "+16045550100", + OtherEmails: []string{"ALEX@example.com", "alex.work@example.com", "not-an-email"}, + OtherPhones: []string{"+16045550100", "+442079460000", "6045550100"}, + Anchor: " notion-user:abc ", + }, + want: Person{ + Name: "Alex Example", Email: "alex@example.com", Phone: "+16045550100", + OtherEmails: []string{"alex.work@example.com"}, + OtherPhones: []string{"+442079460000"}, + Anchor: "notion-user:abc", + }, + }, + { + name: "invalid primaries are dropped", + in: Person{Email: "nobody", Phone: "12345"}, + want: Person{}, + }, + { + name: "a zero after the plus is not a country code", + in: Person{Phone: "+04420794600"}, + want: Person{}, + }, + } { + assert.Equal(t, tt.want, tt.in.Normalized(), tt.name) + } +} + +func TestPersonPrimaryKey(t *testing.T) { + assert.Equal(t, "email:alex@example.com", Person{Email: "alex@example.com", Phone: "+16045550100"}.PrimaryKey()) + assert.Equal(t, "phone:+16045550100", Person{Phone: "+16045550100"}.PrimaryKey()) + assert.Empty(t, Person{Name: "Name Only"}.PrimaryKey()) +} + +func TestArchiverRecordsPhoneOnlyAttendee(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("meeting_import", "phones") + require.NoError(err) + snapshot := testSnapshot(source.ID) + snapshot.Attendees = []Person{ + {Name: "Phone Example", Phone: "+16045550100"}, + {Name: "Duplicate Row", Phone: "+16045550100"}, + {Name: "Name Only"}, + } + + result, err := New(st).Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + + var count int + var email string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT count(*), COALESCE(MAX(mr.email_address), '') + FROM message_recipients mr + JOIN participants p ON p.id = mr.participant_id + WHERE mr.message_id = ? AND mr.recipient_type = 'to' AND p.phone_number = ?`), + result.MessageID, "+16045550100").Scan(&count, &email)) + assert.Equal(1, count, "duplicate attendee rows resolve to one recipient") + assert.Empty(email) + var toCount int + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT count(*) FROM message_recipients WHERE message_id = ? AND recipient_type = 'to'`), + result.MessageID).Scan(&toCount)) + assert.Equal(1, toCount, "name-only attendees are not recipients") + if st.FTS5Available() && !st.IsPostgreSQL() { + var toAddr string + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT to_addr FROM messages_fts WHERE message_id = ?`), result.MessageID).Scan(&toAddr)) + assert.Contains(toAddr, "+16045550100") + } +} + +func TestArchiverAttributesPhoneOrganizerToOwner(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("meeting_import", "phone-owner") + require.NoError(err) + require.NoError(st.AddAccountIdentity(source.ID, "+16045550199", "phone-e164")) + snapshot := testSnapshot(source.ID) + snapshot.Organizer = &Person{Name: "Owner", Phone: "+16045550199"} + archiver := New(st) + + first, err := archiver.Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + fromMe, err := st.GetMessageIsFromMe(first.MessageID) + require.NoError(err) + assert.True(fromMe) + + second, err := archiver.Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + assert.False(second.Changed, "a phone organizer must not force a rewrite on every sync") + assert.Equal(first.MessageID, second.MessageID) +} + +func TestMeetingContextIncludesPhoneAttendee(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("meeting_import", "packet-phones") + require.NoError(err) + snapshot := testSnapshot(source.ID) + snapshot.RawFormat = "meeting_json" + snapshot.Raw = []byte(`{"summary_markdown":"Decision","attendees":[{"name":"Phone Example","phone":"+16045550100"}]}`) + snapshot.Attendees = []Person{{Name: "Phone Example", Phone: "+16045550100"}} + result, err := New(st).Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + + ids := []int64{result.MessageID} + packet, err := st.GetMeetingContextContext(t.Context(), store.MeetingQueryScope{MessageIDs: &ids}, + meetingcontent.PacketOptions{Format: meetingcontent.FormatJSON, MaxBytes: 65536}) + require.NoError(err) + var decoded meetingcontent.Packet + require.NoError(json.Unmarshal([]byte(packet.Content), &decoded)) + require.Len(decoded.Meetings, 1) + var phones []meetingcontent.Participant + for _, participant := range decoded.Meetings[0].Participants { + if participant.Phone != "" { + phones = append(phones, participant) + } + } + require.Len(phones, 1, "archive and raw evidence merge into one phone attendee") + assert.Equal("Phone Example", phones[0].Name) + assert.NotNil(phones[0].ParticipantID) +} diff --git a/internal/meetingarchive/links.go b/internal/meetingarchive/links.go new file mode 100644 index 000000000..d310b5c8e --- /dev/null +++ b/internal/meetingarchive/links.go @@ -0,0 +1,298 @@ +package meetingarchive + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "slices" + "strconv" + "strings" + "time" + + "go.kenn.io/msgvault/internal/store" +) + +// evidenceStableProviderID names identity-match evidence that two participants +// carry the same stable provider anchor. +const evidenceStableProviderID = "stable_provider_id" + +// Anchor builds a stable, provider-scoped identifier for one human. The parts +// are length-prefixed before hashing, so ("a:b","c") and ("a","b:c") differ, +// and hashing keeps device-local identifiers such as an Apple Contacts ID out +// of the archive. The namespace prefix stays readable: identities asserted by +// two different anchors of one namespace are treated as contradictory. +func Anchor(namespace string, parts ...string) string { + hash := sha256.New() + for _, part := range parts { + _, _ = hash.Write([]byte(strconv.Itoa(len(part)) + ":" + part)) + } + return namespace + ":" + hex.EncodeToString(hash.Sum(nil)) +} + +func anchorNamespace(anchor string) string { + namespace, _, _ := strings.Cut(anchor, ":") + return namespace +} + +// LinkResult counts what one LinkIdentities call did. +type LinkResult struct { + // Linked counts identity pairs linked or resumed by this call. + Linked int + // Conflicts counts people or pairs left for user review instead of being + // linked: two different curated persons, or an address that another + // anchor of the same provider already claims. + Conflicts int + // Settled counts anchored people that needed no writes. + Settled int +} + +// LinkIdentities records every anchored person's emails and phones as +// observations of the person's stable anchor and links the participants that +// share it. This is msgvault's stable-provider-ID policy (see +// internal/beeper/matching.go): only a shared stable ID links automatically, +// a link that would join two different persons becomes a conflict, and a user +// rejection is kept. Names never match anything. +func (a *Archiver) LinkIdentities(ctx context.Context, sourceID int64, people []Person) (LinkResult, error) { + var result LinkResult + if a == nil || a.store == nil { + return result, ErrUnavailable + } + for _, raw := range people { + if err := ctx.Err(); err != nil { + return result, err + } + person := raw.Normalized() + identities := person.identities() + if person.Anchor == "" || len(identities) == 0 { + continue + } + if err := a.linkPerson(ctx, sourceID, person, identities, &result); err != nil { + return result, err + } + } + return result, nil +} + +func (a *Archiver) linkPerson( + ctx context.Context, sourceID int64, person Person, identities []identity, result *LinkResult, +) error { + anchor := person.Anchor + // The archive owner's own contact card or record can list a shared number + // or inbox. Linking it would put the owner into someone else's person. + for _, id := range identities { + owner, err := a.store.IsAccountIdentityAddressContext(ctx, id.value) + if err != nil { + return err + } + if owner { + return nil + } + } + identities = slices.DeleteFunc(identities, func(id identity) bool { + return slices.Contains(person.LinkExcludedAddresses, id.value) + }) + if len(identities) == 0 { + return nil + } + settled, err := a.store.StableAnchorSettledContext(ctx, sourceID, anchor, anchorIdentities(identities)) + if err != nil { + return err + } + if settled { + result.Settled++ + return nil + } + + participantIDs := make([]int64, len(identities)) + for i, id := range identities { + participantID, err := a.ensureIdentityParticipant(ctx, person.Name, id) + if err != nil { + return err + } + participantIDs[i] = participantID + } + + contradicted, err := a.contradictedParticipants(ctx, sourceID, anchor, identities, participantIDs) + if err != nil { + return err + } + if len(contradicted) > 0 { + return a.recordContradiction(ctx, sourceID, anchor, contradicted, participantIDs, result) + } + + sourceRef := "meeting-person:" + anchor + observedAt := time.Now().UTC() + for i, id := range identities { + kind := store.ContactAddressEmail + if id.kind == identityPhone { + kind = store.ContactAddressPhone + } + if _, err := a.store.RecordContactObservationContext(ctx, participantIDs[i], + store.ParticipantContactObservationInput{ + SourceID: &sourceID, AddressKind: kind, ProviderUserID: &anchor, + OriginalValue: id.value, ObservedAt: &observedAt, + Envelope: store.ValueEnvelopeInput{ + Source: store.ProvenanceArchiveObservation, SourceRef: &sourceRef, + }, + }); err != nil { + return fmt.Errorf("record meeting attendee observation: %w", err) + } + } + + observations, err := a.store.FindObservationsByProviderUserIDContext(ctx, anchor, 0) + if err != nil { + return err + } + primary := participantIDs[0] + seen := map[int64]bool{primary: true} + for _, observation := range observations { + other := observation.ParticipantID + if seen[other] || slices.Contains(person.LinkExcludedAddresses, observation.NormalizedValue) { + continue + } + seen[other] = true + if err := a.linkPair(ctx, sourceID, anchor, sourceRef, primary, other, result); err != nil { + return err + } + } + return nil +} + +func (a *Archiver) ensureIdentityParticipant(ctx context.Context, name string, id identity) (int64, error) { + if id.kind == identityPhone { + participantID, err := a.store.EnsurePhoneParticipantContext(ctx, id.value, name) + if err != nil { + return 0, fmt.Errorf("ensure attendee phone participant: %w", err) + } + return participantID, nil + } + participantID, err := a.store.EnsureParticipantContext(ctx, id.value, name, emailDomain(id.value)) + if err != nil { + return 0, fmt.Errorf("ensure attendee email participant: %w", err) + } + return participantID, nil +} + +// contradictedParticipants returns the participants whose address another +// anchor of the same namespace already claims in the same source. Two +// Contacts cards, or two import ids, sharing one household phone must not +// expand each other's person automatically. Another source's anchor is +// independent evidence, like another provider's. +func (a *Archiver) contradictedParticipants( + ctx context.Context, sourceID int64, anchor string, identities []identity, participantIDs []int64, +) ([]int64, error) { + namespace := anchorNamespace(anchor) + var contradicted []int64 + for i, id := range identities { + observations, err := a.store.ListParticipantObservationsContext(ctx, participantIDs[i], true) + if err != nil { + return nil, err + } + for _, observation := range observations { + if observation.ProviderUserID == nil || *observation.ProviderUserID == anchor || + observation.SourceID == nil || *observation.SourceID != sourceID || + anchorNamespace(*observation.ProviderUserID) != namespace || + observation.NormalizedValue != id.value { + continue + } + contradicted = append(contradicted, participantIDs[i]) + break + } + } + return contradicted, nil +} + +func (a *Archiver) recordContradiction( + ctx context.Context, sourceID int64, anchor string, + contradicted, participantIDs []int64, result *LinkResult, +) error { + sourceRef := "meeting-person:" + anchor + for _, conflicted := range contradicted { + for _, other := range participantIDs { + if other == conflicted { + continue + } + if _, _, err := a.store.UpsertIdentityMatchCandidateContext(ctx, store.IdentityMatchCandidateInput{ + LeftKind: store.IdentityMatchParticipant, LeftID: conflicted, + RightKind: store.IdentityMatchParticipant, RightID: other, + Basis: store.IdentityMatchStableProviderID, NormalizedValue: &anchor, + State: store.IdentityMatchStateConflict, Source: store.ProvenanceArchiveObservation, + SourceRef: &sourceRef, SourceID: &sourceID, + }); err != nil && !errors.Is(err, store.ErrIdentityMatchSelfLink) { + return fmt.Errorf("record contradictory attendee identity: %w", err) + } + } + } + result.Conflicts++ + return nil +} + +func (a *Archiver) linkPair( + ctx context.Context, sourceID int64, anchor, sourceRef string, left, right int64, result *LinkResult, +) error { + candidate, _, err := a.store.UpsertIdentityMatchCandidateContext(ctx, store.IdentityMatchCandidateInput{ + LeftKind: store.IdentityMatchParticipant, LeftID: left, + RightKind: store.IdentityMatchParticipant, RightID: right, + Basis: store.IdentityMatchStableProviderID, NormalizedValue: &anchor, + State: store.IdentityMatchStateCandidate, Source: store.ProvenanceArchiveObservation, + SourceRef: &sourceRef, SourceID: &sourceID, + }) + if err != nil { + return fmt.Errorf("record attendee identity match: %w", err) + } + if candidate.State == store.IdentityMatchStateRejected { + return nil + } + if candidate.State == store.IdentityMatchStateConflict { + result.Conflicts++ + return nil + } + if err := a.store.AttachIdentityMatchCandidateSourceContext(ctx, candidate.ID, sourceID); err != nil { + return err + } + if _, err := a.store.AddIdentityMatchEvidenceContext(ctx, candidate.ID, store.IdentityMatchEvidenceInput{ + EvidenceKind: evidenceStableProviderID, Detail: &anchor, + Source: store.ProvenanceArchiveObservation, SourceID: &sourceID, + }); err != nil { + return err + } + + linked := true + if candidate.State == store.IdentityMatchStateAccepted { + _, _, linked, err = a.store.ResumeAcceptedIdentityMatchCandidateContext(ctx, candidate.ID) + } else { + _, _, err = a.store.AcceptIdentityMatchCandidateContext(ctx, candidate.ID, "system", nil) + } + switch { + case err == nil: + if linked { + result.Linked++ + } + case errors.Is(err, store.ErrPersonBindingConflict): + slog.Warn("meeting attendee identities belong to different persons; left for review", + "candidate_id", candidate.ID) + result.Conflicts++ + case errors.Is(err, store.ErrIdentityMatchRejected), + errors.Is(err, store.ErrIdentityMatchNotAccepted), + errors.Is(err, store.ErrIdentityMatchNotFound): + // A concurrent user decision or participant merge won; it is durable. + default: + return fmt.Errorf("link attendee identities: %w", err) + } + return nil +} + +func anchorIdentities(identities []identity) []store.AnchorIdentity { + out := make([]store.AnchorIdentity, len(identities)) + for i, id := range identities { + kind := store.ContactAddressEmail + if id.kind == identityPhone { + kind = store.ContactAddressPhone + } + out[i] = store.AnchorIdentity{Kind: kind, Value: id.value} + } + return out +} diff --git a/internal/meetingarchive/links_test.go b/internal/meetingarchive/links_test.go new file mode 100644 index 000000000..ad71176d4 --- /dev/null +++ b/internal/meetingarchive/links_test.go @@ -0,0 +1,260 @@ +package meetingarchive + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type linkFixture struct { + st *store.Store + archiver *Archiver + sourceID int64 +} + +func newLinkFixture(t *testing.T) linkFixture { + t.Helper() + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("meeting_import", "links") + require.NoError(t, err) + return linkFixture{st: st, archiver: New(st), sourceID: source.ID} +} + +func (f linkFixture) emailParticipant(t *testing.T, email string) int64 { + t.Helper() + id, err := f.st.EnsureParticipant(email, "", emailDomain(email)) + require.NoError(t, err) + return id +} + +func (f linkFixture) phoneParticipant(t *testing.T, phone string) int64 { + t.Helper() + id, err := f.st.EnsureParticipantByPhone(phone, "", "imessage") + require.NoError(t, err) + return id +} + +func (f linkFixture) person(t *testing.T, participantID int64) int64 { + t.Helper() + person, _, err := f.st.CreatePersonFromParticipantContext(t.Context(), participantID) + require.NoError(t, err) + return person.ID +} + +func (f linkFixture) linked(t *testing.T, a, b int64) bool { + t.Helper() + members, err := f.st.ClusterMembers(a) + require.NoError(t, err) + return slices.Contains(members, b) +} + +func (f linkFixture) boundPerson(t *testing.T, participantID int64) int64 { + t.Helper() + var personID int64 + err := f.st.DB().QueryRow(f.st.Rebind( + `SELECT person_id FROM person_participants WHERE participant_id = ?`), participantID).Scan(&personID) + if err != nil { + return 0 + } + return personID +} + +func (f linkFixture) candidates(t *testing.T) []store.IdentityMatchCandidate { + t.Helper() + candidates, err := f.st.ListIdentityMatchCandidatesContext(t.Context(), nil, 500, 0) + require.NoError(t, err) + return candidates +} + +func (f linkFixture) link(t *testing.T, people ...Person) LinkResult { + t.Helper() + result, err := f.archiver.LinkIdentities(t.Context(), f.sourceID, people) + require.NoError(t, err) + return result +} + +func TestAnchorIsStableAndLengthPrefixed(t *testing.T) { + assert := assert.New(t) + + first := Anchor("meeting-import", "a:b", "c") + assert.Equal(first, Anchor("meeting-import", "a:b", "c")) + assert.NotEqual(first, Anchor("meeting-import", "a", "b:c")) + assert.NotEqual(first, Anchor("notion-user", "a:b", "c")) + assert.Regexp(`^meeting-import:[0-9a-f]{64}$`, first) + assert.NotContains(first, "a:b") +} + +func TestLinkIdentitiesJoinsPhoneToExistingPerson(t *testing.T) { + assert := assert.New(t) + f := newLinkFixture(t) + email := f.emailParticipant(t, "alex@example.com") + personID := f.person(t, email) + + result := f.link(t, Person{ + Name: "Alex Example", Email: "alex@example.com", Phone: "+16045550100", + OtherEmails: []string{"alex.work@example.com"}, + Anchor: Anchor("apple-contact", "card-1"), + }) + + phone := f.phoneParticipant(t, "+16045550100") + work := f.emailParticipant(t, "alex.work@example.com") + assert.Equal(2, result.Linked) + assert.True(f.linked(t, email, phone)) + assert.True(f.linked(t, email, work)) + assert.Equal(personID, f.boundPerson(t, phone), "the phone joins the person automatically") + assert.Equal(personID, f.boundPerson(t, work)) +} + +func TestLinkIdentitiesLinksSightingsAcrossMeetings(t *testing.T) { + f := newLinkFixture(t) + anchor := Anchor("meeting-import", "crm", "42") + + f.link(t, Person{Email: "sam@example.com", Anchor: anchor}) + f.link(t, Person{Phone: "+16045550101", Anchor: anchor}) + + assert.True(t, f.linked(t, + f.emailParticipant(t, "sam@example.com"), f.phoneParticipant(t, "+16045550101"))) +} + +func TestLinkIdentitiesNeverJoinsTwoPeople(t *testing.T) { + assert := assert.New(t) + f := newLinkFixture(t) + email := f.emailParticipant(t, "one@example.com") + phone := f.phoneParticipant(t, "+16045550102") + first := f.person(t, email) + second := f.person(t, phone) + + result := f.link(t, Person{ + Email: "one@example.com", Phone: "+16045550102", Anchor: Anchor("notion-user", "u1"), + }) + + assert.Equal(1, result.Conflicts) + assert.False(f.linked(t, email, phone)) + assert.Equal(first, f.boundPerson(t, email)) + assert.Equal(second, f.boundPerson(t, phone)) + candidates := f.candidates(t) + require.Len(t, candidates, 1) + assert.Equal(store.IdentityMatchStateConflict, candidates[0].State) +} + +func TestLinkIdentitiesSendsReusedAddressToReview(t *testing.T) { + assert := assert.New(t) + f := newLinkFixture(t) + f.link(t, Person{Email: "desk@example.com", Phone: "+16045550103", Anchor: Anchor("apple-contact", "card-a")}) + desk := f.emailParticipant(t, "desk@example.com") + personID := f.person(t, desk) + + result := f.link(t, Person{Email: "desk@example.com", Phone: "+16045550104", Anchor: Anchor("apple-contact", "card-b")}) + + other := f.phoneParticipant(t, "+16045550104") + assert.Equal(1, result.Conflicts) + assert.False(f.linked(t, desk, other), "a second card claiming the same email must not expand the person") + assert.Zero(f.boundPerson(t, other)) + assert.Equal(personID, f.boundPerson(t, f.phoneParticipant(t, "+16045550103"))) + var conflict bool + for _, candidate := range f.candidates(t) { + if candidate.State == store.IdentityMatchStateConflict && + slices.Contains([]int64{candidate.LeftID, candidate.RightID}, other) { + conflict = true + } + } + assert.True(conflict, "the contradiction is reviewable") + + // A different provider namespace is independent evidence. + f.link(t, Person{Email: "desk@example.com", Phone: "+16045550105", Anchor: Anchor("notion-user", "u9")}) + assert.True(f.linked(t, desk, f.phoneParticipant(t, "+16045550105"))) +} + +func TestLinkIdentitiesRespectsUserRejection(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newLinkFixture(t) + anchor := Anchor("meeting-import", "crm", "7") + person := Person{Email: "kim@example.com", Phone: "+16045550106", Anchor: anchor} + f.link(t, person) + candidates := f.candidates(t) + require.Len(candidates, 1) + _, err := f.st.DecideIdentityMatchCandidateContext( + t.Context(), candidates[0].ID, store.IdentityMatchStateRejected, "user", nil) + require.NoError(err) + + f.link(t, person) + + email := f.emailParticipant(t, "kim@example.com") + phone := f.phoneParticipant(t, "+16045550106") + assert.False(f.linked(t, email, phone)) + current, err := f.st.GetIdentityMatchCandidateContext(t.Context(), candidates[0].ID) + require.NoError(err) + assert.Equal(store.IdentityMatchStateRejected, current.State) + + // Rejection is candidate-scoped: a later sighting led by a third identity + // of the same anchor pairs it with both, which connects them through it. + f.link(t, Person{Email: "kim.home@example.com", Anchor: anchor}) + assert.True(f.linked(t, email, phone)) +} + +func TestUpsertLinksOnUnchangedSnapshotsWithoutRewriting(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newLinkFixture(t) + snapshot := testSnapshot(f.sourceID) + snapshot.Attendees = []Person{{Name: "Pat Example", Email: "pat@example.com", Phone: "+16045550107"}} + + first, err := f.archiver.Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + email := f.emailParticipant(t, "pat@example.com") + assert.False(f.linked(t, email, f.phoneParticipant(t, "+16045550107")), "no anchor, no link") + + // The same raw snapshot now carries a provider anchor, as after an + // upgrade or a failed linking attempt. + snapshot.Attendees[0].Anchor = Anchor("meeting-import", "crm", "9") + second, err := f.archiver.Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + assert.False(second.Changed) + assert.Equal(first.MessageID, second.MessageID) + assert.Equal(1, second.Links.Linked) + assert.True(f.linked(t, email, f.phoneParticipant(t, "+16045550107"))) + + before := len(f.candidates(t)) + third, err := f.archiver.Upsert(t.Context(), snapshot, UpsertOptions{}) + require.NoError(err) + assert.Equal(LinkResult{Settled: 1}, third.Links, "a settled person takes the read-only fast path") + assert.Len(f.candidates(t), before) +} + +func TestLinkIdentitiesNeverLinksTheAccountOwner(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newLinkFixture(t) + require.NoError(f.st.AddAccountIdentity(f.sourceID, "owner@example.com", "account-email")) + household := f.phoneParticipant(t, "+16045550110") + spouse := f.person(t, household) + + f.link(t, Person{Email: "owner@example.com", Phone: "+16045550110", Anchor: Anchor("apple-contact", "my-card")}) + + owner := f.emailParticipant(t, "owner@example.com") + assert.False(f.linked(t, owner, household), "the owner's own card must not tie them to a shared number") + assert.Zero(f.boundPerson(t, owner)) + assert.Equal(spouse, f.boundPerson(t, household)) +} + +func TestLinkIdentitiesAcrossSourcesIsNotAContradiction(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newLinkFixture(t) + f.link(t, Person{Email: "lee@example.com", Phone: "+16045550111", Anchor: Anchor("meeting-import", "crm-a", "1")}) + other, err := f.st.GetOrCreateSource("meeting_import", "crm-b") + require.NoError(err) + + result, err := f.archiver.LinkIdentities(t.Context(), other.ID, []Person{ + {Email: "lee@example.com", Phone: "+16045550112", Anchor: Anchor("meeting-import", "crm-b", "7")}, + }) + require.NoError(err) + + assert.Equal(0, result.Conflicts, "another source's anchor is independent evidence") + assert.True(f.linked(t, f.emailParticipant(t, "lee@example.com"), f.phoneParticipant(t, "+16045550112"))) +} diff --git a/internal/meetingcontent/decode.go b/internal/meetingcontent/decode.go index c37854111..70b3660b2 100644 --- a/internal/meetingcontent/decode.go +++ b/internal/meetingcontent/decode.go @@ -33,6 +33,8 @@ func Decode(rawFormat string, raw, _ []byte) Content { return decodeCircleback(fields) case "notion_meeting_json": return decodeNotion(fields) + case "muesli_json": + return decodeMuesli(fields) case "meeting_json": return decodeGeneric(fields) default: @@ -129,20 +131,19 @@ func providerParticipants(fields map[string]jsontext.Value) []Participant { type personWire struct { Name string `json:"name"` Email string `json:"email"` + Phone string `json:"phone"` } func participantsFromPeople(organizer *personWire, attendees []personWire) []Participant { participants := make([]Participant, 0, len(attendees)+1) if organizer != nil { - name, email := strings.TrimSpace(organizer.Name), normalizeExplicitEmail(organizer.Email) - if name != "" || email != "" { - participants = append(participants, Participant{Name: name, Email: email, Role: "from"}) + if participant, ok := participantFromPerson(*organizer, "from"); ok { + participants = append(participants, participant) } } for _, attendee := range attendees { - name, email := strings.TrimSpace(attendee.Name), normalizeExplicitEmail(attendee.Email) - if name != "" || email != "" { - participants = append(participants, Participant{Name: name, Email: email, Role: "to"}) + if participant, ok := participantFromPerson(attendee, "to"); ok { + participants = append(participants, participant) } } return participants @@ -305,3 +306,11 @@ func isNull(raw jsontext.Value) bool { func finite(value float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) } + +func participantFromPerson(person personWire, role string) (Participant, bool) { + participant := Participant{ + Name: strings.TrimSpace(person.Name), Email: normalizeExplicitEmail(person.Email), + Phone: strings.TrimSpace(person.Phone), Role: role, + } + return participant, participant.Name != "" || participant.Email != "" || participant.Phone != "" +} diff --git a/internal/meetingcontent/decode_test.go b/internal/meetingcontent/decode_test.go index aad819fa0..8e57d06e8 100644 --- a/internal/meetingcontent/decode_test.go +++ b/internal/meetingcontent/decode_test.go @@ -299,3 +299,93 @@ func mustTime(t *testing.T, value string) *time.Time { require.NoError(t, err) return &parsed } + +func TestDecodeMuesliEvidence(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + raw := []byte(`{ + "schema_version":1, + "meeting":{ + "id":42,"title":"Weekly sync","start_time":"2026-09-01T14:00:00Z","end_time":"2026-09-01T14:30:00Z", + "duration_seconds":2700,"status":"completed", + "formatted_notes":"## Decisions\nShip it","notes_state":"structured_notes", + "manual_notes":"typed note","raw_transcript":"[10:00:01] You: hello\n[10:00:04] Speaker 1: hi", + "created_at":"2026-09-01T14:00:03Z" + }, + "participants":[ + {"name":"Carol Example","source":"manual"}, + {"name":"Alice Example","email":"Alice@Example.com","source":"calendar"} + ] + }`) + + content := Decode("muesli_json", raw, nil) + + assertions.Equal(Section{State: StateAvailable, Text: "## Decisions\nShip it"}, content.Summary) + assertions.Equal(Section{State: StateAvailable, Text: "typed note"}, content.Notes) + assertions.Equal(Transcript{State: StateAvailable, Text: "[10:00:01] You: hello\n[10:00:04] Speaker 1: hi"}, content.Transcript) + assertions.Equal(CoverageUnsupported, content.ActionCoverage) + assertions.Equal("no_structured_actions", content.ActionReason) + assertions.Empty(content.Actions) + requirements.NotNil(content.DurationSeconds) + assertions.InDelta(float64(2700), *content.DurationSeconds, 0) + assertions.Equal(DurationProvider, content.DurationBasis) + assertions.Equal([]Participant{ + {Name: "Carol Example", Role: "to"}, + {Name: "Alice Example", Email: "alice@example.com", Role: "to"}, + }, content.SourceParticipants) +} + +func TestDecodeMuesliFallbackNotesAndTimes(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + + fallback := Decode("muesli_json", []byte(`{"schema_version":1,"meeting":{ + "id":1,"start_time":"2026-09-01T14:00:00Z","end_time":"2026-09-01T14:10:00Z", + "formatted_notes":"## Raw Transcript\n\nhello","notes_state":"raw_transcript_fallback", + "raw_transcript":" ","created_at":"2026-09-01T14:00:03Z"}}`), nil) + assertions.Equal(StateEmpty, fallback.Summary.State) + assertions.Equal(StateEmpty, fallback.Notes.State) + assertions.Equal(StateEmpty, fallback.Transcript.State) + requirements.NotNil(fallback.DurationSeconds) + assertions.InDelta(float64(600), *fallback.DurationSeconds, 0) + assertions.Equal(DurationProvider, fallback.DurationBasis) + assertions.Empty(fallback.SourceParticipants) + + failed := Decode("muesli_json", []byte(`{"schema_version":1,"meeting":{ + "id":2,"start_time":"2026-09-01T14:00:00Z","formatted_notes":"## Summary failed\n\ntimeout", + "notes_state":"summary_failed","raw_transcript":"x","created_at":"2026-09-01T14:00:03Z"}}`), nil) + assertions.Equal(StateEmpty, failed.Summary.State) + assertions.Nil(failed.DurationSeconds) + + missing := Decode("muesli_json", []byte(`{"schema_version":1}`), nil) + assertions.Equal(StateUnavailable, missing.Summary.State) + assertions.Equal(StateUnavailable, missing.Transcript.State) + assertions.Equal(CoverageUnavailable, missing.ActionCoverage) +} + +func TestDecodeCarriesAttendeePhones(t *testing.T) { + generic := Decode("meeting_json", []byte(`{"summary_markdown":"x", + "organizer":{"name":"Owner","phone":"+16045550199"}, + "attendees":[{"name":"Pat Example","phone":"+16045550100"},{"email":"sam@example.com"}]}`), nil) + assert.Equal(t, []Participant{ + {Name: "Owner", Phone: "+16045550199", Role: "from"}, + {Name: "Pat Example", Phone: "+16045550100", Role: "to"}, + {Email: "sam@example.com", Role: "to"}, + }, generic.SourceParticipants) + + muesli := Decode("muesli_json", []byte(`{"schema_version":1,"meeting":{"raw_transcript":"x"}, + "participants":[{"name":"Pat Example","phone":"+16045550100","phones":["+16045550100"]}]}`), nil) + assert.Equal(t, []Participant{{Name: "Pat Example", Phone: "+16045550100", Role: "to"}}, muesli.SourceParticipants) +} + +func TestDecodeMuesliFallsBackToContactsIdentities(t *testing.T) { + content := Decode("muesli_json", []byte(`{"schema_version":1,"meeting":{"raw_transcript":"x"}, + "participants":[ + {"name":"Alex Example","emails":["alex@example.com"],"phones":["+16045550100"]}, + {"name":"Phone Example","phone":"+16045550101","phones":["+16045550101"]} + ]}`), nil) + assert.Equal(t, []Participant{ + {Name: "Alex Example", Email: "alex@example.com", Role: "to"}, + {Name: "Phone Example", Phone: "+16045550101", Role: "to"}, + }, content.SourceParticipants, "packets merge these with the archived recipients instead of listing them twice") +} diff --git a/internal/meetingcontent/muesli.go b/internal/meetingcontent/muesli.go new file mode 100644 index 000000000..d476493f1 --- /dev/null +++ b/internal/meetingcontent/muesli.go @@ -0,0 +1,74 @@ +package meetingcontent + +import ( + "encoding/json/jsontext" + "encoding/json/v2" + "strings" +) + +// decodeMuesli reads msgvault's muesli_json evidence. Muesli has no +// structured action items, and its transcript is plain text whose line stamps +// are local wall-clock times, so the transcript stays unsegmented. +func decodeMuesli(fields map[string]jsontext.Value) Content { + var meeting map[string]jsontext.Value + raw, ok := fields["meeting"] + if !ok || isNull(raw) || json.Unmarshal(raw, &meeting) != nil || meeting == nil { + return unavailableContent(reasonMissingField) + } + content := baseRecognizedContent() + // Fallback and failure notices are not summaries; the transcript and the + // raw evidence still carry their text. + content.Summary = Section{State: StateEmpty} + if state, _ := optionalString(meeting["notes_state"]); state == "structured_notes" { + content.Summary = optionalSection(meeting, "formatted_notes") + } + content.Notes = optionalSection(meeting, "manual_notes") + content.Transcript = Transcript{State: StateEmpty} + if text, _ := optionalString(meeting["raw_transcript"]); strings.TrimSpace(text) != "" { + content.Transcript = Transcript{State: StateAvailable, Text: strings.TrimSpace(text)} + } + content.ActionCoverage = CoverageUnsupported + content.ActionReason = "no_structured_actions" + + // Archived recipients use the Contacts email (or phone) when Muesli has + // none, so the source participants must too, or packets list them twice. + var participants []struct { + personWire + + Emails []string `json:"emails"` + Phones []string `json:"phones"` + } + _ = json.Unmarshal(fields["participants"], &participants) + people := make([]personWire, 0, len(participants)) + for _, participant := range participants { + person := participant.personWire + if strings.TrimSpace(person.Email) == "" && len(participant.Emails) > 0 { + person.Email = participant.Emails[0] + } + if strings.TrimSpace(person.Email) == "" && strings.TrimSpace(person.Phone) == "" && len(participant.Phones) > 0 { + person.Phone = participant.Phones[0] + } + people = append(people, person) + } + content.SourceParticipants = participantsFromPeople(nil, people) + + if seconds, ok := rawPositiveFloat(meeting["duration_seconds"]); ok { + setDuration(&content, seconds, DurationProvider) + } else { + start, startOK := rawTime(meeting["start_time"], false) + end, endOK := rawTime(meeting["end_time"], false) + if startOK && endOK && end.After(start) { + setDuration(&content, end.Sub(start).Seconds(), DurationProvider) + } + } + return content +} + +// optionalSection treats an omitted field as empty: muesli_json omits blank +// text fields rather than writing empty strings. +func optionalSection(fields map[string]jsontext.Value, key string) Section { + if _, present := fields[key]; !present { + return Section{State: StateEmpty} + } + return decodeStringField(fields, key) +} diff --git a/internal/meetingcontent/render.go b/internal/meetingcontent/render.go index 1d6062b50..675d034b9 100644 --- a/internal/meetingcontent/render.go +++ b/internal/meetingcontent/render.go @@ -119,6 +119,7 @@ func normalizeEntry(entry Entry) Entry { for index := range participants { participants[index].Name = strings.TrimSpace(participants[index].Name) participants[index].Email = strings.TrimSpace(participants[index].Email) + participants[index].Phone = strings.TrimSpace(participants[index].Phone) participants[index].Role = strings.TrimSpace(participants[index].Role) } sort.SliceStable(participants, func(left, right int) bool { @@ -146,12 +147,18 @@ func entryLess(left, right Entry) bool { } } +// participantEmailRoleKey identifies an archived or source participant by role +// plus email, else role plus phone. Participants with neither are never +// merged: two name-only attendees can be different people. func participantEmailRoleKey(participant Participant) string { - email := strings.ToLower(strings.TrimSpace(participant.Email)) - if email == "" { - return "" + role := strings.TrimSpace(participant.Role) + if email := strings.ToLower(strings.TrimSpace(participant.Email)); email != "" { + return role + "\x00email\x00" + email } - return strings.TrimSpace(participant.Role) + "\x00" + email + if phone := strings.TrimSpace(participant.Phone); phone != "" { + return role + "\x00phone\x00" + phone + } + return "" } func participantLess(left, right Participant) bool { @@ -163,6 +170,9 @@ func participantLess(left, right Participant) bool { if leftEmail != rightEmail { return leftEmail < rightEmail } + if left.Phone != right.Phone { + return left.Phone < right.Phone + } if left.Name != right.Name { return left.Name < right.Name } @@ -540,9 +550,9 @@ func writeMarkdownParticipants(builder *strings.Builder, participants []Particip builder.WriteString(": ") builder.WriteString(markdownSingleLine(participant.Name)) } - if participant.Email != "" { + if address := participantAddress(participant); address != "" { builder.WriteString(" <") - builder.WriteString(participant.Email) + builder.WriteString(address) builder.WriteByte('>') } if participant.ParticipantID != nil { @@ -689,3 +699,10 @@ func cloneTranscript(transcript Transcript) Transcript { clone.Segments = append([]Segment(nil), transcript.Segments...) return clone } + +func participantAddress(participant Participant) string { + if participant.Email != "" { + return participant.Email + } + return participant.Phone +} diff --git a/internal/meetingcontent/render_test.go b/internal/meetingcontent/render_test.go index e29df08bc..07384fd92 100644 --- a/internal/meetingcontent/render_test.go +++ b/internal/meetingcontent/render_test.go @@ -315,3 +315,37 @@ func mustRenderTime(t *testing.T, value string) *time.Time { func jsonNumber(value int64) string { return strconv.FormatInt(value, 10) } + +func TestRenderMergesParticipantsByPhone(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + entry := Entry{ + Meeting: MeetingRef{MessageID: 4, Title: "Phone", ArchivePath: "/api/v1/messages/4"}, + Participants: []Participant{ + {ParticipantID: new(int64(5)), Phone: "+16045550100", Role: "to"}, + }, + Content: Content{ + Summary: Section{State: StateAvailable, Text: "x"}, Notes: Section{State: StateEmpty}, + Transcript: Transcript{State: StateEmpty}, Actions: []Action{}, ActionCoverage: CoverageUnsupported, + SourceParticipants: []Participant{ + {Name: "Pat Example", Phone: "+16045550100", Role: "to"}, + {Name: "First Guest", Role: "to"}, + {Name: "Second Guest", Role: "to"}, + }, + }, + } + + result, err := Render("archive", []Entry{entry}, PacketOptions{Format: FormatJSON, MaxBytes: 8192}) + requirements.NoError(err) + var packet Packet + requirements.NoError(json.Unmarshal([]byte(result.Content), &packet)) + assertions.Equal([]Participant{ + {Name: "First Guest", Role: "to"}, + {Name: "Second Guest", Role: "to"}, + {ParticipantID: new(int64(5)), Name: "Pat Example", Phone: "+16045550100", Role: "to"}, + }, packet.Meetings[0].Participants) + + markdown, err := Render("archive", []Entry{entry}, PacketOptions{Format: FormatMarkdown, MaxBytes: 8192}) + requirements.NoError(err) + assertions.Contains(markdown.Content, "- to: Pat Example <+16045550100> (participant 5)") +} diff --git a/internal/meetingcontent/types.go b/internal/meetingcontent/types.go index 7021d966e..16a3160a8 100644 --- a/internal/meetingcontent/types.go +++ b/internal/meetingcontent/types.go @@ -102,6 +102,7 @@ type Participant struct { ParticipantID *int64 `json:"participant_id,omitempty"` Name string `json:"name,omitempty"` Email string `json:"email,omitempty"` + Phone string `json:"phone,omitempty"` Role string `json:"role"` } diff --git a/internal/meetingimport/identities_test.go b/internal/meetingimport/identities_test.go new file mode 100644 index 000000000..12c305dfd --- /dev/null +++ b/internal/meetingimport/identities_test.go @@ -0,0 +1,142 @@ +package meetingimport + +import ( + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/testutil" +) + +// emailOnlyCanonicalGolden is the canonical raw meeting that the email-only +// fixture produced before attendees gained phones and ids. Existing archives +// must keep byte-identical snapshots so re-imports stay unchanged. +const emailOnlyCanonicalGolden = `{"external_id":"42","title":"Weekly planning","started_at":"2026-07-23T18:00:00Z","ended_at":"2026-07-23T18:30:00Z","summary_markdown":"## Summary\n\nReviewed the launch plan.","transcript_segments":[{"speaker":"Test Speaker","text":"Let's review the launch plan.","offset_seconds":4}],"organizer":{"name":"Test Organizer","email":"organizer@example.com"},"attendees":[{"name":"Test Attendee","email":"attendee@example.com"}],"metadata":{"calendar_event_id":"synthetic-event-42","nested":{"accepted":true}}}` + +func TestEmailOnlyCanonicalRawIsUnchanged(t *testing.T) { + normalized, err := decodedValidRequest(t).Normalize() + require.NoError(t, err) + snapshot, err := BuildSnapshot(normalized) + require.NoError(t, err) + + //nolint:testifylint // byte-for-byte equality is the compatibility contract; JSONEq would hide changes + assert.Equal(t, emailOnlyCanonicalGolden, string(snapshot.Raw)) +} + +func requestWithAttendees(t *testing.T, attendees string) Request { + t.Helper() + body := strings.Replace(validRequestJSON, + `"attendees": [`, `"attendees": [`+attendees+`, `, 1) + req, err := DecodeRequest(strings.NewReader(body), MaxRequestBytes) + require.NoError(t, err) + return req +} + +func TestNormalizeAttendeePhonesAndIDs(t *testing.T) { + for _, tt := range []struct { + name string + attendee string + wantPhone string + wantID string + wantErr string + }{ + {name: "formatted E.164", attendee: `{"phone":"+1 (604) 555-0100"}`, wantPhone: "+16045550100"}, + {name: "00 prefix", attendee: `{"phone":"0044 20 7946 0000","id":" crm-7 "}`, wantPhone: "+442079460000", wantID: "crm-7"}, + {name: "national number", attendee: `{"phone":"(604) 555-0100"}`, wantErr: "meeting.attendees[0].phone"}, + {name: "trunk marker", attendee: `{"phone":"+44 (0)20 7946 0000"}`, wantPhone: "+442079460000"}, + {name: "plus and double zero", attendee: `{"phone":"+0044 20 7946 0000"}`, wantErr: "meeting.attendees[0].phone"}, + {name: "letters", attendee: `{"phone":"+1 604 CALL NOW"}`, wantErr: "meeting.attendees[0].phone"}, + {name: "no identity", attendee: `{"name":"Name Only"}`, wantErr: "meeting.attendees[0] requires an email or phone"}, + {name: "control character id", attendee: `{"email":"x@example.com","id":"a\u0007b"}`, wantErr: "meeting.attendees[0].id"}, + {name: "long id", attendee: `{"email":"x@example.com","id":"` + strings.Repeat("a", 201) + `"}`, wantErr: "meeting.attendees[0].id"}, + } { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + normalized, err := requestWithAttendees(t, tt.attendee).Normalize() + if tt.wantErr != "" { + require.ErrorIs(err, ErrValidation) + assert.Contains(err.Error(), tt.wantErr) + assert.NotContains(err.Error(), "555", "validation errors never echo phone numbers") + return + } + require.NoError(err) + attendee := normalized.Meeting.Attendees[0] + assert.Equal(tt.wantPhone, attendee.Phone) + assert.Equal(tt.wantID, attendee.ID) + }) + } +} + +func TestNormalizeAttendeesCollapsesOnlyExactRepeats(t *testing.T) { + normalized, err := requestWithAttendees(t, + `{"email":"Attendee@example.com","name":"Again"}, {"email":"attendee@example.com","phone":"+16045550100","id":"crm-1"}`, + ).Normalize() + require.NoError(t, err) + + emails := make([]string, 0, len(normalized.Meeting.Attendees)) + for _, attendee := range normalized.Meeting.Attendees { + emails = append(emails, attendee.Email+"|"+attendee.Phone+"|"+attendee.ID) + } + assert.Equal(t, []string{ + "attendee@example.com||", + "attendee@example.com|+16045550100|crm-1", + }, emails, "an email-only repeat collapses as before; a different identity set stays its own assertion") +} + +func TestImporterLinksAttendeeIdentitiesThroughID(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + email, err := st.EnsureParticipant("pat@example.com", "", "example.com") + require.NoError(err) + person, _, err := st.CreatePersonFromParticipantContext(t.Context(), email) + require.NoError(err) + + _, err = NewImporter(st, Hooks{}).Import(t.Context(), requestWithAttendees(t, + `{"name":"Pat Example","email":"pat@example.com","phone":"+1 604 555 0100","id":"crm-9"}`)) + require.NoError(err) + + phone, err := st.EnsureParticipantByPhone("+16045550100", "", "imessage") + require.NoError(err) + members, err := st.ClusterMembers(email) + require.NoError(err) + assert.True(slices.Contains(members, phone)) + var bound int64 + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT person_id FROM person_participants WHERE participant_id = ?`), phone).Scan(&bound)) + assert.Equal(person.ID, bound) +} + +func TestImporterArchivesPhoneOnlyAttendee(t *testing.T) { + require := require.New(t) + st := testutil.NewTestStore(t) + result, err := NewImporter(st, Hooks{}).Import(t.Context(), requestWithAttendees(t, + `{"name":"Phone Example","phone":"+16045550101"}`)) + require.NoError(err) + + var count int + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT count(*) FROM message_recipients mr JOIN participants p ON p.id = mr.participant_id + WHERE mr.message_id = ? AND p.phone_number = ?`), result.MessageID, "+16045550101").Scan(&count)) + assert.Equal(t, 1, count) +} + +func TestImporterSendsSharedPhoneAcrossIDsToReview(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + _, err := NewImporter(st, Hooks{}).Import(t.Context(), requestWithAttendees(t, + `{"email":"one@example.com","phone":"+16045550102","id":"crm-1"}, {"email":"two@example.com","phone":"+16045550102","id":"crm-2"}`)) + require.NoError(err) + + one, err := st.EnsureParticipant("one@example.com", "", "example.com") + require.NoError(err) + two, err := st.EnsureParticipant("two@example.com", "", "example.com") + require.NoError(err) + members, err := st.ClusterMembers(one) + require.NoError(err) + assert.False(slices.Contains(members, two), "a household phone must not merge two people") +} diff --git a/internal/meetingimport/importer.go b/internal/meetingimport/importer.go index 2a252922f..d9ed4f2a8 100644 --- a/internal/meetingimport/importer.go +++ b/internal/meetingimport/importer.go @@ -119,17 +119,12 @@ func (i *Importer) Import(ctx context.Context, req Request) (result Result, retE var organizer *meetingarchive.Person if snapshot.Organizer != nil { - organizer = &meetingarchive.Person{ - Name: snapshot.Organizer.Name, - Email: snapshot.Organizer.Email, - } + person := archivePerson(snapshot.SourceIdentifier, *snapshot.Organizer) + organizer = &person } attendees := make([]meetingarchive.Person, 0, len(snapshot.Attendees)) for _, attendee := range snapshot.Attendees { - attendees = append(attendees, meetingarchive.Person{ - Name: attendee.Name, - Email: attendee.Email, - }) + attendees = append(attendees, archivePerson(snapshot.SourceIdentifier, attendee)) } archiveResult, archiveErr := meetingarchive.New(i.store).Upsert(ctx, meetingarchive.Snapshot{ SourceID: source.ID, @@ -200,3 +195,13 @@ func (i *Importer) refreshCache(ctx context.Context, label string, sourceID int6 ) } } + +// archivePerson maps an imported person to the archive. The anchor is scoped +// to the import source, so the same id from two sources names two people. +func archivePerson(sourceIdentifier string, person MeetingPerson) meetingarchive.Person { + archived := meetingarchive.Person{Name: person.Name, Email: person.Email, Phone: person.Phone} + if person.ID != "" { + archived.Anchor = meetingarchive.Anchor("meeting-import", sourceIdentifier, person.ID) + } + return archived +} diff --git a/internal/meetingimport/models.go b/internal/meetingimport/models.go index 801422cf5..dbf487572 100644 --- a/internal/meetingimport/models.go +++ b/internal/meetingimport/models.go @@ -12,6 +12,7 @@ import ( "net/mail" "strings" "time" + "unicode" "unicode/utf8" "go.kenn.io/msgvault/internal/jsonexact" @@ -27,6 +28,7 @@ const ( maxSourceIdentifierChars = 128 maxSourceDisplayNameChars = 256 maxExternalIDChars = 256 + maxPersonIDChars = 200 maxTitleChars = 4096 ) @@ -101,9 +103,14 @@ type MeetingActionItem struct { DueDate string `json:"due_date,omitempty"` } +// MeetingPerson is one organizer or attendee. At least one of Email or Phone +// is required. ID is a stable identifier for this human in the import source; +// with it, the person's email and phone are linked, including across meetings. type MeetingPerson struct { Name string `json:"name,omitempty"` - Email string `json:"email" format:"email"` + Email string `json:"email,omitempty" format:"email"` + Phone string `json:"phone,omitempty" doc:"International phone number starting with + or 00; normalized to E.164"` + ID string `json:"id,omitempty" doc:"Stable identifier for this person in the import source"` } type TranscriptSegment struct { @@ -310,14 +317,67 @@ func normalizeSegments(segments []TranscriptSegment) ([]TranscriptSegment, error } func normalizePerson(field string, person MeetingPerson) (MeetingPerson, error) { - email, err := normalizeEmail(field+".email", person.Email) - if err != nil { + out := MeetingPerson{Name: strings.TrimSpace(person.Name)} + if strings.TrimSpace(person.Email) != "" { + email, err := normalizeEmail(field+".email", person.Email) + if err != nil { + return MeetingPerson{}, err + } + out.Email = email + } + if strings.TrimSpace(person.Phone) != "" { + phone, err := normalizePhone(field+".phone", person.Phone) + if err != nil { + return MeetingPerson{}, err + } + out.Phone = phone + } + if out.Email == "" && out.Phone == "" { + return MeetingPerson{}, validationError("%s requires an email or phone", field) + } + out.ID = strings.TrimSpace(person.ID) + if err := validateBoundedOptional(field+".id", out.ID, maxPersonIDChars); err != nil { return MeetingPerson{}, err } - return MeetingPerson{ - Name: strings.TrimSpace(person.Name), - Email: email, - }, nil + if strings.IndexFunc(out.ID, unicode.IsControl) >= 0 { + return MeetingPerson{}, validationError("%s.id must not contain control characters", field) + } + return out, nil +} + +// normalizePhone accepts international numbers only (a leading + or 00) and +// returns E.164. National numbers are rejected rather than guessed: a wrong +// country code could attach a meeting to someone else. Errors name the field, +// never the number. +func normalizePhone(field, value string) (string, error) { + // "+44 (0)20 …" marks a trunk zero that is dialed only nationally. + value = strings.ReplaceAll(strings.TrimSpace(value), "(0)", "") + var digits strings.Builder + for i, r := range value { + switch { + case r >= '0' && r <= '9': + digits.WriteRune(r) + case r == '+' && i == 0: + case r == ' ' || r == '-' || r == '.' || r == '(' || r == ')': + default: + return "", validationError("%s must contain only digits and phone formatting", field) + } + } + number := digits.String() + switch { + case strings.HasPrefix(value, "+"): + case strings.HasPrefix(number, "00"): + number = number[2:] + default: + return "", validationError("%s must be an international number starting with + or 00", field) + } + if strings.HasPrefix(number, "0") { + return "", validationError("%s must start with a country code", field) + } + if len(number) < 7 || len(number) > 15 { + return "", validationError("%s must have 7 to 15 digits", field) + } + return "+" + number, nil } func normalizeAttendees(attendees []MeetingPerson) ([]MeetingPerson, error) { @@ -331,7 +391,9 @@ func normalizeAttendees(attendees []MeetingPerson) ([]MeetingPerson, error) { if err != nil { return nil, err } - key := strings.ToLower(person.Email) + // Only exact repeats collapse. Entries with different identity sets are + // separate assertions; merging them could link unrelated people. + key := person.Email + "\x00" + person.Phone + "\x00" + person.ID if _, exists := seen[key]; exists { continue } diff --git a/internal/muesli/contacts.go b/internal/muesli/contacts.go new file mode 100644 index 000000000..ef6be72ce --- /dev/null +++ b/internal/muesli/contacts.go @@ -0,0 +1,363 @@ +package muesli + +import ( + "context" + "database/sql" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" +) + +// ContactsState reports how much of the macOS Contacts directory a sync could +// read. +type ContactsState string + +const ( + // ContactsComplete means every Contacts store opened with the expected + // schema, so an email that matches one card is unambiguous. + ContactsComplete ContactsState = "complete" + // ContactsPartial means some stores could not be read. Identifier lookups + // still work; email lookups are disabled because a skipped store could + // hold another card with the same email. + ContactsPartial ContactsState = "partial" + // ContactsUnavailable means no store could be read, for example without + // Full Disk Access. + ContactsUnavailable ContactsState = "unavailable" + // ContactsOff means enrichment is disabled in the configuration. + ContactsOff ContactsState = "off" +) + +const addressBookFile = "AddressBook-v22.abcddb" + +// ContactCard is one person in Contacts: a card, or every card linked to it +// across accounts. Phones are as typed in Contacts; see NormalizeContactPhone. +type ContactCard struct { + // GroupKey is the card's link ID, or its unique ID when it is not linked. + // It never leaves msgvault's memory unhashed. + GroupKey string + Emails []string + Phones []string +} + +// Contacts is an in-memory, read-only snapshot of the macOS Contacts stores. +type Contacts struct { + state ContactsState + byUniqueID map[string]string + byLinkID map[string]string + byEmail map[string]map[string]bool + cards map[string]*ContactCard +} + +// DisabledContacts is the Contacts snapshot used when enrichment is off. +func DisabledContacts() *Contacts { + return &Contacts{state: ContactsOff} +} + +// OpenContacts reads the Contacts stores under root (the root store and each +// Sources// store) into memory. Stores are opened query-only and +// never modified. Unreadable stores lower the state instead of failing, so a +// sync without Full Disk Access still archives meetings. Only context +// cancellation is returned as an error. +func OpenContacts(ctx context.Context, root string) (*Contacts, error) { + contacts := newContacts() + paths, complete := addressBookStores(root) + read := 0 + for _, path := range paths { + if err := ctx.Err(); err != nil { + return nil, err + } + // Each store is read into its own snapshot and merged only when the + // whole store was read, so a store that fails partway contributes + // no half-populated cards. + store := newContacts() + if err := store.readStore(ctx, path); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + complete = false + continue + } + contacts.merge(store) + read++ + } + switch { + case read == 0: + contacts.state = ContactsUnavailable + case complete: + contacts.state = ContactsComplete + default: + contacts.state = ContactsPartial + } + return contacts, nil +} + +func newContacts() *Contacts { + return &Contacts{ + state: ContactsUnavailable, + byUniqueID: map[string]string{}, + byLinkID: map[string]string{}, + byEmail: map[string]map[string]bool{}, + cards: map[string]*ContactCard{}, + } +} + +// merge adds another store's snapshot. Cards linked across stores share a +// group key and combine their identities. +func (c *Contacts) merge(other *Contacts) { + maps.Copy(c.byUniqueID, other.byUniqueID) + maps.Copy(c.byLinkID, other.byLinkID) + for email, groups := range other.byEmail { + if c.byEmail[email] == nil { + c.byEmail[email] = map[string]bool{} + } + maps.Copy(c.byEmail[email], groups) + } + for key, card := range other.cards { + existing := c.cards[key] + if existing == nil { + c.cards[key] = card + continue + } + for _, email := range card.Emails { + if !slices.Contains(existing.Emails, email) { + existing.Emails = append(existing.Emails, email) + } + } + for _, phone := range card.Phones { + if !slices.Contains(existing.Phones, phone) { + existing.Phones = append(existing.Phones, phone) + } + } + slices.Sort(existing.Emails) + slices.Sort(existing.Phones) + } +} + +// sharedAddresses finds addresses claimed by more than one distinct Contacts +// group. Normalize phones with the import's country code before comparing them. +func (c *Contacts) sharedAddresses(countryCode string) map[string]bool { + owners := map[string]string{} + shared := map[string]bool{} + for key, card := range c.cards { + phones, _ := normalizedPhones(card.Phones, countryCode) + for _, address := range append(slices.Clone(card.Emails), phones...) { + if owner, ok := owners[address]; ok && owner != key { + shared[address] = true + } + owners[address] = key + } + } + return shared +} + +// addressBookStores lists the store files under root. The second result is +// false when a directory that should be listed could not be read. +func addressBookStores(root string) ([]string, bool) { + if root == "" { + return nil, false + } + var paths []string + complete := true + rootStore := filepath.Join(root, addressBookFile) + if _, err := os.Stat(rootStore); err == nil { + paths = append(paths, rootStore) + } else if !errors.Is(err, os.ErrNotExist) { + complete = false + } + entries, err := os.ReadDir(filepath.Join(root, "Sources")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + complete = false + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + store := filepath.Join(root, "Sources", entry.Name(), addressBookFile) + if _, err := os.Stat(store); err == nil { + paths = append(paths, store) + } else if !errors.Is(err, os.ErrNotExist) { + complete = false + } + } + return paths, complete +} + +// State reports how much of the Contacts directory was read. +func (c *Contacts) State() ContactsState { + if c == nil { + return ContactsOff + } + return c.state +} + +// Resolve finds the Contacts person for a Muesli participant: first by the +// stored Contacts identifier (a card's unique ID, with or without its +// ":ABPerson" suffix, or a unified contact's link ID), then by exact email. +// Email lookup needs a complete read and a single matching person. Names are +// never used. +func (c *Contacts) Resolve(contactID, email string) (ContactCard, bool) { + if c == nil || c.cards == nil { + return ContactCard{}, false + } + if contactID = strings.TrimSpace(contactID); contactID != "" { + bare := strings.TrimSuffix(contactID, ":ABPerson") + for _, candidate := range []string{contactID, bare + ":ABPerson"} { + if key, ok := c.byUniqueID[candidate]; ok { + return c.card(key), true + } + } + for _, candidate := range []string{contactID, bare} { + if key, ok := c.byLinkID[candidate]; ok { + return c.card(key), true + } + } + } + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" || c.state != ContactsComplete { + return ContactCard{}, false + } + groups := c.byEmail[email] + if len(groups) != 1 { + return ContactCard{}, false + } + for key := range groups { + return c.card(key), true + } + return ContactCard{}, false +} + +func (c *Contacts) card(key string) ContactCard { + card := c.cards[key] + return ContactCard{ + GroupKey: card.GroupKey, + Emails: slices.Clone(card.Emails), + Phones: slices.Clone(card.Phones), + } +} + +func (c *Contacts) readStore(ctx context.Context, path string) error { + db, err := openQueryOnly(path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + for _, required := range []struct{ table, column string }{ + {"Z_PRIMARYKEY", "Z_ENT"}, {"Z_PRIMARYKEY", "Z_NAME"}, + {"ZABCDRECORD", "Z_PK"}, {"ZABCDRECORD", "Z_ENT"}, {"ZABCDRECORD", "ZUNIQUEID"}, + {"ZABCDEMAILADDRESS", "ZOWNER"}, {"ZABCDEMAILADDRESS", "ZADDRESS"}, + {"ZABCDPHONENUMBER", "ZOWNER"}, {"ZABCDPHONENUMBER", "ZFULLNUMBER"}, + } { + columns, err := tableColumns(ctx, db, required.table) + if err != nil { + return err + } + if !columns[required.column] { + return fmt.Errorf("contacts store %s lacks %s.%s", filepath.Base(filepath.Dir(path)), required.table, required.column) + } + } + recordColumns, err := tableColumns(ctx, db, "ZABCDRECORD") + if err != nil { + return err + } + linkColumn := "NULL" + if recordColumns["ZLINKID"] { + linkColumn = "r.ZLINKID" + } + + tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + owners, err := c.readRecords(ctx, tx, linkColumn) + if err != nil { + return err + } + if err := c.readAddresses(ctx, tx, owners, `SELECT ZOWNER, ZADDRESS FROM ZABCDEMAILADDRESS`, true); err != nil { + return err + } + return c.readAddresses(ctx, tx, owners, `SELECT ZOWNER, ZFULLNUMBER FROM ZABCDPHONENUMBER`, false) +} + +// readRecords loads the store's person cards and returns their group keys by +// record primary key. +func (c *Contacts) readRecords(ctx context.Context, tx *sql.Tx, linkColumn string) (map[int64]string, error) { + rows, err := tx.QueryContext(ctx, `SELECT r.Z_PK, r.ZUNIQUEID, `+linkColumn+` + FROM ZABCDRECORD r + WHERE r.Z_ENT IN (SELECT Z_ENT FROM Z_PRIMARYKEY + WHERE Z_NAME IN ('ABCDContact', 'ABCDSubscribedContact'))`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + owners := map[int64]string{} + for rows.Next() { + var pk int64 + var uniqueID, linkID sql.NullString + if err := rows.Scan(&pk, &uniqueID, &linkID); err != nil { + return nil, err + } + unique := strings.TrimSpace(uniqueID.String) + link := strings.TrimSpace(linkID.String) + key := link + if key == "" { + key = unique + } + if key == "" { + continue + } + owners[pk] = key + if unique != "" { + c.byUniqueID[unique] = key + } + if link != "" { + c.byLinkID[link] = key + } + if c.cards[key] == nil { + c.cards[key] = &ContactCard{GroupKey: key} + } + } + return owners, rows.Err() +} + +func (c *Contacts) readAddresses(ctx context.Context, tx *sql.Tx, owners map[int64]string, query string, email bool) error { + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var owner sql.NullInt64 + var value sql.NullString + if err := rows.Scan(&owner, &value); err != nil { + return err + } + key, ok := owners[owner.Int64] + address := strings.TrimSpace(value.String) + if !ok || address == "" { + continue + } + card := c.cards[key] + if email { + address = strings.ToLower(address) + if !slices.Contains(card.Emails, address) { + card.Emails = append(card.Emails, address) + slices.Sort(card.Emails) + } + if c.byEmail[address] == nil { + c.byEmail[address] = map[string]bool{} + } + c.byEmail[address][key] = true + continue + } + if !slices.Contains(card.Phones, address) { + card.Phones = append(card.Phones, address) + slices.Sort(card.Phones) + } + } + return rows.Err() +} diff --git a/internal/muesli/contacts_fixture_test.go b/internal/muesli/contacts_fixture_test.go new file mode 100644 index 000000000..d2899a23e --- /dev/null +++ b/internal/muesli/contacts_fixture_test.go @@ -0,0 +1,65 @@ +package muesli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// addressBookDDL is the subset of the macOS Contacts Core Data store +// (AddressBook-v22.abcddb) that msgvault reads. +const addressBookDDL = ` +CREATE TABLE Z_PRIMARYKEY (Z_ENT INTEGER PRIMARY KEY, Z_NAME VARCHAR, Z_SUPER INTEGER, Z_MAX INTEGER); +INSERT INTO Z_PRIMARYKEY (Z_ENT, Z_NAME) VALUES (19, 'ABCDGroup'), (22, 'ABCDContact'); +CREATE TABLE ZABCDRECORD ( + Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZUNIQUEID VARCHAR, ZLINKID VARCHAR, + ZFIRSTNAME VARCHAR, ZLASTNAME VARCHAR, ZORGANIZATION VARCHAR +); +CREATE TABLE ZABCDEMAILADDRESS ( + Z_PK INTEGER PRIMARY KEY, ZOWNER INTEGER, ZADDRESS VARCHAR, + ZADDRESSNORMALIZED VARCHAR, ZORDERINGINDEX INTEGER +); +CREATE TABLE ZABCDPHONENUMBER ( + Z_PK INTEGER PRIMARY KEY, ZOWNER INTEGER, ZFULLNUMBER VARCHAR, ZORDERINGINDEX INTEGER +); +` + +type fixtureCard struct { + uniqueID string + linkID string + emails []string + phones []string + group bool +} + +// newAddressBookStore writes one Contacts store with the given cards. +func newAddressBookStore(t *testing.T, path string, cards ...fixtureCard) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + db := newFixtureDB(t, path, addressBookDDL) + for _, card := range cards { + entity := 22 + if card.group { + entity = 19 + } + var linkID any + if card.linkID != "" { + linkID = card.linkID + } + owner := insertRow(t, db, "ZABCDRECORD", map[string]any{ + "Z_ENT": entity, "ZUNIQUEID": card.uniqueID, "ZLINKID": linkID, + }) + for i, email := range card.emails { + insertRow(t, db, "ZABCDEMAILADDRESS", map[string]any{ + "ZOWNER": owner, "ZADDRESS": email, "ZORDERINGINDEX": i, + }) + } + for i, phone := range card.phones { + insertRow(t, db, "ZABCDPHONENUMBER", map[string]any{ + "ZOWNER": owner, "ZFULLNUMBER": phone, "ZORDERINGINDEX": i, + }) + } + } +} diff --git a/internal/muesli/contacts_test.go b/internal/muesli/contacts_test.go new file mode 100644 index 000000000..ec60e6068 --- /dev/null +++ b/internal/muesli/contacts_test.go @@ -0,0 +1,167 @@ +package muesli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func contactsRoot(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "AddressBook") + newAddressBookStore(t, filepath.Join(root, "AddressBook-v22.abcddb")) + newAddressBookStore(t, filepath.Join(root, "Sources", "A", "AddressBook-v22.abcddb"), + fixtureCard{uniqueID: "CARD-1:ABPerson", emails: []string{"Alex@Example.com"}, phones: []string{"+1 (604) 555-0100"}}, + fixtureCard{uniqueID: "CARD-2:ABPerson", linkID: "LINK-9", emails: []string{"jo@example.com"}}, + fixtureCard{uniqueID: "SHARED-1:ABPerson", emails: []string{"desk@example.com"}}, + fixtureCard{uniqueID: "GROUP-1:ABGroup", group: true, emails: []string{"team@example.com"}}, + ) + newAddressBookStore(t, filepath.Join(root, "Sources", "B", "AddressBook-v22.abcddb"), + fixtureCard{uniqueID: "CARD-3:ABPerson", linkID: "LINK-9", emails: []string{"jo.work@example.com"}, phones: []string{"0044 20 7946 0000"}}, + fixtureCard{uniqueID: "SHARED-2:ABPerson", emails: []string{"desk@example.com"}}, + ) + return root +} + +func TestContactsResolveByIdentifierAndEmail(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + contacts, err := OpenContacts(t.Context(), contactsRoot(t)) + require.NoError(err) + assert.Equal(ContactsComplete, contacts.State()) + + card, ok := contacts.Resolve("CARD-1:ABPerson", "") + require.True(ok) + assert.Equal([]string{"alex@example.com"}, card.Emails) + assert.Equal([]string{"+1 (604) 555-0100"}, card.Phones) + + bare, ok := contacts.Resolve("CARD-1", "") + require.True(ok, "a bare UUID matches its :ABPerson record") + assert.Equal(card.GroupKey, bare.GroupKey) + + unified, ok := contacts.Resolve("LINK-9", "") + require.True(ok, "a unified identifier resolves every linked card") + assert.Equal([]string{"jo.work@example.com", "jo@example.com"}, unified.Emails) + assert.Equal([]string{"0044 20 7946 0000"}, unified.Phones) + linked, ok := contacts.Resolve("CARD-2:ABPerson", "") + require.True(ok) + assert.Equal(unified.GroupKey, linked.GroupKey) + + byEmail, ok := contacts.Resolve("", "ALEX@example.com") + require.True(ok) + assert.Equal(card.GroupKey, byEmail.GroupKey) + + _, ok = contacts.Resolve("", "desk@example.com") + assert.False(ok, "an email on two unrelated cards is ambiguous") + _, ok = contacts.Resolve("", "team@example.com") + assert.False(ok, "groups are not people") + _, ok = contacts.Resolve("MISSING", "nobody@example.com") + assert.False(ok) +} + +func TestContactsPartialReadDisablesEmailLookup(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + root := contactsRoot(t) + broken := filepath.Join(root, "Sources", "C", "AddressBook-v22.abcddb") + require.NoError(os.MkdirAll(filepath.Dir(broken), 0o755)) + newFixtureDB(t, broken, `CREATE TABLE unrelated (id INTEGER)`) + + contacts, err := OpenContacts(t.Context(), root) + require.NoError(err) + + assert.Equal(ContactsPartial, contacts.State()) + _, ok := contacts.Resolve("", "alex@example.com") + assert.False(ok, "a skipped store could hide another card with the same email") + _, ok = contacts.Resolve("CARD-1:ABPerson", "") + assert.True(ok, "identifier lookups still work") +} + +func TestContactsUnavailable(t *testing.T) { + require := require.New(t) + missing, err := OpenContacts(t.Context(), filepath.Join(t.TempDir(), "absent")) + require.NoError(err) + assert.Equal(t, ContactsUnavailable, missing.State()) + _, ok := missing.Resolve("CARD-1:ABPerson", "alex@example.com") + assert.False(t, ok) + + fileRoot := filepath.Join(t.TempDir(), "not-a-directory") + require.NoError(os.WriteFile(fileRoot, []byte("not a directory"), 0o600)) + denied, err := OpenContacts(t.Context(), fileRoot) + require.NoError(err) + assert.Equal(t, ContactsUnavailable, denied.State()) +} + +func TestContactsReadDoesNotChangeStores(t *testing.T) { + require := require.New(t) + root := contactsRoot(t) + path := filepath.Join(root, "Sources", "A", "AddressBook-v22.abcddb") + db, err := openQueryOnly(path) + require.NoError(err) + var before int + require.NoError(db.QueryRow(`SELECT count(*) FROM ZABCDRECORD`).Scan(&before)) + require.NoError(db.Close()) + + _, err = OpenContacts(t.Context(), root) + require.NoError(err) + + db, err = openQueryOnly(path) + require.NoError(err) + defer func() { _ = db.Close() }() + var after int + require.NoError(db.QueryRow(`SELECT count(*) FROM ZABCDRECORD`).Scan(&after)) + assert.Equal(t, before, after) +} + +func TestNormalizeContactPhone(t *testing.T) { + for _, tt := range []struct { + raw, country, want string + ok bool + }{ + {raw: "+1 (604) 555-0100", want: "+16045550100", ok: true}, + {raw: "0044 20 7946 0000", want: "+442079460000", ok: true}, + {raw: "(604) 555-0100", ok: false}, + {raw: "(604) 555-0100", country: "1", want: "+16045550100", ok: true}, + {raw: "1-604-555-0100", country: "1", want: "+16045550100", ok: true}, + {raw: "555-0100", country: "1", ok: false}, + {raw: "020 7946 0000", country: "44", want: "+442079460000", ok: true}, + {raw: "06 1234 5678", country: "39", want: "+390612345678", ok: true}, + {raw: "+1 604 555 0100 ext. 12", want: "+16045550100", ok: true}, + {raw: "+12", ok: false}, + {raw: "+44 (0)20 7946 0000", want: "+442079460000", ok: true}, + {raw: "+0044 20 7946 0000", ok: false}, + {raw: "+1234567890123456", ok: false}, + {raw: "call me", country: "1", ok: false}, + } { + got, ok := NormalizeContactPhone(tt.raw, tt.country) + assert.Equal(t, tt.ok, ok, "%q/%q", tt.raw, tt.country) + assert.Equal(t, tt.want, got, "%q/%q", tt.raw, tt.country) + } +} + +func TestContactsDropStoreThatFailsMidRead(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + root := contactsRoot(t) + path := filepath.Join(root, "Sources", "D", "AddressBook-v22.abcddb") + require.NoError(os.MkdirAll(filepath.Dir(path), 0o755)) + db := newFixtureDB(t, path, ` +CREATE TABLE Z_PRIMARYKEY (Z_ENT INTEGER PRIMARY KEY, Z_NAME VARCHAR); +INSERT INTO Z_PRIMARYKEY VALUES (22, 'ABCDContact'); +CREATE TABLE ZABCDRECORD (Z_PK INTEGER PRIMARY KEY, Z_ENT INTEGER, ZUNIQUEID VARCHAR); +CREATE TABLE ZABCDEMAILADDRESS (Z_PK INTEGER PRIMARY KEY, ZOWNER INTEGER, ZADDRESS VARCHAR); +CREATE TABLE ZABCDPHONENUMBER (Z_PK INTEGER PRIMARY KEY, ZFULLNUMBER VARCHAR); +`) + insertRow(t, db, "ZABCDRECORD", map[string]any{"Z_ENT": 22, "ZUNIQUEID": "HALF-1:ABPerson"}) + insertRow(t, db, "ZABCDEMAILADDRESS", map[string]any{"ZOWNER": 1, "ZADDRESS": "half@example.com"}) + + contacts, err := OpenContacts(t.Context(), root) + require.NoError(err) + + assert.Equal(ContactsPartial, contacts.State()) + _, ok := contacts.Resolve("HALF-1:ABPerson", "") + assert.False(ok, "a store that could not be read completely contributes no cards") +} diff --git a/internal/muesli/fixture_test.go b/internal/muesli/fixture_test.go new file mode 100644 index 000000000..8ad2f8040 --- /dev/null +++ b/internal/muesli/fixture_test.go @@ -0,0 +1,163 @@ +package muesli + +import ( + "database/sql" + "fmt" + "maps" + "net/url" + "path/filepath" + "sort" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +// currentSchemaDDL is the meeting subset of Muesli 0.8.4's fresh-database +// schema (DictationStore.migrateIfNeeded), including folder_id, which the +// app adds with ALTER TABLE. +const currentSchemaDDL = ` +CREATE TABLE meeting_folders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_order INTEGER NOT NULL DEFAULT 0, + parent_id INTEGER REFERENCES meeting_folders(id), + created_at TEXT DEFAULT (datetime('now')) +); +CREATE TABLE meetings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + calendar_event_id TEXT, + calendar_occurrence_key TEXT, + calendar_source TEXT, + calendar_id TEXT, + calendar_series_id TEXT, + calendar_occurrence_start REAL, + start_time TEXT NOT NULL, + end_time TEXT, + duration_seconds REAL, + raw_transcript TEXT, + formatted_notes TEXT, + mic_audio_path TEXT, + system_audio_path TEXT, + saved_recording_path TEXT, + meeting_status TEXT NOT NULL DEFAULT 'completed', + manual_notes TEXT NOT NULL DEFAULT '', + word_count INTEGER NOT NULL DEFAULT 0, + selected_template_id TEXT, + selected_template_name TEXT, + selected_template_kind TEXT, + selected_template_prompt TEXT, + source TEXT NOT NULL DEFAULT 'meeting', + updated_at REAL NOT NULL DEFAULT 0, + deleted_at REAL, + cloud_record_name TEXT, + cloud_change_tag TEXT, + cloud_system_fields BLOB, + cloud_transcript_record_name TEXT, + last_synced_at REAL, + sync_dirty INTEGER NOT NULL DEFAULT 1, + follow_up_to_id INTEGER REFERENCES meetings(id) ON DELETE SET NULL, + follow_up_to_record_name TEXT, + visual_context TEXT, + created_at TEXT DEFAULT (datetime('now')), + folder_id INTEGER REFERENCES meeting_folders(id) +); +CREATE TABLE meeting_participants ( + meeting_id INTEGER NOT NULL REFERENCES meetings(id) ON DELETE CASCADE, + participant_identifier TEXT NOT NULL, + display_name TEXT NOT NULL, + email_address TEXT, + insertion_order INTEGER NOT NULL, + source TEXT NOT NULL DEFAULT 'manual', + is_suppressed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (meeting_id, participant_identifier) +); +` + +// legacySchemaDDL models an early Muesli database: no status, manual notes, +// deletion, folder, or template columns; suppressions in their own table. +const legacySchemaDDL = ` +CREATE TABLE meetings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + calendar_event_id TEXT, + start_time TEXT NOT NULL, + end_time TEXT, + duration_seconds REAL, + raw_transcript TEXT, + formatted_notes TEXT, + word_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')) +); +CREATE TABLE meeting_participants ( + meeting_id INTEGER NOT NULL REFERENCES meetings(id) ON DELETE CASCADE, + participant_identifier TEXT NOT NULL, + display_name TEXT NOT NULL, + email_address TEXT, + insertion_order INTEGER NOT NULL, + PRIMARY KEY (meeting_id, participant_identifier) +); +CREATE TABLE meeting_participant_suppressions ( + meeting_id INTEGER NOT NULL, + participant_identifier TEXT NOT NULL, + PRIMARY KEY (meeting_id, participant_identifier) +); +` + +// newFixtureDB creates a WAL-mode Muesli-shaped database at path and returns +// a read-write connection the test can keep mutating. +func newFixtureDB(t *testing.T, path, ddl string) *sql.DB { + t.Helper() + abs, err := filepath.Abs(path) + require.NoError(t, err) + dsn := (&url.URL{Scheme: "file", Path: sqliteURIPath(abs), RawQuery: "_journal_mode=WAL&_busy_timeout=5000"}).String() + db, err := sql.Open("sqlite3", dsn) + require.NoError(t, err) + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec(ddl) + require.NoError(t, err) + return db +} + +// insertRow inserts one row with the given column values and returns its +// rowid. +func insertRow(t *testing.T, db *sql.DB, table string, values map[string]any) int64 { + t.Helper() + columns := make([]string, 0, len(values)) + for column := range values { + columns = append(columns, column) + } + sort.Strings(columns) + args := make([]any, len(columns)) + for i, column := range columns { + args[i] = values[column] + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(columns)), ",") + result, err := db.Exec(fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + table, strings.Join(columns, ","), placeholders), args...) + require.NoError(t, err) + id, err := result.LastInsertId() + require.NoError(t, err) + return id +} + +// completedMeeting returns column values for an ordinary finished meeting. +func completedMeeting(overrides map[string]any) map[string]any { + values := map[string]any{ + "title": "Weekly sync", + "start_time": "2026-09-01T14:00:00Z", + "end_time": "2026-09-01T14:45:00Z", + "duration_seconds": 2700.0, + "raw_transcript": "[10:00:01] You: hello\n[10:00:04] Speaker 1: hi there", + "formatted_notes": "## Decisions\nShip it", + "manual_notes": "typed note", + "word_count": 5, + "meeting_status": "completed", + "created_at": "2026-09-01 14:00:03", + } + maps.Copy(values, overrides) + return values +} diff --git a/internal/muesli/format.go b/internal/muesli/format.go new file mode 100644 index 000000000..2d4a82d94 --- /dev/null +++ b/internal/muesli/format.go @@ -0,0 +1,365 @@ +package muesli + +import ( + "encoding/json/v2" + "errors" + "fmt" + "strings" + "time" + + "go.kenn.io/msgvault/internal/meetingarchive" +) + +const rawSchemaVersion = 1 + +// SkipReason explains why a Muesli meeting is not archived; empty means it +// is archived. +type SkipReason string + +const ( + SkipDeleted SkipReason = "deleted" + SkipInProgress SkipReason = "in_progress" + SkipEmpty SkipReason = "empty" +) + +// Notes states. The first three are Muesli's MeetingNotesState values; +// summary_failed is msgvault's addition for Muesli's failure notice. +const ( + NotesMissing = "missing" + NotesRawTranscriptFallback = "raw_transcript_fallback" + NotesSummaryFailed = "summary_failed" + NotesStructured = "structured_notes" +) + +// Eligibility reports whether the meeting should be archived. Deleted +// meetings are never archived, so a Muesli tombstone cannot overwrite an +// archived copy. Meetings still recording or processing wait for a later +// sync. +func (m Meeting) Eligibility() SkipReason { + switch { + case m.Deleted: + return SkipDeleted + case m.Status == "recording" || m.Status == "processing": + return SkipInProgress + case strings.TrimSpace(m.RawTranscript) == "" && + NotesState(m.FormattedNotes) != NotesStructured && + strings.TrimSpace(m.ManualNotes) == "": + return SkipEmpty + default: + return "" + } +} + +// NotesState classifies Muesli's formatted notes the way Muesli's own +// MeetingRecord.notesState does, adding summary_failed. +func NotesState(notes string) string { + normalized := strings.ToLower(strings.TrimSpace(notes)) + switch { + case normalized == "": + return NotesMissing + case headingOnlyOrFirst(normalized, "## raw transcript"): + return NotesRawTranscriptFallback + case headingOnlyOrFirst(normalized, "## summary failed"): + return NotesSummaryFailed + default: + return NotesStructured + } +} + +func headingOnlyOrFirst(normalized, heading string) bool { + return normalized == heading || strings.HasPrefix(normalized, heading+"\n") +} + +// SourceMessageID is the stable archive key. Muesli's id restarts after a +// database reset and its start time changes when a meeting is resumed, so the +// key pairs the id with the row's insert time. +func (m Meeting) SourceMessageID() (string, error) { + created, err := parseCreatedAt(m.CreatedAt) + if err != nil { + return "", err + } + return fmt.Sprintf("meeting:%d:%s", m.ID, created.Format("20060102T150405Z")), nil +} + +func parseCreatedAt(value string) (time.Time, error) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, errors.New("created_at is missing") + } + // SQLite's datetime('now') is UTC without a zone suffix. + if parsed, err := time.Parse(time.DateTime, value); err == nil { + return parsed.UTC(), nil + } + if parsed, err := time.Parse(time.RFC3339Nano, value); err == nil { + return parsed.UTC(), nil + } + return time.Time{}, fmt.Errorf("created_at %q is not a timestamp", value) +} + +type rawParticipant struct { + // Ref is a short hash of Muesli's participant identifier. It lets a later + // sync retain this participant's identities when its Contacts card cannot + // be resolved, without storing the Contacts identifier. + Ref string `json:"ref,omitempty"` + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Phone string `json:"phone,omitempty"` + Source string `json:"source,omitempty"` + // Emails and Phones are the participant's Contacts identities. + Emails []string `json:"emails,omitempty"` + Phones []string `json:"phones,omitempty"` +} + +type rawMeeting struct { + ID int64 `json:"id"` + Title string `json:"title"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time,omitempty"` + DurationSeconds float64 `json:"duration_seconds,omitzero"` + Status string `json:"status,omitempty"` + Source string `json:"source,omitempty"` + FormattedNotes string `json:"formatted_notes,omitempty"` + NotesState string `json:"notes_state"` + ManualNotes string `json:"manual_notes,omitempty"` + RawTranscript string `json:"raw_transcript,omitempty"` + WordCount int64 `json:"word_count,omitzero"` + TemplateName string `json:"template_name,omitempty"` + TemplateKind string `json:"template_kind,omitempty"` + CalendarEventID string `json:"calendar_event_id,omitempty"` + CalendarSource string `json:"calendar_source,omitempty"` + CalendarSeriesID string `json:"calendar_series_id,omitempty"` + Folder string `json:"folder,omitempty"` + FollowUpToID int64 `json:"follow_up_to_id,omitzero"` + CreatedAt string `json:"created_at"` +} + +// rawEvidence is the archived muesli_json document. It holds only content +// that describes the meeting: no audio paths, template prompts, screen text, +// sync bookkeeping, or Contacts identifiers. +type rawEvidence struct { + SchemaVersion int `json:"schema_version"` + Meeting rawMeeting `json:"meeting"` + Participants []rawParticipant `json:"participants,omitempty"` +} + +type meetingMetadata struct { + Platform string `json:"platform"` + SourceIdentifier string `json:"source_identifier,omitempty"` + MuesliID int64 `json:"muesli_id"` + Status string `json:"status,omitempty"` + Source string `json:"source,omitempty"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at,omitempty"` + DurationSeconds float64 `json:"duration_seconds,omitzero"` + NotesState string `json:"notes_state"` + HasSummary bool `json:"has_summary"` + HasNotes bool `json:"has_notes"` + HasTranscript bool `json:"has_transcript"` + TemplateName string `json:"template_name,omitempty"` + Folder string `json:"folder,omitempty"` + CalendarEventID string `json:"calendar_event_id,omitempty"` + CalendarSource string `json:"calendar_source,omitempty"` + FollowUpToID int64 `json:"follow_up_to_id,omitzero"` + NameOnlyParticipants int `json:"name_only_participants,omitzero"` + Contacts string `json:"contacts,omitempty"` + ContactsResolved int `json:"contacts_resolved,omitzero"` + ContactsCarried int `json:"contacts_carried_forward,omitzero"` + ContactsUnresolved int `json:"contacts_unresolved,omitzero"` + ContactsPhoneSkipped int `json:"contacts_phones_skipped,omitzero"` +} + +// ArchiveSnapshot converts the meeting into the canonical archive shape. +// Muesli records no organizer, so the configured account (the person who +// recorded the meeting) is the organizer. +func (m Meeting) ArchiveSnapshot(sourceID int64, identifier, accountEmail string) (meetingarchive.Snapshot, error) { + key, err := m.SourceMessageID() + if err != nil { + return meetingarchive.Snapshot{}, err + } + created, _ := parseCreatedAt(m.CreatedAt) + started, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(m.StartTime)) + if err != nil { + return meetingarchive.Snapshot{}, fmt.Errorf("start_time %q is not a timestamp", m.StartTime) + } + started = started.UTC() + ended := m.endedAt(started) + + title := strings.Join(strings.Fields(m.Title), " ") + if title == "" { + title = "Meeting on " + started.Format(time.DateOnly) + } + notesState := NotesState(m.FormattedNotes) + summary := "" + if notesState == NotesStructured { + summary = strings.TrimSpace(m.FormattedNotes) + } + notes := strings.TrimSpace(m.ManualNotes) + transcript := strings.TrimSpace(m.RawTranscript) + + people := dedupeParticipants(m.Participants) + var attendees []meetingarchive.Person + var labels []string + nameOnly := 0 + var resolved, carried, unresolved, skippedPhones int + rawPeople := make([]rawParticipant, 0, len(people)) + for _, participant := range people { + person := participant.archivePerson() + rawPeople = append(rawPeople, participant.raw(person)) + label := person.Name + if label == "" { + label = person.Email + } + if label == "" { + label = person.Phone + } + if label != "" { + labels = append(labels, label) + } + skippedPhones += participant.SkippedPhones + switch participant.Resolution { + case resolutionResolved: + resolved++ + case resolutionCarried: + carried++ + default: + unresolved++ + } + if person.PrimaryKey() == "" { + nameOnly++ + continue + } + attendees = append(attendees, person) + } + contactsState := "" + if m.ContactsState != "" { + contactsState = string(m.ContactsState) + } + if contactsState == "" || m.ContactsState == ContactsOff { + resolved, carried, unresolved, skippedPhones = 0, 0, 0, 0 + } + + body := meetingBody(title, started, ended, labels, summary, notes, transcript) + raw, err := json.Marshal(rawEvidence{ + SchemaVersion: rawSchemaVersion, + Meeting: rawMeeting{ + ID: m.ID, Title: m.Title, StartTime: m.StartTime, EndTime: m.EndTime, + DurationSeconds: m.DurationSeconds, Status: m.Status, Source: m.Source, + FormattedNotes: m.FormattedNotes, NotesState: notesState, ManualNotes: m.ManualNotes, + RawTranscript: m.RawTranscript, WordCount: m.WordCount, + TemplateName: m.TemplateName, TemplateKind: m.TemplateKind, + CalendarEventID: m.CalendarEventID, CalendarSource: m.CalendarSource, + CalendarSeriesID: m.CalendarSeriesID, Folder: m.Folder, FollowUpToID: m.FollowUpToID, + CreatedAt: created.Format(time.RFC3339), + }, + Participants: rawPeople, + }, json.Deterministic(true)) + if err != nil { + return meetingarchive.Snapshot{}, fmt.Errorf("marshal Muesli raw evidence: %w", err) + } + metadata, err := json.Marshal(meetingMetadata{ + Platform: SourceType, SourceIdentifier: identifier, MuesliID: m.ID, + Status: m.Status, Source: m.Source, + StartedAt: started.Format(time.RFC3339Nano), EndedAt: formatOptionalTime(ended), + DurationSeconds: m.DurationSeconds, NotesState: notesState, + HasSummary: summary != "", HasNotes: notes != "", HasTranscript: transcript != "", + TemplateName: m.TemplateName, Folder: m.Folder, + CalendarEventID: m.CalendarEventID, CalendarSource: m.CalendarSource, + FollowUpToID: m.FollowUpToID, NameOnlyParticipants: nameOnly, + Contacts: contactsState, ContactsResolved: resolved, + ContactsCarried: carried, ContactsUnresolved: unresolved, + ContactsPhoneSkipped: skippedPhones, + }, json.Deterministic(true)) + if err != nil { + return meetingarchive.Snapshot{}, fmt.Errorf("marshal Muesli meeting metadata: %w", err) + } + + var organizer *meetingarchive.Person + if email := strings.ToLower(strings.TrimSpace(accountEmail)); email != "" { + organizer = &meetingarchive.Person{Email: email} + } + return meetingarchive.Snapshot{ + SourceID: sourceID, AccountEmail: accountEmail, + SourceMessageID: key, SourceConversationID: key, + Title: title, StartedAt: started, Body: body, Snippet: snippet(body), + Metadata: metadata, Raw: raw, RawFormat: RawFormat, + Organizer: organizer, Attendees: attendees, + }, nil +} + +func (m Meeting) endedAt(started time.Time) time.Time { + if value := strings.TrimSpace(m.EndTime); value != "" { + if ended, err := time.Parse(time.RFC3339Nano, value); err == nil && ended.After(started) { + return ended.UTC() + } + } + if m.DurationSeconds > 0 { + return started.Add(time.Duration(m.DurationSeconds * float64(time.Second))) + } + return time.Time{} +} + +// dedupeParticipants drops repeated emails (Muesli can list the same person +// from the calendar and from Contacts) and fully blank rows. The first row +// keeps its place; a later duplicate only supplies a name the first lacks. +func dedupeParticipants(participants []Participant) []Participant { + out := make([]Participant, 0, len(participants)) + index := map[string]int{} + for _, person := range participants { + person.Name = strings.TrimSpace(person.Name) + person.Email = strings.ToLower(strings.TrimSpace(person.Email)) + if person.Name == "" && person.Email == "" && len(person.ContactEmails) == 0 && len(person.ContactPhones) == 0 { + continue + } + if person.Email != "" { + if at, seen := index[person.Email]; seen { + if out[at].Name == "" { + out[at].Name = person.Name + } + if out[at].Resolution == "" && person.Resolution != "" { + out[at].ContactEmails, out[at].ContactPhones = person.ContactEmails, person.ContactPhones + out[at].Anchor, out[at].Resolution = person.Anchor, person.Resolution + out[at].LinkExcludedAddresses = person.LinkExcludedAddresses + } + continue + } + index[person.Email] = len(out) + } + out = append(out, person) + } + return out +} + +func meetingBody(title string, start, end time.Time, attendees []string, summary, notes, transcript string) string { + lines := []string{title} + when := "When: " + start.Format("2006-01-02 15:04") + if !end.IsZero() { + when += " - " + end.UTC().Format("15:04") + } + lines = append(lines, when) + if len(attendees) > 0 { + lines = append(lines, "Attendees: "+strings.Join(attendees, ", ")) + } + for _, section := range []struct{ label, content string }{ + {"Summary", summary}, {"Notes", notes}, {"Transcript", transcript}, + } { + if section.content != "" { + lines = append(lines, "", section.label+":", section.content) + } + } + return strings.Join(lines, "\n") +} + +func formatOptionalTime(value time.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} + +func snippet(body string) string { + runes := []rune(strings.TrimSpace(body)) + if len(runes) > 200 { + runes = runes[:200] + } + return string(runes) +} diff --git a/internal/muesli/format_test.go b/internal/muesli/format_test.go new file mode 100644 index 000000000..2acd3a518 --- /dev/null +++ b/internal/muesli/format_test.go @@ -0,0 +1,214 @@ +package muesli + +import ( + "encoding/json/v2" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/meetingarchive" +) + +func sampleMeeting() Meeting { + return Meeting{ + ID: 42, Title: " Weekly sync ", StartTime: "2026-09-01T14:00:00Z", EndTime: "2026-09-01T14:45:00Z", + CreatedAt: "2026-09-01 14:00:03", DurationSeconds: 2700, Status: "completed", Source: "meeting", + RawTranscript: "[10:00:01] You: hello\n", + FormattedNotes: "## Decisions\nShip it", ManualNotes: "typed note", WordCount: 2, + TemplateName: "Auto", TemplateKind: "auto", + CalendarEventID: "event-1", CalendarSource: "eventKit", CalendarSeriesID: "series-1", + Folder: "Clients/Acme", FollowUpToID: 41, + Participants: []Participant{ + {Name: "Carol Example", Source: "manual"}, + {Name: "Alice Example", Email: "alice@example.com", Source: "calendar"}, + {Name: "Alice (manual)", Email: "Alice@Example.com", Source: "manual"}, + }, + } +} + +func TestNotesState(t *testing.T) { + for _, tt := range []struct { + notes string + want string + }{ + {notes: " \n ", want: "missing"}, + {notes: " ## Raw Transcript\n\n[10:00:01] You: hi", want: "raw_transcript_fallback"}, + {notes: "## raw transcript", want: "raw_transcript_fallback"}, + {notes: "## Raw Transcript Analysis\nfindings", want: "structured_notes"}, + {notes: "## Summary failed\n\ntimeout", want: "summary_failed"}, + {notes: "## Summary\nfine", want: "structured_notes"}, + } { + assert.Equal(t, tt.want, NotesState(tt.notes), "notes %q", tt.notes) + } +} + +func TestSourceMessageID(t *testing.T) { + for _, tt := range []struct { + createdAt string + want string + wantErr bool + }{ + {createdAt: "2026-09-01 14:00:03", want: "meeting:42:20260901T140003Z"}, + {createdAt: "2026-09-01T16:00:03+02:00", want: "meeting:42:20260901T140003Z"}, + {createdAt: "", wantErr: true}, + {createdAt: "yesterday", wantErr: true}, + } { + got, err := Meeting{ID: 42, CreatedAt: tt.createdAt}.SourceMessageID() + if tt.wantErr { + require.Error(t, err, "created_at %q", tt.createdAt) + continue + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } +} + +func TestEligibility(t *testing.T) { + for _, tt := range []struct { + name string + meeting Meeting + want SkipReason + }{ + {name: "completed", meeting: sampleMeeting(), want: ""}, + {name: "deleted", meeting: Meeting{Deleted: true, RawTranscript: "x"}, want: SkipDeleted}, + {name: "recording", meeting: Meeting{Status: "recording", RawTranscript: "x"}, want: SkipInProgress}, + {name: "processing", meeting: Meeting{Status: "processing", RawTranscript: "x"}, want: SkipInProgress}, + {name: "empty", meeting: Meeting{Status: "completed", FormattedNotes: " "}, want: SkipEmpty}, + {name: "failed with transcript", meeting: Meeting{Status: "failed", RawTranscript: "x"}, want: ""}, + {name: "unknown future status", meeting: Meeting{Status: "archived", ManualNotes: "x"}, want: ""}, + } { + assert.Equal(t, tt.want, tt.meeting.Eligibility(), tt.name) + } +} + +func TestArchiveSnapshot(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + snapshot, err := sampleMeeting().ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + + assert.Equal(int64(7), snapshot.SourceID) + assert.Equal("you@example.com", snapshot.AccountEmail) + assert.Equal("meeting:42:20260901T140003Z", snapshot.SourceMessageID) + assert.Equal(snapshot.SourceMessageID, snapshot.SourceConversationID) + assert.Equal("Weekly sync", snapshot.Title) + assert.Equal(time.Date(2026, 9, 1, 14, 0, 0, 0, time.UTC), snapshot.StartedAt) + assert.Equal(&meetingarchive.Person{Email: "you@example.com"}, snapshot.Organizer) + assert.Equal([]meetingarchive.Person{{Name: "Alice Example", Email: "alice@example.com"}}, snapshot.Attendees) + assert.Equal(RawFormat, snapshot.RawFormat) + assert.Equal("Weekly sync\n"+ + "When: 2026-09-01 14:00 - 14:45\n"+ + "Attendees: Carol Example, Alice Example\n"+ + "\n"+ + "Summary:\n"+ + "## Decisions\nShip it\n"+ + "\n"+ + "Notes:\n"+ + "typed note\n"+ + "\n"+ + "Transcript:\n"+ + "[10:00:01] You: hello", snapshot.Body) + assert.Equal(snapshot.Body, snapshot.Snippet, "bodies under 200 runes are their own snippet") + + raw := string(snapshot.Raw) + for _, forbidden := range []string{"contact:", "ABC", "updated_at", "recording", "visual_context", "prompt", "muesli.db"} { + assert.NotContains(raw, forbidden) + } + var evidence map[string]any + require.NoError(json.Unmarshal(snapshot.Raw, &evidence)) + assert.InDelta(float64(1), evidence["schema_version"], 0) + meeting, ok := evidence["meeting"].(map[string]any) + require.True(ok, "meeting is an object") + assert.Equal("structured_notes", meeting["notes_state"]) + assert.Equal("Clients/Acme", meeting["folder"]) + assert.Equal("2026-09-01T14:00:03Z", meeting["created_at"]) + assert.Equal([]any{ + map[string]any{"name": "Carol Example", "source": "manual"}, + map[string]any{"name": "Alice Example", "email": "alice@example.com", "source": "calendar"}, + }, evidence["participants"]) + + var metadata map[string]any + require.NoError(json.Unmarshal(snapshot.Metadata, &metadata)) + assert.Equal("muesli", metadata["platform"]) + assert.Equal("mac", metadata["source_identifier"]) + assert.InDelta(float64(42), metadata["muesli_id"], 0) + assert.Equal(true, metadata["has_summary"]) + assert.InDelta(float64(1), metadata["name_only_participants"], 0) + + again, err := sampleMeeting().ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + assert.Equal(snapshot.Raw, again.Raw, "raw evidence is deterministic") + assert.Equal(snapshot.Metadata, again.Metadata) +} + +func TestArchiveSnapshotOmitsFallbackNotesFromSummary(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + meeting := sampleMeeting() + meeting.FormattedNotes = "## Raw Transcript\n\n[10:00:01] You: hello" + meeting.ManualNotes = "" + meeting.Participants = nil + + snapshot, err := meeting.ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + + assert.Equal("Weekly sync\nWhen: 2026-09-01 14:00 - 14:45\n\nTranscript:\n[10:00:01] You: hello", snapshot.Body) + var metadata map[string]any + require.NoError(json.Unmarshal(snapshot.Metadata, &metadata)) + assert.Equal(false, metadata["has_summary"]) + assert.Equal("raw_transcript_fallback", metadata["notes_state"]) +} + +func TestArchiveSnapshotTimes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + byDuration := sampleMeeting() + byDuration.EndTime = "" + byDuration.DurationSeconds = 600 + snapshot, err := byDuration.ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + assert.Contains(snapshot.Body, "When: 2026-09-01 14:00 - 14:10\n") + + backwards := sampleMeeting() + backwards.EndTime = "2026-09-01T13:00:00Z" + backwards.DurationSeconds = 0 + snapshot, err = backwards.ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + assert.Contains(snapshot.Body, "When: 2026-09-01 14:00\n") + + untitled := sampleMeeting() + untitled.Title = " " + snapshot, err = untitled.ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + assert.Equal("Meeting on 2026-09-01", snapshot.Title) + + unparseable := sampleMeeting() + unparseable.StartTime = "soon" + _, err = unparseable.ArchiveSnapshot(7, "mac", "you@example.com") + require.ErrorContains(err, "start_time") + + noCreatedAt := sampleMeeting() + noCreatedAt.CreatedAt = "" + _, err = noCreatedAt.ArchiveSnapshot(7, "mac", "you@example.com") + assert.ErrorContains(err, "created_at") +} + +func TestArchiveSnapshotFillsBlankNameFromDuplicateParticipant(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + meeting := sampleMeeting() + meeting.Participants = []Participant{ + {Email: "alice@example.com", Source: "calendar"}, + {Name: "Alice Example", Email: "ALICE@example.com", Source: "manual"}, + } + + snapshot, err := meeting.ArchiveSnapshot(7, "mac", "you@example.com") + require.NoError(err) + + assert.Equal([]meetingarchive.Person{{Name: "Alice Example", Email: "alice@example.com"}}, snapshot.Attendees) + assert.Contains(snapshot.Body, "Attendees: Alice Example\n") + assert.Contains(string(snapshot.Raw), `{"name":"Alice Example","email":"alice@example.com","source":"calendar"}`) +} diff --git a/internal/muesli/importer.go b/internal/muesli/importer.go new file mode 100644 index 000000000..375fc0c50 --- /dev/null +++ b/internal/muesli/importer.go @@ -0,0 +1,233 @@ +package muesli + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "path/filepath" + "strings" + "time" + + "go.kenn.io/msgvault/internal/meetingarchive" + "go.kenn.io/msgvault/internal/store" +) + +const ( + syncStateVersion = 1 + checkpointInterval = 50 +) + +// Importer archives the meetings in one Muesli database. +type Importer struct { + store *store.Store + now func() time.Time +} + +func NewImporter(st *store.Store) *Importer { + return &Importer{store: st, now: time.Now} +} + +type ImportOptions struct { + Identifier string + AccountEmail string + DBPath string + // Full rewrites every archived meeting even when its evidence matches, + // which refreshes attribution after identity changes. + Full bool + // Limit caps the eligible meetings processed; 0 means all. + Limit int + // StartedAfter keeps meetings that start on or after this time. + StartedAfter time.Time + // ContactsEnabled resolves attendees through the Mac's Contacts stores + // under ContactsPath. PhoneCountryCode converts national-format phones. + ContactsEnabled bool + ContactsPath string + PhoneCountryCode string + Progress func(string) +} + +type ImportSummary struct { + SourceID int64 + MeetingsProcessed int64 + MeetingsAdded int64 + MeetingsUpdated int64 + SkippedDeleted int64 + SkippedInProgress int64 + SkippedEmpty int64 + Errors int64 + // ContactsState is how much of the Mac's Contacts the sync could read. + ContactsState ContactsState + Duration time.Duration +} + +// syncState is recorded for diagnostics only. Every run rescans the whole +// database because Muesli does not bump updated_at for participant or folder +// edits; the archiver skips meetings whose evidence is unchanged. +type syncState struct { + Version int `json:"version"` + ScannedAt string `json:"scanned_at"` +} + +// Import runs one sync of the Muesli database at opts.DBPath into the +// registered source opts.Identifier. +func (imp *Importer) Import(ctx context.Context, opts ImportOptions) (sum *ImportSummary, retErr error) { + if imp == nil || imp.store == nil { + return nil, errors.New("muesli importer is unavailable") + } + started := imp.now() + identifier := strings.TrimSpace(opts.Identifier) + source, err := imp.store.GetSourceByTypeAndIdentifier(SourceType, identifier) + if err != nil { + if errors.Is(err, store.ErrSourceNotFound) { + return nil, fmt.Errorf("muesli source %q is not registered; run msgvault add-muesli %s first", + identifier, identifier) + } + return nil, fmt.Errorf("look up Muesli source %q: %w", identifier, err) + } + sum = &ImportSummary{SourceID: source.ID} + if strings.TrimSpace(opts.AccountEmail) != "" { + if err := imp.store.AddAccountIdentityContext(ctx, source.ID, opts.AccountEmail, "account-email"); err != nil { + return sum, fmt.Errorf("confirm Muesli account identity: %w", err) + } + } + + syncID, err := imp.store.StartSync(source.ID, SourceType) + if err != nil { + return sum, err + } + // A superseded or failed run must not keep writing, so every write below + // goes through the sync-scoped store. + scoped := imp.store.ScopedToSync(source.ID, syncID) + checkpoint := func() *store.Checkpoint { + return &store.Checkpoint{ + MessagesProcessed: sum.MeetingsProcessed, + MessagesAdded: sum.MeetingsAdded, + MessagesUpdated: sum.MeetingsUpdated, + ErrorsCount: sum.Errors, + } + } + defer func() { + sum.Duration = imp.now().Sub(started) + if retErr != nil { + _ = scoped.FailSyncWithCheckpoint(syncID, withoutPath(retErr.Error(), opts.DBPath), checkpoint()) + } + }() + + reader, err := Open(ctx, opts.DBPath) + if err != nil { + sum.Errors++ + return sum, err + } + defer func() { _ = reader.Close() }() + + contacts := DisabledContacts() + if opts.ContactsEnabled { + contacts, err = OpenContacts(ctx, opts.ContactsPath) + if err != nil { + return sum, err + } + } + sum.ContactsState = contacts.State() + sharedAddresses := contacts.sharedAddresses(opts.PhoneCountryCode) + + meetings, err := reader.ListMeetings(ctx) + if err != nil { + sum.Errors++ + return sum, err + } + progress := opts.Progress + if progress == nil { + progress = func(string) {} + } + archiver := meetingarchive.New(scoped) + var meetingErrors []error + for _, meeting := range meetings { + if err := ctx.Err(); err != nil { + return sum, err + } + switch meeting.Eligibility() { + case SkipDeleted: + sum.SkippedDeleted++ + continue + case SkipInProgress: + sum.SkippedInProgress++ + continue + case SkipEmpty: + sum.SkippedEmpty++ + continue + } + if !opts.StartedAfter.IsZero() { + startedAt, parseErr := time.Parse(time.RFC3339Nano, strings.TrimSpace(meeting.StartTime)) + if parseErr == nil && startedAt.Before(opts.StartedAfter) { + continue + } + } + if opts.Limit > 0 && sum.MeetingsProcessed >= int64(opts.Limit) { + break + } + sum.MeetingsProcessed++ + + if err := imp.resolveParticipants(source.ID, &meeting, contacts, opts.PhoneCountryCode, sharedAddresses); err != nil { + return sum, err + } + snapshot, err := meeting.ArchiveSnapshot(source.ID, identifier, opts.AccountEmail) + if err != nil { + sum.Errors++ + meetingErrors = append(meetingErrors, fmt.Errorf("muesli meeting %d: %w", meeting.ID, err)) + continue + } + result, err := archiver.Upsert(ctx, snapshot, meetingarchive.UpsertOptions{Force: opts.Full}) + // Upsert can commit and then fail conversation-stat maintenance, so + // count the write before looking at the error. + switch { + case result.Created: + sum.MeetingsAdded++ + progress(fmt.Sprintf("added Muesli meeting %d", meeting.ID)) + case result.Changed: + sum.MeetingsUpdated++ + progress(fmt.Sprintf("updated Muesli meeting %d", meeting.ID)) + } + if err != nil { + sum.Errors++ + meetingErrors = append(meetingErrors, fmt.Errorf("muesli meeting %d: %w", meeting.ID, err)) + continue + } + if sum.MeetingsProcessed%checkpointInterval == 0 { + if err := scoped.UpdateSyncCheckpoint(syncID, checkpoint()); err != nil { + return sum, err + } + } + } + if len(meetingErrors) > 0 { + return sum, errors.Join(meetingErrors...) + } + + if err := scoped.UpdateSyncCheckpoint(syncID, checkpoint()); err != nil { + return sum, err + } + cursor, err := json.Marshal(syncState{ + Version: syncStateVersion, + ScannedAt: imp.now().UTC().Format(time.RFC3339), + }, json.Deterministic(true)) + if err != nil { + return sum, fmt.Errorf("marshal Muesli sync cursor: %w", err) + } + if err := scoped.CompleteSync(syncID, string(cursor)); err != nil { + return sum, err + } + return sum, nil +} + +// withoutPath keeps the database location, which names the local user and +// folder layout, out of the sync history stored in the archive. The error +// returned to the caller still names it. +func withoutPath(message, dbPath string) string { + if abs, err := filepath.Abs(dbPath); err == nil && abs != "" { + message = strings.ReplaceAll(message, abs, "") + } + if dbPath != "" { + message = strings.ReplaceAll(message, dbPath, "") + } + return message +} diff --git a/internal/muesli/importer_test.go b/internal/muesli/importer_test.go new file mode 100644 index 000000000..d1fc0b0d9 --- /dev/null +++ b/internal/muesli/importer_test.go @@ -0,0 +1,298 @@ +package muesli + +import ( + "context" + "database/sql" + "encoding/json/v2" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type importerFixture struct { + st *store.Store + muesli *sql.DB + path string + source *store.Source + imp *Importer +} + +func newImporterFixture(t *testing.T) *importerFixture { + t.Helper() + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource(SourceType, "mac") + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "muesli.db") + return &importerFixture{ + st: st, muesli: newFixtureDB(t, path, currentSchemaDDL), path: path, + source: source, imp: NewImporter(st), + } +} + +func (f *importerFixture) run(t *testing.T, opts ImportOptions) (*ImportSummary, error) { + t.Helper() + if opts.Identifier == "" { + opts.Identifier = "mac" + } + if opts.AccountEmail == "" { + opts.AccountEmail = "you@example.com" + } + if opts.DBPath == "" { + opts.DBPath = f.path + } + return f.imp.Import(context.Background(), opts) +} + +func (f *importerFixture) body(t *testing.T, sourceMessageID string) string { + t.Helper() + var messageID int64 + require.NoError(t, f.st.DB().QueryRow(f.st.Rebind( + `SELECT id FROM messages WHERE source_id = ? AND source_message_id = ?`), + f.source.ID, sourceMessageID).Scan(&messageID)) + body, err := f.st.GetMessageBodyText(messageID) + require.NoError(t, err) + return body +} + +func (f *importerFixture) messageCount(t *testing.T) int { + t.Helper() + var count int + require.NoError(t, f.st.DB().QueryRow(f.st.Rebind( + `SELECT count(*) FROM messages WHERE source_id = ?`), f.source.ID).Scan(&count)) + return count +} + +func TestImportArchivesMeetingsAndIsIdempotent(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + first := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "title": "Planning", "created_at": "2026-09-02 09:00:00", "start_time": "2026-09-02T09:00:00Z", + })) + insertRow(t, f.muesli, "meeting_participants", map[string]any{ + "meeting_id": first, "participant_identifier": "email:alice@example.com", + "display_name": "Alice Example", "email_address": "alice@example.com", + "insertion_order": 0, "source": "calendar", + }) + + summary, err := f.run(t, ImportOptions{}) + require.NoError(err) + assert.Equal(int64(2), summary.MeetingsProcessed) + assert.Equal(int64(2), summary.MeetingsAdded) + assert.Equal(2, f.messageCount(t)) + + var messageType string + var fromMe bool + require.NoError(f.st.DB().QueryRow(f.st.Rebind(` + SELECT message_type, is_from_me FROM messages + WHERE source_id = ? AND source_message_id = ?`), + f.source.ID, "meeting:1:20260901T140003Z").Scan(&messageType, &fromMe)) + assert.Equal("meeting_transcript", messageType) + assert.True(fromMe, "the configured account recorded the meeting") + var recipients int + require.NoError(f.st.DB().QueryRow(f.st.Rebind(` + SELECT count(*) FROM message_recipients mr + JOIN participants p ON p.id = mr.participant_id + JOIN messages m ON m.id = mr.message_id + WHERE m.source_id = ? AND mr.recipient_type = 'to' AND p.email_address = ?`), + f.source.ID, "alice@example.com").Scan(&recipients)) + assert.Equal(1, recipients) + + again, err := f.run(t, ImportOptions{}) + require.NoError(err) + assert.Equal(int64(2), again.MeetingsProcessed) + assert.Equal(int64(0), again.MeetingsAdded) + assert.Equal(int64(0), again.MeetingsUpdated) +} + +func TestImportUpdatesEditedMeetingInPlace(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + id := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + _, err := f.run(t, ImportOptions{}) + require.NoError(err) + + _, err = f.muesli.Exec(`UPDATE meetings SET raw_transcript = ? WHERE id = ?`, + "[10:00:01] You: revised words", id) + require.NoError(err) + insertRow(t, f.muesli, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "contact:XYZ", + "display_name": "Carol Example", "insertion_order": 0, + }) + summary, err := f.run(t, ImportOptions{}) + require.NoError(err) + + assert.Equal(int64(0), summary.MeetingsAdded) + assert.Equal(int64(1), summary.MeetingsUpdated) + assert.Equal(1, f.messageCount(t)) + body := f.body(t, "meeting:1:20260901T140003Z") + assert.Contains(body, "revised words") + assert.Contains(body, "Attendees: Carol Example") +} + +func TestImportSkipsDeletedInProgressAndEmptyMeetings(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + archived := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "created_at": "2026-09-01 15:00:00", "deleted_at": 1788271503.0, + })) + recording := insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "created_at": "2026-09-01 16:00:00", "meeting_status": "recording", + })) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "created_at": "2026-09-01 17:00:00", "raw_transcript": "", "formatted_notes": "", "manual_notes": "", + })) + + summary, err := f.run(t, ImportOptions{}) + require.NoError(err) + assert.Equal(int64(1), summary.MeetingsAdded) + assert.Equal(int64(1), summary.SkippedDeleted) + assert.Equal(int64(1), summary.SkippedInProgress) + assert.Equal(int64(1), summary.SkippedEmpty) + + // Muesli's soft delete blanks the row; the archived copy must survive. + before := f.body(t, "meeting:1:20260901T140003Z") + _, err = f.muesli.Exec(`UPDATE meetings SET title = 'Deleted Meeting', raw_transcript = NULL, + formatted_notes = NULL, manual_notes = '', deleted_at = 1788271600 WHERE id = ?`, archived) + require.NoError(err) + _, err = f.muesli.Exec(`UPDATE meetings SET meeting_status = 'completed' WHERE id = ?`, recording) + require.NoError(err) + + next, err := f.run(t, ImportOptions{}) + require.NoError(err) + assert.Equal(int64(1), next.MeetingsAdded, "the finished recording is archived") + assert.Equal(int64(2), next.SkippedDeleted) + assert.Equal(before, f.body(t, "meeting:1:20260901T140003Z")) +} + +func TestImportDatabaseResetDoesNotOverwriteArchivedMeetings(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{"title": "Before reset"})) + _, err := f.run(t, ImportOptions{}) + require.NoError(err) + + resetPath := filepath.Join(t.TempDir(), "muesli.db") + reset := newFixtureDB(t, resetPath, currentSchemaDDL) + insertRow(t, reset, "meetings", completedMeeting(map[string]any{ + "title": "After reset", "created_at": "2026-10-01 08:00:00", + })) + summary, err := f.run(t, ImportOptions{DBPath: resetPath}) + require.NoError(err) + + assert.Equal(int64(1), summary.MeetingsAdded) + assert.Equal(2, f.messageCount(t)) + assert.Contains(f.body(t, "meeting:1:20260901T140003Z"), "Before reset") + assert.Contains(f.body(t, "meeting:1:20261001T080000Z"), "After reset") +} + +func TestImportCountsUnkeyableMeetingsAndContinues(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{"created_at": nil})) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{"created_at": "2026-09-02 09:00:00"})) + + summary, err := f.run(t, ImportOptions{}) + + require.Error(err) + assert.Contains(err.Error(), "muesli meeting 1") + assert.Equal(int64(1), summary.MeetingsAdded) + assert.Equal(int64(1), summary.Errors) + run, runErr := f.st.GetLatestSync(f.source.ID) + require.NoError(runErr) + assert.Equal(store.SyncStatusFailed, run.Status) + assert.Equal(int64(1), run.MessagesAdded) + assert.Equal(int64(1), run.ErrorsCount) +} + +func TestImportLimitAndStartedAfter(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "created_at": "2026-09-10 09:00:00", "start_time": "2026-09-10T09:00:00Z", + })) + insertRow(t, f.muesli, "meetings", completedMeeting(map[string]any{ + "created_at": "2026-09-11 09:00:00", "start_time": "2026-09-11T09:00:00Z", + })) + + limited, err := f.run(t, ImportOptions{Limit: 1}) + require.NoError(err) + assert.Equal(int64(1), limited.MeetingsProcessed) + + after, err := f.run(t, ImportOptions{StartedAfter: time.Date(2026, 9, 10, 0, 0, 0, 0, time.UTC)}) + require.NoError(err) + assert.Equal(int64(2), after.MeetingsProcessed) + assert.Equal(int64(2), after.MeetingsAdded) +} + +func TestImportFullRewritesUnchangedMeetings(t *testing.T) { + require := require.New(t) + f := newImporterFixture(t) + insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + _, err := f.run(t, ImportOptions{}) + require.NoError(err) + + summary, err := f.run(t, ImportOptions{Full: true}) + require.NoError(err) + + assert.Equal(t, int64(1), summary.MeetingsUpdated) +} + +func TestImportRecordsCursorWithoutDatabasePath(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + + _, err := f.run(t, ImportOptions{}) + require.NoError(err) + + run, err := f.st.GetLastSuccessfulSync(f.source.ID) + require.NoError(err) + require.True(run.CursorAfter.Valid) + var cursor map[string]any + require.NoError(json.Unmarshal([]byte(run.CursorAfter.String), &cursor)) + assert.InDelta(float64(1), cursor["version"], 0) + assert.NotEmpty(cursor["scanned_at"]) + assert.NotContains(run.CursorAfter.String, "muesli.db") +} + +func TestImportRequiresRegisteredSource(t *testing.T) { + f := newImporterFixture(t) + + _, err := f.run(t, ImportOptions{Identifier: "laptop"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), `muesli source "laptop" is not registered`) +} + +func TestImportRecordsFailedSyncWhenDatabaseCannotOpen(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + missing := filepath.Join(t.TempDir(), "private-folder", "muesli.db") + + _, err := f.run(t, ImportOptions{DBPath: missing}) + + require.Error(err) + assert.Contains(err.Error(), missing, "the CLI error names the path it tried") + run, runErr := f.st.GetLatestSync(f.source.ID) + require.NoError(runErr) + assert.Equal(store.SyncStatusFailed, run.Status) + require.True(run.ErrorMessage.Valid) + assert.NotContains(run.ErrorMessage.String, "private-folder", "the archive does not keep local paths") + assert.Contains(run.ErrorMessage.String, "db_path") +} diff --git a/internal/muesli/metadata_test.go b/internal/muesli/metadata_test.go new file mode 100644 index 000000000..c8919dd2e --- /dev/null +++ b/internal/muesli/metadata_test.go @@ -0,0 +1,29 @@ +package muesli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContactsMetadataRecoversWithoutMeetingEdits(t *testing.T) { + assert := assert.New(t) + f := newResolveFixture(t, fixtureCard{ + uniqueID: "CARD-METADATA:ABPerson", phones: []string{"+16045550100"}, + }) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-METADATA:ABPerson", "Alex Example") + f.sync(t, ImportOptions{}) + + f.sync(t, ImportOptions{ContactsPath: filepath.Join(t.TempDir(), "missing")}) + metadata := messageMetadata(t, f) + assert.Equal("unavailable", metadata["contacts"]) + assert.InDelta(float64(1), metadata["contacts_carried_forward"], 0) + + recovered := f.sync(t, ImportOptions{}) + assert.Equal(ContactsComplete, recovered.ContactsState) + metadata = messageMetadata(t, f) + assert.Equal("complete", metadata["contacts"]) + assert.NotContains(metadata, "contacts_carried_forward") +} diff --git a/internal/muesli/models.go b/internal/muesli/models.go new file mode 100644 index 000000000..48835845f --- /dev/null +++ b/internal/muesli/models.go @@ -0,0 +1,60 @@ +// Package muesli archives meetings recorded by the Muesli macOS app +// (github.com/Muesli-HQ/muesli) by reading its local SQLite database +// read-only. +package muesli + +const ( + SourceType = "muesli" + RawFormat = "muesli_json" +) + +// Participant is one person Muesli attached to a meeting, either from the +// calendar event or picked manually from Contacts. Muesli's Contacts and +// calendar identifiers are deliberately not carried. +type Participant struct { + Name string + Email string // lowercased; empty when Muesli has none + Source string // "calendar", "manual", or empty on old databases + // Identifier is Muesli's participant_identifier ("email:…", "contact:…", + // "calendar:…"). It stays in memory; raw evidence carries only a hash. + Identifier string + // ContactEmails and ContactPhones (E.164) come from the Mac's Contacts + // card for this participant; Anchor links them. Resolution is "resolved" + // or "carried_forward" when they are set. + ContactEmails []string + ContactPhones []string + // LinkExcludedAddresses remain meeting evidence but cannot assert ownership. + LinkExcludedAddresses []string + Anchor string + Resolution string + // SkippedPhones counts Contacts phones that could not become E.164. + SkippedPhones int +} + +// Meeting is one row of Muesli's meetings table. Text columns hold Muesli's +// values verbatim; columns an older database lacks are left empty. +type Meeting struct { + ID int64 + Title string + StartTime string + EndTime string + CreatedAt string + DurationSeconds float64 + Status string + Source string + RawTranscript string + FormattedNotes string + ManualNotes string + WordCount int64 + TemplateName string + TemplateKind string + CalendarEventID string + CalendarSource string + CalendarSeriesID string + Folder string // folder path such as "Clients/Acme" + FollowUpToID int64 + Deleted bool + Participants []Participant // not suppressed, in Muesli's display order + // ContactsState is how much of Contacts the sync could read. + ContactsState ContactsState +} diff --git a/internal/muesli/person_link_test.go b/internal/muesli/person_link_test.go new file mode 100644 index 000000000..785810d0c --- /dev/null +++ b/internal/muesli/person_link_test.go @@ -0,0 +1,53 @@ +package muesli + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/activity" + "go.kenn.io/msgvault/internal/store" +) + +// TestImportedMeetingReachesExistingPerson proves the end-to-end contract a +// user relies on: a Muesli attendee whose email already belongs to a person +// shows up as that person's meeting after activity projection, without any +// name matching. +func TestImportedMeetingReachesExistingPerson(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newImporterFixture(t) + ctx := t.Context() + + participantID, err := f.st.EnsureParticipant("alice@example.com", "Alice Example", "example.com") + require.NoError(err) + person, _, err := f.st.CreatePersonFromParticipantContext(ctx, participantID) + require.NoError(err) + + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + insertRow(t, f.muesli, "meeting_participants", map[string]any{ + "meeting_id": meetingID, "participant_identifier": "email:alice@example.com", + "display_name": "Alice (Contacts)", "email_address": "ALICE@example.com", + "insertion_order": 0, "source": "manual", + }) + _, err = f.run(t, ImportOptions{}) + require.NoError(err) + + projector, err := activity.NewProjector(f.st, activity.Options{ + Timezone: "UTC", BatchSize: 10, MaxDirectCounterparts: 25, + }) + require.NoError(err) + _, err = projector.RunOnce(ctx) + require.NoError(err) + + var role, refKind string + require.NoError(f.st.DB().QueryRow(f.st.Rebind(` + SELECT aep.role, ae.ref_kind + FROM activity_event_persons aep + JOIN activity_events ae ON ae.message_id = aep.message_id + JOIN messages m ON m.id = aep.message_id + WHERE m.source_id = ? AND aep.person_id = ?`), + f.source.ID, person.ID).Scan(&role, &refKind)) + assert.Equal(string(store.RoleAttendee), role) + assert.Equal(string(store.RefKindMeeting), refKind) +} diff --git a/internal/muesli/phone.go b/internal/muesli/phone.go new file mode 100644 index 000000000..1feb61347 --- /dev/null +++ b/internal/muesli/phone.go @@ -0,0 +1,58 @@ +package muesli + +import "strings" + +// NormalizeContactPhone converts a phone number as typed in Contacts to E.164. +// Numbers written with + or 00 always convert. A national number converts only +// when countryCode (for example "1" or "44") is configured; msgvault never +// guesses a country, because a wrong one could match somebody else's phone. +// An extension is dropped. The result has 7 to 15 digits. +func NormalizeContactPhone(raw, countryCode string) (string, bool) { + // "+44 (0)20 …" marks a trunk zero that is dialed only nationally. + value := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(raw)), "(0)", "") + for _, marker := range []string{"ext", "x", ";", ","} { + if index := strings.Index(value, marker); index > 0 { + value = strings.TrimSpace(value[:index]) + } + } + var digits strings.Builder + for i, r := range value { + switch { + case r >= '0' && r <= '9': + digits.WriteRune(r) + case r == '+' && i == 0: + case strings.ContainsRune(" -.()/", r): + default: + return "", false + } + } + number := digits.String() + switch { + case strings.HasPrefix(value, "+") && strings.HasPrefix(number, "0"): + return "", false + case strings.HasPrefix(value, "+"): + case strings.HasPrefix(number, "00"): + number = number[2:] + case countryCode == "": + return "", false + case countryCode == "1": + switch { + case len(number) == 10: + number = "1" + number + case len(number) == 11 && number[0] == '1': + default: + return "", false + } + default: + // Most countries drop a national trunk 0 after the country code; Italy + // keeps it. + if countryCode != "39" { + number = strings.TrimPrefix(number, "0") + } + number = countryCode + number + } + if len(number) < 7 || len(number) > 15 { + return "", false + } + return "+" + number, true +} diff --git a/internal/muesli/reader.go b/internal/muesli/reader.go new file mode 100644 index 000000000..4724b6b86 --- /dev/null +++ b/internal/muesli/reader.go @@ -0,0 +1,333 @@ +package muesli + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + _ "github.com/mattn/go-sqlite3" // registers the "sqlite3" database/sql driver +) + +// maxFolderDepth bounds folder-path resolution so a corrupt parent cycle +// cannot loop forever. +const maxFolderDepth = 32 + +// Reader reads meetings from a Muesli database without changing it. Muesli +// runs its schema migration and tombstone purge whenever it (or muesli-cli) +// opens the database, so msgvault reads the file directly instead. +type Reader struct { + db *sql.DB + path string + columns map[string]map[string]bool +} + +// Open opens the Muesli database at path and checks that it looks like a +// Muesli database. SQLite may use the WAL sidecar files to coordinate with a +// running Muesli app; query_only prevents this connection from changing data. +func Open(ctx context.Context, path string) (*Reader, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("open Muesli database %s: %w", path, err) + } + if _, err := os.Stat(abs); err != nil { + return nil, fmt.Errorf("open Muesli database %s: %w", abs, err) + } + db, err := openQueryOnly(abs) + if err != nil { + return nil, fmt.Errorf("open Muesli database %s: %w", abs, err) + } + reader := &Reader{db: db, path: abs} + if err := reader.inspect(ctx); err != nil { + _ = db.Close() + return nil, err + } + return reader, nil +} + +// openQueryOnly opens another app's live SQLite store without changing it. +// url.URL escapes '#', '?', and '%' in the path so they cannot end the +// filename early. mode=ro prevents checkpointing on close; query_only rejects +// write SQL. SQLite can still manage WAL sidecars. immutable=1 would ignore +// rows the owning app has committed to the WAL. +func openQueryOnly(abs string) (*sql.DB, error) { + dsn := (&url.URL{ + Scheme: "file", + Path: sqliteURIPath(abs), + RawQuery: "mode=ro&_busy_timeout=5000&_query_only=1", + }).String() + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + return db, nil +} + +// sqliteURIPath returns an absolute path in the form expected in a file URI. +// Windows drive letters need a leading slash so url.URL produces file:///C:/… +// instead of treating C: as the URI authority. +func sqliteURIPath(path string) string { + slashed := filepath.ToSlash(path) + volume := filepath.VolumeName(path) + if len(volume) == 2 && volume[1] == ':' { + return "/" + slashed + } + return slashed +} + +// Close releases the database handle. +func (r *Reader) Close() error { + if r == nil || r.db == nil { + return nil + } + return r.db.Close() +} + +func (r *Reader) inspect(ctx context.Context) error { + if err := r.db.PingContext(ctx); err != nil { + return fmt.Errorf("open Muesli database %s: %w", r.path, err) + } + r.columns = map[string]map[string]bool{} + for _, table := range []string{"meetings", "meeting_participants", "meeting_folders", "meeting_participant_suppressions"} { + columns, err := tableColumns(ctx, r.db, table) + if err != nil { + return fmt.Errorf("inspect Muesli database %s: %w", r.path, err) + } + r.columns[table] = columns + } + for _, required := range []string{"id", "title", "start_time"} { + if !r.columns["meetings"][required] { + return fmt.Errorf("%s is not a Muesli database: meetings.%s is missing", r.path, required) + } + } + return nil +} + +// tableColumns returns the column names of table, or an empty set when the +// table does not exist. +func tableColumns(ctx context.Context, db *sql.DB, table string) (map[string]bool, error) { + rows, err := db.QueryContext(ctx, `SELECT name FROM pragma_table_info(?)`, table) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + columns := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + columns[name] = true + } + return columns, rows.Err() +} + +func (r *Reader) has(table, column string) bool { + return r.columns[table][column] +} + +// column selects alias.column when the table has it and NULL otherwise, so +// the same query works across Muesli schema generations. +func (r *Reader) column(table, alias, column string) string { + if r.has(table, column) { + return alias + "." + column + } + return "NULL" +} + +// ListMeetings returns every meeting in ascending id order, including +// deleted and in-progress rows; callers decide what to archive. All reads +// share one transaction so they see a single WAL snapshot. +func (r *Reader) ListMeetings(ctx context.Context) ([]Meeting, error) { + tx, err := r.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, fmt.Errorf("read Muesli meetings: %w", err) + } + defer func() { _ = tx.Rollback() }() + + folders, err := r.folderPaths(ctx, tx) + if err != nil { + return nil, err + } + meetings, err := r.meetings(ctx, tx, folders) + if err != nil { + return nil, err + } + participants, err := r.participants(ctx, tx) + if err != nil { + return nil, err + } + for i := range meetings { + meetings[i].Participants = participants[meetings[i].ID] + } + return meetings, nil +} + +func (r *Reader) meetings(ctx context.Context, tx *sql.Tx, folders map[int64]string) ([]Meeting, error) { + col := func(name string) string { return r.column("meetings", "m", name) } + query := `SELECT m.id, m.title, m.start_time, ` + + col("end_time") + `, ` + + col("created_at") + `, ` + + col("duration_seconds") + `, ` + + col("meeting_status") + `, ` + + col("source") + `, ` + + col("raw_transcript") + `, ` + + col("formatted_notes") + `, ` + + col("manual_notes") + `, ` + + col("word_count") + `, ` + + col("selected_template_name") + `, ` + + col("selected_template_kind") + `, ` + + col("calendar_event_id") + `, ` + + col("calendar_source") + `, ` + + col("calendar_series_id") + `, ` + + col("folder_id") + `, ` + + col("follow_up_to_id") + `, ` + + col("deleted_at") + ` IS NOT NULL + FROM meetings m ORDER BY m.id` + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("read Muesli meetings: %w", err) + } + defer func() { _ = rows.Close() }() + + var meetings []Meeting + for rows.Next() { + var ( + m Meeting + title, start sql.NullString + end, created, status, source, transcript, notes sql.NullString + manual, templateName, templateKind, eventID sql.NullString + calendarSource, seriesID sql.NullString + duration sql.NullFloat64 + wordCount, folderID, followUpID sql.NullInt64 + ) + if err := rows.Scan(&m.ID, &title, &start, &end, &created, &duration, &status, &source, + &transcript, ¬es, &manual, &wordCount, &templateName, &templateKind, + &eventID, &calendarSource, &seriesID, &folderID, &followUpID, &m.Deleted); err != nil { + return nil, fmt.Errorf("read Muesli meeting row: %w", err) + } + m.Title, m.StartTime, m.EndTime, m.CreatedAt = title.String, start.String, end.String, created.String + m.DurationSeconds, m.Status, m.Source = duration.Float64, status.String, source.String + m.RawTranscript, m.FormattedNotes, m.ManualNotes = transcript.String, notes.String, manual.String + m.WordCount, m.TemplateName, m.TemplateKind = wordCount.Int64, templateName.String, templateKind.String + m.CalendarEventID, m.CalendarSource, m.CalendarSeriesID = eventID.String, calendarSource.String, seriesID.String + m.FollowUpToID = followUpID.Int64 + if folderID.Valid { + m.Folder = folders[folderID.Int64] + } + meetings = append(meetings, m) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read Muesli meetings: %w", err) + } + return meetings, nil +} + +// participants returns non-suppressed participants keyed by meeting id, in +// Muesli's display order. Older databases kept suppressions in a separate +// table that Muesli later folded into is_suppressed. +func (r *Reader) participants(ctx context.Context, tx *sql.Tx) (map[int64][]Participant, error) { + out := map[int64][]Participant{} + if !r.has("meeting_participants", "meeting_id") { + return out, nil + } + suppressed := "1 = 1" + switch { + case r.has("meeting_participants", "is_suppressed"): + suppressed = "COALESCE(p.is_suppressed, 0) = 0" + case r.has("meeting_participant_suppressions", "participant_identifier"): + suppressed = `NOT EXISTS ( + SELECT 1 FROM meeting_participant_suppressions s + WHERE s.meeting_id = p.meeting_id + AND s.participant_identifier = p.participant_identifier)` + } + query := `SELECT p.meeting_id, p.participant_identifier, p.display_name, ` + + r.column("meeting_participants", "p", "email_address") + `, ` + + r.column("meeting_participants", "p", "source") + ` + FROM meeting_participants p + WHERE ` + suppressed + ` + ORDER BY p.meeting_id, p.insertion_order, p.participant_identifier` + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("read Muesli participants: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var ( + meetingID int64 + identifier, name sql.NullString + emailAddress, participantOf sql.NullString + ) + if err := rows.Scan(&meetingID, &identifier, &name, &emailAddress, &participantOf); err != nil { + return nil, fmt.Errorf("read Muesli participant row: %w", err) + } + email := strings.ToLower(strings.TrimSpace(emailAddress.String)) + if email == "" { + if rest, ok := strings.CutPrefix(identifier.String, "email:"); ok { + email = strings.ToLower(strings.TrimSpace(rest)) + } + } + out[meetingID] = append(out[meetingID], Participant{ + Name: strings.TrimSpace(name.String), + Email: email, + Source: strings.TrimSpace(participantOf.String), + Identifier: strings.TrimSpace(identifier.String), + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read Muesli participants: %w", err) + } + return out, nil +} + +// folderPaths resolves every folder to its "Parent/Child" path. +func (r *Reader) folderPaths(ctx context.Context, tx *sql.Tx) (map[int64]string, error) { + paths := map[int64]string{} + if !r.has("meeting_folders", "id") || !r.has("meeting_folders", "name") { + return paths, nil + } + query := `SELECT f.id, f.name, ` + r.column("meeting_folders", "f", "parent_id") + + ` FROM meeting_folders f` + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("read Muesli folders: %w", err) + } + defer func() { _ = rows.Close() }() + type folder struct { + name string + parent sql.NullInt64 + } + folders := map[int64]folder{} + for rows.Next() { + var ( + id int64 + name sql.NullString + parent sql.NullInt64 + ) + if err := rows.Scan(&id, &name, &parent); err != nil { + return nil, fmt.Errorf("read Muesli folder row: %w", err) + } + folders[id] = folder{name: strings.TrimSpace(name.String), parent: parent} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read Muesli folders: %w", err) + } + for id := range folders { + var parts []string + current, ok := folders[id], true + for depth := 0; ok && depth < maxFolderDepth; depth++ { + parts = append([]string{current.name}, parts...) + if !current.parent.Valid { + break + } + current, ok = folders[current.parent.Int64] + } + paths[id] = strings.Join(parts, "/") + } + return paths, nil +} diff --git a/internal/muesli/reader_test.go b/internal/muesli/reader_test.go new file mode 100644 index 000000000..8306ee051 --- /dev/null +++ b/internal/muesli/reader_test.go @@ -0,0 +1,263 @@ +package muesli + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReaderReadsCurrentSchema(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + path := filepath.Join(t.TempDir(), "muesli.db") + db := newFixtureDB(t, path, currentSchemaDDL) + clients := insertRow(t, db, "meeting_folders", map[string]any{"name": "Clients"}) + acme := insertRow(t, db, "meeting_folders", map[string]any{"name": "Acme", "parent_id": clients}) + id := insertRow(t, db, "meetings", completedMeeting(map[string]any{ + "folder_id": acme, + "calendar_event_id": "event-1", + "calendar_source": "eventKit", + "calendar_series_id": "series-1", + "selected_template_name": "Auto", + "selected_template_kind": "auto", + "source": "meeting", + "updated_at": 1788271503.0, + })) + followUp := insertRow(t, db, "meetings", completedMeeting(map[string]any{ + "title": "Follow-up", "follow_up_to_id": id, "created_at": "2026-09-02 09:00:00", + })) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "email:alice@example.com", + "display_name": "Alice Example", "email_address": "Alice@Example.com", + "insertion_order": 1, "source": "calendar", + }) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "contact:ABC-123", + "display_name": "Carol Example", "insertion_order": 0, "source": "manual", + }) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "email:bob@example.com", + "display_name": "Bob Example", "email_address": "bob@example.com", + "insertion_order": 2, "source": "calendar", "is_suppressed": 1, + }) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": followUp, "participant_identifier": "email:dana@example.com", + "display_name": " Dana Example ", "insertion_order": 0, "source": "manual", + }) + var beforeCount int + var beforeUpdated float64 + require.NoError(db.QueryRow(`SELECT count(*), max(updated_at) FROM meetings`).Scan(&beforeCount, &beforeUpdated)) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + + require.Len(meetings, 2) + assert.Equal(Meeting{ + ID: id, Title: "Weekly sync", + StartTime: "2026-09-01T14:00:00Z", EndTime: "2026-09-01T14:45:00Z", + CreatedAt: "2026-09-01 14:00:03", DurationSeconds: 2700, + Status: "completed", Source: "meeting", + RawTranscript: "[10:00:01] You: hello\n[10:00:04] Speaker 1: hi there", + FormattedNotes: "## Decisions\nShip it", ManualNotes: "typed note", WordCount: 5, + TemplateName: "Auto", TemplateKind: "auto", + CalendarEventID: "event-1", CalendarSource: "eventKit", CalendarSeriesID: "series-1", + Folder: "Clients/Acme", + Participants: []Participant{ + {Name: "Carol Example", Source: "manual", Identifier: "contact:ABC-123"}, + {Name: "Alice Example", Email: "alice@example.com", Source: "calendar", Identifier: "email:alice@example.com"}, + }, + }, meetings[0]) + assert.Equal(id, meetings[1].FollowUpToID) + assert.Equal([]Participant{{Name: "Dana Example", Email: "dana@example.com", Source: "manual", Identifier: "email:dana@example.com"}}, + meetings[1].Participants, "an email identifier supplies a missing email address") + + var afterCount int + var afterUpdated float64 + require.NoError(db.QueryRow(`SELECT count(*), max(updated_at) FROM meetings`).Scan(&afterCount, &afterUpdated)) + assert.Equal(beforeCount, afterCount) + assert.InDelta(beforeUpdated, afterUpdated, 0) +} + +func TestReaderReadsLegacySchema(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + path := filepath.Join(t.TempDir(), "muesli.db") + db := newFixtureDB(t, path, legacySchemaDDL) + id := insertRow(t, db, "meetings", map[string]any{ + "title": "Old meeting", "start_time": "2025-01-02T03:04:05Z", + "raw_transcript": "[03:04:06] You: hi", "created_at": "2025-01-02 03:04:05", + }) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "email:alice@example.com", + "display_name": "Alice Example", "email_address": "alice@example.com", "insertion_order": 0, + }) + insertRow(t, db, "meeting_participants", map[string]any{ + "meeting_id": id, "participant_identifier": "email:hidden@example.com", + "display_name": "Hidden Example", "email_address": "hidden@example.com", "insertion_order": 1, + }) + insertRow(t, db, "meeting_participant_suppressions", map[string]any{ + "meeting_id": id, "participant_identifier": "email:hidden@example.com", + }) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + + require.Len(meetings, 1) + got := meetings[0] + assert.Equal("Old meeting", got.Title) + assert.Empty(got.Status) + assert.Empty(got.ManualNotes) + assert.Empty(got.Folder) + assert.False(got.Deleted) + assert.Equal([]Participant{{Name: "Alice Example", Email: "alice@example.com", Identifier: "email:alice@example.com"}}, got.Participants) +} + +func TestReaderMarksDeletedMeetings(t *testing.T) { + require := require.New(t) + path := filepath.Join(t.TempDir(), "muesli.db") + db := newFixtureDB(t, path, currentSchemaDDL) + insertRow(t, db, "meetings", completedMeeting(map[string]any{"deleted_at": 1788271503.0})) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + + require.Len(meetings, 1) + assert.True(t, meetings[0].Deleted) +} + +func TestReaderReadsCommittedRowsStillInTheWAL(t *testing.T) { + require := require.New(t) + path := filepath.Join(t.TempDir(), "muesli.db") + db := newFixtureDB(t, path, currentSchemaDDL) + _, err := db.Exec(`PRAGMA wal_autocheckpoint=0`) + require.NoError(err) + insertRow(t, db, "meetings", completedMeeting(map[string]any{"title": "Only in WAL"})) + walInfo, err := os.Stat(path + "-wal") + require.NoError(err) + require.Positive(walInfo.Size(), "the fixture row must still be in the WAL") + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + + require.Len(meetings, 1) + assert.Equal(t, "Only in WAL", meetings[0].Title) +} + +func TestReaderOpensWALWithoutSharedMemorySidecar(t *testing.T) { + require := require.New(t) + const writerDBEnv = "MSGVAULT_TEST_WAL_WRITER_DB" + if path := os.Getenv(writerDBEnv); path != "" { + db := newFixtureDB(t, path, currentSchemaDDL) + _, err := db.Exec(`PRAGMA wal_autocheckpoint=0`) + require.NoError(err) + insertRow(t, db, "meetings", completedMeeting(map[string]any{"title": "Only in WAL"})) + // Exit without closing the database so the committed WAL remains for the + // parent process to open after removing the shared-memory sidecar. + os.Exit(0) + } + + path := filepath.Join(t.TempDir(), "muesli.db") + executable, err := os.Executable() + require.NoError(err) + cmd := exec.Command(executable, "-test.run=^TestReaderOpensWALWithoutSharedMemorySidecar$") + cmd.Env = append(os.Environ(), writerDBEnv+"="+path) + output, err := cmd.CombinedOutput() + require.NoError(err, string(output)) + require.FileExists(path + "-wal") + require.FileExists(path + "-shm") + require.NoError(os.Remove(path + "-shm")) + before, err := os.ReadFile(path) + require.NoError(err) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + require.Len(meetings, 1) + assert.Equal(t, "Only in WAL", meetings[0].Title) + require.NoError(reader.Close()) + after, err := os.ReadFile(path) + require.NoError(err) + assert.True(t, bytes.Equal(before, after), "closing the reader must not checkpoint the source database") +} + +func TestReaderConnectionRejectsWriteQueries(t *testing.T) { + require := require.New(t) + path := filepath.Join(t.TempDir(), "muesli.db") + db := newFixtureDB(t, path, currentSchemaDDL) + insertRow(t, db, "meetings", completedMeeting(nil)) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + _, err = reader.db.ExecContext(context.Background(), `UPDATE meetings SET title = 'Changed'`) + require.Error(err) + + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + require.Len(meetings, 1) + assert.Equal(t, "Weekly sync", meetings[0].Title) +} + +func TestReaderOpensPathsWithURISpecialCharacters(t *testing.T) { + require := require.New(t) + dir := filepath.Join(t.TempDir(), "Application Support #1%41") + require.NoError(os.MkdirAll(dir, 0o755)) + path := filepath.Join(dir, "muesli.db") + db := newFixtureDB(t, path, currentSchemaDDL) + insertRow(t, db, "meetings", completedMeeting(nil)) + + reader, err := Open(context.Background(), path) + require.NoError(err) + t.Cleanup(func() { _ = reader.Close() }) + meetings, err := reader.ListMeetings(context.Background()) + require.NoError(err) + + assert.Len(t, meetings, 1) +} + +func TestReaderRejectsMissingOrForeignDatabases(t *testing.T) { + dir := t.TempDir() + foreign := filepath.Join(dir, "foreign.db") + newFixtureDB(t, foreign, `CREATE TABLE notes (id INTEGER PRIMARY KEY)`) + incomplete := filepath.Join(dir, "incomplete.db") + newFixtureDB(t, incomplete, `CREATE TABLE meetings (id INTEGER PRIMARY KEY, title TEXT)`) + + for _, tt := range []struct { + name string + path string + want string + }{ + {name: "missing file", path: filepath.Join(dir, "absent.db"), want: "open Muesli database"}, + {name: "no meetings table", path: foreign, want: "not a Muesli database"}, + {name: "missing start_time", path: incomplete, want: "not a Muesli database"}, + } { + t.Run(tt.name, func(t *testing.T) { + reader, err := Open(context.Background(), tt.path) + if reader != nil { + _ = reader.Close() + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.want) + }) + } +} diff --git a/internal/muesli/resolve.go b/internal/muesli/resolve.go new file mode 100644 index 000000000..cecc7ebe2 --- /dev/null +++ b/internal/muesli/resolve.go @@ -0,0 +1,174 @@ +package muesli + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json/v2" + "fmt" + "slices" + "strings" + + "go.kenn.io/msgvault/internal/meetingarchive" +) + +const ( + resolutionResolved = "resolved" + resolutionCarried = "carried_forward" +) + +// participantRef hashes Muesli's participant identifier into a stable, +// non-reversible key for carrying identities forward between syncs. +func participantRef(identifier string) string { + if identifier == "" { + return "" + } + sum := sha256.Sum256([]byte("muesli-participant:" + identifier)) + return hex.EncodeToString(sum[:])[:16] +} + +// archivePerson turns a participant and its Contacts identities into an +// archive identity set. Muesli's own email stays primary. +func (p Participant) archivePerson() meetingarchive.Person { + var emails []string + for _, email := range append([]string{p.Email}, p.ContactEmails...) { + if email != "" && !slices.Contains(emails, email) { + emails = append(emails, email) + } + } + person := meetingarchive.Person{ + Name: p.Name, Anchor: p.Anchor, LinkExcludedAddresses: p.LinkExcludedAddresses, + } + phones := slices.Clone(p.ContactPhones) + if len(emails) > 0 { + person.Email = emails[0] + person.OtherEmails = emails[1:] + } else if len(phones) > 0 { + person.Phone = phones[0] + phones = phones[1:] + } + person.OtherPhones = phones + return person.Normalized() +} + +func (p Participant) raw(person meetingarchive.Person) rawParticipant { + raw := rawParticipant{ + Ref: participantRef(p.Identifier), Name: p.Name, Email: p.Email, Source: p.Source, + } + if p.Resolution == "" { + return raw + } + if person.Email == "" { + raw.Phone = person.Phone + } + raw.Emails = slices.Clone(p.ContactEmails) + raw.Phones = slices.Clone(p.ContactPhones) + return raw +} + +// resolveParticipants fills each participant's Contacts identities. A still +// present participant whose card cannot be resolved keeps its earlier archived +// identities, without asserting current ownership of those addresses. +func (imp *Importer) resolveParticipants( + sourceID int64, meeting *Meeting, contacts *Contacts, countryCode string, sharedAddresses map[string]bool, +) error { + state := contacts.State() + meeting.ContactsState = state + if state == ContactsOff { + return nil + } + var previous map[string]rawParticipant + loaded := false + for i := range meeting.Participants { + participant := &meeting.Participants[i] + // Only contact: identifiers are Contacts IDs; email: and calendar: + // participants resolve by email alone. + contactID, isContact := strings.CutPrefix(participant.Identifier, "contact:") + if !isContact { + contactID = "" + } + if card, ok := contacts.Resolve(contactID, participant.Email); ok { + participant.ContactEmails = card.Emails + participant.ContactPhones, participant.SkippedPhones = normalizedPhones(card.Phones, countryCode) + addresses := append([]string{participant.Email}, participant.ContactEmails...) + addresses = append(addresses, participant.ContactPhones...) + for _, address := range addresses { + if sharedAddresses[address] { + participant.LinkExcludedAddresses = append(participant.LinkExcludedAddresses, address) + } + } + // An unreadable store may hide another card sharing these addresses. + if state == ContactsComplete { + participant.Anchor = meetingarchive.Anchor("apple-contact", card.GroupKey) + } + participant.Resolution = resolutionResolved + continue + } + if !loaded { + loaded = true + // An unkeyable meeting is reported when its snapshot is built. + if key, keyErr := meeting.SourceMessageID(); keyErr == nil { + var err error + previous, err = imp.previousParticipants(sourceID, key) + if err != nil { + return err + } + } + } + earlier, ok := previous[participantRef(participant.Identifier)] + if !ok || (len(earlier.Emails) == 0 && len(earlier.Phones) == 0) { + continue + } + participant.ContactEmails = earlier.Emails + participant.ContactPhones = earlier.Phones + participant.Resolution = resolutionCarried + } + return nil +} + +// normalizedPhones converts Contacts phones to E.164 and reports how many +// could not be converted, such as national numbers without a configured +// phone_country_code. +func normalizedPhones(raw []string, countryCode string) ([]string, int) { + var phones []string + skipped := 0 + for _, value := range raw { + phone, ok := NormalizeContactPhone(value, countryCode) + if !ok { + skipped++ + continue + } + if !slices.Contains(phones, phone) { + phones = append(phones, phone) + } + } + slices.Sort(phones) + return phones, skipped +} + +// previousParticipants reads the meeting's archived participants by ref. A +// meeting that is not archived yet, or whose evidence cannot be read, has none. +func (imp *Importer) previousParticipants(sourceID int64, key string) (map[string]rawParticipant, error) { + out := map[string]rawParticipant{} + existing, err := imp.store.MessageExistsBatch(sourceID, []string{key}) + if err != nil { + return nil, fmt.Errorf("look up archived Muesli meeting: %w", err) + } + messageID, ok := existing[key] + if !ok { + return out, nil + } + raw, err := imp.store.GetMessageRaw(messageID) + if err != nil { + return nil, fmt.Errorf("read archived Muesli meeting: %w", err) + } + // Evidence this version cannot read has nothing to carry forward. + var evidence rawEvidence + if decodeErr := json.Unmarshal(raw, &evidence); decodeErr == nil { + for _, participant := range evidence.Participants { + if participant.Ref != "" { + out[participant.Ref] = participant + } + } + } + return out, nil +} diff --git a/internal/muesli/resolve_test.go b/internal/muesli/resolve_test.go new file mode 100644 index 000000000..b930bec3e --- /dev/null +++ b/internal/muesli/resolve_test.go @@ -0,0 +1,397 @@ +package muesli + +import ( + "encoding/json/v2" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/activity" +) + +// resolveFixture is a Muesli database, a Contacts folder, and an archive +// where one person is already known by the phone number of an iMessage chat. +type resolveFixture struct { + *importerFixture + + contacts string + personID int64 + phoneID int64 +} + +func newResolveFixture(t *testing.T, cards ...fixtureCard) resolveFixture { + t.Helper() + f := newImporterFixture(t) + root := filepath.Join(t.TempDir(), "AddressBook") + newAddressBookStore(t, filepath.Join(root, "Sources", "A", "AddressBook-v22.abcddb"), cards...) + phoneID, err := f.st.EnsureParticipantByPhone("+16045550100", "Alex Chat", "imessage") + require.NoError(t, err) + person, _, err := f.st.CreatePersonFromParticipantContext(t.Context(), phoneID) + require.NoError(t, err) + return resolveFixture{importerFixture: f, contacts: root, personID: person.ID, phoneID: phoneID} +} + +func (f resolveFixture) sync(t *testing.T, opts ImportOptions) *ImportSummary { + t.Helper() + opts.ContactsEnabled = true + if opts.ContactsPath == "" { + opts.ContactsPath = f.contacts + } + summary, err := f.run(t, opts) + require.NoError(t, err) + return summary +} + +func (f resolveFixture) project(t *testing.T) { + t.Helper() + projector, err := activity.NewProjector(f.st, activity.Options{Timezone: "UTC", BatchSize: 10, MaxDirectCounterparts: 25}) + require.NoError(t, err) + _, err = projector.RunOnce(t.Context()) + require.NoError(t, err) +} + +func (f resolveFixture) meetingPersons(t *testing.T) []int64 { + t.Helper() + rows, err := f.st.DB().Query(f.st.Rebind(` + SELECT aep.person_id FROM activity_event_persons aep + JOIN messages m ON m.id = aep.message_id WHERE m.source_id = ?`), f.source.ID) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + var ids []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + ids = append(ids, id) + } + require.NoError(t, rows.Err()) + return ids +} + +func (f resolveFixture) addContactParticipant(t *testing.T, meetingID int64, identifier, name string) { + t.Helper() + insertRow(t, f.muesli, "meeting_participants", map[string]any{ + "meeting_id": meetingID, "participant_identifier": identifier, + "display_name": name, "insertion_order": 10, "source": "manual", + }) +} + +func TestContactOnlyParticipantReachesPersonKnownByPhone(t *testing.T) { + f := newResolveFixture(t, fixtureCard{ + uniqueID: "CARD-1:ABPerson", emails: []string{"alex@example.com"}, phones: []string{"+1 604 555 0100"}, + }) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-1:ABPerson", "Alex Example") + + summary := f.sync(t, ImportOptions{}) + f.project(t) + + assert.Equal(t, ContactsComplete, summary.ContactsState) + assert.Contains(t, f.meetingPersons(t), f.personID, + "the Contacts card ties the attendee's email to the person known by phone") +} + +func TestPartialContactsKeepsEvidenceWithoutLinking(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newResolveFixture(t, fixtureCard{ + uniqueID: "CARD-A:ABPerson", emails: []string{"alice@example.com"}, phones: []string{"+16045550100"}, + }) + path := filepath.Join(f.contacts, "Sources", "B", addressBookFile) + newAddressBookStore(t, path, fixtureCard{ + uniqueID: "CARD-B:ABPerson", emails: []string{"bob@example.com"}, phones: []string{"+16045550100"}, + }) + // An unsupported store can hide another card with the same address. + db := newFixtureDB(t, path, "") + _, err := db.Exec(`ALTER TABLE Z_PRIMARYKEY RENAME COLUMN Z_NAME TO Z_FUTURE_NAME`) + require.NoError(err) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-A:ABPerson", "Alice Example") + + summary := f.sync(t, ImportOptions{}) + assert.Equal(ContactsPartial, summary.ContactsState) + alice, err := f.st.EnsureParticipant("alice@example.com", "", "example.com") + require.NoError(err) + members, err := f.st.ClusterMembers(f.phoneID) + require.NoError(err) + assert.NotContains(members, alice, "an unreadable store prevents new automatic links") + observations, err := f.st.ListParticipantObservationsContext(t.Context(), alice, true) + require.NoError(err) + assert.Empty(observations, "partial Contacts evidence must not assert ownership") + var messageID int64 + require.NoError(f.st.DB().QueryRow(f.st.Rebind(`SELECT id FROM messages WHERE source_id=?`), f.source.ID).Scan(&messageID)) + raw, err := f.st.GetMessageRaw(messageID) + require.NoError(err) + var evidence rawEvidence + require.NoError(json.Unmarshal(raw, &evidence)) + require.Len(evidence.Participants, 1) + assert.Equal([]string{"alice@example.com"}, evidence.Participants[0].Emails) + assert.Equal([]string{"+16045550100"}, evidence.Participants[0].Phones) + + // Once every store is readable and the phone is unshared, linking resumes. + _, err = db.Exec(`ALTER TABLE Z_PRIMARYKEY RENAME COLUMN Z_FUTURE_NAME TO Z_NAME`) + require.NoError(err) + _, err = db.Exec(`DELETE FROM ZABCDPHONENUMBER`) + require.NoError(err) + summary = f.sync(t, ImportOptions{}) + assert.Equal(ContactsComplete, summary.ContactsState) + members, err = f.st.ClusterMembers(f.phoneID) + require.NoError(err) + assert.Contains(members, alice) +} + +func TestPhoneOnlyContactBecomesPhoneAttendee(t *testing.T) { + f := newResolveFixture(t, fixtureCard{uniqueID: "CARD-2:ABPerson", phones: []string{"(604) 555-0100"}}) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-2:ABPerson", "Alex Example") + + f.sync(t, ImportOptions{PhoneCountryCode: "1"}) + f.project(t) + + var recipients int + require.NoError(t, f.st.DB().QueryRow(f.st.Rebind(` + SELECT count(*) FROM message_recipients mr JOIN messages m ON m.id = mr.message_id + WHERE m.source_id = ? AND mr.participant_id = ?`), f.source.ID, f.phoneID).Scan(&recipients)) + assert.Equal(t, 1, recipients) + assert.Contains(t, f.meetingPersons(t), f.personID) +} + +func TestContactEnrichmentShapesRawAndMetadata(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newResolveFixture(t, fixtureCard{ + uniqueID: "CARD-3:ABPerson", emails: []string{"jo@example.com", "jo.work@example.com"}, + phones: []string{"+44 20 7946 0000"}, + }) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + insertRow(t, f.muesli, "meeting_participants", map[string]any{ + "meeting_id": meetingID, "participant_identifier": "email:jo@example.com", + "display_name": "Jo Example", "email_address": "jo@example.com", "insertion_order": 0, "source": "calendar", + }) + f.addContactParticipant(t, meetingID, "contact:UNKNOWN:ABPerson", "Nobody Example") + + f.sync(t, ImportOptions{}) + + var messageID int64 + require.NoError(f.st.DB().QueryRow(f.st.Rebind( + `SELECT id FROM messages WHERE source_id = ?`), f.source.ID).Scan(&messageID)) + raw, err := f.st.GetMessageRaw(messageID) + require.NoError(err) + assert.NotContains(string(raw), "CARD-3") + assert.NotContains(string(raw), "UNKNOWN") + var evidence struct { + Participants []map[string]any `json:"participants"` + } + require.NoError(json.Unmarshal(raw, &evidence)) + require.Len(evidence.Participants, 2) + jo := evidence.Participants[0] + assert.Regexp(`^[0-9a-f]{16}$`, jo["ref"]) + assert.Equal("jo@example.com", jo["email"]) + assert.Equal([]any{"jo.work@example.com", "jo@example.com"}, jo["emails"]) + assert.Equal([]any{"+442079460000"}, jo["phones"]) + assert.NotContains(evidence.Participants[1], "emails") + + var metadata map[string]any + var metadataText string + require.NoError(f.st.DB().QueryRow(f.st.Rebind( + `SELECT metadata FROM messages WHERE id = ?`), messageID).Scan(&metadataText)) + require.NoError(json.Unmarshal([]byte(metadataText), &metadata)) + assert.Equal("complete", metadata["contacts"]) + assert.InDelta(float64(1), metadata["contacts_resolved"], 0) + assert.InDelta(float64(1), metadata["contacts_unresolved"], 0) + + work, err := f.st.EnsureParticipant("jo.work@example.com", "", "example.com") + require.NoError(err) + primary, err := f.st.EnsureParticipant("jo@example.com", "", "example.com") + require.NoError(err) + members, err := f.st.ClusterMembers(primary) + require.NoError(err) + assert.True(slices.Contains(members, work), "the card's second email is linked") +} + +func TestContactsOutageKeepsAttendees(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newResolveFixture(t, fixtureCard{uniqueID: "CARD-4:ABPerson", phones: []string{"+16045550100"}}) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-4:ABPerson", "Alex Example") + f.sync(t, ImportOptions{}) + + outage := f.sync(t, ImportOptions{ContactsPath: filepath.Join(t.TempDir(), "missing")}) + assert.Equal(ContactsUnavailable, outage.ContactsState) + assert.Equal(int64(1), outage.MeetingsUpdated, "Contacts status changes even when carried-forward identities are unchanged") + f.project(t) + assert.Contains(f.meetingPersons(t), f.personID) + + recovered := f.sync(t, ImportOptions{}) + assert.Equal(int64(1), recovered.MeetingsUpdated, "recovery refreshes the archived Contacts status") + + off, err := f.run(t, ImportOptions{ContactsEnabled: false}) + require.NoError(err) + assert.Equal(ContactsOff, off.ContactsState) + assert.Equal(int64(1), off.MeetingsUpdated, "turning Contacts off drops the enrichment") +} + +func messageMetadata(t *testing.T, f resolveFixture) map[string]any { + t.Helper() + var text string + require.NoError(t, f.st.DB().QueryRow(f.st.Rebind( + `SELECT metadata FROM messages WHERE source_id = ?`), f.source.ID).Scan(&text)) + var metadata map[string]any + require.NoError(t, json.Unmarshal([]byte(text), &metadata)) + return metadata +} + +func TestContactPhonesThatCannotConvertAreCounted(t *testing.T) { + f := newResolveFixture(t, fixtureCard{ + uniqueID: "CARD-5:ABPerson", emails: []string{"sam@example.com"}, + phones: []string{"(604) 555-0130", "+1 604 555 0131"}, + }) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-5:ABPerson", "Sam Example") + + f.sync(t, ImportOptions{}) + + assert.InDelta(t, float64(1), messageMetadata(t, f)["contacts_phones_skipped"], 0, + "a national number without phone_country_code is skipped, and the count says so") +} + +func TestOnlyContactIdentifiersUseContactsLookup(t *testing.T) { + f := newResolveFixture(t, fixtureCard{uniqueID: "calendar:room-1", emails: []string{"room@example.com"}}) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "calendar:room-1", "Board Room") + + f.sync(t, ImportOptions{}) + + metadata := messageMetadata(t, f) + assert.Nil(t, metadata["contacts_resolved"], "a calendar identifier is not a Contacts ID") + assert.InDelta(t, float64(1), metadata["contacts_unresolved"], 0) +} + +func TestSharedContactsAddressesStayInMeetingEvidenceWithoutLinking(t *testing.T) { + for _, tt := range []struct { + name string + sharedEmail bool + linkedCards bool + }{ + {name: "shared phone"}, + {name: "shared email", sharedEmail: true}, + {name: "phone on linked cards", linkedCards: true}, + {name: "email on linked cards", sharedEmail: true, linkedCards: true}, + } { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + first := fixtureCard{uniqueID: "CARD-A:ABPerson", emails: []string{"alice@example.com", "work@example.com"}, phones: []string{"+16045550100"}} + second := fixtureCard{uniqueID: "CARD-B:ABPerson", emails: []string{"bob@example.com"}} + if tt.sharedEmail { + second.emails = []string{"ALICE@example.com"} + } else { + second.phones = []string{"(604) 555-0100"} + } + if tt.linkedCards { + first.linkID, second.linkID = "ONE-PERSON", "ONE-PERSON" + } + f := newResolveFixture(t, first, second) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-A:ABPerson", "Alice Example") + f.sync(t, ImportOptions{PhoneCountryCode: "1"}) + + alice, err := f.st.EnsureParticipant("alice@example.com", "", "example.com") + require.NoError(err) + work, err := f.st.EnsureParticipant("work@example.com", "", "example.com") + require.NoError(err) + members, err := f.st.ClusterMembers(work) + require.NoError(err) + shared, unique := f.phoneID, alice + if tt.sharedEmail { + shared, unique = alice, f.phoneID + } + assert.Contains(members, unique, "unshared addresses still link") + if tt.linkedCards { + assert.Contains(members, shared, "linked cards are one Contacts group") + } else { + assert.NotContains(members, shared, "a second card need not appear in any meeting to prevent linking") + observations, err := f.st.ListParticipantObservationsContext(t.Context(), shared, true) + require.NoError(err) + assert.Empty(observations, "shared addresses must not assert ownership of this card") + } + var messageID, recipient int64 + require.NoError(f.st.DB().QueryRow(f.st.Rebind(`SELECT id FROM messages WHERE source_id=?`), f.source.ID).Scan(&messageID)) + require.NoError(f.st.DB().QueryRow(f.st.Rebind(`SELECT participant_id FROM message_recipients WHERE message_id=? AND recipient_type='to'`), messageID).Scan(&recipient)) + assert.Equal(alice, recipient, "shared primary emails remain meeting evidence") + raw, err := f.st.GetMessageRaw(messageID) + require.NoError(err) + var evidence rawEvidence + require.NoError(json.Unmarshal(raw, &evidence)) + require.Len(evidence.Participants, 1) + assert.Contains(evidence.Participants[0].Emails, "alice@example.com") + assert.Contains(evidence.Participants[0].Emails, "work@example.com") + assert.Equal([]string{"+16045550100"}, evidence.Participants[0].Phones) + }) + } +} + +func TestDeletedContactsCardKeepsPresentAttendeeHistory(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newResolveFixture(t, fixtureCard{uniqueID: "CARD-A:ABPerson", emails: []string{"alice@example.com"}, phones: []string{"+16045550100"}}) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-A:ABPerson", "Alice Example") + f.sync(t, ImportOptions{}) + f.project(t) + require.Contains(f.meetingPersons(t), f.personID) + var messageID int64 + require.NoError(f.st.DB().QueryRow(f.st.Rebind(`SELECT id FROM messages WHERE source_id=?`), f.source.ID).Scan(&messageID)) + beforeRaw, err := f.st.GetMessageRaw(messageID) + require.NoError(err) + alice, err := f.st.EnsureParticipant("alice@example.com", "", "example.com") + require.NoError(err) + // Close the prior observation so a fresh assertion would be observable. + observations, err := f.st.ListParticipantObservationsContext(t.Context(), alice, true) + require.NoError(err) + require.Len(observations, 1) + require.NoError(f.st.SupersedeParticipantObservationContext(t.Context(), alice, observations[0].Envelope.ID, nil)) + contactsDB := newFixtureDB(t, filepath.Join(f.contacts, "Sources", "A", addressBookFile), "") + _, err = contactsDB.Exec(`DELETE FROM ZABCDRECORD WHERE ZUNIQUEID='CARD-A:ABPerson'`) + require.NoError(err) + + summary := f.sync(t, ImportOptions{}) + f.project(t) + assert.Equal(ContactsComplete, summary.ContactsState) + afterRaw, err := f.st.GetMessageRaw(messageID) + require.NoError(err) + assert.Equal(beforeRaw, afterRaw, "losing the lookup does not rewrite historical evidence") + assert.Contains(f.meetingPersons(t), f.personID) + observations, err = f.st.ListParticipantObservationsContext(t.Context(), alice, true) + require.NoError(err) + assert.Empty(observations, "carried evidence makes no fresh ownership assertion") + + _, err = f.muesli.Exec(`DELETE FROM meeting_participants WHERE meeting_id=?`, meetingID) + require.NoError(err) + f.sync(t, ImportOptions{}) + f.project(t) + assert.NotContains(f.meetingPersons(t), f.personID, "removing the source attendee still removes its activity association") +} + +func TestPreviouslyObservedSharedPhoneDoesNotGainNewLinks(t *testing.T) { + require := require.New(t) + f := newResolveFixture(t, fixtureCard{uniqueID: "CARD-A:ABPerson", phones: []string{"+16045550100"}}) + meetingID := insertRow(t, f.muesli, "meetings", completedMeeting(nil)) + f.addContactParticipant(t, meetingID, "contact:CARD-A:ABPerson", "Alice Example") + f.sync(t, ImportOptions{}) + + contactsDB := newFixtureDB(t, filepath.Join(f.contacts, "Sources", "A", addressBookFile), "") + _, err := contactsDB.Exec(`INSERT INTO ZABCDEMAILADDRESS (ZOWNER, ZADDRESS) SELECT Z_PK, 'alice@example.com' FROM ZABCDRECORD WHERE ZUNIQUEID='CARD-A:ABPerson'`) + require.NoError(err) + newAddressBookStore(t, filepath.Join(f.contacts, "Sources", "B", addressBookFile), + fixtureCard{uniqueID: "CARD-B:ABPerson", phones: []string{"+1 604 555 0100"}}) + f.sync(t, ImportOptions{}) + + alice, err := f.st.EnsureParticipant("alice@example.com", "", "example.com") + require.NoError(err) + members, err := f.st.ClusterMembers(alice) + require.NoError(err) + assert.NotContains(t, members, f.phoneID, "an earlier observation cannot link a currently shared address") +} diff --git a/internal/notionmeetings/format.go b/internal/notionmeetings/format.go index 12467fda3..09895ae70 100644 --- a/internal/notionmeetings/format.go +++ b/internal/notionmeetings/format.go @@ -195,3 +195,13 @@ func meetingSnippet(body string) string { } return string(runes) } + +// userAnchor is the stable identity of a Notion user. The provider verifies +// the user's email, so every email a user has had links to the same person. +func userAnchor(userID string) string { + userID = strings.TrimSpace(userID) + if userID == "" { + return "" + } + return meetingarchive.Anchor("notion-user", userID) +} diff --git a/internal/notionmeetings/hydrate.go b/internal/notionmeetings/hydrate.go index adaa3a5c5..ddde8d997 100644 --- a/internal/notionmeetings/hydrate.go +++ b/internal/notionmeetings/hydrate.go @@ -304,8 +304,9 @@ func (h *Hydrator) resolveAttendees(ctx context.Context, result *HydratedMeeting continue } result.Attendees = append(result.Attendees, meetingarchive.Person{ - Name: strings.TrimSpace(user.Name), - Email: strings.ToLower(strings.TrimSpace(user.Person.Email)), + Name: strings.TrimSpace(user.Name), + Email: strings.ToLower(strings.TrimSpace(user.Person.Email)), + Anchor: userAnchor(id), }) result.ResolvedUsers = append(result.ResolvedUsers, resolvedUser{ ID: id, Name: strings.TrimSpace(user.Name), diff --git a/internal/notionmeetings/importer.go b/internal/notionmeetings/importer.go index a9e56d42e..af6cc5465 100644 --- a/internal/notionmeetings/importer.go +++ b/internal/notionmeetings/importer.go @@ -379,6 +379,13 @@ func (imp *Importer) Import(ctx context.Context, opts ImportOptions) (sum *Impor known, exists := state.Known[id] if archived.HasEvidence && exists && !opts.Full && known.SnapshotVersion == notionSnapshotVersion && known.SnapshotSHA256 == checksum { + // The archived meeting is current, but attendee linking may not + // have finished before an earlier run stopped. + if _, linkErr := archiver.LinkIdentities(ctx, source.ID, snapshot.Attendees); linkErr != nil { + sum.Errors++ + hardErrors = append(hardErrors, fmt.Errorf("meeting %s: link attendee identities: %w", id, linkErr)) + continue + } next.Known[id] = knownMeeting{ LastEditedTime: meeting.LastEditedTime, SnapshotSHA256: checksum, @@ -621,7 +628,9 @@ func preserveArchivedAttendees(meeting *HydratedMeeting, archived []resolvedUser if _, seen := seenEmails[user.Email]; seen { continue } - attendees = append(attendees, meetingarchive.Person{Name: user.Name, Email: user.Email}) + attendees = append(attendees, meetingarchive.Person{ + Name: user.Name, Email: user.Email, Anchor: userAnchor(user.ID), + }) seenEmails[user.Email] = struct{}{} } if restored == 0 { diff --git a/internal/notionmeetings/person_link_test.go b/internal/notionmeetings/person_link_test.go new file mode 100644 index 000000000..ac49f8ee8 --- /dev/null +++ b/internal/notionmeetings/person_link_test.go @@ -0,0 +1,73 @@ +package notionmeetings + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/meetingarchive" + "go.kenn.io/msgvault/internal/store" +) + +func boundPersonForEmail(t *testing.T, st *store.Store, email string) int64 { + t.Helper() + var personID int64 + err := st.DB().QueryRow(st.Rebind(` + SELECT pp.person_id FROM person_participants pp + JOIN participants p ON p.id = pp.participant_id + WHERE p.email_address = ?`), email).Scan(&personID) + if err != nil { + return 0 + } + return personID +} + +func TestHydratorAnchorsOnlyVerifiedNotionUsers(t *testing.T) { + require := require.New(t) + hydrated, err := NewHydrator(completeHydrationSource()).Hydrate(context.Background(), hydrationMeeting()) + require.NoError(err) + + require.Len(hydrated.Attendees, 1) + assert.Equal(t, meetingarchive.Anchor("notion-user", "user-1"), hydrated.Attendees[0].Anchor) +} + +func TestNotionUserKeepsPersonAcrossEmailChange(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st, source, imp := newImporterFixture(t) + _, err := imp.Import(context.Background(), ImportOptions{Identifier: "work"}) + require.NoError(err) + participantID, err := st.EnsureParticipant("attendee@example.com", "", "example.com") + require.NoError(err) + person, _, err := st.CreatePersonFromParticipantContext(t.Context(), participantID) + require.NoError(err) + + source.users[""].Results[0].Person.Email = "attendee.new@example.com" + _, err = imp.Import(context.Background(), ImportOptions{Identifier: "work"}) + require.NoError(err) + + assert.Equal(person.ID, boundPersonForEmail(t, st, "attendee.new@example.com"), + "the same Notion user under a new email joins the person") +} + +func TestNotionChecksumSkipStillLinksAttendees(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st, _, imp := newImporterFixture(t) + _, err := imp.Import(context.Background(), ImportOptions{Identifier: "work"}) + require.NoError(err) + // Simulate linking that never completed after the meeting was written. + _, err = st.DB().Exec(`DELETE FROM participant_contact_observations`) + require.NoError(err) + + second, err := imp.Import(context.Background(), ImportOptions{Identifier: "work"}) + require.NoError(err) + assert.Equal(int64(0), second.MeetingsUpdated, "the unchanged meeting is not rewritten") + + var observed int + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT count(*) FROM participant_contact_observations WHERE provider_user_id = ?`), + meetingarchive.Anchor("notion-user", "user-1")).Scan(&observed)) + assert.Equal(1, observed) +} diff --git a/internal/store/docbank_media_test.go b/internal/store/docbank_media_test.go index f24152350..ded4afd02 100644 --- a/internal/store/docbank_media_test.go +++ b/internal/store/docbank_media_test.go @@ -108,11 +108,11 @@ func TestBeeperMediaKeyChangeKeepsSharedDelivery(t *testing.T) { require.NoError(err) a.ProcessingKey = "new-key" require.NoError(f.Store.ReconcileBeeperMediaMapping(t.Context(), a)) - operation, ready, err := f.Store.NextBeeperMediaOperation(t.Context(), "shared", time.Now().UTC()) + mappings, err := f.Store.ListLiveBeeperMediaMappings(t.Context(), "shared", "old-key", 10) require.NoError(err) - require.True(ready) - assert.Equal("old-key", operation.ProcessingKey) - assert.Equal(store.BeeperMediaOperationArtifact, operation.Kind) + require.Len(mappings, 1) + assert.Equal(b.OccurrenceRef, mappings[0].OccurrenceRef) + assert.Equal("pending-artifact", mappings[0].ProcessingPhase) } func liveMessageIDs(t *testing.T, st *store.Store) []string { diff --git a/internal/store/meeting_phone_participants_test.go b/internal/store/meeting_phone_participants_test.go new file mode 100644 index 000000000..d41aebc8f --- /dev/null +++ b/internal/store/meeting_phone_participants_test.go @@ -0,0 +1,213 @@ +package store_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +type phonePersistFixture struct { + st *store.Store + sourceID int64 + conversationID int64 +} + +func newPhonePersistFixture(t *testing.T) phonePersistFixture { + t.Helper() + st := storetest.New(t).Store + source, err := st.GetOrCreateSource("meeting_import", "phone-fixture") + require.NoError(t, err) + conversationID, err := st.EnsureConversationWithType( + source.ID, "phone-fixture-meeting", "meeting", "Phone Fixture", + ) + require.NoError(t, err) + return phonePersistFixture{st: st, sourceID: source.ID, conversationID: conversationID} +} + +func (f phonePersistFixture) persist( + ctx context.Context, messageID string, participants []store.ParticipantPersistData, +) ([]int64, error) { + var resolved []int64 + _, err := f.st.PersistMessageWithParticipantsContext(ctx, participants, + func(ids []int64) *store.MessagePersistData { + resolved = append([]int64(nil), ids...) + return &store.MessagePersistData{Message: &store.Message{ + SourceID: f.sourceID, SourceMessageID: messageID, + ConversationID: f.conversationID, MessageType: "meeting_transcript", + }} + }) + return resolved, err +} + +func phoneIdentifierOwner(t *testing.T, st *store.Store, phone string) int64 { + t.Helper() + var owner int64 + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT participant_id FROM participant_identifiers + WHERE identifier_type = ? AND identifier_value = ?`), + store.PhoneIdentifierType, phone).Scan(&owner)) + return owner +} + +func TestPersistMessageCreatesPhoneParticipantInsideTransaction(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newPhonePersistFixture(t) + + ids, err := f.persist(t.Context(), "phone-only", []store.ParticipantPersistData{ + {EmailAddress: "owner@example.com", DisplayName: "Owner", Domain: "example.com"}, + {PhoneNumber: "+16045550100", DisplayName: "Phone Example"}, + }) + require.NoError(err) + require.Len(ids, 2) + + var phone, name string + require.NoError(f.st.DB().QueryRow(f.st.Rebind( + `SELECT phone_number, display_name FROM participants WHERE id = ?`), ids[1]). + Scan(&phone, &name)) + assert.Equal("+16045550100", phone) + assert.Equal("Phone Example", name) + assert.Equal(ids[1], phoneIdentifierOwner(t, f.st, "+16045550100"), + "owner attribution matches identifier rows, so the phone needs one") +} + +func TestPersistMessageReusesExistingPhoneParticipant(t *testing.T) { + require := require.New(t) + f := newPhonePersistFixture(t) + existing, err := f.st.EnsureParticipantByPhone("+16045550101", "Chat Example", "imessage") + require.NoError(err) + + ids, err := f.persist(t.Context(), "phone-reuse", []store.ParticipantPersistData{ + {PhoneNumber: "+16045550101", DisplayName: "Meeting Label"}, + }) + require.NoError(err) + + assert.Equal(t, []int64{existing}, ids) +} + +func TestPersistMessageRejectsParticipantWithoutIdentity(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newPhonePersistFixture(t) + + _, err := f.persist(t.Context(), "no-identity", []store.ParticipantPersistData{ + {DisplayName: "Name Only"}, + }) + require.Error(err) + + var count int + require.NoError(f.st.DB().QueryRow(f.st.Rebind( + `SELECT count(*) FROM messages WHERE source_id = ?`), f.sourceID).Scan(&count)) + assert.Equal(0, count) +} + +func TestPersistMessageRollsBackPhoneParticipantOnFailure(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newPhonePersistFixture(t) + + _, err := f.st.PersistMessageWithParticipantsContext(t.Context(), + []store.ParticipantPersistData{{PhoneNumber: "+16045550102"}}, + func([]int64) *store.MessagePersistData { + // A message without a source is rejected after participants resolve. + return &store.MessagePersistData{Message: &store.Message{ + SourceMessageID: "rolled-back", ConversationID: f.conversationID, + }} + }) + require.Error(err) + + var count int + require.NoError(f.st.DB().QueryRow(f.st.Rebind( + `SELECT count(*) FROM participants WHERE phone_number = ?`), "+16045550102").Scan(&count)) + assert.Equal(0, count) +} + +func TestEnsurePhoneParticipantIsIdempotent(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := storetest.New(t).Store + + first, err := st.EnsurePhoneParticipantContext(t.Context(), "+16045550103", "Phone Example") + require.NoError(err) + second, err := st.EnsurePhoneParticipantContext(t.Context(), "+16045550103", "") + require.NoError(err) + + assert.Equal(first, second) + assert.Equal(first, phoneIdentifierOwner(t, st, "+16045550103")) + _, err = st.EnsurePhoneParticipantContext(t.Context(), "6045550103", "") + assert.Error(err, "only E.164 phones are accepted") +} + +func TestPhoneParticipantErrorsDoNotRepeatTheNumber(t *testing.T) { + st := storetest.New(t).Store + + _, err := st.EnsurePhoneParticipantContext(t.Context(), "6045550188", "") + + require.Error(t, err) + assert.NotContains(t, err.Error(), "6045550188") +} + +func identifierTypesForValue(t *testing.T, st *store.Store, value string) []string { + t.Helper() + rows, err := st.DB().Query(st.Rebind(` + SELECT identifier_type FROM participant_identifiers + WHERE identifier_value = ? ORDER BY identifier_type`), value) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + var types []string + for rows.Next() { + var kind string + require.NoError(t, rows.Scan(&kind)) + types = append(types, kind) + } + require.NoError(t, rows.Err()) + return types +} + +func TestPhoneIdentifierIsNotDuplicatedByMessagingIdentifiers(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := storetest.New(t).Store + + // A chat importer saw the number first: the meeting adds no second row. + _, err := st.EnsureParticipantByPhone("+16045550120", "", "imessage") + require.NoError(err) + _, err = st.EnsurePhoneParticipantContext(t.Context(), "+16045550120", "") + require.NoError(err) + assert.Equal([]string{"imessage"}, identifierTypesForValue(t, st, "+16045550120")) + + // The meeting saw the number first: the chat importer's row replaces it. + _, err = st.EnsurePhoneParticipantContext(t.Context(), "+16045550121", "") + require.NoError(err) + assert.Equal([]string{store.PhoneIdentifierType}, identifierTypesForValue(t, st, "+16045550121"), + "alone, the phone row is what owner attribution matches") + _, err = st.EnsureParticipantByPhone("+16045550121", "", "whatsapp") + require.NoError(err) + assert.Equal([]string{"whatsapp"}, identifierTypesForValue(t, st, "+16045550121")) +} + +func TestPhoneIdentifierStaysWhenServiceIdentifierBelongsElsewhere(t *testing.T) { + require := require.New(t) + st := storetest.New(t).Store + meetingPhone, err := st.EnsurePhoneParticipantContext(t.Context(), "+16045550122", "") + require.NoError(err) + // Another participant already owns the service identifier for this value. + other, err := st.EnsureParticipant("elsewhere@example.com", "", "example.com") + require.NoError(err) + _, err = st.DB().Exec(st.Rebind(`INSERT INTO participant_identifiers + (participant_id, identifier_type, identifier_value, is_primary) VALUES (?, 'whatsapp', ?, FALSE)`), + other, "+16045550122") + require.NoError(err) + + _, err = st.EnsureParticipantByPhone("+16045550122", "", "whatsapp") + require.NoError(err) + + var owner int64 + require.NoError(st.DB().QueryRow(st.Rebind(`SELECT participant_id FROM participant_identifiers + WHERE identifier_type = ? AND identifier_value = ?`), store.PhoneIdentifierType, "+16045550122").Scan(&owner)) + assert.Equal(t, meetingPhone, owner, "the phone participant must keep an identifier for its own number") +} diff --git a/internal/store/meetings.go b/internal/store/meetings.go index 698dd6fbc..f0bf91db2 100644 --- a/internal/store/meetings.go +++ b/internal/store/meetings.go @@ -317,6 +317,7 @@ func loadMeetingContextParticipants( SELECT mr.participant_id, COALESCE(NULLIF(mr.display_name, ''), NULLIF(p.display_name, ''), ''), COALESCE(NULLIF(mr.email_address, ''), NULLIF(p.email_address, ''), ''), + COALESCE(p.phone_number, ''), LOWER(mr.recipient_type) FROM message_recipients mr JOIN participants p ON p.id = mr.participant_id @@ -328,7 +329,7 @@ func loadMeetingContextParticipants( for rows.Next() { var participant meetingcontent.Participant var participantID int64 - if scanErr := rows.Scan(&participantID, &participant.Name, &participant.Email, &participant.Role); scanErr != nil { + if scanErr := rows.Scan(&participantID, &participant.Name, &participant.Email, &participant.Phone, &participant.Role); scanErr != nil { _ = rows.Close() return nil, fmt.Errorf("scan meeting recipient %d: %w", id, scanErr) } @@ -352,10 +353,11 @@ func loadMeetingContextParticipants( var sender meetingcontent.Participant var senderID int64 err = tx.QueryRowContext(ctx, ` - SELECT p.id, COALESCE(p.display_name, ''), COALESCE(p.email_address, '') + SELECT p.id, COALESCE(p.display_name, ''), COALESCE(p.email_address, ''), + COALESCE(p.phone_number, '') FROM messages m JOIN participants p ON p.id = m.sender_id - WHERE m.id = ?`, id).Scan(&senderID, &sender.Name, &sender.Email) + WHERE m.id = ?`, id).Scan(&senderID, &sender.Name, &sender.Email, &sender.Phone) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("load meeting sender %d: %w", id, err) } @@ -380,7 +382,8 @@ func meetingParticipantKey(participant meetingcontent.Participant) string { if participant.ParticipantID != nil { id = *participant.ParticipantID } - return fmt.Sprintf("%d\x00%s\x00%s", id, strings.ToLower(participant.Role), strings.ToLower(participant.Email)) + return fmt.Sprintf("%d\x00%s\x00%s\x00%s", id, strings.ToLower(participant.Role), + strings.ToLower(participant.Email), participant.Phone) } func (s *Store) meetingIDsMembership(column string, ids []int64) (string, []any, error) { diff --git a/internal/store/messages.go b/internal/store/messages.go index 522a7087a..80ecaf928 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -68,10 +68,18 @@ type RecipientSet struct { // message persistence transaction. type ParticipantPersistData struct { EmailAddress string - DisplayName string - Domain string + // PhoneNumber is an E.164 phone that identifies the participant when + // EmailAddress is empty. It is backed by a participant_identifiers row of + // type PhoneIdentifierType so owner attribution can match it. + PhoneNumber string + DisplayName string + Domain string } +// PhoneIdentifierType is the participant_identifiers type for a phone number +// observed without a messaging service, such as a meeting attendee. +const PhoneIdentifierType = "phone" + // MessageDeliveryEvidence is provider-observed delivery state. A valid field // enriches the archived message; an invalid field preserves existing evidence. // Local read state is deliberately outside this type. @@ -1954,7 +1962,24 @@ func (s *Store) persistMessageWithParticipantsTx( return 0, fmt.Errorf("lock message persistence: %w", err) } } - if len(participants) > 1 { + hasPhoneParticipant := false + for idx, participant := range participants { + if participant.EmailAddress == "" && participant.PhoneNumber == "" { + return 0, fmt.Errorf("ensure participant %d: an email address or phone number is required", idx) + } + if participant.EmailAddress == "" { + hasPhoneParticipant = true + } + } + if hasPhoneParticipant { + // Phone participants write participant_identifiers, which participant + // merges rewrite under the identity lock. Take it before the directory + // lock, the same order MergeParticipants uses. + if err := s.lockIdentityMutationTxContext(ctx, tx); err != nil { + return 0, err + } + } + if len(participants) > 1 || hasPhoneParticipant { // Participant merges take the directory lock before rewriting // message rows. Keep the same order when a repair's preflight // callback locks its target message for identity revalidation. @@ -1974,17 +1999,25 @@ func (s *Store) persistMessageWithParticipantsTx( if err := ctx.Err(); err != nil { return 0, err } - participantID, err := ensureParticipantWith( - q, - s.dialect, - participant.EmailAddress, - participant.DisplayName, - participant.Domain, - func() error { - participantInserted = true - return nil - }, - ) + var participantID int64 + var err error + if participant.EmailAddress == "" { + participantID, err = s.ensureParticipantByPhoneTx( + ctx, tx, participant.PhoneNumber, participant.DisplayName, PhoneIdentifierType, + ) + } else { + participantID, err = ensureParticipantWith( + q, + s.dialect, + participant.EmailAddress, + participant.DisplayName, + participant.Domain, + func() error { + participantInserted = true + return nil + }, + ) + } if err != nil { return 0, fmt.Errorf("ensure participant %d: %w", idx, err) } @@ -4015,11 +4048,50 @@ func ensureConversationWithTypePolicy( // Also creates a participant_identifiers row with the given identifierType // (e.g., "whatsapp", "imessage", "google_voice"). func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType string) (int64, error) { + var id int64 + err := s.withTx(func(tx *loggedTx) error { + if err := s.lockIdentityMutationTx(tx); err != nil { + return err + } + var err error + id, err = s.ensureParticipantByPhoneTx(context.Background(), tx, phone, displayName, identifierType) + return err + }) + if err != nil { + return 0, err + } + return id, nil +} + +// EnsurePhoneParticipantContext gets or creates a participant for an E.164 +// phone observed without a messaging service, such as a meeting attendee. +func (s *Store) EnsurePhoneParticipantContext(ctx context.Context, phone, displayName string) (int64, error) { + var id int64 + err := s.withTxContext(ctx, func(tx *loggedTx) error { + if err := s.lockIdentityMutationTxContext(ctx, tx); err != nil { + return err + } + var err error + id, err = s.ensureParticipantByPhoneTx(ctx, tx, phone, displayName, PhoneIdentifierType) + return err + }) + if err != nil { + return 0, err + } + return id, nil +} + +// ensureParticipantByPhoneTx is EnsureParticipantByPhone inside a caller's +// transaction. The caller holds the identity-mutation lock. +func (s *Store) ensureParticipantByPhoneTx( + ctx context.Context, tx *loggedTx, phone, displayName, identifierType string, +) (int64, error) { if phone == "" { return 0, errors.New("phone number is required") } if !strings.HasPrefix(phone, "+") { - return 0, fmt.Errorf("phone number must be in E.164 format (starting with +), got %q", phone) + // Name the rule, not the number: errors reach logs and API responses. + return 0, errors.New("phone number must be in E.164 format (starting with +)") } // The conflict target mirrors the partial unique index on @@ -4029,14 +4101,11 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri // from an existing participant; a guarded UPDATE then reports whether an // existing blank display name was really filled. var id int64 - err := s.withTx(func(tx *loggedTx) error { - if err := s.lockIdentityMutationTx(tx); err != nil { - return err - } + err := func() error { displayNameChanged := false now := s.dialect.Now() for range 3 { - insertResult, err := tx.Exec(fmt.Sprintf(` + insertResult, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO participants (phone_number, display_name, created_at, updated_at) VALUES (?, ?, %s, %s) ON CONFLICT (phone_number) WHERE phone_number IS NOT NULL @@ -4050,12 +4119,12 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri return fmt.Errorf("check participant by phone insert: %w", err) } if inserted > 0 { - if err := s.bumpParticipantDisplayNameRevision(tx); err != nil { + if err := s.bumpParticipantDisplayNameRevisionContext(ctx, tx); err != nil { return err } } if inserted == 0 && displayName != "" { - updateResult, err := tx.Exec(` + updateResult, err := tx.ExecContext(ctx, ` UPDATE participants SET display_name = ? WHERE phone_number = ? AND COALESCE(NULLIF(TRIM(display_name), ''), '') = '' @@ -4065,12 +4134,12 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri if err != nil { return fmt.Errorf("backfill participant by phone: %w", err) } - displayNameChanged, err = s.bumpParticipantDisplayNameRevisionIfChanged(tx, updateResult) + displayNameChanged, err = s.bumpParticipantDisplayNameRevisionIfChangedContext(ctx, tx, updateResult) if err != nil { return err } } - lookupErr := tx.QueryRow( + lookupErr := tx.QueryRowContext(ctx, `SELECT id FROM participants WHERE phone_number = ?`+s.dialect.SelectForUpdate(), phone, ).Scan(&id) @@ -4082,29 +4151,62 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri } } if id == 0 { - return fmt.Errorf("ensure participant by phone %q after concurrent deletion", phone) + return errors.New("ensure participant by phone after concurrent deletion") } // Ensure a participant_identifiers row exists for this identifierType // and attach service/scope metadata whenever the importer namespace is // unambiguous. A repeat call repairs metadata but does not repoint the // identifier away from its existing participant. - classificationColumns, err := s.participantIdentifierClassificationColumnsTx(tx) + classificationColumns, err := s.participantIdentifierClassificationColumnsTx(ctx, tx) if err != nil { return err } finish := func(result sql.Result) error { - if err := s.bumpParticipantIdentifierRevisionIfChanged(tx, result); err != nil { - return err + if result != nil { + if err := s.bumpParticipantIdentifierRevisionIfChanged(ctx, tx, result); err != nil { + return err + } + } + if identifierType != PhoneIdentifierType { + // Drop the generic row only once this participant really + // holds another row for the number; the insert above can + // yield to another participant's identifier. + removed, err := tx.ExecContext(ctx, `DELETE FROM participant_identifiers + WHERE participant_id = ? AND identifier_type = ? AND identifier_value = ? + AND EXISTS (SELECT 1 FROM participant_identifiers other + WHERE other.participant_id = ? AND other.identifier_value = ? + AND other.identifier_type <> ?)`, + id, PhoneIdentifierType, phone, id, phone, PhoneIdentifierType) + if err != nil { + return fmt.Errorf("replace generic phone identifier: %w", err) + } + if err := s.bumpParticipantIdentifierRevisionIfChanged(ctx, tx, removed); err != nil { + return err + } } if !displayNameChanged { return nil } return s.invalidateParticipantPersonEnrichmentTx( - context.Background(), tx, id) + ctx, tx, id) + } + // One identifier row per number keeps profiles from listing it twice. + // Owner attribution matches any identifier value, so the generic + // phone row is written only when no service row already carries the + // number, and a service row replaces it (see finish). + if identifierType == PhoneIdentifierType { + var existing int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM participant_identifiers + WHERE participant_id = ? AND identifier_value = ?`, id, phone).Scan(&existing); err != nil { + return fmt.Errorf("check participant identifiers: %w", err) + } + if existing > 0 { + return finish(nil) + } } if !classificationColumns { - result, err := tx.Exec(`INSERT INTO participant_identifiers ( + result, err := tx.ExecContext(ctx, `INSERT INTO participant_identifiers ( participant_id, identifier_type, identifier_value, is_primary ) VALUES (?, ?, ?, TRUE) ON CONFLICT (identifier_type, identifier_value) DO NOTHING`, @@ -4117,7 +4219,7 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri serviceSlug, scopeKind, scopeValue := participantIdentifierClassificationValues( identifierType, phone, ) - result, err := tx.Exec(`INSERT INTO participant_identifiers ( + result, err := tx.ExecContext(ctx, `INSERT INTO participant_identifiers ( participant_id, identifier_type, identifier_value, is_primary, service_id, scope_kind, scope_value ) VALUES (?, ?, ?, TRUE, @@ -4147,7 +4249,7 @@ func (s *Store) EnsureParticipantByPhone(phone, displayName, identifierType stri return fmt.Errorf("insert participant identifier: %w", err) } return finish(result) - }) + }() if err != nil { return 0, err } @@ -4318,7 +4420,7 @@ func (s *Store) MergeParticipants(oldID, newID int64) error { if err := s.bumpAccountIdentityRevision(tx); err != nil { return err } - if err := s.bumpParticipantIdentifierRevision(tx); err != nil { + if err := s.bumpParticipantIdentifierRevision(context.Background(), tx); err != nil { return err } if err := rewritePersonMergeParticipantLineageTx( @@ -4452,7 +4554,7 @@ func (s *Store) AdoptLegacyParticipantIdentifier( identifierType, legacyValue); err != nil { return fmt.Errorf("remove ambiguous legacy participant identifier: %w", err) } - if err := s.bumpParticipantIdentifierRevision(tx); err != nil { + if err := s.bumpParticipantIdentifierRevision(context.Background(), tx); err != nil { return err } return nil @@ -4464,7 +4566,7 @@ func (s *Store) AdoptLegacyParticipantIdentifier( scopedValue, identifierType, legacyValue); err != nil { return fmt.Errorf("migrate legacy participant identifier: %w", err) } - if err := s.bumpParticipantIdentifierRevision(tx); err != nil { + if err := s.bumpParticipantIdentifierRevision(context.Background(), tx); err != nil { return err } adoptedID = legacyID @@ -4527,7 +4629,7 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id serviceSlug, scopeKind, scopeValue := participantIdentifierClassificationValues( identifierType, identifierValue, ) - classificationColumns, err := s.participantIdentifierClassificationColumnsTx(tx) + classificationColumns, err := s.participantIdentifierClassificationColumnsTx(context.Background(), tx) if err != nil { return err } @@ -4563,7 +4665,7 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id if setErr != nil { return fmt.Errorf("set participant identifier: %w", setErr) } - if err := s.bumpParticipantIdentifierRevision(tx); err != nil { + if err := s.bumpParticipantIdentifierRevision(context.Background(), tx); err != nil { return err } var ownerEvidence bool @@ -4656,7 +4758,7 @@ func (s *Store) RepairParticipantEmailAddresses(repairs []ParticipantEmailRepair } func (s *Store) participantIdentifierClassificationColumnsTx( - tx *loggedTx, + ctx context.Context, tx *loggedTx, ) (bool, error) { var count int query := `SELECT COUNT(*) FROM pragma_table_info('participant_identifiers') @@ -4667,7 +4769,7 @@ func (s *Store) participantIdentifierClassificationColumnsTx( AND table_name = 'participant_identifiers' AND column_name IN ('service_id', 'scope_kind', 'scope_value')` } - if err := tx.QueryRow(query).Scan(&count); err != nil { + if err := tx.QueryRowContext(ctx, query).Scan(&count); err != nil { return false, fmt.Errorf("inspect participant identifier classification schema: %w", err) } return count == 3, nil @@ -4745,7 +4847,7 @@ func (s *Store) EnsureParticipantByIdentifier(identifierType, identifierValue, d if err := s.bumpParticipantDisplayNameRevision(tx); err != nil { return err } - classificationColumns, err := s.participantIdentifierClassificationColumnsTx(tx) + classificationColumns, err := s.participantIdentifierClassificationColumnsTx(context.Background(), tx) if err != nil { return err } @@ -4774,7 +4876,7 @@ func (s *Store) EnsureParticipantByIdentifier(identifierType, identifierValue, d return fmt.Errorf("insert participant identifier: %w", err) } } - return s.bumpParticipantIdentifierRevision(tx) + return s.bumpParticipantIdentifierRevision(context.Background(), tx) }) if err != nil { return 0, err diff --git a/internal/store/messages_test.go b/internal/store/messages_test.go index 4c9b0dd43..67993a0f8 100644 --- a/internal/store/messages_test.go +++ b/internal/store/messages_test.go @@ -2274,3 +2274,56 @@ func TestCountMessagesPerMailbox(t *testing.T) { assert.Equal(int64(1), counts["Sent"], "Sent count") assert.Equal(int64(1), counts["Drafts"], "Drafts count") } + +func TestEnsurePhoneParticipantContextCancelsBlockedNameBackfill(t *testing.T) { + require := require.New(t) + st := testutil.NewTestStore(t) + if !st.IsPostgreSQL() { + t.Skip("PostgreSQL embedding-clock lock required") + } + const phone = "+12025550100" + participantID, err := st.EnsurePhoneParticipantContext(t.Context(), phone, "") + require.NoError(err) + + // Embedding publication holds this exclusive lock; the production + // participant-update trigger requests its shared form. + blocker, err := st.DB().BeginTx(t.Context(), nil) + require.NoError(err) + t.Cleanup(func() { _ = blocker.Rollback() }) + _, err = blocker.ExecContext(t.Context(), `SELECT pg_advisory_xact_lock( + hashtextextended('msgvault.embedding_change_clock', 0))`) + require.NoError(err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + result := make(chan error, 1) + done := make(chan struct{}) + go func() { + defer close(done) + _, err := st.EnsurePhoneParticipantContext(ctx, phone, "Taylor Example") + result <- err + }() + t.Cleanup(func() { + cancel() + _ = blocker.Rollback() + select { + case <-done: + case <-time.After(5 * time.Second): + assert.Fail(t, "phone participant write did not stop after releasing its blocker") + } + }) + waitForPostgreSQLLockWait(t, st, "%UPDATE participants SET display_name =%") + cancel() + select { + case err := <-result: + require.ErrorIs(err, context.Canceled) + case <-time.After(5 * time.Second): + require.FailNow("cancelled phone participant write kept waiting for the embedding lock") + } + + var name string + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT COALESCE(display_name, '') FROM participants WHERE id = ?`), + participantID).Scan(&name)) + assert.Empty(t, name, "cancelled name backfill must roll back") +} diff --git a/internal/store/participant_identifier_revision.go b/internal/store/participant_identifier_revision.go index 6ac0c620d..0568c3b95 100644 --- a/internal/store/participant_identifier_revision.go +++ b/internal/store/participant_identifier_revision.go @@ -45,8 +45,7 @@ func (s *Store) ParticipantIdentifierRevisionContext(ctx context.Context) (int64 // bumpParticipantIdentifierRevision increments the participant-identifier // revision inside tx, seeding the row with 0 first if it does not exist yet, // following bumpAccountIdentityRevision's approach. -func (s *Store) bumpParticipantIdentifierRevision(tx *loggedTx) error { - ctx := context.Background() +func (s *Store) bumpParticipantIdentifierRevision(ctx context.Context, tx *loggedTx) error { if _, err := tx.ExecContext(ctx, s.dialect.InsertOrIgnore( `INSERT OR IGNORE INTO archive_metadata (key, value) VALUES (?, '0')`), participantIdentifierRevisionKey); err != nil { @@ -62,7 +61,7 @@ func (s *Store) bumpParticipantIdentifierRevision(tx *loggedTx) error { } func (s *Store) bumpParticipantIdentifierRevisionIfChanged( - tx *loggedTx, result sql.Result, + ctx context.Context, tx *loggedTx, result sql.Result, ) error { changed, err := result.RowsAffected() if err != nil { @@ -71,5 +70,5 @@ func (s *Store) bumpParticipantIdentifierRevisionIfChanged( if changed == 0 { return nil } - return s.bumpParticipantIdentifierRevision(tx) + return s.bumpParticipantIdentifierRevision(ctx, tx) } diff --git a/internal/store/participant_observations.go b/internal/store/participant_observations.go index 4201ee591..fe4c03c6f 100644 --- a/internal/store/participant_observations.go +++ b/internal/store/participant_observations.go @@ -320,7 +320,7 @@ func (s *Store) RecordContactObservationContext( return err } result.Created = true - if err := s.bumpParticipantIdentifierRevision(tx); err != nil { + if err := s.bumpParticipantIdentifierRevision(ctx, tx); err != nil { return err } if providerContradicted { @@ -490,7 +490,7 @@ func (s *Store) SupersedeParticipantObservationContext( ); err != nil { return err } - return s.bumpParticipantIdentifierRevision(tx) + return s.bumpParticipantIdentifierRevision(ctx, tx) }) } diff --git a/internal/store/stable_anchor.go b/internal/store/stable_anchor.go new file mode 100644 index 000000000..6caea7342 --- /dev/null +++ b/internal/store/stable_anchor.go @@ -0,0 +1,128 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" +) + +// AnchorIdentity is one email or E.164 phone that an importer asserts for a +// stable provider anchor. +type AnchorIdentity struct { + Kind ContactAddressKind // ContactAddressEmail or ContactAddressPhone + Value string +} + +// StableAnchorSettledContext reports whether linking the identities of one +// anchor would change nothing. It is true when every identity already exists +// as a participant carrying a current observation of the anchor from +// sourceID, and every other participant carrying the anchor has a decided +// stable_provider_id candidate with the first identity's participant: an +// applied acceptance, a rejection, or a conflict. It only reads, so importers +// can call it on every sync. +func (s *Store) StableAnchorSettledContext( + ctx context.Context, sourceID int64, anchor string, identities []AnchorIdentity, +) (bool, error) { + if anchor == "" || len(identities) == 0 { + return true, nil + } + participantIDs := make([]int64, 0, len(identities)) + for _, identity := range identities { + column := "email_address" + if identity.Kind == ContactAddressPhone { + column = "phone_number" + } + var id int64 + err := s.db.QueryRowContext(ctx, + `SELECT id FROM participants WHERE `+column+` = ?`, identity.Value).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("look up anchored participant: %w", err) + } + var observed int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) + FROM participant_contact_observations + WHERE participant_id = ? AND source_id = ? AND provider_user_id = ? + AND active_until IS NULL AND superseded_at IS NULL`, + id, sourceID, anchor).Scan(&observed); err != nil { + return false, fmt.Errorf("check anchored observation: %w", err) + } + if observed == 0 { + return false, nil + } + participantIDs = append(participantIDs, id) + } + + rows, err := s.db.QueryContext(ctx, `SELECT participant_id + FROM participant_contact_observations + WHERE provider_user_id = ? AND active_until IS NULL AND superseded_at IS NULL + GROUP BY participant_id`, anchor) + if err != nil { + return false, fmt.Errorf("list anchored participants: %w", err) + } + var anchored []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + _ = rows.Close() + return false, err + } + anchored = append(anchored, id) + } + if err := rows.Close(); err != nil { + return false, err + } + if err := rows.Err(); err != nil { + return false, err + } + + primary := participantIDs[0] + for _, other := range anchored { + if other == primary { + continue + } + left, right := primary, other + if left > right { + left, right = right, left + } + var state IdentityMatchState + var pending bool + err := s.db.QueryRowContext(ctx, `SELECT state, application_pending + FROM identity_match_candidates + WHERE left_kind = ? AND left_id = ? AND right_kind = ? AND right_id = ? + AND basis = ? AND normalized_value = ?`, + IdentityMatchParticipant, left, IdentityMatchParticipant, right, + IdentityMatchStableProviderID, anchor).Scan(&state, &pending) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check anchored candidate: %w", err) + } + switch { + case state == IdentityMatchStateRejected, state == IdentityMatchStateConflict: + case state == IdentityMatchStateAccepted && !pending: + default: + return false, nil + } + } + return true, nil +} + +// IsAccountIdentityAddressContext reports whether address is a confirmed +// account identity (one of the archive owner's own addresses) on any source. +func (s *Store) IsAccountIdentityAddressContext(ctx context.Context, address string) (bool, error) { + var found int + err := s.db.QueryRowContext(ctx, `SELECT 1 FROM account_identities + WHERE LOWER(address) = LOWER(?) LIMIT 1`, address).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check account identity: %w", err) + } + return true, nil +} diff --git a/internal/tui/meeting_mode_test.go b/internal/tui/meeting_mode_test.go index 9714e6c7e..4a7ac6bc3 100644 --- a/internal/tui/meeting_mode_test.go +++ b/internal/tui/meeting_mode_test.go @@ -99,16 +99,18 @@ func TestMeetingAccountsExcludeUnrelatedSources(t *testing.T) { query.AccountInfo{ID: 4, SourceType: "teams", Identifier: "team-chat"}, query.AccountInfo{ID: 5, SourceType: meetingSourceImported, Identifier: "local-meetings"}, query.AccountInfo{ID: 6, SourceType: meetingSourceNotion, Identifier: "notion-notes"}, + query.AccountInfo{ID: 7, SourceType: "muesli", Identifier: "mac"}, ).Build() accounts := model.meetingAccounts() - require.Len(t, accounts, 4) - assert.Equal(t, []string{"work-notes", "team-meetings", "local-meetings", "notion-notes"}, []string{ + require.Len(t, accounts, 5) + assert.Equal(t, []string{"work-notes", "team-meetings", "local-meetings", "notion-notes", "mac"}, []string{ accounts[0].Identifier, accounts[1].Identifier, accounts[2].Identifier, accounts[3].Identifier, + accounts[4].Identifier, }) } diff --git a/internal/tui/meeting_state.go b/internal/tui/meeting_state.go index d92ce6826..baab5c19e 100644 --- a/internal/tui/meeting_state.go +++ b/internal/tui/meeting_state.go @@ -13,6 +13,7 @@ const ( meetingSourceGranola = "granola" meetingSourceCircleback = "circleback" meetingSourceNotion = "notion_meetings" + meetingSourceMuesli = "muesli" meetingSourceImported = "meeting_import" ) @@ -78,7 +79,7 @@ func (m Model) meetingAccounts() []query.AccountInfo { accounts := make([]query.AccountInfo, 0, len(m.accounts)) for _, account := range m.accounts { switch strings.ToLower(strings.TrimSpace(account.SourceType)) { - case meetingSourceGranola, meetingSourceCircleback, meetingSourceNotion, meetingSourceImported: + case meetingSourceGranola, meetingSourceCircleback, meetingSourceNotion, meetingSourceMuesli, meetingSourceImported: accounts = append(accounts, account) } } diff --git a/internal/tui/meeting_view.go b/internal/tui/meeting_view.go index 58378dc8d..59624f1e3 100644 --- a/internal/tui/meeting_view.go +++ b/internal/tui/meeting_view.go @@ -62,6 +62,8 @@ func (m Model) meetingSourceLabel(sourceID int64) string { return "Circleback" case meetingSourceNotion: return "Notion" + case meetingSourceMuesli: + return "Muesli" case meetingSourceImported: if account.DisplayName != "" { return textutil.SanitizeTerminal(account.DisplayName) diff --git a/internal/tui/meeting_view_test.go b/internal/tui/meeting_view_test.go index dd5726bdf..16c2f98c1 100644 --- a/internal/tui/meeting_view_test.go +++ b/internal/tui/meeting_view_test.go @@ -211,3 +211,18 @@ func TestMeetingListFitsNarrowTerminal(t *testing.T) { assert.LessOrEqual(lipgloss.Width(line), 32, "line exceeds terminal width: %q", stripANSI(line)) } } + +func TestMeetingViewLabelsMuesliSource(t *testing.T) { + model := NewBuilder().WithAccounts( + query.AccountInfo{ID: 9, SourceType: "muesli", Identifier: "mac"}, + ).WithSize(100, 24).Build() + model.mode = modeMeetings + model.loading = false + model.meetingState.messages = []query.MessageSummary{{ + ID: 12, SourceID: 9, Subject: "Muesli meeting", SentAt: time.Now(), + }} + + view := stripANSI(model.renderView()) + assert.Contains(t, view, "Muesli") + assert.Equal(t, "Muesli", model.meetingSourceLabel(9)) +} diff --git a/pkg/client/generated/types.go b/pkg/client/generated/types.go index e1748a035..3970039af 100644 --- a/pkg/client/generated/types.go +++ b/pkg/client/generated/types.go @@ -5489,8 +5489,14 @@ func (m MeetingMetricsRequest) Validate() error { } type MeetingPerson struct { - Email string `json:"email" validate:"required"` - Name *string `json:"name,omitzero"` + Email string `json:"email,omitempty" validate:"required_without=Phone"` + + // ID Stable identifier for this person in the import source + ID *string `json:"id,omitzero" validate:"omitempty,max=200"` + Name *string `json:"name,omitzero"` + + // Phone International phone number starting with + or 00; normalized to E.164 + Phone *string `json:"phone,omitzero" validate:"omitempty,max=64"` } func (m MeetingPerson) Validate() error { diff --git a/pkg/client/meeting_person_test.go b/pkg/client/meeting_person_test.go new file mode 100644 index 000000000..290b8713f --- /dev/null +++ b/pkg/client/meeting_person_test.go @@ -0,0 +1,45 @@ +package client + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func TestMeetingPersonOmitsEmptyEmail(t *testing.T) { + phone := "+16045550100" + + encoded, err := json.Marshal(generated.MeetingPerson{Phone: &phone}) + + require.NoError(t, err) + assert.JSONEq(t, `{"phone":"+16045550100"}`, string(encoded), + "a phone-only attendee must not send an empty email that violates format: email") +} + +func TestMeetingPersonRequiresIdentity(t *testing.T) { + t.Parallel() + phone := "+16045550100" + for _, tt := range []struct { + name string + person generated.MeetingPerson + wantErr bool + }{ + {name: "missing identity", wantErr: true}, + {name: "email only", person: generated.MeetingPerson{Email: "attendee@example.com"}}, + {name: "phone only", person: generated.MeetingPerson{Phone: &phone}}, + {name: "both", person: generated.MeetingPerson{Email: "attendee@example.com", Phone: &phone}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := tt.person.Validate() + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/client/openapi.yaml b/pkg/client/openapi.yaml index 65e3cb5af..214d6b8e4 100644 --- a/pkg/client/openapi.yaml +++ b/pkg/client/openapi.yaml @@ -5826,10 +5826,20 @@ components: format: email type: string x-go-type: string + x-go-type-skip-optional-pointer: true + x-oapi-codegen-extra-tags: + validate: required_without=Phone + x-omitempty: true + id: + description: Stable identifier for this person in the import source + maxLength: 200 + type: string name: type: string - required: - - email + phone: + description: International phone number starting with + or 00; normalized to E.164 + maxLength: 64 + type: string type: object MeetingRef: properties: diff --git a/web/src/lib/api/generated/models/meetingPerson.ts b/web/src/lib/api/generated/models/meetingPerson.ts index dff00f0ec..9efe8d057 100644 --- a/web/src/lib/api/generated/models/meetingPerson.ts +++ b/web/src/lib/api/generated/models/meetingPerson.ts @@ -2,7 +2,37 @@ * Generated by Orval. Do not edit manually. */ -export interface MeetingPerson { - email: string; - name?: string; -} +export type MeetingPerson = + | ({ + email: string; + } & { + email?: string; + /** + * Stable identifier for this person in the import source + * @maxLength 200 + */ + id?: string; + name?: string; + /** + * International phone number starting with + or 00; normalized to E.164 + * @maxLength 64 + */ + phone?: string; + }) + | ({ + /** @maxLength 64 */ + phone: string; + } & { + email?: string; + /** + * Stable identifier for this person in the import source + * @maxLength 200 + */ + id?: string; + name?: string; + /** + * International phone number starting with + or 00; normalized to E.164 + * @maxLength 64 + */ + phone?: string; + }); diff --git a/website/index.html b/website/index.html index 6089b4aff..d6ac85af8 100644 --- a/website/index.html +++ b/website/index.html @@ -150,7 +150,7 @@

Bring your accounts together.

  1. MailGmail, IMAP, and Microsoft 365 sync; MBOX, Maildir, Apple Mail, PST, and EML imports.
  2. ChatSlack, Teams, Discord, and chats available through Beeper Desktop; WhatsApp, iMessage, Google Voice, Messenger, and SMS imports.
    3. -
  3. MeetingsGranola, Circleback, and Notion AI Meeting Notes in the same searchable record.
    4. +
  4. MeetingsGranola, Circleback, Notion AI Meeting Notes, and Muesli in the same searchable record.
  5. CalendarGoogle Calendar events, organizers, and attendees, read-only.
  6. ContactsImport contacts from a CardDAV address book and choose which saved profiles to sync back.
diff --git a/website/index.md b/website/index.md index 2f79519ba..83a56e00d 100644 --- a/website/index.md +++ b/website/index.md @@ -50,7 +50,7 @@ attachments alongside the records you browse. EML imports. - **Chat** — Slack, Teams, Discord, and chats available through Beeper Desktop; WhatsApp, iMessage, Google Voice, Messenger, and SMS imports. -- **Meetings** — Granola, Circleback, and Notion AI Meeting Notes in the same +- **Meetings** — Granola, Circleback, Notion AI Meeting Notes, and Muesli in the same searchable record. - **Calendar** — Google Calendar events, organizers, and attendees, read-only. - **Contacts** — import contacts from a CardDAV address book and choose which From 386d024250f51cc467cf77e3d530deeac8fe11c3 Mon Sep 17 00:00:00 2001 From: Rusty Shackleford <182032677+salmonumbrella@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:44:31 -0700 Subject: [PATCH 082/125] feat(export): export original emails and threads (#953) MCP clients can download stored original emails and attachments without shell access to the archive host. The CLI can export visible messages in a conversation as numbered `.eml` files. - `export_eml` and chunked `get_attachment` return base64 chunks with total size and a whole-object SHA-256 digest. Later chunks pass that digest and reuse a snapshot for up to five minutes. Chunks default to 1 MiB and cap at 4 MiB; snapshot storage is bounded to eight objects and 256 MiB total. Original MIME reads enforce the 256 MiB limit before a full oversized message is decoded. Whole-file attachment responses retain their 50 MiB limit. - `list_thread` returns visible archived messages oldest first, with participants, original-MIME availability, and pagination. Hidden duplicate copies are excluded; empty or hidden-only conversations do not cause ambiguous thread lookups. Account sync time describes freshness, not conversation completeness. - `msgvault export-eml --thread -o DIR` captures membership once, writes stored MIME, and reports messages that become unavailable. Numeric references try an internal ID first, then a provider ID. `--account` disambiguates provider IDs. Filenames support IMAP mailbox IDs on Windows. Requires daemon API schema 2.33.0. SQLite and PostgreSQL serve the new routes; DuckDB reads originals through its SQLite engine. Internal-ID raw reads remain strict after deduplication purge. Gmail, IMAP, and MIME-file imports preserve received bytes; PST exports contain reconstructed MIME. Non-MIME raw data is not exported as email. Closes #952 Co-authored-by: Rusty Shackleford --- api/openapi.yaml | 299 ++++++++++- cmd/msgvault/cmd/export_eml.go | 111 +++- cmd/msgvault/cmd/export_eml_test.go | 234 ++++++++- cmd/msgvault/cmd/mcp.go | 2 +- docs/api-server.md | 15 +- docs/changelog.md | 13 + docs/cli-reference.md | 15 +- docs/usage/chat.md | 60 ++- docs/usage/exporting.md | 4 + internal/api/cli_original.go | 136 +++++ internal/api/cli_original_test.go | 234 +++++++++ internal/api/handlers_test.go | 2 + internal/api/openapi.go | 6 +- internal/api/openapi_test.go | 30 +- internal/api/person_brief_test.go | 2 +- internal/api/relationship_calendar_test.go | 2 +- internal/api/routes.go | 21 + internal/api/server.go | 2 + internal/api/server_test.go | 2 + internal/daemonclient/cli.go | 5 + internal/daemonclient/engine_adapter.go | 20 +- .../daemonclient/engine_adapter_full_test.go | 19 - internal/daemonclient/original_message.go | 187 +++++++ .../daemonclient/original_message_test.go | 250 +++++++++ internal/mcp/catalog.go | 82 ++- internal/mcp/catalog_test.go | 2 + internal/mcp/contract_integration_test.go | 47 +- internal/mcp/download.go | 123 +++++ internal/mcp/download_test.go | 204 +++++++ internal/mcp/export.go | 247 +++++++++ internal/mcp/export_test.go | 362 +++++++++++++ internal/mcp/handlers.go | 39 +- internal/mcp/protocol_test.go | 15 +- internal/mcp/resources.go | 26 +- internal/mcp/server.go | 15 + internal/mcp/server_test.go | 23 +- internal/query/duckdb.go | 27 +- internal/query/models.go | 4 + internal/query/original_message.go | 496 ++++++++++++++++++ internal/query/original_message_test.go | 432 +++++++++++++++ internal/query/postgres.go | 21 + internal/query/shared.go | 20 +- internal/query/sqlite.go | 9 +- internal/query/sqlite_crud_test.go | 21 + internal/testutil/pg_template_test.go | 20 +- pkg/client/generated/client.go | 134 +++++ pkg/client/generated/client_options.go | 88 ++++ pkg/client/generated/client_with_response.go | 238 +++++++++ pkg/client/generated/queries.go | 37 ++ pkg/client/generated/responses.go | 45 ++ pkg/client/generated/types.go | 123 +++++ pkg/client/openapi.yaml | 291 +++++++++- web/src/lib/api/generated/api/api.ts | 28 + .../models/cliOriginalMessageResponse.ts | 10 + .../models/getCLIMessageOriginalParams.ts | 22 + .../models/getCLIMessageThreadParams.ts | 34 ++ web/src/lib/api/generated/models/index.ts | 6 + .../lib/api/generated/models/messageRecord.ts | 16 + .../lib/api/generated/models/threadMessage.ts | 20 + .../lib/api/generated/models/threadPage.ts | 21 + 60 files changed, 4910 insertions(+), 109 deletions(-) create mode 100644 internal/api/cli_original.go create mode 100644 internal/api/cli_original_test.go create mode 100644 internal/daemonclient/original_message.go create mode 100644 internal/daemonclient/original_message_test.go create mode 100644 internal/mcp/download.go create mode 100644 internal/mcp/download_test.go create mode 100644 internal/mcp/export.go create mode 100644 internal/mcp/export_test.go create mode 100644 internal/query/original_message.go create mode 100644 internal/query/original_message_test.go create mode 100644 web/src/lib/api/generated/models/cliOriginalMessageResponse.ts create mode 100644 web/src/lib/api/generated/models/getCLIMessageOriginalParams.ts create mode 100644 web/src/lib/api/generated/models/getCLIMessageThreadParams.ts create mode 100644 web/src/lib/api/generated/models/messageRecord.ts create mode 100644 web/src/lib/api/generated/models/threadMessage.ts create mode 100644 web/src/lib/api/generated/models/threadPage.ts diff --git a/api/openapi.yaml b/api/openapi.yaml index 4411da732..9d437a8c6 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2333,6 +2333,18 @@ components: - body_text - body_html type: object + CliOriginalMessageResponse: + additionalProperties: true + properties: + message: + $ref: "#/components/schemas/MessageRecord" + mime: + contentEncoding: base64 + type: string + required: + - message + - mime + type: object CliRebuildFTSEvent: additionalProperties: true properties: @@ -6175,6 +6187,38 @@ components: - page_size - messages type: object + MessageRecord: + additionalProperties: true + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + last_sync_at: + format: date-time + type: + - string + - "null" + message_id: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + required: + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object MessageSummary: additionalProperties: true properties: @@ -13110,6 +13154,101 @@ components: - limit - messages type: object + ThreadMessage: + additionalProperties: true + properties: + attachment_count: + format: int64 + type: integer + cc: + items: + $ref: "#/components/schemas/Address" + type: array + deleted_from_source_at: + format: date-time + type: + - string + - "null" + from: + items: + $ref: "#/components/schemas/Address" + type: array + has_raw: + type: boolean + id: + format: int64 + type: integer + sent_at: + format: date-time + type: + - string + - "null" + source_message_id: + type: string + subject: + type: string + to: + items: + $ref: "#/components/schemas/Address" + type: array + required: + - id + - source_message_id + - subject + - from + - to + - cc + - has_raw + - attachment_count + type: object + ThreadPage: + additionalProperties: true + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + has_more: + type: boolean + last_sync_at: + format: date-time + type: + - string + - "null" + message_id: + format: int64 + type: integer + messages: + items: + $ref: "#/components/schemas/ThreadMessage" + type: array + offset: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + total: + format: int64 + type: integer + required: + - total + - offset + - has_more + - messages + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object TimelineRow: additionalProperties: true properties: @@ -13509,7 +13648,7 @@ components: type: apiKey info: title: msgvault API - version: 2.32.0 + version: 2.33.0 openapi: 3.1.0 paths: /api/ping: @@ -16436,6 +16575,80 @@ paths: summary: Get one message for CLI output tags: - API + /api/v1/cli/message/original: + get: + operationId: getCLIMessageOriginal + parameters: + - description: Internal message ID + in: query + name: id + schema: + format: int64 + type: integer + - description: Maximum decoded MIME bytes; omit for an unrestricted export + in: query + name: max_bytes + schema: + format: int64 + type: integer + - description: Provider message ID + in: query + name: source_message_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CliOriginalMessageResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get one message's original MIME for export + tags: + - API /api/v1/cli/message/raw: get: operationId: getCLIMessageRaw @@ -16496,6 +16709,90 @@ paths: summary: Get one raw message for CLI export tags: - API + /api/v1/cli/message/thread: + get: + operationId: getCLIMessageThread + parameters: + - description: Internal ID of a message in the conversation + in: query + name: id + schema: + format: int64 + type: integer + - description: Provider ID of a message in the conversation + in: query + name: source_message_id + schema: + type: string + - description: Provider conversation ID + in: query + name: thread_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + - description: Return fixed membership for the entire conversation; cannot be combined with limit or offset + in: query + name: all + schema: + type: boolean + - description: Messages per page (default 100, max 500) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Messages to skip + in: query + name: offset + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadPage" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List one conversation in chronological order for export + tags: + - API /api/v1/cli/rebuild-fts: post: operationId: rebuildCLIFTS diff --git a/cmd/msgvault/cmd/export_eml.go b/cmd/msgvault/cmd/export_eml.go index 2a4881db2..6ea8d63c3 100644 --- a/cmd/msgvault/cmd/export_eml.go +++ b/cmd/msgvault/cmd/export_eml.go @@ -4,10 +4,15 @@ import ( "errors" "fmt" "path/filepath" + "strconv" "strings" + "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/export" "go.kenn.io/msgvault/internal/fileutil" + "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" ) @@ -17,7 +22,9 @@ const ( ) var ( - exportEMLOutput string + exportEMLOutput string + exportEMLThread bool + exportEMLAccount string ) var exportEMLCmd = &cobra.Command{ @@ -28,27 +35,30 @@ var exportEMLCmd = &cobra.Command{ This command retrieves the raw MIME data stored during sync and writes it to a file. The .eml format is compatible with most email clients. +With --thread, it writes every message in the conversation that has stored +MIME into a directory, numbered oldest first, and reports messages it had to +skip and when the account last synced. + Examples: msgvault export-eml 12345 msgvault export-eml 12345 --output message.eml - msgvault export-eml 18f0abc123def -o important.eml`, + msgvault export-eml 18f0abc123def -o important.eml + msgvault export-eml 18f0abc123def --thread -o thread/`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { id, err := resolveMessageIDArg(args[0]) if err != nil { return err } + if exportEMLThread { + return runExportEMLThread(cmd, id, exportEMLAccount, exportEMLOutput) + } return runExportEML(cmd, id, exportEMLOutput) }, } func sanitizeEMLFilename(sourceMessageID string) string { - safe := strings.Map(func(r rune) rune { - if r == '/' || r == '\\' || r == '\x00' { - return '_' - } - return r - }, sourceMessageID) + safe := export.SanitizeFilename(strings.ReplaceAll(sourceMessageID, "\x00", "_")) // Ensure the result is a plain filename with no directory // components, guarding against IMAP mailbox names with // path separators or traversal sequences. @@ -102,7 +112,90 @@ func writeExportedEML(cmd *cobra.Command, sourceMessageID, outputPath string, ra return nil } +// runExportEMLThread writes each archived message of messageRef's +// conversation that has stored MIME to outputDir as -.eml, +// numbered oldest first. Numeric references try an internal message ID first, +// then a provider message ID if no internal message exists. +func runExportEMLThread(cmd *cobra.Command, messageRef, account, outputDir string) error { + if outputDir == stdoutSentinel { + return errors.New("--thread writes one file per message; pass a directory with -o") + } + if outputDir == "" { + outputDir = "." + } + s, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return fmt.Errorf("open store: %w", err) + } + defer func() { _ = s.Close() }() + engine := daemonclient.NewEngineAdapter(s) + + ref := query.MessageRef{Account: account} + if id, parseErr := strconv.ParseInt(messageRef, 10, 64); parseErr == nil && id > 0 { + ref.ID = id + } else { + ref.SourceMessageID = messageRef + } + + header, err := engine.ListThread(cmd.Context(), query.ThreadQuery{MessageRef: ref, All: true}) + if ref.ID != 0 && errors.Is(err, store.ErrMessageNotFound) { + ref.ID, ref.SourceMessageID = 0, messageRef + header, err = engine.ListThread(cmd.Context(), query.ThreadQuery{MessageRef: ref, All: true}) + } + if errors.Is(err, store.ErrMessageNotFound) { + return fmt.Errorf("message not found: %s", messageRef) + } + if errors.Is(err, query.ErrAmbiguousReference) { + return fmt.Errorf("list thread: %w; pass --account", err) + } + if err != nil { + return fmt.Errorf("list thread: %w", err) + } + messages := header.Messages + + if err := fileutil.SecureMkdirAll(outputDir, 0o700); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + width := len(strconv.Itoa(len(messages))) + exported := 0 + for i, msg := range messages { + if !msg.HasRaw { + cmd.Printf("Skipped %s: no original MIME stored\n", msg.SourceMessageID) + continue + } + original, err := engine.ReadOriginalMessage(cmd.Context(), query.MessageRef{ID: msg.ID}, 0) + if errors.Is(err, store.ErrMessageNotFound) { + cmd.Printf("Skipped %s: message no longer available\n", msg.SourceMessageID) + continue + } + if errors.Is(err, query.ErrOriginalMIMEUnavailable) { + cmd.Printf("Skipped %s: no original MIME stored\n", msg.SourceMessageID) + continue + } + if err != nil { + return fmt.Errorf("export message %d: %w", msg.ID, err) + } + name := fmt.Sprintf("%0*d-%s", width, i+1, sanitizeEMLFilename(msg.SourceMessageID)) + if err := fileutil.SecureWriteFile(filepath.Join(outputDir, name), original.MIME, emlFileMode); err != nil { + return fmt.Errorf("write file: %w", err) + } + exported++ + } + + cmd.Printf("Exported %d of %d messages in thread %s to %s\n", + exported, len(messages), header.SourceConversationID, outputDir) + if header.LastSyncAt != nil { + cmd.Printf("%s last synced %s; newer replies may not be archived yet\n", + header.Account, header.LastSyncAt.Format(time.RFC3339)) + } else { + cmd.Printf("%s has never completed a sync; newer replies may not be archived yet\n", header.Account) + } + return nil +} + func init() { rootCmd.AddCommand(exportEMLCmd) - exportEMLCmd.Flags().StringVarP(&exportEMLOutput, "output", "o", "", "Output file path (default: .eml, use - for stdout)") + exportEMLCmd.Flags().StringVarP(&exportEMLOutput, "output", "o", "", "Output file path (default: .eml, use - for stdout); with --thread, the output directory (default: current directory)") + exportEMLCmd.Flags().BoolVar(&exportEMLThread, "thread", false, "Export every message in the conversation that has stored MIME") + exportEMLCmd.Flags().StringVar(&exportEMLAccount, "account", "", "With --thread, the account (email address) that holds a provider message ID found in several accounts") } diff --git a/cmd/msgvault/cmd/export_eml_test.go b/cmd/msgvault/cmd/export_eml_test.go index 44efaa605..2132bf1af 100644 --- a/cmd/msgvault/cmd/export_eml_test.go +++ b/cmd/msgvault/cmd/export_eml_test.go @@ -2,19 +2,29 @@ package cmd import ( "bytes" + "database/sql" + "fmt" + "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" + "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/kit/daemon" + "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" ) func TestExportEMLUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) { @@ -112,14 +122,14 @@ func TestWriteExportedEMLDefaultsToSourceMessageIDFilename(t *testing.T) { cmd := &cobra.Command{Use: "export-eml"} cmd.SetOut(&out) - err := writeExportedEML(cmd, "gmail-raw", "", raw) + err := writeExportedEML(cmd, "INBOX|\x00/42", "", raw) require.NoError(err) - outputPath := filepath.Join(outputDir, "gmail-raw.eml") + outputPath := filepath.Join(outputDir, "INBOX___42.eml") got, err := os.ReadFile(outputPath) require.NoError(err) assert.Equal(raw, got) - assert.Contains(out.String(), "Exported message to: gmail-raw.eml") + assert.Contains(out.String(), "Exported message to: INBOX___42.eml") } func TestWriteExportedEMLWritesRawBytesToStdout(t *testing.T) { @@ -175,3 +185,221 @@ func emlHTTPNotFoundDaemon(t *testing.T) *httptest.Server { t.Cleanup(server.Close) return server } + +// TestExportEMLThreadWritesEveryOriginalInOrder runs against the real API +// server with an IMAP-style source: no provider thread IDs, so the +// conversation key is the root Message-ID msgvault derived from References. +func TestExportEMLThreadWritesEveryOriginalInOrder(t *testing.T) { + must := require.New(t) + checks := assert.New(t) + st := testutil.NewTestStore(t) + + src, err := st.GetOrCreateSource("imap", "owner@example.com") + must.NoError(err) + convID, err := st.EnsureConversation(src.ID, "", "Quarterly report") + must.NoError(err) + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + persist := func(sourceMessageID string, sentAt time.Time, raw []byte) { + _, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: src.ID, ConversationID: convID, SourceMessageID: sourceMessageID, + MessageType: "email", SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, + RawMIME: raw, + }) + must.NoError(err) + } + reply := []byte("Message-ID: \r\nReferences: \r\n\r\nreply \xe9\n") + root := []byte("Message-ID: \r\n\r\nroot\r\n") + persist("INBOX|2", base.Add(time.Hour), reply) + persist("INBOX|1", base, root) + persist("INBOX|3", base.Add(2*time.Hour), nil) + + cmd, out, dataDir := emlArchiveCommand(t, st, nil) + outDir := filepath.Join(dataDir, "thread") + must.NoError(runExportEMLThread(cmd, "INBOX|2", "", outDir)) + + entries, err := os.ReadDir(outDir) + must.NoError(err) + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + checks.Equal([]string{"1-INBOX_1.eml", "2-INBOX_2.eml"}, names) + got, err := os.ReadFile(filepath.Join(outDir, "1-INBOX_1.eml")) + must.NoError(err) + checks.Equal(root, got) + got, err = os.ReadFile(filepath.Join(outDir, "2-INBOX_2.eml")) + must.NoError(err) + checks.Equal(reply, got) + checks.Contains(out.String(), "Skipped INBOX|3: no original MIME stored") + checks.Contains(out.String(), "Exported 2 of 3 messages") + checks.Contains(out.String(), "owner@example.com has never completed a sync") + + err = runExportEMLThread(cmd, "INBOX|2", "", "-") + checks.ErrorContains(err, "--thread writes one file per message") +} + +// emlArchiveCommand uses the real daemon handlers and a private runtime record. +// beforeRequest lets tests coordinate actual store mutations between reads. +func emlArchiveCommand(t *testing.T, st *store.Store, beforeRequest func(*http.Request)) (*cobra.Command, *bytes.Buffer, string) { + t.Helper() + dataDir := t.TempDir() + engine := query.NewEngine(st.DB(), st.IsPostgreSQL()) + t.Cleanup(func() { _ = engine.Close() }) + router := api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{}, Store: st, Engine: engine, Logger: slog.New(slog.DiscardHandler), + }).Router() + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{Service: daemonService, Version: Version})) + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if beforeRequest != nil { + beforeRequest(r) + } + router.ServeHTTP(w, r) + })) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + writeStatsHTTPDaemonRuntime(t, dataDir, server) + cfg := &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + cmd := &cobra.Command{Use: "export-eml"} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{useLocal: true})) + out := &bytes.Buffer{} + cmd.SetOut(out) + return cmd, out, dataDir +} + +func TestExportEMLThreadResolvesNumericReferencesAndAccount(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "wanted-thread", "Wanted") + require.NoError(err) + persist := func(source, conversation int64, provider, body string) int64 { + id, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: source, ConversationID: conversation, SourceMessageID: provider, MessageType: "email", + }, RawMIME: []byte(body)}) + require.NoError(err) + return id + } + wantedID := persist(src.ID, conv, "999999", "wanted MIME") + other, err := st.GetOrCreateSource("gmail", "other@example.com") + require.NoError(err) + otherConv, err := st.EnsureConversation(other.ID, "other-thread", "Other") + require.NoError(err) + persist(other.ID, otherConv, strconv.FormatInt(wantedID, 10), "unrelated MIME") + cmd, out, dataDir := emlArchiveCommand(t, st, nil) + for _, ref := range []string{"999999", strconv.FormatInt(wantedID, 10)} { + outDir := filepath.Join(dataDir, ref) + require.NoError(runExportEMLThread(cmd, ref, "", outDir)) + got, err := os.ReadFile(filepath.Join(outDir, "1-999999.eml")) + require.NoError(err) + assert.Equal("wanted MIME", string(got)) + } + persist(other.ID, otherConv, "999999", "other MIME") + err = runExportEMLThread(cmd, "999999", "", filepath.Join(dataDir, "ambiguous")) + require.ErrorIs(err, query.ErrAmbiguousReference) + assert.Equal(1, strings.Count(err.Error(), "--account")) + assert.NotContains(err.Error(), "; pass account") + out.Reset() + require.NoError(runExportEMLThread(cmd, "999999", "owner@example.com", filepath.Join(dataDir, "scoped"))) + assert.Contains(out.String(), "thread wanted-thread") +} + +func TestExportEMLThreadHandlesOriginalReadFailures(t *testing.T) { + for _, missing := range []string{"message", "MIME", "corrupt MIME"} { + t.Run(missing, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "partial-thread", "Partial") + require.NoError(err) + var ids []int64 + for _, name := range []string{"first", "middle", "last"} { + id, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: src.ID, ConversationID: conv, SourceMessageID: name, MessageType: "email", + }, RawMIME: []byte("original MIME")}) + require.NoError(err) + ids = append(ids, id) + } + cmd, out, dataDir := emlArchiveCommand(t, st, func(r *http.Request) { + if r.URL.Path != "/api/v1/cli/message/original" || r.URL.Query().Get("id") != strconv.FormatInt(ids[1], 10) { + return + } + switch missing { + case "message": + _, err := st.MergeDuplicates(ids[0], []int64{ids[1]}, "test-dedup") + if !assert.NoError(err) { //nolint:testifylint // HTTP callback cannot call FailNow on the test goroutine. + return + } + case "MIME": + _, err := st.DB().Exec(st.Rebind("DELETE FROM message_raw WHERE message_id = ?"), ids[1]) + if !assert.NoError(err) { //nolint:testifylint // HTTP callback cannot call FailNow on the test goroutine. + return + } + case "corrupt MIME": + _, err := st.DB().Exec(st.Rebind("UPDATE message_raw SET raw_data = ? WHERE message_id = ?"), []byte("invalid zlib data"), ids[1]) + assert.NoError(err) + } + }) + outDir := filepath.Join(dataDir, "thread") + err = runExportEMLThread(cmd, "first", "", outDir) + if missing == "corrupt MIME" { + require.ErrorContains(err, "export message") + return + } + require.NoError(err) + entries, err := os.ReadDir(outDir) + require.NoError(err) + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + assert.Equal([]string{"1-first.eml", "3-last.eml"}, names) + assert.Contains(out.String(), "Skipped middle:") + assert.Contains(out.String(), "Exported 2 of 3 messages") + assert.Contains(out.String(), "owner@example.com has never completed a sync") + }) + } +} + +func TestExportEMLThreadKeepsInitialMembershipDuringInsert(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "large-thread", "Large") + require.NoError(err) + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + persist := func(name string, sentAt time.Time) error { + _, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: src.ID, ConversationID: conv, SourceMessageID: name, MessageType: "email", + SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, RawMIME: []byte(name)}) + return err + } + for i := 1; i <= 501; i++ { + require.NoError(persist(fmt.Sprintf("original-%03d", i), base.Add(time.Duration(i)*time.Minute))) + } + var inserted sync.Once + cmd, out, dataDir := emlArchiveCommand(t, st, func(r *http.Request) { + if r.URL.Path == "/api/v1/cli/message/original" || + (r.URL.Path == "/api/v1/cli/message/thread" && r.URL.Query().Get("offset") != "") { + inserted.Do(func() { assert.NoError(persist("new-older-message", base)) }) + } + }) + outDir := filepath.Join(dataDir, "thread") + require.NoError(runExportEMLThread(cmd, "original-001", "", outDir)) + entries, err := os.ReadDir(outDir) + require.NoError(err) + require.Len(entries, 501) + for i, entry := range entries { + assert.Equal(fmt.Sprintf("%03d-original-%03d.eml", i+1, i+1), entry.Name()) + } + assert.Contains(out.String(), "Exported 501 of 501 messages") +} diff --git a/cmd/msgvault/cmd/mcp.go b/cmd/msgvault/cmd/mcp.go index 673715c68..4c25f1d0e 100644 --- a/cmd/msgvault/cmd/mcp.go +++ b/cmd/msgvault/cmd/mcp.go @@ -34,7 +34,7 @@ var mcpCmd = &cobra.Command{ Long: `Start an MCP (Model Context Protocol) server over stdio. This allows Claude Desktop (or any MCP client) to query your archive -using tools like search_metadata, search_message_bodies, search_document_attachments, semantic_search_messages, get_message, list_messages, get_stats, +using tools like search_metadata, search_message_bodies, search_document_attachments, semantic_search_messages, get_message, list_messages, list_thread, export_eml, get_stats, aggregate, get_person_agenda, list_saved_views, run_saved_view, and stage_deletion. Add to Claude Desktop config: diff --git a/docs/api-server.md b/docs/api-server.md index 8b5641fd3..3d4dfce6c 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-26" +last_edited: "2026-09-27" title: Web UI & API Server description: Daemon-served analytical Web UI and REST API for your msgvault archive, with optional background sync scheduling. --- @@ -29,10 +29,21 @@ browser login, secure remote deployment, search states, and keyboard controls. The API publishes its generated OpenAPI contract at `/openapi.json`. `msgvault openapi` prints the checked-in contract without starting a daemon or opening an archive. OpenAPI `info.version` is the **API schema version**; -it is separate from the binary release version. The current schema is **2.31.0**. +it is separate from the binary release version. The current schema is **2.33.0**. Upgrade clients and daemon together across incompatible schema versions, including remote deployments. +Schema 2.33.0 adds `GET /api/v1/cli/message/original`, which returns one +message's original MIME with its account and conversation, and +`GET /api/v1/cli/message/thread`, which lists visible archived conversation +members in chronological order and reports which have original MIME stored. +The thread route accepts `all=true` to capture membership in one response; +otherwise it uses `limit` and `offset` pagination. Hidden duplicates are excluded. +The original route accepts a positive `max_bytes` below 9223372036854775807 +to bound decoded MIME and its stored payload, allowing for compression overhead. +Exceeding the bound returns `413 original_message_too_large`. Omit `max_bytes` +for an unrestricted export. + Schema 2.31.0 adds analytics query freshness metadata, accepted background cache-build jobs, job status lookup, and restricted archive SQL for MCP. diff --git a/docs/changelog.md b/docs/changelog.md index 0eafe2448..9f8d2359d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,19 @@ All notable changes to msgvault, grouped by release. for a known index gap or rebuild, including rebuilds awaiting other daemon work; an unfinished completeness check alone stays silent. +- **Export original emails over MCP.** `export_eml` returns an email's + original `.eml` bytes in chunks with a whole-message `sha256`, `list_thread` + lists visible archived conversation members oldest first, and + `get_attachment` accepts `offset` and `length` for chunked downloads. + Later chunks pass the first response's `sha256` to reuse the same snapshot. + Responses report the account's + `last_sync_at`, and `get_stats` lists it per account. + `msgvault export-eml --thread -o DIR` writes visible conversation members as + numbered `.eml` files. Works for Gmail, IMAP (including Outlook), and file + imports; PST exports are rebuilt from Outlook data. The daemon serves + these through two new CLI routes; upgrade it for API schema 2.33.0. See + [Export original emails](usage/chat.md#export-original-emails). + - Query published analytics while the daemon refreshes the cache in the background. `query --fresh` waits for current results; HTTP and MCP callers can track refresh jobs. The new `query_sql` MCP tool restricts SQL to archive diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ca051f4c8..954bb4f74 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1799,7 +1799,10 @@ the [Discord export guide](usage/discord.md#export-a-bounded-history-window). ## export-eml -Export a message as a `.eml` file. Accepts either a numeric database ID or a Gmail message ID. +Export a message as a `.eml` file. Accepts an internal message ID or a provider +message ID. A numeric reference selects a live internal ID first, then falls +back to a provider ID if that internal ID does not exist. This also applies +with `--thread`. ```bash msgvault export-eml [flags] @@ -1807,7 +1810,15 @@ msgvault export-eml [flags] | Flag | Description | |---|---| -| `-o`, `--output ` | Output file (default: `.eml`, use `-` for stdout) | +| `-o`, `--output ` | Output file (default: `.eml`, use `-` for stdout). With `--thread`, the output directory (default: current directory) | +| `--thread` | Write visible archived messages in the conversation that have stored MIME as `-.eml`, numbered oldest first. Reports skipped messages and when the account last synced. Needs a daemon with API schema 2.33.0 or newer | +| `--account ` | With `--thread`, the account that holds a provider message ID found in more than one account | + +Thread export captures its message list once. Messages added afterward belong +to the next export. It reports and skips messages that disappear or lose their +stored MIME before they are read; other read or write errors stop the command. +Hidden duplicate copies are excluded. The last-sync time describes account +freshness, not conversation completeness. --- diff --git a/docs/usage/chat.md b/docs/usage/chat.md index 9898cb21f..e14fe85b0 100644 --- a/docs/usage/chat.md +++ b/docs/usage/chat.md @@ -171,9 +171,11 @@ The MCP server exposes the following tools to connected AI clients: | `search_by_domains` | Find messages where any participant (`from`, `to`, or `cc`) belongs to one of several domains, regardless of direction. | `domains` (comma-separated string, required), `limit` (int), `offset` (int), `after` (string), `before` (string) | | `get_message` | Get message details with windowed body paging | `id` (int, required), `offset` (int), `center_at` (int), `max_chars` (int), `body_format` (string: `auto`/`text`/`html`), `full_body` (bool) | | `list_messages` | List messages with filters | `from` (string), `to` (string), `label` (string), `after` (string), `before` (string), `has_attachment` (bool), `conversation_id` (int), `limit` (int), `offset` (int), `account` (string) | -| `get_attachment` | Get attachment content by ID | `attachment_id` (int) | +| `list_thread` | List visible archived messages in one conversation, oldest first, with `has_raw` marking messages whose original `.eml` is stored. See [Export original emails](#export-original-emails). | exactly one of `id` (int), `source_message_id` (string), or `thread_id` (string); `account` (string), `limit` (1–500, default 100), `offset` (int) | +| `export_eml` | Export one email's original `.eml` bytes in verified chunks. See [Export original emails](#export-original-emails). | exactly one of `id` (int) or `source_message_id` (string); `account` (string), `offset` (int), `length` (1–4194304, default 1048576), `sha256` (string, required after offset 0) | +| `get_attachment` | Get attachment content by ID. Pass `offset` or `length` to receive verified chunks instead of one embedded blob. | `attachment_id` (int), `offset` (int), `length` (int), `sha256` (string, required after offset 0) | | `export_attachment` | Save attachment to filesystem | `attachment_id` (int), `destination` (string) | -| `get_stats` | Archive overview statistics. Includes vector index state when configured. | — | +| `get_stats` | Archive overview statistics, plus each account's `LastSyncAt`. Includes vector index state when configured. | — | | `aggregate` | Grouped statistics (top senders, domains, labels, or message volume by calendar year) | `group_by` (string: sender/recipient/domain/label/time), `limit` (int), `after` (string), `before` (string), `account` (string) | | `query_sql` | Advanced read-only SQL over the published analytics cache. Returns rows and freshness metadata, or an accepted refresh job. | `sql` (string, required), `fresh` (bool, default false) | | `list_saved_views` | List persistent reusable Saved Views and their complete definitions. Read-only. | — | @@ -248,6 +250,60 @@ slice of the body plus `body_length`, `body_returned`, `offset`, and `has_more`, so unusually large messages are paged across calls instead of being returned in a single response. +### Export original emails + +`export_eml` returns an email's original MIME exactly as the provider +delivered it, so a client that reaches msgvault only over MCP can save a +byte-identical `.eml`. `get_attachment` in chunk mode and `list_thread` +provide the attachment bytes and visible archived messages in the conversation. + +Each chunk response carries `offset`, `length`, `size`, `sha256` (of the +whole object), `complete`, and `data_base64`. To download: + +1. Call with `offset = 0`. +2. Decode `data_base64`, append it, and call again with `offset += length`. + Pass the first response's `sha256` and the same message or attachment + reference on each later call. +3. Stop when `complete` is true, then check that the file matches `size` + and `sha256` before using it. + +Chunks default to 1 MiB and are capped at 4 MiB, which keeps each response +small enough for tool gateways that reject multi-megabyte strings. The MCP +server keeps each download snapshot for up to five minutes, with at most eight +snapshots and 256 MiB total. Objects larger than 256 MiB cannot use chunk mode. +Email downloads enforce this limit while reading and decompressing the original; +CLI exports remain unrestricted. +If a snapshot expires or is evicted, restart at offset 0 and discard the earlier +partial file. The checksum pins later chunks to the same content even if the +archived original changes. Whole-file attachment responses remain limited to +50 MiB. + +Pass a provider ID as `source_message_id` or `thread_id`, never as `id`. When +the same provider ID exists in more than one account, the tool returns +`message_ambiguous` with the candidate accounts; repeat the call with +`account`. Messages without stored original MIME, such as chat and calendar +items, return `raw_mime_unavailable`. + +The export works for every email source that keeps MIME: + +| Source | What `export_eml` returns | +|---|---| +| Gmail, IMAP (including Outlook and Microsoft 365 over IMAP) | The exact bytes the server delivered | +| mbox, `.eml`, `.emlx`, and Maildir imports | The exact bytes of the imported message | +| PST imports | MIME rebuilt from Outlook data, with the original transport headers when the PST kept them; `source_type` is `pst` | + +Gmail threads use Gmail's `threadId`. IMAP and file imports have no provider +thread ID, so msgvault groups replies by their `References` and `In-Reply-To` +headers and uses the root Message-ID as `thread_id`. Pass any message's `id` +or `source_message_id` to `list_thread` when you don't know that key. + +`last_sync_at` reports the account's most recent sync activity; it does not +prove the conversation is complete. The provider may hold replies that msgvault +has not archived. Listings also omit hidden duplicate copies, including copies +whose surviving message belongs to another account's conversation. See +[Deduplication](/docs/usage/deduplication/) for that visibility policy. Both tools need a daemon with API schema +`2.33.0` or newer. + ### `search_metadata` and `search_message_bodies` / `semantic_search_messages` query syntax Supported operators: `from:`, `to:`, `cc:`, `bcc:`, `subject:`, `label:` (or `l:`), `list:` (or `list-id:`), `has:attachment`, `before:`/`after:` (YYYY-MM-DD), `older_than:`/`newer_than:` (e.g. `7d`, `2w`, `1m`, `1y`), `larger:`/`smaller:` (e.g. `5M`). Bare domains on `from:`/`to:` match any address at that domain. Multiple terms are ANDed; repeated List-Id operators require every literal substring. diff --git a/docs/usage/exporting.md b/docs/usage/exporting.md index 316852aaf..9a10f5686 100644 --- a/docs/usage/exporting.md +++ b/docs/usage/exporting.md @@ -111,6 +111,10 @@ msgvault export-eml 18abc123def --output - | `-o`, `--output` | Output file path (default: `.eml`; use `-` for stdout) | The exported `.eml` contains the original raw MIME bytes preserved during sync. +To write a conversation as numbered `.eml` files, use +`msgvault export-eml --thread --output DIR`. See the +[command reference](/docs/cli-reference/#export-eml) for reference resolution, +account selection, and skipped-message behavior. ## Export a single attachment diff --git a/internal/api/cli_original.go b/internal/api/cli_original.go new file mode 100644 index 000000000..8712bc619 --- /dev/null +++ b/internal/api/cli_original.go @@ -0,0 +1,136 @@ +package api + +import ( + "errors" + "math" + "net/http" + "strconv" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +// cliOriginalMessageResponse carries one message's provenance and its +// original MIME bytes (base64 in JSON) exactly as the provider delivered them. +type cliOriginalMessageResponse struct { + Message query.MessageRecord `json:"message"` + MIME []byte `json:"mime"` +} + +func (s *Server) originalMessageReader(w http.ResponseWriter, r *http.Request) (query.OriginalMessageReader, bool) { + reader, ok := s.queryEngineForContext(r.Context()).(query.OriginalMessageReader) + if !ok { + writeError(w, http.StatusServiceUnavailable, "original_export_unavailable", + "Original message export is not available for this archive engine") + return nil, false + } + return reader, true +} + +// cliMessageRefFromRequest reads id, source_message_id and account. The +// numeric id names only an internal message ID; a provider ID is always +// passed as source_message_id so it can never collide with one. +func cliMessageRefFromRequest(r *http.Request) (query.MessageRef, error) { + values := r.URL.Query() + ref := query.MessageRef{ + SourceMessageID: values.Get("source_message_id"), + Account: values.Get("account"), + } + if raw := values.Get("id"); raw != "" { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil || id <= 0 { + return ref, newParamError("id", "query parameter \"id\" must be a positive integer") + } + ref.ID = id + } + return ref, nil +} + +func (s *Server) handleCLIMessageOriginal(w http.ResponseWriter, r *http.Request) { + reader, ok := s.originalMessageReader(w, r) + if !ok { + return + } + ref, err := cliMessageRefFromRequest(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", err.Error()) + return + } + maxBytes, present, err := queryInt64(r, "max_bytes") + if err != nil || (present && (maxBytes <= 0 || maxBytes == math.MaxInt64)) { + writeError(w, http.StatusBadRequest, "invalid_request", "query parameter max_bytes must be positive and less than 9223372036854775807") + return + } + original, err := reader.ReadOriginalMessage(r.Context(), ref, maxBytes) + if err != nil { + s.writeOriginalExportError(w, "read original message", err) + return + } + writeJSON(w, http.StatusOK, cliOriginalMessageResponse{Message: original.MessageRecord, MIME: original.MIME}) +} + +func (s *Server) handleCLIMessageThread(w http.ResponseWriter, r *http.Request) { + reader, ok := s.originalMessageReader(w, r) + if !ok { + return + } + ref, err := cliMessageRefFromRequest(r) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", err.Error()) + return + } + q := query.ThreadQuery{MessageRef: ref, ThreadID: r.URL.Query().Get("thread_id"), Limit: query.ThreadDefaultLimit} + q.All, _, err = queryBool(r, "all") + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", err.Error()) + return + } + if q.All && (r.URL.Query().Has("limit") || r.URL.Query().Has("offset")) { + writeError(w, http.StatusBadRequest, "invalid_request", "query parameter \"all\" cannot be combined with \"limit\" or \"offset\"") + return + } + if value, present, err := queryInt(r, "limit"); err != nil || (present && (value < 1 || value > query.ThreadMaxLimit)) { + writeError(w, http.StatusBadRequest, "invalid_request", "query parameter \"limit\" must be between 1 and 500") + return + } else if present { + q.Limit = value + } + if value, present, err := queryInt(r, "offset"); err != nil || (present && value < 0) { + writeError(w, http.StatusBadRequest, "invalid_request", "query parameter \"offset\" must be a non-negative integer") + return + } else if present { + q.Offset = value + } + page, err := reader.ListThread(r.Context(), q) + if err != nil { + s.writeOriginalExportError(w, "list thread", err) + return + } + writeJSON(w, http.StatusOK, page) +} + +func (s *Server) writeOriginalExportError(w http.ResponseWriter, operation string, err error) { + var ambiguous *query.AmbiguousError + switch { + case errors.Is(err, query.ErrInvalidMessageRef): + writeError(w, http.StatusBadRequest, "invalid_request", + "Provide exactly one of id, source_message_id, or thread_id") + case errors.Is(err, store.ErrMessageNotFound): + writeError(w, http.StatusNotFound, cliErrorMessageNotFound, "Message not found") + case errors.Is(err, query.ErrThreadNotFound): + writeError(w, http.StatusNotFound, "thread_not_found", "Thread not found") + case errors.Is(err, query.ErrOriginalMessageTooLarge): + writeError(w, http.StatusRequestEntityTooLarge, "original_message_too_large", "Original MIME exceeds the requested byte limit") + case errors.Is(err, query.ErrOriginalMIMEUnavailable): + writeError(w, http.StatusNotFound, "original_mime_unavailable", "The archive holds no original MIME for this message") + case errors.Is(err, query.ErrOriginalExportUnsupported): + writeError(w, http.StatusServiceUnavailable, "original_export_unavailable", + "Original message export is not available for this archive engine") + case errors.As(err, &ambiguous): + writeError(w, http.StatusConflict, "message_ambiguous", ambiguous.Error()) + case s.writeIfContextError(w, err): + default: + s.logger.Error(operation, "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Could not "+operation) + } +} diff --git a/internal/api/cli_original_test.go b/internal/api/cli_original_test.go new file mode 100644 index 000000000..4f9fa8046 --- /dev/null +++ b/internal/api/cli_original_test.go @@ -0,0 +1,234 @@ +package api + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type cliOriginalFixture struct { + srv *Server + st *store.Store + withRaw int64 + noRaw int64 + sourceID int64 + rawBytes []byte + threadKey string +} + +func newCLIOriginalFixture(t *testing.T) *cliOriginalFixture { + t.Helper() + st := testutil.NewTestStore(t) + engine := query.NewEngine(st.DB(), st.IsPostgreSQL()) + t.Cleanup(func() { _ = engine.Close() }) + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + Store: st, + Engine: engine, + Logger: testLogger(), + }) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(t, err) + convID, err := st.EnsureConversation(src.ID, "thread-original", "Original") + require.NoError(t, err) + raw := []byte("From: owner@example.com\r\nSubject: Original\r\n\r\nbody \xe9\n") + persist := func(sourceMessageID string, sentAt time.Time, rawMIME []byte) int64 { + id, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: sourceMessageID, + MessageType: "email", + SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, + RawMIME: rawMIME, + }) + require.NoError(t, err) + return id + } + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + withRaw := persist("provider-original", base, raw) + noRaw := persist("provider-no-raw", base.Add(time.Hour), nil) + return &cliOriginalFixture{srv: srv, st: st, withRaw: withRaw, noRaw: noRaw, sourceID: src.ID, rawBytes: raw, threadKey: "thread-original"} +} + +func (f *cliOriginalFixture) get(t *testing.T, target string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + f.srv.Router().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func requireErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) { + t.Helper() + require.Equal(t, status, w.Code, "status for body %s", w.Body.String()) + var resp ErrorResponse + require.NoError(t, json.NewDecoder(w.Body).Decode(&resp)) + assert.Equal(t, code, resp.Error) +} + +func TestHandleCLIMessageOriginal(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + t.Parallel() + f := newCLIOriginalFixture(t) + + for _, target := range []string{ + "/api/v1/cli/message/original?id=" + strconv.FormatInt(f.withRaw, 10), + "/api/v1/cli/message/original?source_message_id=provider-original&account=owner@example.com", + } { + w := f.get(t, target) + must.Equal(http.StatusOK, w.Code, "%s: %s", target, w.Body.String()) + var resp cliOriginalMessageResponse + must.NoError(json.NewDecoder(w.Body).Decode(&resp)) + checks.Equal(f.rawBytes, resp.MIME) + checks.Equal(f.withRaw, resp.Message.MessageID) + checks.Equal("provider-original", resp.Message.SourceMessageID) + checks.Equal("thread-original", resp.Message.SourceConversationID) + checks.Equal("owner@example.com", resp.Message.Account) + checks.Equal(f.sourceID, resp.Message.SourceID) + } + + requireErrorCode(t, f.get(t, "/api/v1/cli/message/original"), http.StatusBadRequest, "invalid_request") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/original?id=abc"), http.StatusBadRequest, "invalid_request") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/original?id=999999"), http.StatusNotFound, "message_not_found") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/original?id="+strconv.FormatInt(f.noRaw, 10)), + http.StatusNotFound, "original_mime_unavailable") + + other, err := f.st.GetOrCreateSource("gmail", "second@example.com") + must.NoError(err) + otherConv, err := f.st.EnsureConversation(other.ID, "thread-original", "Other") + must.NoError(err) + _, err = f.st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: other.ID, ConversationID: otherConv, SourceMessageID: "provider-original", MessageType: "email", + }, RawMIME: []byte("x")}) + must.NoError(err) + w := f.get(t, "/api/v1/cli/message/original?source_message_id=provider-original") + must.Equal(http.StatusConflict, w.Code) + var ambiguous ErrorResponse + must.NoError(json.NewDecoder(w.Body).Decode(&ambiguous)) + checks.Equal("message_ambiguous", ambiguous.Error) + checks.Contains(ambiguous.Message, "owner@example.com") + checks.Contains(ambiguous.Message, "second@example.com") + checks.NotContains(ambiguous.Message, "pass account") +} + +func TestHandleCLIMessageOriginalByteLimit(t *testing.T) { + must := require.New(t) + f := newCLIOriginalFixture(t) + target := "/api/v1/cli/message/original?id=" + strconv.FormatInt(f.withRaw, 10) + requireErrorCode(t, f.get(t, target+"&max_bytes=1"), http.StatusRequestEntityTooLarge, "original_message_too_large") + w := f.get(t, target+"&max_bytes="+strconv.Itoa(len(f.rawBytes))) + must.Equal(http.StatusOK, w.Code, w.Body.String()) + var result cliOriginalMessageResponse + must.NoError(json.NewDecoder(w.Body).Decode(&result)) + assert.Equal(t, f.rawBytes, result.MIME) + for _, limit := range []string{"0", "-1", "9223372036854775807", "9223372036854775808", "bad"} { + requireErrorCode(t, f.get(t, target+"&max_bytes="+limit), http.StatusBadRequest, "invalid_request") + } +} + +func TestHandleCLIMessageOriginalUnsupportedBackend(t *testing.T) { + st := testutil.NewTestStore(t) + engine, err := query.NewDuckDBEngine("", "", nil) + require.NoError(t, err) + t.Cleanup(func() { _ = engine.Close() }) + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + Store: st, + Engine: engine, + Logger: testLogger(), + }) + w := httptest.NewRecorder() + srv.Router().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/v1/cli/message/original?id=1", nil)) + + requireErrorCode(t, w, http.StatusServiceUnavailable, "original_export_unavailable") +} + +func TestHandleCLIMessageThread(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + t.Parallel() + f := newCLIOriginalFixture(t) + + w := f.get(t, "/api/v1/cli/message/thread?id="+strconv.FormatInt(f.noRaw, 10)) + must.Equal(http.StatusOK, w.Code, w.Body.String()) + var page struct { + MessageID int64 `json:"message_id"` + Total int64 `json:"total"` + HasMore bool `json:"has_more"` + Messages []struct { + ID int64 `json:"id"` + HasRaw bool `json:"has_raw"` + } `json:"messages"` + } + must.NoError(json.NewDecoder(w.Body).Decode(&page)) + must.Len(page.Messages, 2) + checks.Equal(f.withRaw, page.Messages[0].ID) + checks.True(page.Messages[0].HasRaw) + checks.Equal(f.noRaw, page.Messages[1].ID) + checks.False(page.Messages[1].HasRaw) + checks.Equal(int64(2), page.Total) + checks.Equal(f.noRaw, page.MessageID) + + w = f.get(t, "/api/v1/cli/message/thread?thread_id=thread-original&account=owner@example.com&limit=1&offset=1") + must.Equal(http.StatusOK, w.Code, w.Body.String()) + must.NoError(json.NewDecoder(w.Body).Decode(&page)) + must.Len(page.Messages, 1) + checks.Equal(f.noRaw, page.Messages[0].ID) + checks.False(page.HasMore) + + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread"), http.StatusBadRequest, "invalid_request") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?thread_id=x&limit=501"), http.StatusBadRequest, "invalid_request") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?thread_id=x&offset=-1"), http.StatusBadRequest, "invalid_request") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?thread_id=missing"), http.StatusNotFound, "thread_not_found") + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?id=999999"), http.StatusNotFound, "message_not_found") +} + +func TestHandleCLIMessageThreadAll(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + t.Parallel() + f := newCLIOriginalFixture(t) + convID, err := f.st.EnsureConversation(f.sourceID, f.threadKey, "Original") + must.NoError(err) + for i := range 99 { + _, err := f.st.UpsertMessage(&store.Message{ + SourceID: f.sourceID, ConversationID: convID, + SourceMessageID: "additional-" + strconv.Itoa(i), MessageType: "email", + }) + must.NoError(err) + } + + w := f.get(t, "/api/v1/cli/message/thread?thread_id=thread-original&all=true") + must.Equal(http.StatusOK, w.Code, w.Body.String()) + var page struct { + Total int64 `json:"total"` + Offset int `json:"offset"` + HasMore bool `json:"has_more"` + Messages []json.RawMessage `json:"messages"` + } + must.NoError(json.NewDecoder(w.Body).Decode(&page)) + checks.Len(page.Messages, 101) + checks.Equal(int64(101), page.Total) + checks.Zero(page.Offset) + checks.False(page.HasMore) + + for _, suffix := range []string{"&limit=1", "&offset=0", "&limit=", "&offset="} { + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?thread_id=thread-original&all=true"+suffix), + http.StatusBadRequest, "invalid_request") + } + requireErrorCode(t, f.get(t, "/api/v1/cli/message/thread?thread_id=thread-original&all=invalid"), + http.StatusBadRequest, "invalid_request") +} diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go index e60145f56..461f0466c 100644 --- a/internal/api/handlers_test.go +++ b/internal/api/handlers_test.go @@ -743,6 +743,8 @@ func TestOpenAPIExportsServerRouteTable(t *testing.T) { "/api/v1/cli/repair-encoding": {"post"}, "/api/v1/cli/message": {"get"}, "/api/v1/cli/message/raw": {"get"}, + "/api/v1/cli/message/original": {"get"}, + "/api/v1/cli/message/thread": {"get"}, "/api/v1/cli/attachment": {"get"}, "/api/v1/cli/collections": {"get", "post"}, "/api/v1/cli/collections/{name}": {"delete"}, diff --git a/internal/api/openapi.go b/internal/api/openapi.go index 266e75d28..2bd57aa4d 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -331,7 +331,11 @@ import ( // requests remain valid. // 2.32.0 adds frozen sender keys to agent-token source views and optional // sender selections to owner token issuance. -const APISchemaVersion = "2.32.0" +// 2.33.0 adds GET /api/v1/cli/message/original (a message's original MIME +// and provenance) and GET /api/v1/cli/message/thread (a complete, +// chronological conversation listing with original-MIME availability). +// Additive (minor bump): existing routes are unchanged. +const APISchemaVersion = "2.33.0" // OpenAPIDocument builds the API schema from the same Huma route registration // used by the daemon. It binds no socket and needs no database. diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go index c1a342788..68e10cf30 100644 --- a/internal/api/openapi_test.go +++ b/internal/api/openapi_test.go @@ -41,7 +41,7 @@ func TestMeetingIntelligenceOpenAPIContract(t *testing.T) { t.Parallel() requirements := require.New(t) assertions := assert.New(t) - assertions.Equal("2.32.0", APISchemaVersion) + assertions.Equal("2.33.0", APISchemaVersion) doc := OpenAPIDocument() for path, operationID := range map[string]string{ "/api/v1/meetings/context": "getMeetingContext", @@ -187,7 +187,7 @@ func TestOpenAPISchemaVersionSavedViewRun(t *testing.T) { t.Parallel() assertions := assert.New(t) requirements := require.New(t) - assertions.Equal("2.32.0", APISchemaVersion) + assertions.Equal("2.33.0", APISchemaVersion) doc := OpenAPIDocument() run := doc.Paths["/api/v1/saved-views/{id}/run"] requirements.NotNil(run, "Saved View run path") @@ -206,13 +206,13 @@ func TestOpenAPISchemaVersionSavedViewRun(t *testing.T) { func TestDeletionSubsetSchemaVersion(t *testing.T) { t.Parallel() - assert.Equal(t, "2.32.0", APISchemaVersion) + assert.Equal(t, "2.33.0", APISchemaVersion) } func TestOperationsWorkspaceSchemaVersion(t *testing.T) { t.Parallel() for _, doc := range []*huma.OpenAPI{OpenAPIDocument(), openAPIClientDocument()} { - assert.Equal(t, "2.32.0", doc.Info.Version) + assert.Equal(t, "2.33.0", doc.Info.Version) } } @@ -220,7 +220,7 @@ func TestSearchTimingFieldsUseAdditiveSchemaVersion(t *testing.T) { t.Parallel() assertions := assert.New(t) requirements := require.New(t) - assertions.Equal("2.32.0", APISchemaVersion) + assertions.Equal("2.33.0", APISchemaVersion) for _, document := range []*huma.OpenAPI{OpenAPIDocument(), openAPIClientDocument()} { schemas := document.Components.Schemas.Map() @@ -240,7 +240,7 @@ func TestSearchTimingFieldsUseAdditiveSchemaVersion(t *testing.T) { func TestOpenAPISchemaVersionPersonBrief(t *testing.T) { t.Parallel() - assert.Equal(t, "2.32.0", APISchemaVersion) + assert.Equal(t, "2.33.0", APISchemaVersion) } func TestOpenAPIImportJobContract(t *testing.T) { @@ -405,7 +405,7 @@ func TestOpenAPISeparatesParticipantAnalyticsFromDurablePeople(t *testing.T) { assertions := assert.New(t) doc := OpenAPIDocument() - assertions.Equal("2.32.0", APISchemaVersion) + assertions.Equal("2.33.0", APISchemaVersion) for _, path := range []string{ "/api/v1/participants/search", "/api/v1/participants/{id}", @@ -428,12 +428,12 @@ func TestOpenAPISeparatesParticipantAnalyticsFromDurablePeople(t *testing.T) { func TestAnalyticsCacheReadinessUsesAdditiveSchemaVersion(t *testing.T) { t.Parallel() - assert.Equal(t, "2.32.0", APISchemaVersion) + assert.Equal(t, "2.33.0", APISchemaVersion) } func TestPersonFilesUseAdditiveSchemaVersion(t *testing.T) { t.Parallel() - assert.Equal(t, "2.32.0", APISchemaVersion) + assert.Equal(t, "2.33.0", APISchemaVersion) } func TestPersonFileRoutesPublishTypedPathIDs(t *testing.T) { @@ -459,7 +459,7 @@ func TestPersonFileRoutesPublishTypedPathIDs(t *testing.T) { func TestOrganizationCreateOpenAPIDocumentsLocationHeader(t *testing.T) { t.Parallel() requirements := require.New(t) - assert.Equal(t, "2.32.0", APISchemaVersion, + assert.Equal(t, "2.33.0", APISchemaVersion, "document and person-file search preserve the organization and employment contract") for _, document := range []*huma.OpenAPI{ OpenAPIDocument(), @@ -798,7 +798,7 @@ func TestOpenAPIPersonAttributeContract(t *testing.T) { t.Parallel() requirements := require.New(t) assertions := assert.New(t) - assertions.Equal("2.32.0", APISchemaVersion, + assertions.Equal("2.33.0", APISchemaVersion, "activity, identity match review, document search, and person files preserve the structured profile contract") doc := OpenAPIDocument() @@ -915,7 +915,7 @@ func TestOpenAPIOrganizationProfilePutDocumentsLimits(t *testing.T) { t.Parallel() assertions := assert.New(t) requirements := require.New(t) - assertions.Equal("2.32.0", APISchemaVersion, + assertions.Equal("2.33.0", APISchemaVersion, "organization profile write limits advance the published contract") doc := OpenAPIDocument() path := doc.Paths["/api/v1/organizations/{id}/profile"] @@ -936,7 +936,7 @@ func TestOpenAPIPersonProfileMediaContentContract(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) - assertions.Equal("2.32.0", APISchemaVersion, + assertions.Equal("2.33.0", APISchemaVersion, "activity, identity match review, document search, and person files preserve the raw profile media contract") doc := OpenAPIDocument() path := doc.Paths["/api/v1/people/{id}/profile/media/{media_id}/content"] @@ -965,7 +965,7 @@ func TestOpenAPIIdentityMatchReviewContract(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) - assertions.Equal("2.32.0", APISchemaVersion, + assertions.Equal("2.33.0", APISchemaVersion, "document and person-file search preserve the identity match review contract") doc := OpenAPIDocument() @@ -1019,7 +1019,7 @@ func TestOpenAPIMeetingImportContract(t *testing.T) { // historical import jobs in 2.16.0, collection source scopes in 2.17.0, // deletion subset counts in 2.18.0, Operations in 2.19.0, person briefs // in 2.20.0, and Saved View execution in 2.21.0 did not touch it. - assertions.Equal("2.32.0", APISchemaVersion, "meeting import is an additive schema release") + assertions.Equal("2.33.0", APISchemaVersion, "meeting import is an additive schema release") doc := OpenAPIDocument() path := doc.Paths["/api/v1/import/meeting"] diff --git a/internal/api/person_brief_test.go b/internal/api/person_brief_test.go index 4ca21902d..fdcd5f7ae 100644 --- a/internal/api/person_brief_test.go +++ b/internal/api/person_brief_test.go @@ -754,7 +754,7 @@ func TestPersonBriefOpenAPIContract(t *testing.T) { assertions.Equal("getPersonBriefEnrollment", enrollment.Get.OperationID) assertions.Equal("setPersonBriefEnrollment", enrollment.Put.OperationID) - assertions.Equal("2.32.0", APISchemaVersion) + assertions.Equal("2.33.0", APISchemaVersion) // The handlers accept an absent reason and an absent track, so the schema // generated clients are built from must not demand either. diff --git a/internal/api/relationship_calendar_test.go b/internal/api/relationship_calendar_test.go index c179f5a15..926d0f97a 100644 --- a/internal/api/relationship_calendar_test.go +++ b/internal/api/relationship_calendar_test.go @@ -184,7 +184,7 @@ func TestRelationshipCalendarOpenAPIContract(t *testing.T) { t.Parallel() assert := assert.New(t) require := require.New(t) - assert.Equal("2.32.0", APISchemaVersion) + assert.Equal("2.33.0", APISchemaVersion) document := OpenAPIDocument() path := document.Paths["/api/v1/relationships/{id}/calendar"] require.NotNil(path) diff --git a/internal/api/routes.go b/internal/api/routes.go index baf0a5b01..1bcca5b8a 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -358,6 +358,10 @@ func (s *Server) registerHumaRoutes(api huma.API, apiV1 huma.API) { registerAPIV1RawHumaJSONRouteWithRequest[CLIEmbeddingsPlanRequest, CLIEmbeddingsPlanResponse](apiV1, "planCLIEmbeddings", http.MethodPost, "/cli/embeddings/plan", "Plan CLI embeddings management", s.handleCLIEmbeddingsPlan) registerAPIV1RawHumaNDJSONRouteWithRequest[CLIRunRequest, CLIRunEvent](apiV1, "runCLI", http.MethodPost, "/cli/run", "Run an allowlisted CLI command", s.handleCLIRun) registerAPIV1RawHumaJSONRoute[cliMessageResponse](apiV1, "getCLIMessage", http.MethodGet, "/cli/message", "Get one message for CLI output", s.handleCLIMessage) + registerAPIV1RawHumaJSONRouteWithErrors[cliOriginalMessageResponse](apiV1, "getCLIMessageOriginal", http.MethodGet, "/cli/message/original", "Get one message's original MIME for export", s.handleCLIMessageOriginal, + http.StatusBadRequest, http.StatusNotFound, http.StatusConflict, http.StatusRequestEntityTooLarge, http.StatusServiceUnavailable) + registerAPIV1RawHumaJSONRouteWithErrors[query.ThreadPage](apiV1, "getCLIMessageThread", http.MethodGet, "/cli/message/thread", "List one conversation in chronological order for export", s.handleCLIMessageThread, + http.StatusBadRequest, http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable) // Agent-token management routes: owner API key required. registerAPIV1RawHumaJSONRouteWithRequest[agentTokenIssueRequest, agentTokenIssueResponse](apiV1, "issueAgentToken", http.MethodPost, "/agent-tokens", "Issue a restricted agent grant", s.handleIssueAgentToken, http.StatusCreated) registerAPIV1RawHumaJSONRoute[agentTokenListResponse](apiV1, "listAgentTokens", http.MethodGet, "/agent-tokens", "List active agent grants", s.handleListAgentTokens) @@ -807,6 +811,23 @@ func rawRouteParameters(operationID string) []*huma.Param { } case "getCLIMessage", "getCLIMessageRaw": return []*huma.Param{queryStringParam("id", "Message numeric ID or source message ID", true)} + case "getCLIMessageOriginal": + return []*huma.Param{ + queryIntegerParam("id", "Internal message ID"), + queryIntegerParam("max_bytes", "Maximum decoded MIME bytes; omit for an unrestricted export"), + queryStringParam("source_message_id", "Provider message ID", false), + queryStringParam("account", "Source identifier that narrows the lookup", false), + } + case "getCLIMessageThread": + return []*huma.Param{ + queryIntegerParam("id", "Internal ID of a message in the conversation"), + queryStringParam("source_message_id", "Provider ID of a message in the conversation", false), + queryStringParam("thread_id", "Provider conversation ID", false), + queryStringParam("account", "Source identifier that narrows the lookup", false), + queryBooleanParam("all", "Return fixed membership for the entire conversation; cannot be combined with limit or offset"), + queryIntegerParam(limitParam, "Messages per page (default 100, max 500)"), + queryIntegerParam("offset", "Messages to skip"), + } case "getCLIAttachment": return []*huma.Param{queryStringParam("content_hash", "Attachment SHA-256 content hash", true)} case "getCLICollection": diff --git a/internal/api/server.go b/internal/api/server.go index 641237c24..a43b764b4 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1177,6 +1177,8 @@ func cliRequestNeedsProtectiveCeiling(r *http.Request) bool { "POST /api/v1/cli/embeddings/plan", "GET /api/v1/cli/message", "GET /api/v1/cli/message/raw", + "GET /api/v1/cli/message/original", + "GET /api/v1/cli/message/thread", "GET /api/v1/cli/attachment", "GET /api/v1/cli/search", "POST /api/v1/cli/deduplicate/plan", diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 596aa4317..9a22991ad 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -1827,6 +1827,8 @@ func TestMarkedCLIProtectiveCeilingInventory(t *testing.T) { {method: http.MethodPost, path: "/api/v1/cli/embeddings/plan"}, {method: http.MethodGet, path: "/api/v1/cli/message"}, {method: http.MethodGet, path: "/api/v1/cli/message/raw"}, + {method: http.MethodGet, path: "/api/v1/cli/message/original"}, + {method: http.MethodGet, path: "/api/v1/cli/message/thread"}, {method: http.MethodGet, path: "/api/v1/cli/attachment"}, {method: http.MethodGet, path: "/api/v1/cli/search"}, {method: http.MethodPost, path: "/api/v1/cli/deduplicate/plan"}, diff --git a/internal/daemonclient/cli.go b/internal/daemonclient/cli.go index d7c9e5552..ff979f956 100644 --- a/internal/daemonclient/cli.go +++ b/internal/daemonclient/cli.go @@ -380,6 +380,8 @@ type cliStreamEvent struct { const ( apiErrorCodeMessageNotFound = "message_not_found" apiErrorCodeLegacyNotFound = "not_found" + // apiErrorCodeRawMessageNotFound means the message exists without raw data. + apiErrorCodeRawMessageNotFound = "raw_message_not_found" ) // InitCLIArchive runs setup-style startup work through the CLI-compatible API. @@ -1391,6 +1393,9 @@ func handleCLIMessageRawNotFound(resp *generated.GetCLIMessageRawResp, id string (resp.JSON404.ErrorData == apiErrorCodeLegacyNotFound && message == "Message not found") { return fmt.Errorf("message %s: %w", id, store.ErrMessageNotFound) } + if resp.JSON404.ErrorData == apiErrorCodeRawMessageNotFound { + return fmt.Errorf("message %s: %w", id, ErrMessageRawNotFound) + } if message != "" { return fmt.Errorf("API error (%d): %s", http.StatusNotFound, message) } diff --git a/internal/daemonclient/engine_adapter.go b/internal/daemonclient/engine_adapter.go index d415a8bea..6129bd99c 100644 --- a/internal/daemonclient/engine_adapter.go +++ b/internal/daemonclient/engine_adapter.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" "slices" - "strconv" "strings" + "sync" "time" "go.kenn.io/msgvault/internal/query" @@ -19,6 +19,9 @@ import ( // ErrNotSupported is returned for operations not available through the daemon API. var ErrNotSupported = errors.New("operation not supported through daemon API") +// ErrMessageRawNotFound means the daemon holds the message but no raw data. +var ErrMessageRawNotFound = errors.New("message raw data not found") + const ( apiValueCount = "count" apiValueLabels = "labels" @@ -31,7 +34,9 @@ const ( // Engine implements query.Engine by making HTTP calls to a msgvault daemon. type Engine struct { - store *Client + store *Client + originalCapabilityMu sync.Mutex + originalCapabilityOK bool } // QueryArchiveSQL uses the daemon's archive-only SQL endpoint. It never falls @@ -908,8 +913,14 @@ func (e *Engine) GetMessageSummariesByIDs(ctx context.Context, ids []int64) ([]q // GetMessageRaw returns raw MIME data for a message. func (e *Engine) GetMessageRaw(ctx context.Context, id int64) ([]byte, error) { - raw, _, err := e.store.GetCLIMessageRaw(ctx, strconv.FormatInt(id, 10)) - return raw, err + original, err := e.ReadOriginalMessage(ctx, query.MessageRef{ID: id}, 0) + if errors.Is(err, query.ErrOriginalMIMEUnavailable) { + return nil, nil + } + if err != nil { + return nil, err + } + return original.MIME, nil } // GetAttachment returns attachment metadata by ID. @@ -1344,6 +1355,7 @@ func (e *Engine) ListAccounts(ctx context.Context) ([]query.AccountInfo, error) SourceType: acc.Type, Identifier: acc.Email, DisplayName: acc.DisplayName, + LastSyncAt: copyTime(acc.LastSync), } } return result, nil diff --git a/internal/daemonclient/engine_adapter_full_test.go b/internal/daemonclient/engine_adapter_full_test.go index a08121ffd..fdda6c7b7 100644 --- a/internal/daemonclient/engine_adapter_full_test.go +++ b/internal/daemonclient/engine_adapter_full_test.go @@ -799,25 +799,6 @@ func TestEngineGetMessagePreservesPhoneOnlyGeneratedSender(t *testing.T) { assert.Equal("Alice", msg.From[0].Name, "From[0].Name") } -func TestEngineGetMessageRawUsesGeneratedCLIEndpoint(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - raw := []byte("From: alice@example.com\r\nSubject: Raw\r\n\r\nBody") - - store := newGeneratedClientAdapterStore(t, func(w http.ResponseWriter, r *http.Request) { - assert.Equal("/api/v1/cli/message/raw", r.URL.Path, "path") - assert.Equal("42", r.URL.Query().Get("id"), "id") - w.Header().Set("Content-Type", "message/rfc822") - _, _ = w.Write(raw) - }) - - engine := NewEngineAdapter(store) - - got, err := engine.GetMessageRaw(context.Background(), 42) - require.NoError(err, "GetMessageRaw") - assert.Equal(raw, got, "raw") -} - func TestEngineGetAttachmentUsesGeneratedClientAdapter(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/internal/daemonclient/original_message.go b/internal/daemonclient/original_message.go new file mode 100644 index 000000000..9fab6a383 --- /dev/null +++ b/internal/daemonclient/original_message.go @@ -0,0 +1,187 @@ +package daemonclient + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "math" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + apiclient "go.kenn.io/msgvault/pkg/client" + "go.kenn.io/msgvault/pkg/client/generated" +) + +// originalMessageMinAPISchemaVersion is the first daemon API schema that +// serves /api/v1/cli/message/original and /api/v1/cli/message/thread. +const originalMessageMinAPISchemaVersion = "2.33.0" + +var _ query.OriginalMessageReader = (*Engine)(nil) + +// ReadOriginalMessage fetches a message's original MIME and provenance. +func (e *Engine) ReadOriginalMessage(ctx context.Context, ref query.MessageRef, maxBytes int64) (*query.OriginalMessage, error) { + if maxBytes < 0 || maxBytes == math.MaxInt64 { + return nil, fmt.Errorf("max_bytes must be between 0 and %d", int64(math.MaxInt64-1)) + } + if (ref.ID == 0) == (ref.SourceMessageID == "") { + return nil, query.ErrInvalidMessageRef + } + if err := e.requireOriginalMessageCapability(ctx); err != nil { + return nil, err + } + resp, err := APIResponse(e.store, func(client *apiclient.Client) (*generated.GetCLIMessageOriginalResp, error) { + return client.GetCLIMessageOriginalWithResponse(ctx, &generated.GetCLIMessageOriginalRequestOptions{ + Query: &generated.GetCLIMessageOriginalQuery{ + ID: optionalPositiveInt64Value(ref.ID), + MaxBytes: optionalPositiveInt64Value(maxBytes), + SourceMessageID: optionalString(ref.SourceMessageID), + Account: optionalString(ref.Account), + }, + }) + }) + if err != nil { + return nil, originalExportError(err) + } + body := resp.JSON200 + mime, err := base64.StdEncoding.DecodeString(body.Mime) + if err != nil { + return nil, fmt.Errorf("decode original MIME: %w", err) + } + return &query.OriginalMessage{MessageRecord: messageRecordFromGenerated(body.Message), MIME: mime}, nil +} + +// ListThread fetches a page or the fixed membership of an entire conversation. +func (e *Engine) ListThread(ctx context.Context, q query.ThreadQuery) (*query.ThreadPage, error) { + if q.All { + q.Limit, q.Offset = 0, 0 + } + if err := e.requireOriginalMessageCapability(ctx); err != nil { + return nil, err + } + resp, err := APIResponse(e.store, func(client *apiclient.Client) (*generated.GetCLIMessageThreadResp, error) { + return client.GetCLIMessageThreadWithResponse(ctx, &generated.GetCLIMessageThreadRequestOptions{ + Query: &generated.GetCLIMessageThreadQuery{ + ID: optionalPositiveInt64Value(q.ID), + SourceMessageID: optionalString(q.SourceMessageID), + ThreadID: optionalString(q.ThreadID), + Account: optionalString(q.Account), + Limit: optionalPositiveInt64(q.Limit), + Offset: optionalPositiveInt64(q.Offset), + All: optionalBool(q.All), + }, + }) + }) + if err != nil { + return nil, originalExportError(err) + } + return threadPageFromGenerated(resp.JSON200), nil +} + +func (e *Engine) requireOriginalMessageCapability(ctx context.Context) error { + e.originalCapabilityMu.Lock() + defer e.originalCapabilityMu.Unlock() + if e.originalCapabilityOK { + return nil + } + supported, err := e.store.SupportsAPISchemaVersion(ctx, originalMessageMinAPISchemaVersion) + if err != nil { + return fmt.Errorf("check daemon original-message capability: %w", err) + } + if !supported { + return fmt.Errorf("original message export needs daemon API schema %s or newer: %w: %w", + originalMessageMinAPISchemaVersion, ErrNotSupported, query.ErrOriginalExportUnsupported) + } + e.originalCapabilityOK = true + return nil +} + +// originalExportError maps the routes' stable error codes back to the +// query package's sentinels so callers branch the same way locally and +// through the daemon. +func originalExportError(err error) error { + var apiErr *APIError + if !errors.As(err, &apiErr) { + return err + } + switch apiErr.Code { + case "invalid_request": + return &originalMessageError{message: apiErr.Message, cause: query.ErrInvalidMessageRef} + case apiErrorCodeMessageNotFound: + return &originalMessageError{message: apiErr.Message, cause: store.ErrMessageNotFound} + case "thread_not_found": + return &originalMessageError{message: apiErr.Message, cause: query.ErrThreadNotFound} + case "original_message_too_large": + return &originalMessageError{message: apiErr.Message, cause: query.ErrOriginalMessageTooLarge} + case "original_mime_unavailable": + return &originalMessageError{message: apiErr.Message, cause: query.ErrOriginalMIMEUnavailable} + case "original_export_unavailable": + return &originalMessageError{message: apiErr.Message, cause: query.ErrOriginalExportUnsupported} + case "message_ambiguous": + return &originalMessageError{message: apiErr.Message, cause: query.ErrAmbiguousReference} + } + return err +} + +// originalMessageError keeps the daemon's explanation without appending the +// sentinel's generic text a second time. +type originalMessageError struct { + message string + cause error +} + +func (e *originalMessageError) Error() string { return e.message } +func (e *originalMessageError) Unwrap() error { return e.cause } + +func messageRecordFromGenerated(record generated.MessageRecord) query.MessageRecord { + return query.MessageRecord{ + MessageID: int64Value(record.MessageID), + SourceMessageID: stringValue(record.SourceMessageID), + ConversationID: record.ConversationID, + SourceConversationID: record.SourceConversationID, + SourceID: record.SourceID, + Account: record.Account, + SourceType: record.SourceType, + LastSyncAt: copyTime(record.LastSyncAt), + } +} + +func threadPageFromGenerated(page *generated.ThreadPage) *query.ThreadPage { + out := &query.ThreadPage{ + MessageID: int64Value(page.MessageID), + SourceMessageID: stringValue(page.SourceMessageID), + ConversationID: page.ConversationID, + SourceConversationID: page.SourceConversationID, + SourceID: page.SourceID, + Account: page.Account, + SourceType: page.SourceType, + LastSyncAt: copyTime(page.LastSyncAt), + Total: page.Total, + Offset: int(page.Offset), + HasMore: page.HasMore, + Messages: make([]query.ThreadMessage, 0, len(page.Messages)), + } + for _, msg := range page.Messages { + out.Messages = append(out.Messages, query.ThreadMessage{ + ID: msg.ID, + SourceMessageID: msg.SourceMessageID, + Subject: msg.Subject, + SentAt: copyTime(msg.SentAt), + From: addressesFromGenerated(msg.From), + To: addressesFromGenerated(msg.To), + Cc: addressesFromGenerated(msg.Cc), + HasRaw: msg.HasRaw, + AttachmentCount: int(msg.AttachmentCount), + DeletedFromSourceAt: copyTime(msg.DeletedFromSourceAt), + }) + } + return out +} + +func addressesFromGenerated(addresses []generated.Address) []query.Address { + out := make([]query.Address, 0, len(addresses)) + for _, addr := range addresses { + out = append(out, query.Address{Email: addr.Email, Name: addr.Name}) + } + return out +} diff --git a/internal/daemonclient/original_message_test.go b/internal/daemonclient/original_message_test.go new file mode 100644 index 000000000..05d5e9149 --- /dev/null +++ b/internal/daemonclient/original_message_test.go @@ -0,0 +1,250 @@ +package daemonclient_test + +import ( + "database/sql" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type originalDaemon struct { + engine *daemonclient.Engine + st *store.Store + withRaw int64 + noRaw int64 + raw []byte + healthRequests *atomic.Int64 + failHealth *atomic.Bool +} + +func newOriginalDaemon(t *testing.T) originalDaemon { + t.Helper() + st := testutil.NewTestStore(t) + engine := query.NewEngine(st.DB(), st.IsPostgreSQL()) + t.Cleanup(func() { _ = engine.Close() }) + router := api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{}, Store: st, Engine: engine, Logger: slog.New(slog.DiscardHandler), + }).Router() + healthRequests := new(atomic.Int64) + failHealth := new(atomic.Bool) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/health" { + healthRequests.Add(1) + if failHealth.Swap(false) { + http.Error(w, "temporary failure", http.StatusServiceUnavailable) + return + } + } + router.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{URL: server.URL, AllowInsecure: true, HTTPClient: server.Client()}) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, client.Close()) }) + + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(t, err) + convID, err := st.EnsureConversation(src.ID, "thread-daemon", "Daemon") + require.NoError(t, err) + raw := []byte("Subject: daemon\r\n\r\n\x00\xff bytes\n") + persist := func(sourceMessageID string, sentAt time.Time, rawMIME []byte) int64 { + id, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: src.ID, ConversationID: convID, SourceMessageID: sourceMessageID, + MessageType: "email", SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, + RawMIME: rawMIME, + }) + require.NoError(t, err) + return id + } + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + return originalDaemon{ + engine: daemonclient.NewEngineAdapter(client), + st: st, + withRaw: persist("daemon-raw", base, raw), + noRaw: persist("daemon-no-raw", base.Add(time.Hour), nil), + raw: raw, + healthRequests: healthRequests, + failHealth: failHealth, + } +} + +func TestEngineReadsOriginalMessagesThroughDaemon(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + d := newOriginalDaemon(t) + ctx := t.Context() + + got, err := d.engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: "daemon-raw"}, 0) + must.NoError(err) + checks.Equal(d.raw, got.MIME) + checks.Equal(d.withRaw, got.MessageID) + checks.Equal("owner@example.com", got.Account) + checks.Equal("thread-daemon", got.SourceConversationID) + + _, err = d.engine.ReadOriginalMessage(ctx, query.MessageRef{ID: d.noRaw}, 0) + must.ErrorIs(err, query.ErrOriginalMIMEUnavailable) + _, err = d.engine.ReadOriginalMessage(ctx, query.MessageRef{ID: 999999}, 0) + must.ErrorIs(err, store.ErrMessageNotFound) + _, err = d.engine.ReadOriginalMessage(ctx, query.MessageRef{}, 0) + must.ErrorIs(err, query.ErrInvalidMessageRef) + + raw, err := d.engine.GetMessageRaw(ctx, d.withRaw) + must.NoError(err) + checks.Equal(d.raw, raw) + + raw, err = d.engine.GetMessageRaw(ctx, d.noRaw) + must.NoError(err, "missing raw follows the engine's nil, nil contract") + checks.Nil(raw) +} + +func TestEngineOriginalMessageByteLimitThroughDaemon(t *testing.T) { + must := require.New(t) + checks := assert.New(t) + d := newOriginalDaemon(t) + ref := query.MessageRef{ID: d.withRaw} + _, err := d.engine.ReadOriginalMessage(t.Context(), ref, int64(len(d.raw)-1)) + must.ErrorIs(err, query.ErrOriginalMessageTooLarge) + got, err := d.engine.ReadOriginalMessage(t.Context(), ref, int64(len(d.raw))) + must.NoError(err) + checks.Equal(d.raw, got.MIME) + got, err = d.engine.ReadOriginalMessage(t.Context(), ref, 0) + must.NoError(err) + checks.Equal(d.raw, got.MIME, "unrestricted CLI reads remain available") +} + +func TestEngineListsThreadThroughDaemon(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + d := newOriginalDaemon(t) + ctx := t.Context() + + page, err := d.engine.ListThread(ctx, query.ThreadQuery{ThreadID: "thread-daemon", Limit: 1, Offset: 1}) + must.NoError(err) + must.Len(page.Messages, 1) + checks.Equal(d.noRaw, page.Messages[0].ID) + checks.False(page.Messages[0].HasRaw) + checks.Equal(int64(2), page.Total) + checks.Equal(1, page.Offset) + checks.False(page.HasMore) + + all, err := d.engine.ListThread(ctx, query.ThreadQuery{ThreadID: "thread-daemon", All: true, Limit: 1, Offset: 1}) + must.NoError(err) + must.Len(all.Messages, 2) + checks.Equal(d.withRaw, all.Messages[0].ID) + checks.Equal(d.noRaw, all.Messages[1].ID) + checks.Equal(int64(2), all.Total) + checks.False(all.HasMore) + + _, err = d.engine.ListThread(ctx, query.ThreadQuery{ThreadID: "missing"}) + must.ErrorIs(err, query.ErrThreadNotFound) + + other, err := d.st.GetOrCreateSource("gmail", "second@example.com") + must.NoError(err) + otherConv, err := d.st.EnsureConversation(other.ID, "thread-daemon", "Other") + must.NoError(err) + _, err = d.st.UpsertMessage(&store.Message{ + SourceID: other.ID, ConversationID: otherConv, SourceMessageID: "other-message", MessageType: "email", + }) + must.NoError(err) + _, err = d.engine.ListThread(ctx, query.ThreadQuery{ThreadID: "thread-daemon"}) + must.ErrorIs(err, query.ErrAmbiguousReference) + checks.Contains(err.Error(), "second@example.com") + checks.Equal(1, strings.Count(err.Error(), "matches several accounts")) + checks.NotContains(err.Error(), "pass account", "presentation layers supply their own hint") +} + +func TestEngineOriginalCapabilityRetriesFailureAndCachesSuccess(t *testing.T) { + must := require.New(t) + checks := assert.New(t) + d := newOriginalDaemon(t) + d.failHealth.Store(true) + _, err := d.engine.ReadOriginalMessage(t.Context(), query.MessageRef{ID: d.withRaw}, 0) + must.Error(err) + _, err = d.engine.ListThread(t.Context(), query.ThreadQuery{ID: d.withRaw}) + must.NoError(err) + for range 3 { + original, err := d.engine.ReadOriginalMessage(t.Context(), query.MessageRef{ID: d.withRaw}, 0) + must.NoError(err) + checks.Equal(d.raw, original.MIME) + } + checks.Equal(int64(2), d.healthRequests.Load(), "failed capability checks retry; successful ones are reused") +} + +func TestEngineRawIDDoesNotFallBackAfterDedupPurge(t *testing.T) { + must := require.New(t) + d := newOriginalDaemon(t) + _, err := d.st.MergeDuplicates(d.noRaw, []int64{d.withRaw}, "strict-id-test") + must.NoError(err) + _, err = d.st.DeleteDedupedBatch("strict-id-test") + must.NoError(err) + src, err := d.st.GetOrCreateSource("gmail", "other@example.com") + must.NoError(err) + conv, err := d.st.EnsureConversation(src.ID, "unrelated-thread", "Unrelated") + must.NoError(err) + _, err = d.st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{SourceID: src.ID, ConversationID: conv, SourceMessageID: strconv.FormatInt(d.withRaw, 10), MessageType: "email"}, + RawMIME: []byte("Subject: unrelated\r\n\r\nOther message"), + }) + must.NoError(err) + raw, err := d.engine.GetMessageRaw(t.Context(), d.withRaw) + must.ErrorIs(err, store.ErrMessageNotFound) + assert.Empty(t, raw) +} + +func TestEngineOriginalMessageNeedsCurrentDaemon(t *testing.T) { + must := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v1/health", r.URL.Path, "no export request reaches an old daemon") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok","api_schema_version":"2.32.0"}`)) + })) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{URL: server.URL, AllowInsecure: true, HTTPClient: server.Client()}) + must.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + engine := daemonclient.NewEngineAdapter(client) + + _, err = engine.ReadOriginalMessage(t.Context(), query.MessageRef{ID: 1}, 0) + must.ErrorIs(err, daemonclient.ErrNotSupported) + _, err = engine.GetMessageRaw(t.Context(), 1) + must.ErrorIs(err, daemonclient.ErrNotSupported) + _, err = engine.ListThread(t.Context(), query.ThreadQuery{ThreadID: "x"}) + must.ErrorIs(err, daemonclient.ErrNotSupported) +} + +func TestEngineMapsUnsupportedOriginalExportThroughDaemon(t *testing.T) { + must := require.New(t) + st := testutil.NewTestStore(t) + engine, err := query.NewDuckDBEngine("", "", nil) + must.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{}, Store: st, Engine: engine, Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{URL: server.URL, AllowInsecure: true, HTTPClient: server.Client()}) + must.NoError(err) + t.Cleanup(func() { must.NoError(client.Close()) }) + daemonEngine := daemonclient.NewEngineAdapter(client) + + _, err = daemonEngine.ReadOriginalMessage(t.Context(), query.MessageRef{ID: 1}, 0) + must.ErrorIs(err, query.ErrOriginalExportUnsupported) + _, err = daemonEngine.ListThread(t.Context(), query.ThreadQuery{ThreadID: "thread"}) + must.ErrorIs(err, query.ErrOriginalExportUnsupported) +} diff --git a/internal/mcp/catalog.go b/internal/mcp/catalog.go index 9af9814b7..2da3da25b 100644 --- a/internal/mcp/catalog.go +++ b/internal/mcp/catalog.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json/jsontext" "encoding/json/v2" + "fmt" "slices" "sort" "sync" @@ -166,6 +167,7 @@ func buildOperationCatalog(capabilities catalogCapabilities) []toolDefinition { createSavedViewDefinition(nil), deleteSavedViewDefinition(nil), exportAttachmentDefinition(nil), + exportEMLDefinition(), findSimilarMessagesDefinition(nil), getAttachmentDefinition(nil), getMessageDefinition(nil), @@ -178,6 +180,7 @@ func buildOperationCatalog(capabilities catalogCapabilities) []toolDefinition { getSavedViewDefinition(nil), getStatsDefinition(nil), listMessagesDefinition(nil), + listThreadDefinition(), listMeetingActionItemsDefinition(nil), listDirectoryPeopleDefinition(nil), listSavedViewsDefinition(nil), @@ -584,12 +587,83 @@ func getMessageDefinition(_ *handlers) toolDefinition { ) } +func chunkInputSchemas(object string) (offset, length *jsonschema.Schema) { + offset = nonNegativeIntegerSchema("Byte offset of the "+object+" to start this chunk at (default 0). Request offset += length until complete is true.", 0) + length = boundedIntegerSchema(fmt.Sprintf("Maximum bytes in this chunk (1-%d, default %d)", maxChunkBytes, defaultChunkBytes), 1, maxChunkBytes) + length.Default = jsontext.Value(defaultValueString(defaultChunkBytes)) + return offset, length +} + +const chunkDownloadDescription = " Start at offset 0, then pass the returned sha256 with every nonzero offset and keep the same identifier and account. " + + "Downloads retain a snapshot for 5 minutes, with at most 8 snapshots and 256 MiB total per server (256 MiB per object). " + + "If the snapshot expires or is evicted, restart at offset 0 without sha256." + +func exportEMLDefinition() toolDefinition { + offset, length := chunkInputSchemas("message") + return readDefinition( + ToolExportEML, + "Export one archived email as its original .eml bytes, exactly as the provider delivered them, in base64 chunks. "+ + "Pass exactly one of id (msgvault message ID) or source_message_id (provider ID, such as a Gmail message ID); "+ + "add account when a provider ID exists in more than one account. "+ + "Call from offset 0, then offset += length until complete is true; concatenate the decoded chunks and verify them against size and sha256 (of the whole message). "+ + "last_sync_at is the account's latest sync activity: provider messages newer than it may not be archived yet. "+ + "Messages without stored original MIME (chat and calendar sources, some imports) return raw_mime_unavailable. "+ + "Gmail, IMAP (including Outlook over IMAP), and mbox/eml/emlx/maildir imports hold the bytes msgvault received; "+ + "PST imports hold MIME rebuilt from Outlook data (source_type pst)."+chunkDownloadDescription, + closedObject(map[string]*jsonschema.Schema{ + "id": safeIDSchema("msgvault message ID"), + toolArgSourceMsgID: stringSchema("Provider message ID"), + toolArgAccount: stringSchema("Account identifier (email address) that narrows a source_message_id lookup"), + toolArgOffset: offset, + toolArgLength: length, + "sha256": stringSchema("Whole-object sha256 returned by the first chunk; required for offset greater than zero"), + }), + outputSchemaFor[exportEMLResponse](), + (*handlers).exportEML, + ) +} + +func listThreadDefinition() toolDefinition { + limit := boundedIntegerSchema(fmt.Sprintf("Messages per page (1-%d, default %d)", query.ThreadMaxLimit, query.ThreadDefaultLimit), 1, query.ThreadMaxLimit) + limit.Default = jsontext.Value(defaultValueString(query.ThreadDefaultLimit)) + return readDefinition( + ToolListThread, + "List visible archived messages in one conversation in chronological order (sent_at, undated last, then id). "+ + "Pass exactly one of id or source_message_id (any message in the thread) or thread_id (provider conversation ID, such as a Gmail threadId); "+ + "add account when a provider ID exists in more than one account. "+ + "Page with offset until has_more is false; total counts the whole conversation. "+ + "has_raw marks messages whose original .eml can be fetched with export_eml. "+ + "Messages deleted from the provider stay listed with deleted_from_source_at. "+ + "last_sync_at is the account's latest sync activity: newer replies may not be archived yet. "+ + "Hidden duplicates are excluded; last_sync_at does not establish conversation completeness.", + closedObject(map[string]*jsonschema.Schema{ + "id": safeIDSchema("msgvault ID of any message in the conversation"), + toolArgSourceMsgID: stringSchema("Provider ID of any message in the conversation"), + toolArgThreadID: stringSchema("Provider conversation ID"), + toolArgAccount: stringSchema("Account identifier (email address) that narrows a provider-ID lookup"), + toolArgOffset: nonNegativeIntegerSchema("Messages to skip (default 0)", 0), + toolArgLimit: limit, + }), + outputSchemaFor[query.ThreadPage](), + (*handlers).listThread, + ) +} + func getAttachmentDefinition(_ *handlers) toolDefinition { + // No defaults here: the SDK applies schema defaults to arguments, and + // either argument being present selects chunk mode. + offset, length := chunkInputSchemas("attachment") + offset.Default, length.Default = nil, nil return readDefinition( ToolGetAttachment, - "Get attachment content by attachment ID. Returns metadata as text and the file content as an embedded resource blob. Use get_message first to find attachment IDs.", + "Get attachment content by attachment ID. Returns metadata as text and the file content as an embedded resource blob. Use get_message first to find attachment IDs. "+ + "Pass offset or length to download in base64 chunks instead (no embedded resource): call from offset 0, then offset += length until complete is true, "+ + "and verify the decoded concatenation against size and sha256. Full embedded responses are limited to 50 MiB."+chunkDownloadDescription, closedObject(map[string]*jsonschema.Schema{ toolArgAttachmentID: safeIDSchema("Attachment ID (from get_message response)"), + toolArgOffset: offset, + toolArgLength: length, + "sha256": stringSchema("Whole-object sha256 returned by the first chunk; required for offset greater than zero"), }, toolArgAttachmentID), outputSchemaFor[getAttachmentResponse](), (*handlers).getAttachment, @@ -836,6 +910,12 @@ type getAttachmentResponse struct { Filename string `json:"filename"` MIMEType string `json:"mime_type"` Size int64 `json:"size"` + // Chunk fields are present only when the call passed offset or length. + Offset *int64 `json:"offset,omitzero"` + Length *int64 `json:"length,omitzero"` + SHA256 *string `json:"sha256,omitzero"` + Complete *bool `json:"complete,omitzero"` + DataBase64 *string `json:"data_base64,omitzero"` } type exportAttachmentResponse struct { diff --git a/internal/mcp/catalog_test.go b/internal/mcp/catalog_test.go index fd5c9e864..8112b2bb1 100644 --- a/internal/mcp/catalog_test.go +++ b/internal/mcp/catalog_test.go @@ -287,10 +287,12 @@ func TestCatalogSchemas(t *testing.T) { } expectedNames := []string{ "aggregate", + ToolExportEML, "get_attachment", "get_message", "get_stats", "list_messages", + ToolListThread, "search_by_domains", "search_in_message", "search_message_bodies", diff --git a/internal/mcp/contract_integration_test.go b/internal/mcp/contract_integration_test.go index 85b9adff5..f8697634a 100644 --- a/internal/mcp/contract_integration_test.go +++ b/internal/mcp/contract_integration_test.go @@ -28,6 +28,7 @@ var task5StableToolNames = []string{ ToolCreateSavedView, ToolDeleteSavedView, ToolExportAttachment, + ToolExportEML, ToolFindSimilarMessages, ToolGetAttachment, ToolGetMessage, @@ -35,6 +36,7 @@ var task5StableToolNames = []string{ ToolGetStats, ToolListMessages, ToolListSavedViews, + ToolListThread, ToolRunSavedView, ToolSearchByDomains, ToolSearchInMessage, @@ -47,6 +49,45 @@ var task5StableToolNames = []string{ ToolUpdateSavedView, } +// task5OriginalEngine adds original-message export to the mock engine so +// the export tools have a deterministic success path. +type task5OriginalEngine struct { + *querytest.MockEngine + + message *query.MessageDetail +} + +func (e task5OriginalEngine) record() query.MessageRecord { + return query.MessageRecord{ + MessageID: e.message.ID, SourceMessageID: e.message.SourceMessageID, + ConversationID: e.message.ConversationID, SourceConversationID: e.message.SourceConversationID, + SourceID: e.message.SourceID, Account: "alice@example.com", SourceType: "gmail", + } +} + +func (e task5OriginalEngine) ReadOriginalMessage(_ context.Context, ref query.MessageRef, _ int64) (*query.OriginalMessage, error) { + if ref.ID != e.message.ID { + return nil, store.ErrMessageNotFound + } + return &query.OriginalMessage{ + MessageRecord: e.record(), + MIME: []byte("Subject: Deterministic archive note\r\n\r\nneedle appears\r\n"), + }, nil +} + +func (e task5OriginalEngine) ListThread(_ context.Context, _ query.ThreadQuery) (*query.ThreadPage, error) { + sentAt := e.message.SentAt + return &query.ThreadPage{ + MessageRecord: e.record(), + Total: 1, + Messages: []query.ThreadMessage{{ + ID: e.message.ID, SourceMessageID: e.message.SourceMessageID, Subject: e.message.Subject, + SentAt: &sentAt, From: e.message.From, To: e.message.To, Cc: []query.Address{}, HasRaw: true, + AttachmentCount: len(e.message.Attachments), + }}, + }, nil +} + type task5Fixture struct { opts ServeOptions exportDir string @@ -215,7 +256,7 @@ func newTask5Fixture(t *testing.T, shape string) task5Fixture { saver := &captureDeletionManifestSaver{} opts := ServeOptions{ - Engine: engine, + Engine: task5OriginalEngine{MockEngine: engine, message: message}, AttachmentReader: attachmentReaderFunc(func(_ context.Context, contentHash string) ([]byte, error) { if contentHash != attachment.ContentHash { return nil, fmt.Errorf("attachment content hash = %q, want %q", contentHash, attachment.ContentHash) @@ -336,6 +377,10 @@ func task5ToolArguments(name, shape, exportDir string) (map[string]any, bool) { return map[string]any{"id": 7}, true case ToolGetAttachment: return map[string]any{"attachment_id": 7}, true + case ToolExportEML: + return map[string]any{"id": 42, "length": 16}, true + case ToolListThread: + return map[string]any{"id": 42, "limit": 10}, true case ToolExportAttachment: return map[string]any{"attachment_id": 7, "destination": exportDir}, true case ToolListMessages: diff --git a/internal/mcp/download.go b/internal/mcp/download.go new file mode 100644 index 000000000..c57197549 --- /dev/null +++ b/internal/mcp/download.go @@ -0,0 +1,123 @@ +package mcp + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "sync" + "time" + + "go.kenn.io/msgvault/internal/query" +) + +const ( + maxDownloadBytes = 256 << 20 + maxDownloads = 8 + downloadLifetime = 5 * time.Minute +) + +var ( + errDownloadExpired = errors.New("download expired or sha256 does not match; restart at offset 0 without sha256") + errDownloadTooLarge = errors.New("chunk downloads are limited to 256 MiB per object") +) + +type downloadKey struct { + message query.MessageRef + attachment int64 +} + +type downloadSnapshot struct { + data []byte + digest string + original *query.OriginalMessage + attachment *attachmentPayload + expires time.Time + timer *time.Timer +} + +// downloadCache retains at most 256 MiB across eight downloads. A digest pins +// continuations to their initial bytes even if the archive changes meanwhile. +type downloadCache struct { + mu sync.Mutex + entries map[downloadKey]*downloadSnapshot + bytes int + closed bool +} + +func (c *downloadCache) get(ctx context.Context, key downloadKey, req chunkRequest, load func() (*downloadSnapshot, error)) (*downloadSnapshot, error) { + // ponytail: serialize snapshot loads; per-download locks if concurrent + // large exports need to avoid waiting behind another download. + c.mu.Lock() + defer c.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + if c.closed { + return nil, errDownloadExpired + } + if entry := c.entries[key]; entry != nil { + if time.Now().Before(entry.expires) { + if req.digest != "" && req.digest != entry.digest { + return nil, errDownloadExpired + } + return entry, nil + } + c.remove(key) + } + if req.offset != 0 { + return nil, errDownloadExpired + } + entry, err := load() + if err != nil { + return nil, err + } + if len(entry.data) > maxDownloadBytes { + return nil, errDownloadTooLarge + } + sum := sha256.Sum256(entry.data) + entry.digest = hex.EncodeToString(sum[:]) + if req.digest != "" && req.digest != entry.digest { + return nil, errDownloadExpired + } + for len(c.entries) >= maxDownloads || c.bytes+len(entry.data) > maxDownloadBytes { + var oldest downloadKey + var expires time.Time + for key, cached := range c.entries { + if expires.IsZero() || cached.expires.Before(expires) { + oldest, expires = key, cached.expires + } + } + c.remove(oldest) + } + if c.entries == nil { + c.entries = make(map[downloadKey]*downloadSnapshot) + } + entry.expires = time.Now().Add(downloadLifetime) + c.entries[key] = entry + c.bytes += len(entry.data) + entry.timer = time.AfterFunc(downloadLifetime, func() { + c.mu.Lock() + defer c.mu.Unlock() + if c.entries[key] == entry { + c.remove(key) + } + }) + return entry, nil +} + +func (c *downloadCache) remove(key downloadKey) { + entry := c.entries[key] + entry.timer.Stop() + c.bytes -= len(entry.data) + delete(c.entries, key) +} + +func (c *downloadCache) close() { + c.mu.Lock() + defer c.mu.Unlock() + c.closed = true + for key := range c.entries { + c.remove(key) + } +} diff --git a/internal/mcp/download_test.go b/internal/mcp/download_test.go new file mode 100644 index 000000000..80d587018 --- /dev/null +++ b/internal/mcp/download_test.go @@ -0,0 +1,204 @@ +package mcp + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" + msgexport "go.kenn.io/msgvault/internal/export" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestChunkDownloadsThroughDaemon(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + st := testutil.NewSQLiteTestStore(t) + engine := query.NewSQLiteEngine(st.DB()) + cfg := &config.Config{} + cfg.Data.DataDir = t.TempDir() + source, err := st.GetOrCreateSource("gmail", "owner@example.com") + must.NoError(err) + conversation, err := st.EnsureConversation(source.ID, "download-thread", "Download") + must.NoError(err) + id, err := st.UpsertMessage(&store.Message{SourceID: source.ID, ConversationID: conversation, SourceMessageID: "download-message", MessageType: "email"}) + must.NoError(err) + var originalReads, attachmentReads atomic.Int64 + router := api.NewServerWithOptions(api.ServerOptions{Config: cfg, Store: st, Engine: engine, Logger: slog.New(slog.DiscardHandler)}).Router() + daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/cli/message/original": + checks.Equal("268435456", r.URL.Query().Get("max_bytes"), "bound the daemon read before allocating original MIME") + originalReads.Add(1) + case "/api/v1/cli/attachment": + attachmentReads.Add(1) + } + router.ServeHTTP(w, r) + })) + t.Cleanup(daemon.Close) + client, err := daemonclient.New(daemonclient.Config{URL: daemon.URL, AllowInsecure: true, HTTPClient: daemon.Client()}) + must.NoError(err) + t.Cleanup(func() { checks.NoError(client.Close()) }) + server := newMCPHTTPServer(ServeOptions{Engine: daemonclient.NewEngineAdapter(client), AttachmentReader: client}, HTTPOptions{}) + t.Cleanup(func() { checks.NoError(server.Shutdown(t.Context())) }) + chunk := func(t *testing.T, name string, args map[string]any) exportChunkResp { + t.Helper() + must := require.New(t) + response, raw := task4RawRequest(t, server.Handler, "tools/call", map[string]any{"name": name, "arguments": args}, nil) + must.Empty(response.Error, "%s", raw) + must.NotEqual(true, response.Result["isError"], "%s", raw) + encoded, err := json.Marshal(response.Result["structuredContent"]) + must.NoError(err) + var result exportChunkResp + must.NoError(json.Unmarshal(encoded, &result)) + return result + } + + t.Run("original snapshot survives raw replacement", func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + data := bytes.Repeat([]byte("original"), (40<<20)/8) + must.NoError(st.UpsertMessageRaw(id, data)) + digest := fmt.Sprintf("%x", sha256.Sum256(data)) + for offset := 0; offset < len(data); offset += 1 << 20 { + args := map[string]any{"id": id, "offset": offset, "length": 1 << 20} + if offset > 0 { + args["sha256"] = digest + } + got := chunk(t, ToolExportEML, args) + part, err := base64.StdEncoding.DecodeString(got.DataBase64) + must.NoError(err) + must.Equal(data[offset:offset+(1<<20)], part) + checks.Equal(digest, got.SHA256) + checks.Equal(offset+(1<<20) == len(data), got.Complete) + if offset == 0 { + must.NoError(st.UpsertMessageRaw(id, []byte("replacement MIME"))) + response, raw := task4RawRequest(t, server.Handler, "tools/call", map[string]any{ + "name": ToolExportEML, + "arguments": map[string]any{"id": id, "offset": 1, "sha256": fmt.Sprintf("%064x", 0)}, + }, nil) + must.Empty(response.Error) + checks.Equal(true, response.Result["isError"]) + checks.Contains(raw, "restart at offset 0") + } + } + checks.Equal(int64(1), originalReads.Load()) + }) + + t.Run("large attachment chunks retain full response cap", func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + data := bytes.Repeat([]byte("attachment"), (80<<20)/10+1)[:80<<20] + digest := fmt.Sprintf("%x", sha256.Sum256(data)) + path, err := msgexport.StoragePath(cfg.AttachmentsDir(), digest) + must.NoError(err) + must.NoError(os.MkdirAll(filepath.Dir(path), 0o700)) + must.NoError(os.WriteFile(path, data, 0o600)) + must.NoError(st.UpsertAttachmentRecord(t.Context(), id, store.AttachmentWrite{Filename: "large.bin", MIMEType: "application/octet-stream", ContentHash: digest, StoragePath: path, Size: int64(len(data))})) + attachments, err := engine.GetAttachmentsByHash(t.Context(), digest) + must.NoError(err) + must.Len(attachments, 1) + attachmentID := attachments[0].ID + for offset := 0; offset < 2<<20; offset += 1 << 20 { + got := chunk(t, ToolGetAttachment, map[string]any{"attachment_id": attachmentID, "offset": offset, "length": 1 << 20, "sha256": digest}) + part, err := base64.StdEncoding.DecodeString(got.DataBase64) + must.NoError(err) + checks.Equal(data[offset:offset+(1<<20)], part) + checks.Equal(int64(80<<20), got.Size) + checks.Equal(digest, got.SHA256) + } + checks.Equal(int64(1), attachmentReads.Load()) + response, raw := task4RawRequest(t, server.Handler, "tools/call", map[string]any{"name": ToolGetAttachment, "arguments": map[string]any{"attachment_id": attachmentID}}, nil) + must.Empty(response.Error) + checks.Equal(true, response.Result["isError"]) + checks.Contains(raw, "attachment too large: 83886080 bytes (max 52428800)") + response, _ = task4RawRequest(t, server.Handler, "resources/read", map[string]any{"uri": attachmentResourceURI(attachmentID)}, nil) + checks.NotEmpty(response.Error) + }) +} + +func TestDownloadSnapshotExpiryAndShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + var cache downloadCache + defer cache.close() + key := downloadKey{attachment: 1} + load := func() (*downloadSnapshot, error) { + return &downloadSnapshot{data: []byte("first version")}, nil + } + first, err := cache.get(t.Context(), key, chunkRequest{}, load) + must.NoError(err) + time.Sleep(downloadLifetime) + synctest.Wait() + _, err = cache.get(t.Context(), key, chunkRequest{offset: 1, digest: first.digest}, load) + must.ErrorIs(err, errDownloadExpired) + checks.Zero(cache.bytes, "expiry releases the retained download budget") + _, err = cache.get(t.Context(), key, chunkRequest{}, load) + must.NoError(err, "offset zero can start a new download") + cache.close() + _, err = cache.get(t.Context(), key, chunkRequest{}, load) + must.ErrorIs(err, errDownloadExpired) + checks.Zero(cache.bytes, "shutdown releases snapshots without waiting for expiry") + }) +} + +func TestDownloadSnapshotEviction(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + var cache downloadCache + defer cache.close() + load := func() (*downloadSnapshot, error) { + return &downloadSnapshot{data: []byte("a")}, nil + } + var first *downloadSnapshot + for i := int64(1); i <= maxDownloads+1; i++ { + snapshot, err := cache.get(t.Context(), downloadKey{attachment: i}, chunkRequest{}, load) + must.NoError(err) + if i == 1 { + first = snapshot + } + // Distinct virtual timestamps identify the oldest snapshot on every OS. + time.Sleep(time.Second) + } + _, err := cache.get(t.Context(), downloadKey{attachment: 1}, chunkRequest{offset: 1, digest: first.digest}, load) + must.ErrorIs(err, errDownloadExpired) + checks.Equal(maxDownloads, cache.bytes) + }) +} + +func TestDownloadSnapshotByteBudget(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + var cache downloadCache + defer cache.close() + data := make([]byte, maxDownloadBytes/2+1) + load := func() (*downloadSnapshot, error) { + return &downloadSnapshot{data: data}, nil + } + first, err := cache.get(t.Context(), downloadKey{attachment: 1}, chunkRequest{}, load) + must.NoError(err) + _, err = cache.get(t.Context(), downloadKey{attachment: 2}, chunkRequest{}, load) + must.NoError(err) + _, err = cache.get(t.Context(), downloadKey{attachment: 1}, chunkRequest{offset: 1, digest: first.digest}, load) + must.ErrorIs(err, errDownloadExpired, "byte budget evicts before the entry limit") + checks.Equal(len(data), cache.bytes) +} diff --git a/internal/mcp/export.go b/internal/mcp/export.go new file mode 100644 index 000000000..b8f894616 --- /dev/null +++ b/internal/mcp/export.go @@ -0,0 +1,247 @@ +package mcp + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "math" + "strings" + "time" + + "go.kenn.io/msgvault/internal/export" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +// Chunk bounds. Each result carries its JSON as both text and structured +// content, so a 1 MiB chunk is about 2.8 MB on the wire; gateway sandboxes +// abort on much larger strings. +const ( + defaultChunkBytes = 1 << 20 + maxChunkBytes = 4 << 20 +) + +// byteChunk is one verified slice of a larger object. The field names +// match the chunk contract other archive bridges use, so one client can +// download from any of them: fetch from offset 0 until complete, then +// check the concatenation against size and sha256. +type byteChunk struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Complete bool `json:"complete"` + DataBase64 string `json:"data_base64"` +} + +type chunkRequest struct { + digest string + offset int64 + length int64 +} + +// chunkArgs reads the optional offset, length, and sha256 arguments. +// present reports whether the caller supplied any of them. +func chunkArgs(args map[string]any) (req chunkRequest, present bool, err error) { + req.length = defaultChunkBytes + if raw, ok := args[toolArgOffset]; ok { + present = true + v, isNumber := raw.(float64) + if !isNumber || v != math.Trunc(v) || v < 0 || v > maxJSONSafeInteger { + return req, present, errors.New("offset must be a non-negative integer") + } + req.offset = int64(v) + } + if raw, ok := args[toolArgLength]; ok { + present = true + v, isNumber := raw.(float64) + if !isNumber || v != math.Trunc(v) || v < 1 || v > maxChunkBytes { + return req, present, fmt.Errorf("length must be between 1 and %d", maxChunkBytes) + } + req.length = int64(v) + } + if raw, ok := args["sha256"]; ok { + present = true + value, ok := raw.(string) + if !ok || export.ValidateContentHash(value) != nil { + return req, present, errors.New("sha256 must be a 64-character hexadecimal digest") + } + req.digest = strings.ToLower(value) + } + if req.offset > 0 && req.digest == "" { + return req, present, errors.New("sha256 from the first chunk is required when offset is greater than zero") + } + return req, present, nil +} + +func sliceChunk(data []byte, digest string, req chunkRequest) (byteChunk, error) { + size := int64(len(data)) + if req.offset > size { + return byteChunk{}, fmt.Errorf("offset %d is past the end (size %d)", req.offset, size) + } + end := min(req.offset+req.length, size) + return byteChunk{ + Offset: req.offset, + Length: end - req.offset, + Size: size, + SHA256: digest, + Complete: end == size, + DataBase64: base64.StdEncoding.EncodeToString(data[req.offset:end]), + }, nil +} + +type exportEMLResponse struct { + MessageID int64 `json:"message_id"` + SourceMessageID string `json:"source_message_id"` + ConversationID int64 `json:"conversation_id"` + SourceConversationID string `json:"source_conversation_id"` + SourceID int64 `json:"source_id"` + Account string `json:"account"` + SourceType string `json:"source_type"` + LastSyncAt *time.Time `json:"last_sync_at,omitzero"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Complete bool `json:"complete"` + DataBase64 string `json:"data_base64"` +} + +func (h *handlers) originalMessageReader() (query.OriginalMessageReader, *toolResult) { + reader, ok := h.engine.(query.OriginalMessageReader) + if !ok { + return nil, toolErrorResult(query.ErrOriginalExportUnsupported.Error()) + } + return reader, nil +} + +// messageRefArgs reads exactly one of id or source_message_id, plus account. +func messageRefArgs(args map[string]any) (query.MessageRef, error) { + var ref query.MessageRef + ref.Account, _ = args[toolArgAccount].(string) + ref.SourceMessageID, _ = args[toolArgSourceMsgID].(string) + if _, ok := args["id"]; ok { + id, err := getIDArg(args, "id") + if err != nil { + return ref, err + } + ref.ID = id + } + return ref, nil +} + +func (h *handlers) exportEML(ctx context.Context, req toolRequest) (*toolResult, error) { + reader, unsupported := h.originalMessageReader() + if unsupported != nil { + return unsupported, nil + } + args := req.GetArguments() + ref, err := messageRefArgs(args) + if err != nil { + return toolErrorResult(err.Error()), nil + } + if (ref.ID == 0) == (ref.SourceMessageID == "") { + return toolErrorResult(query.ErrInvalidMessageRef.Error()), nil + } + chunkReq, _, err := chunkArgs(args) + if err != nil { + return toolErrorResult(err.Error()), nil + } + snapshot, err := h.downloads.get(ctx, downloadKey{message: ref}, chunkReq, func() (*downloadSnapshot, error) { + original, err := reader.ReadOriginalMessage(ctx, ref, maxDownloadBytes) + if err != nil { + return nil, err + } + return &downloadSnapshot{data: original.MIME, original: original}, nil + }) + if err != nil { + return originalExportError("read original message", err) + } + chunk, err := sliceChunk(snapshot.data, snapshot.digest, chunkReq) + if err != nil { + return toolErrorResult(err.Error()), nil + } + record := snapshot.original.MessageRecord + return jsonResult(exportEMLResponse{ + MessageID: record.MessageID, + SourceMessageID: record.SourceMessageID, + ConversationID: record.ConversationID, + SourceConversationID: record.SourceConversationID, + SourceID: record.SourceID, + Account: record.Account, + SourceType: record.SourceType, + LastSyncAt: record.LastSyncAt, + Offset: chunk.Offset, + Length: chunk.Length, + Size: chunk.Size, + SHA256: chunk.SHA256, + Complete: chunk.Complete, + DataBase64: chunk.DataBase64, + }) +} + +// originalExportError turns the export sentinels into caller-facing tool +// errors and leaves anything else to the shared dependency mapping. +func originalExportError(operation string, err error) (*toolResult, error) { + switch { + case errors.Is(err, errDownloadExpired), errors.Is(err, errDownloadTooLarge), errors.Is(err, query.ErrInvalidMessageRef): + return toolErrorResult(err.Error()), nil + case errors.Is(err, query.ErrOriginalMessageTooLarge): + return toolErrorResult(errDownloadTooLarge.Error()), nil + case errors.Is(err, store.ErrMessageNotFound): + return toolErrorResult("message not found"), nil + case errors.Is(err, query.ErrThreadNotFound): + return toolErrorResult("thread not found"), nil + case errors.Is(err, query.ErrOriginalMIMEUnavailable): + return toolErrorResult("raw_mime_unavailable: the archive holds no original MIME for this message " + + "(chat and calendar sources and some imports store none)"), nil + case errors.Is(err, query.ErrAmbiguousReference): + return toolErrorResult("message_ambiguous: " + err.Error() + "; pass account"), nil + case errors.Is(err, query.ErrOriginalExportUnsupported): + return toolErrorResult(query.ErrOriginalExportUnsupported.Error()), nil + } + return dependencyError(operation, err) +} + +func (h *handlers) listThread(ctx context.Context, req toolRequest) (*toolResult, error) { + reader, unsupported := h.originalMessageReader() + if unsupported != nil { + return unsupported, nil + } + args := req.GetArguments() + ref, err := messageRefArgs(args) + if err != nil { + return toolErrorResult(err.Error()), nil + } + q := query.ThreadQuery{MessageRef: ref, Limit: query.ThreadDefaultLimit} + q.ThreadID, _ = args[toolArgThreadID].(string) + anchors := 0 + for _, set := range []bool{ref.ID != 0, ref.SourceMessageID != "", q.ThreadID != ""} { + if set { + anchors++ + } + } + if anchors != 1 { + return toolErrorResult("provide exactly one of id, source_message_id, or thread_id"), nil + } + if raw, ok := args[toolArgLimit]; ok { + v, isNumber := raw.(float64) + if !isNumber || v != math.Trunc(v) || v < 1 || v > query.ThreadMaxLimit { + return toolErrorResult(fmt.Sprintf("limit must be between 1 and %d", query.ThreadMaxLimit)), nil + } + q.Limit = int(v) + } + if raw, ok := args[toolArgOffset]; ok { + v, isNumber := raw.(float64) + if !isNumber || v != math.Trunc(v) || v < 0 || v > math.MaxInt32 { + return toolErrorResult("offset must be a non-negative integer"), nil + } + q.Offset = int(v) + } + page, err := reader.ListThread(ctx, q) + if err != nil { + return originalExportError("list thread", err) + } + return jsonResult(page) +} diff --git a/internal/mcp/export_test.go b/internal/mcp/export_test.go new file mode 100644 index 000000000..392681717 --- /dev/null +++ b/internal/mcp/export_test.go @@ -0,0 +1,362 @@ +package mcp + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "maps" + "testing" + "time" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/query/querytest" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +type exportChunkResp struct { + MessageID int64 `json:"message_id"` + SourceMessageID string `json:"source_message_id"` + ConversationID int64 `json:"conversation_id"` + SourceConversationID string `json:"source_conversation_id"` + SourceID int64 `json:"source_id"` + Account string `json:"account"` + SourceType string `json:"source_type"` + LastSyncAt *string `json:"last_sync_at"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Complete bool `json:"complete"` + DataBase64 string `json:"data_base64"` +} + +// exportMIME is 1,000 bytes of CRLF, bare LF, NUL and 8-bit content so a +// small chunk length splits it across many calls, including mid-line. +var exportMIME = func() []byte { + var b bytes.Buffer + b.WriteString("From: sender@example.com\r\nSubject: chunked\r\n\r\n") + for b.Len() < 1000 { + b.Write([]byte{'a', 0x00, 0xe9, '\r', '\n', 'b', '\n', 0xff}) + } + return b.Bytes()[:1000] +}() + +type exportFixture struct { + f *storetest.Fixture + engine *query.SQLiteEngine + h *handlers + withRaw int64 + noRaw int64 +} + +func newExportFixture(t *testing.T) exportFixture { + t.Helper() + f := storetest.New(t) + engine := query.NewSQLiteEngine(f.Store.DB()) + if f.Store.IsPostgreSQL() { + engine = query.NewEngineWithDialect(f.Store.DB(), query.PostgreSQLQueryDialect{}) + } + withRaw := f.NewMessage().WithSourceMessageID("export-provider-id"). + WithSentAt(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)).Create(t, f.Store) + require.NoError(t, f.Store.UpsertMessageRaw(withRaw, exportMIME)) + noRaw := f.NewMessage().WithSourceMessageID("export-no-raw"). + WithSentAt(time.Date(2026, 1, 2, 4, 4, 5, 0, time.UTC)).Create(t, f.Store) + h := newTestHandlers(engine) + t.Cleanup(h.downloads.close) + return exportFixture{f: f, engine: engine, h: h, withRaw: withRaw, noRaw: noRaw} +} + +func downloadEML(t *testing.T, h *handlers, args map[string]any, length int) ([]byte, exportChunkResp, int) { + t.Helper() + var out []byte + var last exportChunkResp + calls := 0 + for { + call := map[string]any{"offset": float64(len(out)), "length": float64(length)} + if last.SHA256 != "" { + call["sha256"] = last.SHA256 + } + maps.Copy(call, args) + last = runTool[exportChunkResp](t, ToolExportEML, h.exportEML, call) + calls++ + require.Equal(t, int64(len(out)), last.Offset) + part, err := base64.StdEncoding.DecodeString(last.DataBase64) + require.NoError(t, err) + require.Equal(t, last.Length, int64(len(part))) + require.LessOrEqual(t, len(part), length) + out = append(out, part...) + if last.Complete { + return out, last, calls + } + require.NotEmpty(t, part, "empty chunk before completion") + } +} + +func TestExportEMLChunksAreByteExact(t *testing.T) { + checks := assert.New(t) + x := newExportFixture(t) + sum := sha256.Sum256(exportMIME) + + got, last, calls := downloadEML(t, x.h, map[string]any{"id": float64(x.withRaw)}, 97) + checks.Equal(exportMIME, got) + checks.Equal(hex.EncodeToString(sum[:]), last.SHA256) + checks.Equal(int64(len(exportMIME)), last.Size) + checks.Equal(11, calls, "1000 bytes in 97-byte chunks") + checks.Equal(x.withRaw, last.MessageID) + checks.Equal("export-provider-id", last.SourceMessageID) + checks.Equal("default-thread", last.SourceConversationID) + checks.Equal("test@example.com", last.Account) + checks.Equal("gmail", last.SourceType) + checks.Nil(last.LastSyncAt, "never-synced account omits last_sync_at") + + byProviderID, _, _ := downloadEML(t, x.h, map[string]any{"source_message_id": "export-provider-id"}, 4194304) + checks.Equal(exportMIME, byProviderID) +} + +func TestExportEMLDefaultChunkAndEndOffset(t *testing.T) { + checks := assert.New(t) + x := newExportFixture(t) + + whole := runTool[exportChunkResp](t, ToolExportEML, x.h.exportEML, map[string]any{"id": float64(x.withRaw)}) + checks.True(whole.Complete) + checks.Equal(int64(1000), whole.Length) + + end := runTool[exportChunkResp](t, ToolExportEML, x.h.exportEML, + map[string]any{"id": float64(x.withRaw), "offset": float64(1000), "sha256": whole.SHA256}) + checks.True(end.Complete) + checks.Zero(end.Length) + checks.Empty(end.DataBase64) +} + +func TestExportEMLErrors(t *testing.T) { + x := newExportFixture(t) + first := runTool[exportChunkResp](t, ToolExportEML, x.h.exportEML, map[string]any{"id": float64(x.withRaw)}) + tests := []struct { + name string + args map[string]any + want string + }{ + {"no identifier", map[string]any{}, "provide exactly one of id or source_message_id"}, + {"both identifiers", map[string]any{"id": float64(x.withRaw), "source_message_id": "export-provider-id"}, "provide exactly one of id or source_message_id"}, + {"unknown", map[string]any{"id": float64(999999)}, "message not found"}, + {"no raw", map[string]any{"id": float64(x.noRaw)}, "raw_mime_unavailable"}, + {"offset past end", map[string]any{"id": float64(x.withRaw), "offset": float64(1001), "sha256": first.SHA256}, "offset 1001 is past the end"}, + {"zero length", map[string]any{"id": float64(x.withRaw), "length": float64(0)}, "length must be between 1 and 4194304"}, + {"length too large", map[string]any{"id": float64(x.withRaw), "length": float64(4194305)}, "length must be between 1 and 4194304"}, + {"fractional offset", map[string]any{"id": float64(x.withRaw), "offset": 1.5}, "offset must be a non-negative integer"}, + {"continuation without digest", map[string]any{"id": float64(x.withRaw), "offset": float64(1)}, "sha256 from the first chunk is required"}, + {"invalid digest", map[string]any{"id": float64(x.withRaw), "sha256": "invalid"}, "sha256 must be a 64-character hexadecimal digest"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := runToolExpectError(t, ToolExportEML, x.h.exportEML, tc.args) + assert.Contains(t, resultText(t, r), tc.want) + }) + } + + unsupported := runToolExpectError(t, ToolExportEML, newTestHandlers(&querytest.MockEngine{}).exportEML, + map[string]any{"id": float64(1)}) + assert.Contains(t, resultText(t, unsupported), "not supported") +} + +func TestExportEMLOversizeError(t *testing.T) { + result, err := originalExportError("read original message", query.ErrOriginalMessageTooLarge) + require.NoError(t, err) + require.True(t, result.isError) + assert.Equal(t, "chunk downloads are limited to 256 MiB per object", resultText(t, result)) +} + +func TestExportEMLThroughOfficialSDK(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + x := newExportFixture(t) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(cancel) + + clientTransport, serverTransport := sdkmcp.NewInMemoryTransports() + serverSession, err := newMCPServer(ServeOptions{Engine: x.engine}, false).Connect(ctx, serverTransport, nil) + must.NoError(err) + t.Cleanup(func() { checks.NoError(serverSession.Close()) }) + client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "export-test", Version: "1.0.0"}, nil) + session, err := client.Connect(ctx, clientTransport, nil) + must.NoError(err) + t.Cleanup(func() { checks.NoError(session.Close()) }) + + var got []byte + var last exportChunkResp + for { + args := map[string]any{"id": x.withRaw, "offset": len(got), "length": 400} + if last.SHA256 != "" { + args["sha256"] = last.SHA256 + } + result, err := session.CallTool(ctx, &sdkmcp.CallToolParams{ + Name: ToolExportEML, + Arguments: args, + }) + must.NoError(err) + must.False(result.IsError, "%v", result.Content) + text, ok := result.Content[0].(*sdkmcp.TextContent) + must.True(ok, "first content is the JSON text a gateway client reads") + must.NoError(json.Unmarshal([]byte(text.Text), &last)) + part, err := base64.StdEncoding.DecodeString(last.DataBase64) + must.NoError(err) + got = append(got, part...) + if last.Complete { + break + } + } + sum := sha256.Sum256(got) + checks.Equal(exportMIME, got) + checks.Equal(hex.EncodeToString(sum[:]), last.SHA256) +} + +type listThreadResp struct { + ConversationID int64 `json:"conversation_id"` + SourceConversationID string `json:"source_conversation_id"` + Account string `json:"account"` + MessageID int64 `json:"message_id"` + Total int64 `json:"total"` + Offset int `json:"offset"` + HasMore bool `json:"has_more"` + LastSyncAt *string `json:"last_sync_at"` + Messages []struct { + ID int64 `json:"id"` + SourceMessageID string `json:"source_message_id"` + SentAt string `json:"sent_at"` + HasRaw bool `json:"has_raw"` + From []query.Address `json:"from"` + } `json:"messages"` +} + +func TestListThread(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + x := newExportFixture(t) + + byID := runTool[listThreadResp](t, ToolListThread, x.h.listThread, map[string]any{"id": float64(x.noRaw)}) + must.Len(byID.Messages, 2) + checks.Equal(x.withRaw, byID.Messages[0].ID) + checks.True(byID.Messages[0].HasRaw) + checks.Equal("2026-01-02T03:04:05Z", byID.Messages[0].SentAt) + checks.Equal(x.noRaw, byID.Messages[1].ID) + checks.False(byID.Messages[1].HasRaw) + checks.Equal(int64(2), byID.Total) + checks.False(byID.HasMore) + checks.Equal(x.noRaw, byID.MessageID) + checks.Equal("default-thread", byID.SourceConversationID) + checks.Equal("test@example.com", byID.Account) + + page := runTool[listThreadResp](t, ToolListThread, x.h.listThread, + map[string]any{"thread_id": "default-thread", "account": "test@example.com", "limit": float64(1)}) + must.Len(page.Messages, 1) + checks.Equal(x.withRaw, page.Messages[0].ID) + checks.True(page.HasMore) + + byProvider := runTool[listThreadResp](t, ToolListThread, x.h.listThread, + map[string]any{"source_message_id": "export-provider-id", "offset": float64(1)}) + must.Len(byProvider.Messages, 1) + checks.Equal(x.noRaw, byProvider.Messages[0].ID) + checks.Equal(1, byProvider.Offset) + + for name, tc := range map[string]struct { + args map[string]any + want string + }{ + "no anchor": {map[string]any{}, "provide exactly one of id, source_message_id, or thread_id"}, + "two anchors": {map[string]any{"id": float64(x.noRaw), "thread_id": "default-thread"}, "provide exactly one of id, source_message_id, or thread_id"}, + "limit too big": {map[string]any{"thread_id": "default-thread", "limit": float64(501)}, "limit must be between 1 and 500"}, + "bad offset": {map[string]any{"thread_id": "default-thread", "offset": float64(-1)}, "offset must be a non-negative integer"}, + "unknown": {map[string]any{"thread_id": "missing"}, "thread not found"}, + } { + t.Run(name, func(t *testing.T) { + r := runToolExpectError(t, ToolListThread, x.h.listThread, tc.args) + assert.Contains(t, resultText(t, r), tc.want) + }) + } +} + +func TestGetAttachmentChunks(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + tmpDir := t.TempDir() + content := bytes.Repeat([]byte{0x00, 0xff, '\r', '\n', 'x'}, 41) // 205 bytes + sum := sha256.Sum256(content) + hash := hex.EncodeToString(sum[:]) + createAttachmentFixture(t, tmpDir, hash, content) + h := &handlers{ + downloads: &downloadCache{}, + engine: &querytest.MockEngine{Attachments: map[int64]*query.AttachmentInfo{ + 10: {ID: 10, Filename: "sheet.xlsx", MimeType: "application/octet-stream", Size: int64(len(content)), ContentHash: hash}, + }}, + attachmentsDir: tmpDir, + } + + t.Cleanup(h.downloads.close) + var got []byte + for { + r := callToolDirect(t, ToolGetAttachment, h.getAttachment, + map[string]any{"attachment_id": float64(10), "offset": float64(len(got)), "length": float64(64), "sha256": hash}) + must.False(r.isError, "unexpected error: %s", resultText(t, r)) + checks.Nil(r.embeddedResource, "chunk mode carries bytes in data_base64 only") + var chunk struct { + Filename string `json:"filename"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Complete bool `json:"complete"` + DataBase64 string `json:"data_base64"` + } + must.NoError(json.Unmarshal([]byte(resultText(t, r)), &chunk)) + checks.Equal("sheet.xlsx", chunk.Filename) + checks.Equal(int64(len(got)), chunk.Offset) + checks.Equal(hash, chunk.SHA256) + part, err := base64.StdEncoding.DecodeString(chunk.DataBase64) + must.NoError(err) + checks.Equal(chunk.Length, int64(len(part))) + got = append(got, part...) + if chunk.Complete { + break + } + } + checks.Equal(content, got) + + legacy := callToolDirect(t, ToolGetAttachment, h.getAttachment, map[string]any{"attachment_id": float64(10)}) + must.False(legacy.isError) + must.NotNil(legacy.embeddedResource, "without offset or length the blob stays embedded") + var legacyFields map[string]any + must.NoError(json.Unmarshal([]byte(resultText(t, legacy)), &legacyFields)) + checks.NotContains(legacyFields, "data_base64") + checks.NotContains(legacyFields, "offset") + + bad := runToolExpectError(t, ToolGetAttachment, h.getAttachment, + map[string]any{"attachment_id": float64(10), "offset": float64(206), "sha256": hash}) + checks.Contains(resultText(t, bad), "offset 206 is past the end") +} + +func TestGetAttachmentChunkThroughOfficialSDK(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + fixture := newTask5Fixture(t, "000") + client := task5ConnectClient(t, fixture.opts, false) + result, err := client.CallTool(t.Context(), &sdkmcp.CallToolParams{ + Name: ToolGetAttachment, + Arguments: map[string]any{"attachment_id": 7, "offset": 0, "length": 8}, + }) + must.NoError(err) + task5AssertJSONParity(t, ToolGetAttachment, result) + must.Len(result.Content, 1, "chunk mode has no embedded resource") + chunk := task5StructuredAs[map[string]any](t, result) + encoded, ok := chunk["data_base64"].(string) + must.True(ok, "data_base64 is a string") + data, err := base64.StdEncoding.DecodeString(encoded) + must.NoError(err) + checks.Equal(fixture.attachmentBytes[:8], data) + checks.Equal(false, chunk["complete"]) +} diff --git a/internal/mcp/handlers.go b/internal/mcp/handlers.go index 01a473afa..7f229cf60 100644 --- a/internal/mcp/handlers.go +++ b/internal/mcp/handlers.go @@ -54,6 +54,9 @@ const ( toolArgBefore = "before" toolArgAccount = "account" toolArgOffset = "offset" + toolArgLength = "length" + toolArgSourceMsgID = "source_message_id" + toolArgThreadID = "thread_id" toolArgMinScore = "min_score" toolArgMaxChars = "max_chars" toolArgAttachmentID = "attachment_id" @@ -128,6 +131,7 @@ func listLimitArg(args map[string]any) int { } type handlers struct { + downloads *downloadCache engine query.Engine archiveSQLQuerier ArchiveSQLQuerier attachmentsDir string @@ -1871,21 +1875,52 @@ func (h *handlers) getAttachment(ctx context.Context, req toolRequest) (*toolRes if err != nil { return toolErrorResult(err.Error()), nil } + chunkReq, chunked, err := chunkArgs(args) + if err != nil { + return toolErrorResult(err.Error()), nil + } - payload, err := h.attachmentService().load(ctx, id) + var payload *attachmentPayload + var snapshot *downloadSnapshot + if chunked { + snapshot, err = h.downloads.get(ctx, downloadKey{attachment: id}, chunkReq, func() (*downloadSnapshot, error) { + payload, err := h.attachmentService().loadBounded(ctx, id, maxDownloadBytes) + if err != nil { + return nil, err + } + return &downloadSnapshot{data: payload.data, attachment: payload}, nil + }) + if err == nil { + payload = snapshot.attachment + } + } else { + payload, err = h.attachmentService().load(ctx, id) + } if err != nil { if unavailable, ok := errors.AsType[*attachmentUnavailableError](err); ok { return toolErrorResult(unavailable.message), nil } + if errors.Is(err, errDownloadExpired) || errors.Is(err, errDownloadTooLarge) { + return toolErrorResult(err.Error()), nil + } return nil, err } att := payload.metadata - metaObj := getAttachmentResponse{ Filename: att.Filename, MIMEType: payload.mimeType, Size: att.Size, } + if chunked { + chunk, err := sliceChunk(snapshot.data, snapshot.digest, chunkReq) + if err != nil { + return toolErrorResult(err.Error()), nil + } + metaObj.Size = chunk.Size + metaObj.Offset, metaObj.Length = &chunk.Offset, &chunk.Length + metaObj.SHA256, metaObj.Complete, metaObj.DataBase64 = &chunk.SHA256, &chunk.Complete, &chunk.DataBase64 + return jsonResult(metaObj) + } result, err := jsonResult(metaObj) if err != nil { return nil, err diff --git a/internal/mcp/protocol_test.go b/internal/mcp/protocol_test.go index 9ebb0dc89..338e42920 100644 --- a/internal/mcp/protocol_test.go +++ b/internal/mcp/protocol_test.go @@ -102,7 +102,7 @@ func task3ToolErrorText(t *testing.T, response task3RPCResponse) string { type task5RawStdioPeer struct { requestWriter *os.File responseReader *os.File - responseLines *bufio.Scanner + responseLines *bufio.Reader cancel context.CancelFunc done chan error } @@ -131,7 +131,7 @@ func newTask5RawStdioPeerWithServer(t *testing.T, server *sdkmcp.Server) *task5R peer := &task5RawStdioPeer{ requestWriter: clientRequestWriter, responseReader: clientResponseReader, - responseLines: bufio.NewScanner(clientResponseReader), + responseLines: bufio.NewReader(clientResponseReader), cancel: cancel, done: done, } @@ -175,21 +175,16 @@ func (p *task5RawStdioPeer) callRaw(t *testing.T, line string) (task3RPCResponse type scanResult struct { line string err error - ok bool } done := make(chan scanResult, 1) go func() { - ok := p.responseLines.Scan() - done <- scanResult{ - line: p.responseLines.Text(), - err: p.responseLines.Err(), - ok: ok, - } + line, err := p.responseLines.ReadString('\n') + done <- scanResult{line: line, err: err} }() var raw string select { case scanned := <-done: - require.True(t, scanned.ok, "raw stdio response: %v", scanned.err) + require.NoError(t, scanned.err, "raw stdio response") raw = scanned.line case <-time.After(5 * time.Second): require.FailNow(t, "raw stdio response read timed out") diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index e4d65b04c..52a177c24 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -71,6 +71,10 @@ func parseAttachmentResourceURI(rawURI string) (int64, error) { } func (s attachmentService) load(ctx context.Context, id int64) (*attachmentPayload, error) { + return s.loadBounded(ctx, id, maxAttachmentSize) +} + +func (s attachmentService) loadBounded(ctx context.Context, id, maxBytes int64) (*attachmentPayload, error) { attachment, err := s.engine.GetAttachment(ctx, id) if err != nil { return nil, newInternalError("look up attachment", err) @@ -81,13 +85,13 @@ func (s attachmentService) load(ctx context.Context, id int64) (*attachmentPaylo if s.reader == nil && s.attachmentsDir == "" { return nil, &attachmentUnavailableError{message: "attachments directory not configured"} } - if attachment.Size > maxAttachmentSize { + if attachment.Size > maxBytes { return nil, &attachmentUnavailableError{message: fmt.Sprintf( - "attachment too large: %d bytes (max %d)", attachment.Size, maxAttachmentSize, + "attachment too large: %d bytes (max %d)", attachment.Size, maxBytes, )} } - data, err := s.read(ctx, attachment.ContentHash) + data, err := s.read(ctx, attachment.ContentHash, maxBytes) if err != nil { return nil, err } @@ -98,7 +102,7 @@ func (s attachmentService) load(ctx context.Context, id int64) (*attachmentPaylo return &attachmentPayload{metadata: attachment, mimeType: mimeType, data: data}, nil } -func (s attachmentService) read(ctx context.Context, contentHash string) ([]byte, error) { +func (s attachmentService) read(ctx context.Context, contentHash string, maxBytes int64) ([]byte, error) { if err := export.ValidateContentHash(contentHash); err != nil { return nil, &attachmentUnavailableError{message: "attachment has invalid content hash"} } @@ -110,9 +114,9 @@ func (s attachmentService) read(ctx context.Context, contentHash string) ([]byte } return nil, newInternalError("read attachment", err) } - if int64(len(data)) > maxAttachmentSize { + if int64(len(data)) > maxBytes { return nil, &attachmentUnavailableError{message: fmt.Sprintf( - "attachment too large: %d bytes (max %d)", len(data), maxAttachmentSize, + "attachment too large: %d bytes (max %d)", len(data), maxBytes, )} } return data, nil @@ -135,18 +139,18 @@ func (s attachmentService) read(ctx context.Context, contentHash string) ([]byte if err != nil { return nil, newInternalError("stat attachment", err) } - if info.Size() > maxAttachmentSize { + if info.Size() > maxBytes { return nil, &attachmentUnavailableError{message: fmt.Sprintf( - "attachment too large: %d bytes (max %d)", info.Size(), maxAttachmentSize, + "attachment too large: %d bytes (max %d)", info.Size(), maxBytes, )} } - data, err := io.ReadAll(io.LimitReader(file, maxAttachmentSize+1)) + data, err := io.ReadAll(io.LimitReader(file, maxBytes+1)) if err != nil { return nil, newInternalError("read attachment", err) } - if int64(len(data)) > maxAttachmentSize { + if int64(len(data)) > maxBytes { return nil, &attachmentUnavailableError{message: fmt.Sprintf( - "attachment too large: %d bytes (max %d)", len(data), maxAttachmentSize, + "attachment too large: %d bytes (max %d)", len(data), maxBytes, )} } return data, nil diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 911ee57d7..9bb90c388 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -36,6 +36,8 @@ const ( ToolGetMessage = "get_message" ToolGetAttachment = "get_attachment" ToolExportAttachment = "export_attachment" + ToolExportEML = "export_eml" + ToolListThread = "list_thread" ToolListMessages = "list_messages" ToolGetStats = "get_stats" ToolAggregate = "aggregate" @@ -77,6 +79,7 @@ const ( // the search_message_bodies tool, and Backend additionally enables the // find_similar_messages tool. type ServeOptions struct { + downloads *downloadCache Engine query.Engine AttachmentsDir string AttachmentReader AttachmentReader @@ -224,7 +227,11 @@ func newMCPServerWithPolicy( cachePolicyMiddleware, ) + if opts.downloads == nil { + opts.downloads = &downloadCache{} + } h := &handlers{ + downloads: opts.downloads, engine: opts.Engine, archiveSQLQuerier: opts.ArchiveSQLQuerier, attachmentsDir: opts.AttachmentsDir, @@ -272,6 +279,8 @@ func Serve(ctx context.Context, engine query.Engine, attachmentsDir, dataDir str // ServeWithOptions creates an MCP server from opts and serves over stdio. func ServeWithOptions(ctx context.Context, opts ServeOptions) error { + opts.downloads = &downloadCache{} + defer opts.downloads.close() policy := newStdioInvocationPolicy() s := newMCPServerWithPolicy(opts, true, policy) if err := s.Run(ctx, &sdkmcp.StdioTransport{}); err != nil { @@ -295,6 +304,8 @@ func ServeHTTPWithOptions(ctx context.Context, opts ServeOptions, httpOpts HTTPO } defer func() { result = errors.Join(result, cleanup()) }() } + opts.downloads = &downloadCache{} + defer opts.downloads.close() stdlibServer := newMCPHTTPServer(opts, httpOpts) fmt.Fprintf(os.Stderr, "Starting MCP server on %s\n", httpOpts.Addr) @@ -332,6 +343,10 @@ func newMCPHTTPServerWithPolicy( ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 120 * time.Second, } + if opts.downloads == nil { + opts.downloads = &downloadCache{} + } + stdlibServer.RegisterOnShutdown(opts.downloads.close) httpServer := sdkmcp.NewStreamableHTTPHandler( func(*http.Request) *sdkmcp.Server { return newMCPServerWithPolicy(opts, httpOpts.AllowWrites, policy) diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index fc3be43cc..2a8709542 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -142,7 +142,7 @@ type paginatedListMessages struct { // newTestHandlers creates a handlers instance with the given mock engine. func newTestHandlers(eng query.Engine) *handlers { - return &handlers{engine: eng} + return &handlers{engine: eng, downloads: &downloadCache{}} } type listAccountsTrackingEngine struct { @@ -4321,3 +4321,24 @@ func TestGetMessagePreservesBrowserURL(t *testing.T) { result := runTool[map[string]any](t, "get_message", h.getMessage, map[string]any{"id": float64(42)}) assert.Equal(t, link, result["web_url"]) } + +func TestGetStats_AccountLastSyncAt(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + syncedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + eng := &querytest.MockEngine{ + Stats: &query.TotalStats{AccountCount: 2}, + Accounts: []query.AccountInfo{ + {ID: 1, Identifier: "synced@example.com", LastSyncAt: &syncedAt}, + {ID: 2, Identifier: "never@example.com"}, + }, + } + r := callToolDirect(t, "get_stats", newTestHandlers(eng).getStats, map[string]any{}) + var raw struct { + Accounts []map[string]json.RawMessage `json:"accounts"` + } + must.NoError(json.Unmarshal([]byte(resultText(t, r)), &raw)) + must.Len(raw.Accounts, 2) + checks.JSONEq(`"2026-03-04T05:06:07Z"`, string(raw.Accounts[0]["LastSyncAt"])) + checks.NotContains(raw.Accounts[1], "LastSyncAt", "unknown sync time is omitted, not null") +} diff --git a/internal/query/duckdb.go b/internal/query/duckdb.go index 7c659872b..4b35ec35b 100644 --- a/internal/query/duckdb.go +++ b/internal/query/duckdb.go @@ -1865,7 +1865,7 @@ func (e *DuckDBEngine) ListAccounts(ctx context.Context) ([]AccountInfo, error) } rows, err := e.db.QueryContext(ctx, ` - SELECT id, source_type, identifier, COALESCE(display_name, '') + SELECT id, source_type, identifier, COALESCE(display_name, ''), last_sync_at FROM sqlite_db.sources ORDER BY identifier `) @@ -1877,9 +1877,14 @@ func (e *DuckDBEngine) ListAccounts(ctx context.Context) ([]AccountInfo, error) var accounts []AccountInfo for rows.Next() { var acc AccountInfo - if err := rows.Scan(&acc.ID, &acc.SourceType, &acc.Identifier, &acc.DisplayName); err != nil { + var lastSyncAt sql.NullTime + if err := rows.Scan(&acc.ID, &acc.SourceType, &acc.Identifier, &acc.DisplayName, &lastSyncAt); err != nil { return nil, fmt.Errorf("scan account: %w", err) } + if lastSyncAt.Valid { + syncedAt := lastSyncAt.Time.UTC() + acc.LastSyncAt = &syncedAt + } accounts = append(accounts, acc) } @@ -3268,3 +3273,21 @@ func appendDuckDBRecipientSearchCondition( args = append(args, recipientArgs...) return conditions, args } + +var errOriginalMessageNeedsSQLite = fmt.Errorf("original message export requires the SQLite archive: %w", ErrOriginalExportUnsupported) + +// ReadOriginalMessage reads from SQLite; the Parquet cache holds no raw MIME. +func (e *DuckDBEngine) ReadOriginalMessage(ctx context.Context, ref MessageRef, maxBytes int64) (*OriginalMessage, error) { + if e.sqliteEngine == nil { + return nil, errOriginalMessageNeedsSQLite + } + return e.sqliteEngine.ReadOriginalMessage(ctx, ref, maxBytes) +} + +// ListThread reads from SQLite because the Parquet cache can lag recent syncs. +func (e *DuckDBEngine) ListThread(ctx context.Context, q ThreadQuery) (*ThreadPage, error) { + if e.sqliteEngine == nil { + return nil, errOriginalMessageNeedsSQLite + } + return e.sqliteEngine.ListThread(ctx, q) +} diff --git a/internal/query/models.go b/internal/query/models.go index 685c09b0b..08413365d 100644 --- a/internal/query/models.go +++ b/internal/query/models.go @@ -366,6 +366,10 @@ type AccountInfo struct { SourceType string Identifier string // email address DisplayName string + // LastSyncAt is the source's most recent sync activity. Provider + // messages newer than this may not be archived yet. Nil when the + // source has never synced. + LastSyncAt *time.Time `json:",omitzero"` } // StatsOptions configures a stats query. diff --git a/internal/query/original_message.go b/internal/query/original_message.go new file mode 100644 index 000000000..f8fa5c08d --- /dev/null +++ b/internal/query/original_message.go @@ -0,0 +1,496 @@ +package query + +import ( + "bytes" + "compress/zlib" + "context" + "database/sql" + "errors" + "fmt" + "io" + "math" + "slices" + "strconv" + "strings" + "time" + + "go.kenn.io/msgvault/internal/store" +) + +var ( + // ErrOriginalMIMEUnavailable means the message exists but the archive + // holds no original MIME for it (chat or calendar sources, imports + // without raw data). + ErrOriginalMIMEUnavailable = errors.New("original MIME unavailable") + // ErrOriginalMessageTooLarge means a bounded read exceeded its stored or decoded byte budget. + ErrOriginalMessageTooLarge = errors.New("original message exceeds byte limit") + // ErrInvalidMessageRef means a reference did not name exactly one of an + // internal message ID or a provider message ID. + ErrInvalidMessageRef = errors.New("provide exactly one of id or source_message_id") + // ErrAmbiguousReference matches every *AmbiguousError, including ones + // reconstructed from a daemon response. + ErrAmbiguousReference = errors.New("reference matches several accounts") + // ErrOriginalExportUnsupported means this engine or daemon cannot serve + // original messages or thread listings. + ErrOriginalExportUnsupported = errors.New("original message export is not supported by this archive backend") +) + +// AmbiguousError reports that a provider identifier matched archive rows in +// more than one account. Callers retry with MessageRef.Account. +type AmbiguousError struct { + Kind string + Accounts []string +} + +func (e *AmbiguousError) Unwrap() error { return ErrAmbiguousReference } + +func (e *AmbiguousError) Error() string { + return fmt.Sprintf("%s matches several accounts: %s", e.Kind, strings.Join(e.Accounts, ", ")) +} + +// MessageRef names one archived message. Exactly one of ID (internal +// msgvault ID) or SourceMessageID (provider message ID) is set. Account, +// when set, restricts the lookup to the source with that identifier. +type MessageRef struct { + ID int64 + SourceMessageID string + Account string +} + +// MessageRecord is the provenance of one message or conversation. +type MessageRecord struct { + MessageID int64 `json:"message_id,omitzero"` + SourceMessageID string `json:"source_message_id,omitzero"` + ConversationID int64 `json:"conversation_id"` + SourceConversationID string `json:"source_conversation_id"` + SourceID int64 `json:"source_id"` + Account string `json:"account"` + SourceType string `json:"source_type"` + LastSyncAt *time.Time `json:"last_sync_at,omitzero"` +} + +// OriginalMessage is a message's provenance plus its original MIME bytes +// exactly as the provider delivered them. +type OriginalMessage struct { + MessageRecord + + MIME []byte +} + +// OriginalMessageReader reads original MIME and visible conversation members +// for export. A positive maxBytes bounds both the decoded MIME and the stored +// payload (with room for compression overhead); zero permits unlimited reads. +// Engines that cannot serve these omit the interface. +type OriginalMessageReader interface { + ReadOriginalMessage(ctx context.Context, ref MessageRef, maxBytes int64) (*OriginalMessage, error) + ListThread(ctx context.Context, q ThreadQuery) (*ThreadPage, error) +} + +var _ OriginalMessageReader = (*SQLiteEngine)(nil) + +// ReadOriginalMessage resolves ref among live messages (dedup losers are +// not found; source-deleted messages remain archive data) and returns the +// stored MIME. Non-MIME raw formats report ErrOriginalMIMEUnavailable. +func (e *SQLiteEngine) ReadOriginalMessage(ctx context.Context, ref MessageRef, maxBytes int64) (*OriginalMessage, error) { + if maxBytes < 0 || maxBytes == math.MaxInt64 { + return nil, errors.New("maxBytes must be non-negative and less than MaxInt64") + } + record, err := e.resolveMessageRecord(ctx, ref) + if err != nil { + return nil, err + } + + var compressed []byte + var format, compression sql.NullString + var storedBytes int64 + plainLimit, compressedLimit := int64(math.MaxInt64), int64(math.MaxInt64) + if maxBytes > 0 { + plainLimit = maxBytes + // Bound the initial database read at 2x + 1 KiB to allow normal archive + // zlib overhead. Decoded MIME keeps the exact maxBytes limit. Saturate + // before multiplying to avoid overflow. + if maxBytes <= (math.MaxInt64-1024)/2 { + compressedLimit = 2*maxBytes + 1024 + } + } + err = e.queryRowContext(ctx, ` + SELECT CASE + WHEN length(raw_data) <= CASE WHEN compression = 'zlib' + THEN CAST(? AS BIGINT) ELSE CAST(? AS BIGINT) END + THEN raw_data + END, + raw_format, compression, length(raw_data) + FROM message_raw WHERE message_id = ? + `, compressedLimit, plainLimit, record.MessageID).Scan(&compressed, &format, &compression, &storedBytes) + if errors.Is(err, sql.ErrNoRows) || (err == nil && format.String != "mime") { + return nil, fmt.Errorf("message %d: %w", record.MessageID, ErrOriginalMIMEUnavailable) + } + if err != nil { + return nil, fmt.Errorf("read original MIME for message %d: %w", record.MessageID, err) + } + if storedBytes > int64(len(compressed)) { + return nil, fmt.Errorf("message %d: %w", record.MessageID, ErrOriginalMessageTooLarge) + } + mime, err := inflateMessageRaw(compressed, compression, maxBytes) + if err != nil { + return nil, fmt.Errorf("decode original MIME for message %d: %w", record.MessageID, err) + } + return &OriginalMessage{MessageRecord: *record, MIME: mime}, nil +} + +func (e *SQLiteEngine) resolveMessageRecord(ctx context.Context, ref MessageRef) (*MessageRecord, error) { + if (ref.ID == 0) == (ref.SourceMessageID == "") || ref.ID < 0 { + return nil, ErrInvalidMessageRef + } + conditions := []string{store.LiveMessagesWhere("m", false)} + var args []any + if ref.ID != 0 { + conditions = append(conditions, "m.id = ?") + args = append(args, ref.ID) + } else { + conditions = append(conditions, "m.source_message_id = ?") + args = append(args, ref.SourceMessageID) + } + if ref.Account != "" { + conditions = append(conditions, "s.identifier = ?") + args = append(args, ref.Account) + } + rows, err := e.queryContext(ctx, ` + SELECT m.id, COALESCE(m.source_message_id, ''), COALESCE(m.conversation_id, 0), + COALESCE(conv.source_conversation_id, ''), + s.id, s.identifier, s.source_type, s.last_sync_at + FROM messages m + JOIN sources s ON s.id = m.source_id + LEFT JOIN conversations conv ON conv.id = m.conversation_id + WHERE `+strings.Join(conditions, " AND ")+` + ORDER BY m.id`, args...) + if err != nil { + return nil, fmt.Errorf("resolve message: %w", err) + } + defer func() { _ = rows.Close() }() + + var records []MessageRecord + for rows.Next() { + var record MessageRecord + var lastSyncAt sql.NullTime + if err := rows.Scan(&record.MessageID, &record.SourceMessageID, &record.ConversationID, + &record.SourceConversationID, &record.SourceID, &record.Account, &record.SourceType, + &lastSyncAt); err != nil { + return nil, fmt.Errorf("scan message reference: %w", err) + } + record.LastSyncAt = utcTimePtr(lastSyncAt) + records = append(records, record) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("resolve message: %w", err) + } + switch len(records) { + case 0: + return nil, fmt.Errorf("message %s: %w", ref.describe(), store.ErrMessageNotFound) + case 1: + return &records[0], nil + } + accounts := make([]string, 0, len(records)) + for _, record := range records { + accounts = append(accounts, record.Account) + } + return nil, &AmbiguousError{Kind: "source_message_id " + ref.SourceMessageID, Accounts: sortedUnique(accounts)} +} + +func (ref MessageRef) describe() string { + if ref.ID != 0 { + return strconv.FormatInt(ref.ID, 10) + } + return ref.SourceMessageID +} + +func utcTimePtr(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + t := value.Time.UTC() + return &t +} + +func sortedUnique(values []string) []string { + slices.Sort(values) + return slices.Compact(values) +} + +// inflateMessageRaw decodes a message_raw payload by its compression tag. +func inflateMessageRaw(stored []byte, compression sql.NullString, maxBytes int64) ([]byte, error) { + if !compression.Valid || compression.String != "zlib" { + if maxBytes > 0 && int64(len(stored)) > maxBytes { + return nil, ErrOriginalMessageTooLarge + } + return stored, nil + } + r, err := zlib.NewReader(bytes.NewReader(stored)) + if err != nil { + return nil, fmt.Errorf("zlib reader: %w", err) + } + defer func() { _ = r.Close() }() + var reader io.Reader = r + if maxBytes > 0 { + reader = io.LimitReader(r, maxBytes+1) + } + raw, err := io.ReadAll(reader) + if maxBytes > 0 && int64(len(raw)) > maxBytes { + return nil, ErrOriginalMessageTooLarge + } + if err != nil { + return nil, fmt.Errorf("zlib decompress: %w", err) + } + return raw, nil +} + +// ErrThreadNotFound means no archived conversation matched a thread lookup. +var ErrThreadNotFound = errors.New("thread not found") + +// Thread listing page bounds. +const ( + ThreadDefaultLimit = 100 + ThreadMaxLimit = 500 +) + +// ThreadQuery selects one conversation by exactly one of an anchor message +// (MessageRef.ID or MessageRef.SourceMessageID) or a provider conversation +// ID (ThreadID). MessageRef.Account narrows either lookup to one source. +type ThreadQuery struct { + MessageRef + + ThreadID string + Limit int + Offset int + + // All returns fixed membership from one query, ignoring Limit and Offset. + All bool +} + +// ThreadMessage is one archived message in a conversation listing. +type ThreadMessage struct { + ID int64 `json:"id"` + SourceMessageID string `json:"source_message_id"` + Subject string `json:"subject"` + SentAt *time.Time `json:"sent_at,omitzero"` + From []Address `json:"from"` + To []Address `json:"to"` + Cc []Address `json:"cc"` + HasRaw bool `json:"has_raw"` + AttachmentCount int `json:"attachment_count"` + DeletedFromSourceAt *time.Time `json:"deleted_from_source_at,omitzero"` +} + +// ThreadPage is one page of a conversation in chronological order. The +// embedded record describes the conversation; MessageID and +// SourceMessageID name the anchor message when the lookup used one. +type ThreadPage struct { + MessageRecord + + Total int64 `json:"total"` + Offset int `json:"offset"` + HasMore bool `json:"has_more"` + Messages []ThreadMessage `json:"messages"` +} + +// ListThread lists the live messages of one conversation ordered by +// sent_at (undated last), then ID. Source-deleted messages are included +// because they remain archive data. HasRaw reports stored original MIME. +func (e *SQLiteEngine) ListThread(ctx context.Context, q ThreadQuery) (*ThreadPage, error) { + anchors := 0 + for _, set := range []bool{q.ID != 0, q.SourceMessageID != "", q.ThreadID != ""} { + if set { + anchors++ + } + } + if anchors != 1 { + return nil, ErrInvalidMessageRef + } + limit := q.Limit + if limit <= 0 { + limit = ThreadDefaultLimit + } + limit = min(limit, ThreadMaxLimit) + offset := max(q.Offset, 0) + if q.All { + offset = 0 + } + + var header *MessageRecord + var err error + if q.ThreadID != "" { + header, err = e.resolveThreadRecord(ctx, q.ThreadID, q.Account) + } else { + header, err = e.resolveMessageRecord(ctx, q.MessageRef) + } + if err != nil { + return nil, err + } + + live := store.LiveMessagesWhere("m", false) + page := &ThreadPage{MessageRecord: *header, Offset: offset, Messages: []ThreadMessage{}} + if !q.All { + if err := e.queryRowContext(ctx, ` + SELECT COUNT(*) FROM messages m WHERE m.conversation_id = ? AND `+live, + header.ConversationID).Scan(&page.Total); err != nil { + return nil, fmt.Errorf("count thread messages: %w", err) + } + } + + statement := ` + SELECT m.id, COALESCE(m.source_message_id, ''), COALESCE(m.subject, ''), m.sent_at, + COALESCE(m.attachment_count, 0), m.deleted_from_source_at, + CASE WHEN EXISTS ( + SELECT 1 FROM message_raw mr + WHERE mr.message_id = m.id AND mr.raw_format = 'mime' + ) THEN 1 ELSE 0 END + FROM messages m + WHERE m.conversation_id = ? AND ` + live + ` + ORDER BY CASE WHEN m.sent_at IS NULL THEN 1 ELSE 0 END, m.sent_at, m.id` + args := []any{header.ConversationID} + if !q.All { + statement += " LIMIT ? OFFSET ?" + args = append(args, limit, offset) + } + rows, err := e.queryContext(ctx, statement, args...) + if err != nil { + return nil, fmt.Errorf("list thread messages: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var msg ThreadMessage + var sentAt, deletedAt sql.NullTime + var hasRaw int + if err := rows.Scan(&msg.ID, &msg.SourceMessageID, &msg.Subject, &sentAt, + &msg.AttachmentCount, &deletedAt, &hasRaw); err != nil { + return nil, fmt.Errorf("scan thread message: %w", err) + } + msg.SentAt = utcTimePtr(sentAt) + msg.DeletedFromSourceAt = utcTimePtr(deletedAt) + msg.HasRaw = hasRaw == 1 + msg.From, msg.To, msg.Cc = []Address{}, []Address{}, []Address{} + page.Messages = append(page.Messages, msg) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list thread messages: %w", err) + } + if q.All { + page.Total = int64(len(page.Messages)) + } else { + page.HasMore = int64(offset+len(page.Messages)) < page.Total + } + for messages := range slices.Chunk(page.Messages, ThreadMaxLimit) { + index := make(map[int64]int, len(messages)) + for i, message := range messages { + index[message.ID] = i + } + if err := e.fillThreadParticipants(ctx, messages, index); err != nil { + return nil, err + } + } + return page, nil +} + +func (e *SQLiteEngine) resolveThreadRecord(ctx context.Context, threadID, account string) (*MessageRecord, error) { + conditions := `conv.source_conversation_id = ? AND EXISTS ( + SELECT 1 FROM messages m WHERE m.conversation_id = conv.id AND ` + store.LiveMessagesWhere("m", false) + ` + )` + args := []any{threadID} + if account != "" { + conditions += " AND s.identifier = ?" + args = append(args, account) + } + rows, err := e.queryContext(ctx, ` + SELECT conv.id, conv.source_conversation_id, s.id, s.identifier, s.source_type, s.last_sync_at + FROM conversations conv + JOIN sources s ON s.id = conv.source_id + WHERE `+conditions+` + ORDER BY conv.id`, args...) + if err != nil { + return nil, fmt.Errorf("resolve thread: %w", err) + } + defer func() { _ = rows.Close() }() + var records []MessageRecord + for rows.Next() { + var record MessageRecord + var lastSyncAt sql.NullTime + if err := rows.Scan(&record.ConversationID, &record.SourceConversationID, &record.SourceID, + &record.Account, &record.SourceType, &lastSyncAt); err != nil { + return nil, fmt.Errorf("scan thread reference: %w", err) + } + record.LastSyncAt = utcTimePtr(lastSyncAt) + records = append(records, record) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("resolve thread: %w", err) + } + switch len(records) { + case 0: + return nil, fmt.Errorf("thread %s: %w", threadID, ErrThreadNotFound) + case 1: + return &records[0], nil + } + accounts := make([]string, 0, len(records)) + for _, record := range records { + accounts = append(accounts, record.Account) + } + return nil, &AmbiguousError{Kind: "thread_id " + threadID, Accounts: sortedUnique(accounts)} +} + +func (e *SQLiteEngine) fillThreadParticipants(ctx context.Context, messages []ThreadMessage, index map[int64]int) error { + if len(messages) == 0 { + return nil + } + ids := make([]any, len(messages)) + placeholders := make([]string, len(messages)) + for i, msg := range messages { + ids[i] = msg.ID + placeholders[i] = "?" + } + rows, err := e.queryContext(ctx, fmt.Sprintf(` + SELECT message_id, recipient_type, address, display_name + FROM ( + SELECT mr.message_id, mr.recipient_type, + COALESCE(NULLIF(p.email_address, ''), NULLIF(p.phone_number, ''), '') AS address, + %[1]s AS display_name, mr.id AS recipient_id + FROM message_recipients mr + JOIN participants p ON p.id = mr.participant_id + WHERE mr.message_id IN (%[2]s) + AND mr.recipient_type IN ('from', 'to', 'cc') + UNION ALL + SELECT m.id, 'from', + COALESCE(NULLIF(p.email_address, ''), NULLIF(p.phone_number, ''), ''), + COALESCE(%[3]s, ''), 0 + FROM messages m + JOIN participants p ON p.id = m.sender_id + WHERE m.id IN (%[2]s) + AND NOT EXISTS ( + SELECT 1 FROM message_recipients mr + WHERE mr.message_id = m.id AND mr.recipient_type = 'from' + ) + ) participants + ORDER BY message_id, recipient_id + `, recipientNameExpr("mr", "p"), strings.Join(placeholders, ","), participantNameExpr("p")), slices.Concat(ids, ids)...) + if err != nil { + return fmt.Errorf("fetch thread participants: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var messageID int64 + var recipientType string + var addr Address + if err := rows.Scan(&messageID, &recipientType, &addr.Email, &addr.Name); err != nil { + return fmt.Errorf("scan thread participant: %w", err) + } + msg := &messages[index[messageID]] + switch recipientType { + case "from": + msg.From = append(msg.From, addr) + case "to": + msg.To = append(msg.To, addr) + case "cc": + msg.Cc = append(msg.Cc, addr) + } + } + return rows.Err() +} diff --git a/internal/query/original_message_test.go b/internal/query/original_message_test.go new file mode 100644 index 000000000..1bef98378 --- /dev/null +++ b/internal/query/original_message_test.go @@ -0,0 +1,432 @@ +package query_test + +import ( + "bytes" + "compress/zlib" + "context" + "crypto/rand" + "math" + "strconv" + "testing" + "time" + + "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +// originalMIME carries bytes a re-serializer would change: CRLF and bare LF +// line endings, 8-bit octets, a NUL, trailing whitespace and no final newline. +var originalMIME = []byte("From: Sender \r\n" + + "To: Recipient \r\n" + + "Subject: Quarterly report \r\n" + + "Content-Type: text/plain; charset=latin1\r\n" + + "\r\n" + + "caf\xe9 \x00 line\n" + + "trailing spaces \r\n" + + "no final newline") + +func originalEngine(f *storetest.Fixture) *query.SQLiteEngine { + if f.Store.IsPostgreSQL() { + return query.NewEngineWithDialect(f.Store.DB(), query.PostgreSQLQueryDialect{}) + } + return query.NewSQLiteEngine(f.Store.DB()) +} + +func TestReadOriginalMessage(t *testing.T) { + must := require.New(t) + ctx := context.Background() + f := storetest.New(t) + engine := originalEngine(f) + + withRaw := f.NewMessage().WithSourceMessageID("provider-abc").WithSubject("Quarterly report").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(withRaw, originalMIME)) + calendarJSON := f.NewMessage().WithSourceMessageID("calendar-event").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRawWithFormat(calendarJSON, []byte(`{"kind":"event"}`), "gcal_json")) + noRaw := f.NewMessage().WithSourceMessageID("chat-only").Create(t, f.Store) + dedupLoser := f.NewMessage().WithSourceMessageID("dedup-loser").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(dedupLoser, originalMIME)) + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET deleted_at = CURRENT_TIMESTAMP WHERE id = ?`), dedupLoser) + must.NoError(err) + + t.Run("by internal id returns exact bytes and provenance", func(t *testing.T) { + checks := assert.New(t) + got, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: withRaw}, 0) + require.NoError(t, err) + checks.Equal(originalMIME, got.MIME) + checks.Equal(withRaw, got.MessageID) + checks.Equal("provider-abc", got.SourceMessageID) + checks.Equal(f.Source.ID, got.SourceID) + checks.Equal(f.ConvID, got.ConversationID) + checks.Equal("default-thread", got.SourceConversationID) + checks.Equal("test@example.com", got.Account) + checks.Equal("gmail", got.SourceType) + checks.Nil(got.LastSyncAt) + }) + + t.Run("by provider id", func(t *testing.T) { + got, err := engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: "provider-abc"}, 0) + require.NoError(t, err) + assert.Equal(t, withRaw, got.MessageID) + assert.Equal(t, originalMIME, got.MIME) + }) + + t.Run("reports last sync time", func(t *testing.T) { + must := require.New(t) + syncedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE sources SET last_sync_at = ? WHERE id = ?`), syncedAt, f.Source.ID) + must.NoError(err) + got, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: withRaw}, 0) + must.NoError(err) + must.NotNil(got.LastSyncAt) + assert.True(t, syncedAt.Equal(*got.LastSyncAt), "last sync %v", got.LastSyncAt) + }) + + t.Run("numeric provider id never resolves an internal id", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: strconv.FormatInt(withRaw, 10)}, 0) + require.ErrorIs(t, err, store.ErrMessageNotFound) + }) + + t.Run("non-MIME raw is unavailable", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: calendarJSON}, 0) + require.ErrorIs(t, err, query.ErrOriginalMIMEUnavailable) + }) + + t.Run("missing raw is unavailable", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: noRaw}, 0) + require.ErrorIs(t, err, query.ErrOriginalMIMEUnavailable) + }) + + t.Run("dedup loser is not found", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: dedupLoser}, 0) + require.ErrorIs(t, err, store.ErrMessageNotFound) + _, err = engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: "dedup-loser"}, 0) + require.ErrorIs(t, err, store.ErrMessageNotFound) + }) + + t.Run("unknown message is not found", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: 999999}, 0) + require.ErrorIs(t, err, store.ErrMessageNotFound) + }) + + t.Run("reference needs exactly one identifier", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{}, 0) + require.ErrorIs(t, err, query.ErrInvalidMessageRef) + _, err = engine.ReadOriginalMessage(ctx, query.MessageRef{ID: withRaw, SourceMessageID: "provider-abc"}, 0) + require.ErrorIs(t, err, query.ErrInvalidMessageRef) + }) + + t.Run("account narrows an internal id", func(t *testing.T) { + _, err := engine.ReadOriginalMessage(ctx, query.MessageRef{ID: withRaw, Account: "other@example.com"}, 0) + require.ErrorIs(t, err, store.ErrMessageNotFound) + }) +} + +func TestReadOriginalMessageBounded(t *testing.T) { + f := storetest.New(t) + engine := originalEngine(f) + incompressible := make([]byte, 4096) + _, err := rand.Read(incompressible) + require.NoError(t, err) + for _, tc := range []struct { + name string + raw []byte + compressed bool + }{ + {name: "small zlib", raw: []byte("a"), compressed: true}, + {name: "incompressible zlib", raw: incompressible, compressed: true}, + {name: "compressible zlib", raw: bytes.Repeat([]byte("a"), 65536), compressed: true}, + {name: "uncompressed", raw: originalMIME}, + } { + t.Run(tc.name, func(t *testing.T) { + must := require.New(t) + checks := assert.New(t) + id := f.NewMessage().Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(id, tc.raw)) + if !tc.compressed { + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE message_raw SET raw_data = ?, compression = NULL WHERE message_id = ?`), tc.raw, id) + must.NoError(err) + } + ref := query.MessageRef{ID: id} + for _, limit := range []int64{0, int64(len(tc.raw)), math.MaxInt64 - 1} { + got, err := engine.ReadOriginalMessage(t.Context(), ref, limit) + must.NoError(err) + checks.Equal(tc.raw, got.MIME) + } + if len(tc.raw) > 1 { + got, err := engine.ReadOriginalMessage(t.Context(), ref, int64(len(tc.raw)-1)) + must.ErrorIs(err, query.ErrOriginalMessageTooLarge) + checks.Nil(got) + } + for _, limit := range []int64{-1, math.MaxInt64} { + _, err := engine.ReadOriginalMessage(t.Context(), ref, limit) + must.Error(err) + } + }) + } +} + +func TestReadOriginalMessageBoundedCorruption(t *testing.T) { + must := require.New(t) + f := storetest.New(t) + engine := originalEngine(f) + id := f.NewMessage().Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(id, bytes.Repeat([]byte("a"), 65536))) + var compressed []byte + must.NoError(f.Store.DB().QueryRow(f.Store.Rebind(`SELECT raw_data FROM message_raw WHERE message_id = ?`), id).Scan(&compressed)) + compressed[len(compressed)-1] ^= 1 // Invalid checksum after the decoded size limit. + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE message_raw SET raw_data = ? WHERE message_id = ?`), compressed, id) + must.NoError(err) + ref := query.MessageRef{ID: id} + _, err = engine.ReadOriginalMessage(t.Context(), ref, 1024) + must.ErrorIs(err, query.ErrOriginalMessageTooLarge) + _, err = engine.ReadOriginalMessage(t.Context(), ref, 65536) + must.ErrorIs(err, zlib.ErrChecksum) + _, err = engine.ReadOriginalMessage(t.Context(), ref, 0) + must.ErrorIs(err, zlib.ErrChecksum) + + // Reject a stored blob beyond the input budget before attempting to decode it. + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE message_raw SET raw_data = ? WHERE message_id = ?`), bytes.Repeat([]byte("x"), 4096), id) + must.NoError(err) + _, err = engine.ReadOriginalMessage(t.Context(), ref, 1) + must.ErrorIs(err, query.ErrOriginalMessageTooLarge) + _, err = engine.ReadOriginalMessage(t.Context(), ref, 0) + must.ErrorIs(err, zlib.ErrHeader) +} + +func TestReadOriginalMessageAllowsNullSourceMessageID(t *testing.T) { + must := require.New(t) + f := storetest.New(t) + engine := originalEngine(f) + + id := f.NewMessage().WithSourceMessageID("temporary-provider-id").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(id, originalMIME)) + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET source_message_id = NULL WHERE id = ?`), id) + must.NoError(err) + + got, err := engine.ReadOriginalMessage(context.Background(), query.MessageRef{ID: id}, 0) + must.NoError(err) + must.Empty(got.SourceMessageID) +} + +func TestReadOriginalMessageAmbiguousProviderID(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + ctx := context.Background() + f := storetest.New(t) + engine := originalEngine(f) + + first := f.NewMessage().WithSourceMessageID("shared-id").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(first, []byte("first\r\n"))) + + other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") + must.NoError(err) + otherConv, err := f.Store.EnsureConversation(other.ID, "other-thread", "Other") + must.NoError(err) + second := storetest.NewMessage(other.ID, otherConv).WithSourceMessageID("shared-id").Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(second, []byte("second\r\n"))) + + _, err = engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: "shared-id"}, 0) + var ambiguous *query.AmbiguousError + must.ErrorAs(err, &ambiguous) + checks.Equal([]string{"other@example.com", "test@example.com"}, ambiguous.Accounts) + + got, err := engine.ReadOriginalMessage(ctx, query.MessageRef{SourceMessageID: "shared-id", Account: "other@example.com"}, 0) + must.NoError(err) + checks.Equal(second, got.MessageID) + checks.Equal([]byte("second\r\n"), got.MIME) +} + +func TestListThread(t *testing.T) { + must := require.New(t) + ctx := context.Background() + f := storetest.New(t) + engine := originalEngine(f) + + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + undated := f.NewMessage().WithSourceMessageID("undated").Create(t, f.Store) + third := f.NewMessage().WithSourceMessageID("third").WithSentAt(base.Add(2*time.Hour)).Create(t, f.Store) + firstA := f.NewMessage().WithSourceMessageID("first-a").WithSubject("Kickoff").WithSentAt(base).Create(t, f.Store) + firstB := f.NewMessage().WithSourceMessageID("first-b").WithSentAt(base).Create(t, f.Store) + must.NoError(f.Store.UpsertMessageRaw(firstA, []byte("a\r\n"))) + must.NoError(f.Store.UpsertMessageRawWithFormat(third, []byte(`{}`), "gcal_json")) + loser := f.NewMessage().WithSourceMessageID("loser").WithSentAt(base.Add(time.Hour)).Create(t, f.Store) + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET deleted_at = CURRENT_TIMESTAMP WHERE id = ?`), loser) + must.NoError(err) + sourceDeleted := f.NewMessage().WithSourceMessageID("source-deleted").WithSentAt(base.Add(90*time.Minute)).Create(t, f.Store) + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ?`), sourceDeleted) + must.NoError(err) + + sender := f.EnsureParticipant("sender@example.com", "Sender", "example.com") + recipient := f.EnsureParticipant("recipient@example.org", "Recipient", "example.org") + fallbackSender := f.EnsureParticipant("fallback@example.com", "Fallback Sender", "example.com") + _, err = f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET sender_id = ? WHERE id IN (?, ?)`), + fallbackSender, firstA, firstB) + must.NoError(err) + must.NoError(f.Store.ReplaceMessageRecipients(firstA, "from", []int64{sender}, []string{"Sender"})) + must.NoError(f.Store.ReplaceMessageRecipients(firstA, "to", []int64{recipient}, []string{"Recipient"})) + + ids := func(page *query.ThreadPage) []int64 { + out := make([]int64, 0, len(page.Messages)) + for _, m := range page.Messages { + out = append(out, m.ID) + } + return out + } + + t.Run("anchored by message lists live messages chronologically", func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + page, err := engine.ListThread(ctx, query.ThreadQuery{ID: third}) + must.NoError(err) + checks.Equal([]int64{firstA, firstB, sourceDeleted, third, undated}, ids(page)) + checks.Equal(int64(5), page.Total) + checks.False(page.HasMore) + checks.Equal(third, page.MessageID) + checks.Equal(f.ConvID, page.ConversationID) + checks.Equal("default-thread", page.SourceConversationID) + checks.Equal("test@example.com", page.Account) + + first := page.Messages[0] + checks.True(first.HasRaw, "mime raw stored") + checks.Equal("Kickoff", first.Subject) + must.NotNil(first.SentAt) + checks.True(base.Equal(*first.SentAt)) + checks.Equal([]query.Address{{Email: "sender@example.com", Name: "Sender"}}, first.From) + checks.Equal([]query.Address{{Email: "recipient@example.org", Name: "Recipient"}}, first.To) + checks.Equal([]query.Address{{Email: "fallback@example.com", Name: "Fallback Sender"}}, page.Messages[1].From) + checks.False(page.Messages[3].HasRaw, "non-MIME raw is not original MIME") + checks.NotNil(page.Messages[2].DeletedFromSourceAt) + checks.Nil(page.Messages[4].SentAt) + }) + + t.Run("pages", func(t *testing.T) { + checks := assert.New(t) + page, err := engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread", Limit: 2, Offset: 2}) + require.NoError(t, err) + checks.Equal([]int64{sourceDeleted, third}, ids(page)) + checks.Equal(int64(5), page.Total) + checks.True(page.HasMore) + checks.Equal(2, page.Offset) + checks.Zero(page.MessageID, "no anchor for thread lookups") + }) + + t.Run("unknown thread", func(t *testing.T) { + _, err := engine.ListThread(ctx, query.ThreadQuery{ThreadID: "missing"}) + require.ErrorIs(t, err, query.ErrThreadNotFound) + }) + + t.Run("needs exactly one anchor", func(t *testing.T) { + _, err := engine.ListThread(ctx, query.ThreadQuery{}) + require.ErrorIs(t, err, query.ErrInvalidMessageRef) + _, err = engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread", ID: third}) + require.ErrorIs(t, err, query.ErrInvalidMessageRef) + }) + + t.Run("thread id in two accounts is ambiguous", func(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") + must.NoError(err) + otherConv, err := f.Store.EnsureConversation(other.ID, "default-thread", "Other") + must.NoError(err) + otherMsg := storetest.NewMessage(other.ID, otherConv).Create(t, f.Store) + + _, err = engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread"}) + var ambiguous *query.AmbiguousError + must.ErrorAs(err, &ambiguous) + checks.Equal([]string{"other@example.com", "test@example.com"}, ambiguous.Accounts) + + page, err := engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread", Account: "other@example.com"}) + must.NoError(err) + checks.Equal([]int64{otherMsg}, ids(page)) + }) +} + +func TestListThreadAllowsNullSourceMessageID(t *testing.T) { + must := require.New(t) + f := storetest.New(t) + engine := originalEngine(f) + id := f.NewMessage().WithSourceMessageID("temporary-provider-id").Create(t, f.Store) + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET source_message_id = NULL WHERE id = ?`), id) + must.NoError(err) + + page, err := engine.ListThread(context.Background(), query.ThreadQuery{ThreadID: "default-thread"}) + must.NoError(err) + must.Len(page.Messages, 1) + must.Equal(id, page.Messages[0].ID) + must.Empty(page.Messages[0].SourceMessageID) +} + +func TestListThreadRequiresVisibleMessages(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + f := storetest.New(t) + engine := originalEngine(f) + ctx := t.Context() + + _, err := engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread"}) + must.ErrorIs(err, query.ErrThreadNotFound, "empty conversations are not exportable threads") + + survivor := f.CreateMessage("survivor") + other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") + must.NoError(err) + otherConv, err := f.Store.EnsureConversation(other.ID, "default-thread", "Duplicate") + must.NoError(err) + duplicate := storetest.NewMessage(other.ID, otherConv).WithSourceMessageID("duplicate").Create(t, f.Store) + _, err = f.Store.MergeDuplicates(survivor, []int64{duplicate}, "thread-visibility") + must.NoError(err) + + _, err = engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread", Account: "other@example.com"}) + must.ErrorIs(err, query.ErrThreadNotFound, "hidden-only conversations are not exportable threads") + + must.NoError(f.Store.MarkMessageDeleted(f.Source.ID, "survivor")) + page, err := engine.ListThread(ctx, query.ThreadQuery{ThreadID: "default-thread"}) + must.NoError(err, "hidden-only conversations must not make the visible thread ambiguous") + must.Len(page.Messages, 1) + checks.Equal(survivor, page.Messages[0].ID) + checks.NotNil(page.Messages[0].DeletedFromSourceAt, "source-deleted messages remain archived") +} + +func TestListThreadAll(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + f := storetest.New(t) + ids := f.CreateMessages(501) + sender := f.EnsureParticipant("sender@example.com", "Sender", "example.com") + recipient := f.EnsureParticipant("recipient@example.com", "Recipient", "example.com") + _, err := f.Store.DB().Exec(f.Store.Rebind(`UPDATE messages SET sender_id = ? WHERE id = ?`), sender, ids[500]) + must.NoError(err) + must.NoError(f.Store.ReplaceMessageRecipients(ids[500], "to", []int64{recipient}, []string{"Recipient"})) + + if !f.Store.IsPostgreSQL() { + // Keep this archive small while exercising the database's real bind limit. + f.Store.DB().SetMaxOpenConns(1) + conn, err := f.Store.DB().Conn(t.Context()) + must.NoError(err) + must.NoError(conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + must.True(ok) + sqliteConn.SetLimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 1000) + return nil + })) + must.NoError(conn.Close()) + } + + page, err := originalEngine(f).ListThread(t.Context(), query.ThreadQuery{ + ThreadID: "default-thread", All: true, Limit: 1, Offset: 1, + }) + must.NoError(err) + must.Len(page.Messages, 501) + checks.Equal(int64(501), page.Total) + checks.False(page.HasMore) + checks.Zero(page.Offset) + for i, message := range page.Messages { + checks.Equal(ids[i], message.ID) + } + checks.Equal([]query.Address{{Email: "sender@example.com", Name: "Sender"}}, page.Messages[500].From) + checks.Equal([]query.Address{{Email: "recipient@example.com", Name: "Recipient"}}, page.Messages[500].To) +} diff --git a/internal/query/postgres.go b/internal/query/postgres.go index 47cabadf5..d1bc1c032 100644 --- a/internal/query/postgres.go +++ b/internal/query/postgres.go @@ -114,3 +114,24 @@ func NewEngine(db *sql.DB, isPostgres bool) Engine { } return NewSQLiteEngine(db) } + +var _ OriginalMessageReader = (*pgEngine)(nil) + +// ReadOriginalMessage forwards to the dialect-parameterized engine; its SQL +// is portable across SQLite and PostgreSQL. +func (e *pgEngine) ReadOriginalMessage(ctx context.Context, ref MessageRef, maxBytes int64) (*OriginalMessage, error) { + reader, ok := e.Engine.(OriginalMessageReader) + if !ok { + return nil, ErrOriginalExportUnsupported + } + return reader.ReadOriginalMessage(ctx, ref, maxBytes) +} + +// ListThread forwards to the dialect-parameterized engine. +func (e *pgEngine) ListThread(ctx context.Context, q ThreadQuery) (*ThreadPage, error) { + reader, ok := e.Engine.(OriginalMessageReader) + if !ok { + return nil, ErrOriginalExportUnsupported + } + return reader.ListThread(ctx, q) +} diff --git a/internal/query/shared.go b/internal/query/shared.go index 6a4ec29a1..578d1bee2 100644 --- a/internal/query/shared.go +++ b/internal/query/shared.go @@ -7,6 +7,7 @@ import ( "context" "database/sql" "encoding/hex" + "errors" "fmt" "io" "slices" @@ -453,27 +454,18 @@ func getMessageRawShared(ctx context.Context, db *sql.DB, rebind rebindFunc, tab JOIN %smessages m ON m.id = mr.message_id WHERE mr.message_id = ? AND %s `, tablePrefix, tablePrefix, store.LiveMessagesWhere("m", false))), messageID).Scan(&compressed, &compression) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { return nil, fmt.Errorf("query message_raw for id %d: %w", messageID, err) } - if compression.Valid && compression.String == "zlib" { - r, err := zlib.NewReader(bytes.NewReader(compressed)) - if err != nil { - return nil, fmt.Errorf("zlib reader for id %d: %w", messageID, err) - } - defer func() { _ = r.Close() }() - raw, err := io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("zlib decompress message_raw id %d: %w", messageID, err) - } - return raw, nil + raw, err := inflateMessageRaw(compressed, compression, 0) + if err != nil { + return nil, fmt.Errorf("message_raw id %d: %w", messageID, err) } - - return compressed, nil + return raw, nil } // getMessageByQueryShared retrieves a full message detail by an arbitrary WHERE clause. diff --git a/internal/query/sqlite.go b/internal/query/sqlite.go index 24aaca509..92a4805f3 100644 --- a/internal/query/sqlite.go +++ b/internal/query/sqlite.go @@ -1287,7 +1287,7 @@ func (e *SQLiteEngine) GetMessageRaw(ctx context.Context, id int64) ([]byte, err // ListAccounts returns all source accounts. func (e *SQLiteEngine) ListAccounts(ctx context.Context) ([]AccountInfo, error) { rows, err := e.queryContext(ctx, ` - SELECT id, source_type, identifier, COALESCE(display_name, '') + SELECT id, source_type, identifier, COALESCE(display_name, ''), last_sync_at FROM sources ORDER BY identifier `) @@ -1299,9 +1299,14 @@ func (e *SQLiteEngine) ListAccounts(ctx context.Context) ([]AccountInfo, error) var accounts []AccountInfo for rows.Next() { var acc AccountInfo - if err := rows.Scan(&acc.ID, &acc.SourceType, &acc.Identifier, &acc.DisplayName); err != nil { + var lastSyncAt sql.NullTime + if err := rows.Scan(&acc.ID, &acc.SourceType, &acc.Identifier, &acc.DisplayName, &lastSyncAt); err != nil { return nil, fmt.Errorf("scan account: %w", err) } + if lastSyncAt.Valid { + syncedAt := lastSyncAt.Time.UTC() + acc.LastSyncAt = &syncedAt + } accounts = append(accounts, acc) } diff --git a/internal/query/sqlite_crud_test.go b/internal/query/sqlite_crud_test.go index 00805a2d1..735ec353d 100644 --- a/internal/query/sqlite_crud_test.go +++ b/internal/query/sqlite_crud_test.go @@ -394,6 +394,27 @@ func TestListAccounts(t *testing.T) { assert.Equal(t, "test@gmail.com", accounts[0].Identifier) } +func TestListAccountsLastSyncAt(t *testing.T) { + checks := assert.New(t) + must := require.New(t) + env := newTestEnv(t) + + accounts, err := env.Engine.ListAccounts(env.Ctx) + must.NoError(err, "ListAccounts before sync") + must.Len(accounts, 1) + checks.Nil(accounts[0].LastSyncAt, "never-synced source has no last sync time") + + syncedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + _, err = env.DB.Exec(`UPDATE sources SET last_sync_at = ? WHERE id = ?`, syncedAt, accounts[0].ID) + must.NoError(err, "set last_sync_at") + + accounts, err = env.Engine.ListAccounts(env.Ctx) + must.NoError(err, "ListAccounts after sync") + must.Len(accounts, 1) + must.NotNil(accounts[0].LastSyncAt) + checks.True(syncedAt.Equal(*accounts[0].LastSyncAt), "last sync time %v", accounts[0].LastSyncAt) +} + func TestGetTotalStats(t *testing.T) { assert := assert.New(t) env := newTestEnv(t) diff --git a/internal/testutil/pg_template_test.go b/internal/testutil/pg_template_test.go index c14ae8249..9fb38f3d9 100644 --- a/internal/testutil/pg_template_test.go +++ b/internal/testutil/pg_template_test.go @@ -189,14 +189,30 @@ func TestSweepReclaimsOnlyUnownedTemplates(t *testing.T) { require := require.New(t) assert := assert.New(t) dbURL := requirePostgresTestURL(t) + ctx := context.Background() + + // Sibling test binaries sweep the configured database's scope too. Use a + // private database so they cannot claim the staged orphans before this test. + parent := requireTemplate(t, dbURL) + parentAdmin, err := pgAdminDB(dbURL) + require.NoError(err, "open parent admin connection") + privateDB, err := parent.clone(ctx) + require.NoError(err, "clone private lock scope") + t.Cleanup(func() { dropOwnedDatabase(parentAdmin, privateDB) }) + dbURL = withDatabase(dbURL, privateDB) admin, err := pgAdminDB(dbURL) require.NoError(err, "open admin connection") - ctx := context.Background() + t.Cleanup(func() { _ = admin.Close() }) // This binary's own template first: building it sweeps, and the staged // databases below must be judged by the sweep under test, not that one. own := requireTemplate(t, dbURL) + t.Cleanup(func() { + dropOwnedDatabase(admin, own.name()) + _, _ = own.owner.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", templateLockKey(own.token)) + _ = own.owner.Close() + }) deadToken, err := newTemplateToken() require.NoError(err, "dead owner token") @@ -235,8 +251,6 @@ func TestSweepReclaimsOnlyUnownedTemplates(t *testing.T) { dropped, err := sweepOrphanTemplates(ctx, admin, own.scope) require.NoError(err, "sweep") - // Other binaries' leftovers may be reclaimed in the same pass, so the - // verdict is per database rather than over the whole list. assert.Contains(dropped, dead, "sweep reclaims the dead owner's template") assert.Contains(dropped, deadClone, "sweep reclaims the dead owner's clone") assert.False(databaseExists(t, admin, dead), "dead owner's template is gone") diff --git a/pkg/client/generated/client.go b/pkg/client/generated/client.go index e2f9f3d0c..58e03e588 100644 --- a/pkg/client/generated/client.go +++ b/pkg/client/generated/client.go @@ -295,10 +295,18 @@ type ClientInterface interface { GetCLIMessage(ctx context.Context, options *GetCLIMessageRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageResponse, error) GetCLIMessageWithResponse(ctx context.Context, options *GetCLIMessageRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageResp, error) + // GetCLIMessageOriginal Get one message's original MIME for export + GetCLIMessageOriginal(ctx context.Context, options *GetCLIMessageOriginalRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageOriginalResponse, error) + GetCLIMessageOriginalWithResponse(ctx context.Context, options *GetCLIMessageOriginalRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageOriginalResp, error) + // GetCLIMessageRaw Get one raw message for CLI export GetCLIMessageRaw(ctx context.Context, options *GetCLIMessageRawRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageRawResponse, error) GetCLIMessageRawWithResponse(ctx context.Context, options *GetCLIMessageRawRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageRawResp, error) + // GetCLIMessageThread List one conversation in chronological order for export + GetCLIMessageThread(ctx context.Context, options *GetCLIMessageThreadRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageThreadResponse, error) + GetCLIMessageThreadWithResponse(ctx context.Context, options *GetCLIMessageThreadRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageThreadResp, error) + // RebuildCLIFTS Rebuild the CLI full-text search index RebuildCLIFTS(ctx context.Context, reqEditors ...runtime.RequestEditorFn) (*RebuildCLIFTSResponse, error) RebuildCLIFTSWithResponse(ctx context.Context, reqEditors ...runtime.RequestEditorFn) (*RebuildCLIFTSResp, error) @@ -5151,6 +5159,69 @@ func (c *Client) GetCLIMessage(ctx context.Context, options *GetCLIMessageReques return responseParser(ctx, resp) } +// GetCLIMessageOriginal Get one message's original MIME for export +func (c *Client) GetCLIMessageOriginal(ctx context.Context, options *GetCLIMessageOriginalRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageOriginalResponse, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/cli/message/original", + Method: "GET", + Options: options, + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + responseParser := func(ctx context.Context, resp *runtime.Response) (*GetCLIMessageOriginalResponse, error) { + bodyBytes := resp.Content + if resp.StatusCode != 200 { + target := new(GetCLIMessageOriginalErrorResponse) + // Handle empty error response body gracefully - skip unmarshal if no content + if len(bodyBytes) > 0 { + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + // Return error with (possibly empty) target + if errTarget, ok := any(*target).(error); ok { + return nil, runtime.NewClientAPIError(errTarget, runtime.WithStatusCode(resp.StatusCode)) + } + return nil, runtime.NewClientAPIError(fmt.Errorf("API error (status %d): %v", resp.StatusCode, *target), + runtime.WithStatusCode(resp.StatusCode)) + } + target := new(GetCLIMessageOriginalResponse) + // Handle empty response body gracefully + if len(bodyBytes) == 0 { + return target, nil + } + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalResponse", + Body: bodyBytes, + Err: err, + } + } + return target, nil + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/cli/message/original") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + return responseParser(ctx, resp) +} + // GetCLIMessageRaw Get one raw message for CLI export func (c *Client) GetCLIMessageRaw(ctx context.Context, options *GetCLIMessageRawRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageRawResponse, error) { var err error @@ -5200,6 +5271,69 @@ func (c *Client) GetCLIMessageRaw(ctx context.Context, options *GetCLIMessageRaw return responseParser(ctx, resp) } +// GetCLIMessageThread List one conversation in chronological order for export +func (c *Client) GetCLIMessageThread(ctx context.Context, options *GetCLIMessageThreadRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageThreadResponse, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/cli/message/thread", + Method: "GET", + Options: options, + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + responseParser := func(ctx context.Context, resp *runtime.Response) (*GetCLIMessageThreadResponse, error) { + bodyBytes := resp.Content + if resp.StatusCode != 200 { + target := new(GetCLIMessageThreadErrorResponse) + // Handle empty error response body gracefully - skip unmarshal if no content + if len(bodyBytes) > 0 { + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + // Return error with (possibly empty) target + if errTarget, ok := any(*target).(error); ok { + return nil, runtime.NewClientAPIError(errTarget, runtime.WithStatusCode(resp.StatusCode)) + } + return nil, runtime.NewClientAPIError(fmt.Errorf("API error (status %d): %v", resp.StatusCode, *target), + runtime.WithStatusCode(resp.StatusCode)) + } + target := new(GetCLIMessageThreadResponse) + // Handle empty response body gracefully + if len(bodyBytes) == 0 { + return target, nil + } + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadResponse", + Body: bodyBytes, + Err: err, + } + } + return target, nil + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/cli/message/thread") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + return responseParser(ctx, resp) +} + // RebuildCLIFTS Rebuild the CLI full-text search index func (c *Client) RebuildCLIFTS(ctx context.Context, reqEditors ...runtime.RequestEditorFn) (*RebuildCLIFTSResponse, error) { var err error diff --git a/pkg/client/generated/client_options.go b/pkg/client/generated/client_options.go index 3fa8828f6..7ee09451a 100644 --- a/pkg/client/generated/client_options.go +++ b/pkg/client/generated/client_options.go @@ -2340,6 +2340,50 @@ func (o *GetCLIMessageRequestOptions) GetHeader() (map[string]string, error) { return nil, nil } +// GetCLIMessageOriginalRequestOptions is the options needed to make a request to GetCLIMessageOriginal. +type GetCLIMessageOriginalRequestOptions struct { + Query *GetCLIMessageOriginalQuery +} + +// Validate validates all the fields in the options. +// Use it if fields validation was not run. +func (o *GetCLIMessageOriginalRequestOptions) Validate() error { + var errors runtime.ValidationErrors + + if o.Query != nil { + if v, ok := any(o.Query).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Query", err) + } + } + } + if len(errors) == 0 { + return nil + } + + return errors +} + +// GetPathParams returns the path params as a map. +func (o *GetCLIMessageOriginalRequestOptions) GetPathParams() (map[string]any, error) { + return nil, nil +} + +// GetQuery returns the query params as a map. +func (o *GetCLIMessageOriginalRequestOptions) GetQuery() (map[string]any, error) { + return runtime.AsMap[any](o.Query) +} + +// GetBody returns the payload in any type that can be marshalled to JSON by the client. +func (o *GetCLIMessageOriginalRequestOptions) GetBody() any { + return nil +} + +// GetHeader returns the headers as a map. +func (o *GetCLIMessageOriginalRequestOptions) GetHeader() (map[string]string, error) { + return nil, nil +} + // GetCLIMessageRawRequestOptions is the options needed to make a request to GetCLIMessageRaw. type GetCLIMessageRawRequestOptions struct { Query *GetCLIMessageRawQuery @@ -2384,6 +2428,50 @@ func (o *GetCLIMessageRawRequestOptions) GetHeader() (map[string]string, error) return nil, nil } +// GetCLIMessageThreadRequestOptions is the options needed to make a request to GetCLIMessageThread. +type GetCLIMessageThreadRequestOptions struct { + Query *GetCLIMessageThreadQuery +} + +// Validate validates all the fields in the options. +// Use it if fields validation was not run. +func (o *GetCLIMessageThreadRequestOptions) Validate() error { + var errors runtime.ValidationErrors + + if o.Query != nil { + if v, ok := any(o.Query).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Query", err) + } + } + } + if len(errors) == 0 { + return nil + } + + return errors +} + +// GetPathParams returns the path params as a map. +func (o *GetCLIMessageThreadRequestOptions) GetPathParams() (map[string]any, error) { + return nil, nil +} + +// GetQuery returns the query params as a map. +func (o *GetCLIMessageThreadRequestOptions) GetQuery() (map[string]any, error) { + return runtime.AsMap[any](o.Query) +} + +// GetBody returns the payload in any type that can be marshalled to JSON by the client. +func (o *GetCLIMessageThreadRequestOptions) GetBody() any { + return nil +} + +// GetHeader returns the headers as a map. +func (o *GetCLIMessageThreadRequestOptions) GetHeader() (map[string]string, error) { + return nil, nil +} + // RepairMessageCLIRequestOptions is the options needed to make a request to RepairMessageCLI. type RepairMessageCLIRequestOptions struct { Body *RepairMessageCLIBody diff --git a/pkg/client/generated/client_with_response.go b/pkg/client/generated/client_with_response.go index 36312b3b5..db1748afb 100644 --- a/pkg/client/generated/client_with_response.go +++ b/pkg/client/generated/client_with_response.go @@ -5590,6 +5590,133 @@ func (c *Client) GetCLIMessageWithResponse(ctx context.Context, options *GetCLIM } } +// GetCLIMessageOriginal Get one message's original MIME for export +func (c *Client) GetCLIMessageOriginalWithResponse(ctx context.Context, options *GetCLIMessageOriginalRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageOriginalResp, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/cli/message/original", + Method: "GET", + Options: options, + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/cli/message/original") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + + out := &GetCLIMessageOriginalResp{ + HTTPResponse: resp.Raw, + Body: resp.Content, + StatusCode: resp.StatusCode, + } + + switch resp.StatusCode { + case 200: + out.JSON200 = new(GetCLIMessageOriginalResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON200); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 400: + out.JSON400 = new(GetCLIMessageOriginalErrorResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON400); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 404: + out.JSON404 = new(GetCLIMessageOriginalErrorResponseJSON) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON404); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponseJSON", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 409: + out.JSON409 = new(GetCLIMessageOriginalErrorResponseJSON409) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON409); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponseJSON409", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 413: + out.JSON413 = new(GetCLIMessageOriginalErrorResponseJSON413) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON413); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponseJSON413", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 503: + out.JSON503 = new(GetCLIMessageOriginalErrorResponseJSON503) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON503); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageOriginalErrorResponseJSON503", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + default: + return out, runtime.NewClientAPIError(fmt.Errorf("unexpected status code: %d", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + } +} + // GetCLIMessageRaw Get one raw message for CLI export func (c *Client) GetCLIMessageRawWithResponse(ctx context.Context, options *GetCLIMessageRawRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageRawResp, error) { var err error @@ -5703,6 +5830,117 @@ func (c *Client) GetCLIMessageRawWithResponse(ctx context.Context, options *GetC } } +// GetCLIMessageThread List one conversation in chronological order for export +func (c *Client) GetCLIMessageThreadWithResponse(ctx context.Context, options *GetCLIMessageThreadRequestOptions, reqEditors ...runtime.RequestEditorFn) (*GetCLIMessageThreadResp, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/cli/message/thread", + Method: "GET", + Options: options, + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/cli/message/thread") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + + out := &GetCLIMessageThreadResp{ + HTTPResponse: resp.Raw, + Body: resp.Content, + StatusCode: resp.StatusCode, + } + + switch resp.StatusCode { + case 200: + out.JSON200 = new(GetCLIMessageThreadResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON200); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 400: + out.JSON400 = new(GetCLIMessageThreadErrorResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON400); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 404: + out.JSON404 = new(GetCLIMessageThreadErrorResponseJSON) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON404); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadErrorResponseJSON", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 409: + out.JSON409 = new(GetCLIMessageThreadErrorResponseJSON409) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON409); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadErrorResponseJSON409", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + case 503: + out.JSON503 = new(GetCLIMessageThreadErrorResponseJSON503) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON503); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "GetCLIMessageThreadErrorResponseJSON503", + Body: bodyBytes, + Err: err, + } + } + } + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + default: + return out, runtime.NewClientAPIError(fmt.Errorf("unexpected status code: %d", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + } +} + // RebuildCLIFTS Rebuild the CLI full-text search index func (c *Client) RebuildCLIFTSWithResponse(ctx context.Context, reqEditors ...runtime.RequestEditorFn) (*RebuildCLIFTSResp, error) { var err error diff --git a/pkg/client/generated/queries.go b/pkg/client/generated/queries.go index 79d7a1d58..9cbed16a7 100644 --- a/pkg/client/generated/queries.go +++ b/pkg/client/generated/queries.go @@ -190,6 +190,20 @@ func (g GetCLIMessageQuery) Validate() error { return runtime.ConvertValidatorError(typesValidator.Struct(g)) } +type GetCLIMessageOriginalQuery struct { + // ID Internal message ID + ID *int64 `json:"id,omitempty"` + + // MaxBytes Maximum decoded MIME bytes; omit for an unrestricted export + MaxBytes *int64 `json:"max_bytes,omitempty"` + + // SourceMessageID Provider message ID + SourceMessageID *string `json:"source_message_id,omitempty"` + + // Account Source identifier that narrows the lookup + Account *string `json:"account,omitempty"` +} + type GetCLIMessageRawQuery struct { // ID Message numeric ID or source message ID ID string `json:"id" validate:"required"` @@ -199,6 +213,29 @@ func (g GetCLIMessageRawQuery) Validate() error { return runtime.ConvertValidatorError(typesValidator.Struct(g)) } +type GetCLIMessageThreadQuery struct { + // ID Internal ID of a message in the conversation + ID *int64 `json:"id,omitempty"` + + // SourceMessageID Provider ID of a message in the conversation + SourceMessageID *string `json:"source_message_id,omitempty"` + + // ThreadID Provider conversation ID + ThreadID *string `json:"thread_id,omitempty"` + + // Account Source identifier that narrows the lookup + Account *string `json:"account,omitempty"` + + // All Return fixed membership for the entire conversation; cannot be combined with limit or offset + All *bool `json:"all,omitempty"` + + // Limit Messages per page (default 100, max 500) + Limit *int64 `json:"limit,omitempty"` + + // Offset Messages to skip + Offset *int64 `json:"offset,omitempty"` +} + type SearchCLIQuery struct { // Q Search query Q string `json:"q" validate:"required"` diff --git a/pkg/client/generated/responses.go b/pkg/client/generated/responses.go index 3c45e9aff..57c532d12 100644 --- a/pkg/client/generated/responses.go +++ b/pkg/client/generated/responses.go @@ -495,6 +495,18 @@ type GetCLIMessageResponse = CliMessageResponse type GetCLIMessageErrorResponse = ErrorResponse +type GetCLIMessageOriginalResponse = CliOriginalMessageResponse + +type GetCLIMessageOriginalErrorResponse = ErrorResponse + +type GetCLIMessageOriginalErrorResponseJSON = ErrorResponse + +type GetCLIMessageOriginalErrorResponseJSON409 = ErrorResponse + +type GetCLIMessageOriginalErrorResponseJSON413 = ErrorResponse + +type GetCLIMessageOriginalErrorResponseJSON503 = ErrorResponse + type GetCLIMessageRawResponse = []byte type GetCLIMessageRawErrorResponse = ErrorResponse @@ -507,6 +519,16 @@ type GetCLIMessageRawErrorResponseJSON500 = ErrorResponse type GetCLIMessageRawErrorResponseJSON503 = ErrorResponse +type GetCLIMessageThreadResponse = ThreadPage + +type GetCLIMessageThreadErrorResponse = ErrorResponse + +type GetCLIMessageThreadErrorResponseJSON = ErrorResponse + +type GetCLIMessageThreadErrorResponseJSON409 = ErrorResponse + +type GetCLIMessageThreadErrorResponseJSON503 = ErrorResponse + type RebuildCLIFTSResponse = []byte type RebuildCLIFTSErrorResponse = ErrorResponse @@ -3948,6 +3970,18 @@ type GetCLIMessageResp struct { JSON200 *GetCLIMessageResponse } +type GetCLIMessageOriginalResp struct { + HTTPResponse *http.Response + Body []byte + StatusCode int + JSON200 *GetCLIMessageOriginalResponse + JSON400 *GetCLIMessageOriginalErrorResponse + JSON404 *GetCLIMessageOriginalErrorResponseJSON + JSON409 *GetCLIMessageOriginalErrorResponseJSON409 + JSON413 *GetCLIMessageOriginalErrorResponseJSON413 + JSON503 *GetCLIMessageOriginalErrorResponseJSON503 +} + type GetCLIMessageRawResp struct { HTTPResponse *http.Response Body []byte @@ -3959,6 +3993,17 @@ type GetCLIMessageRawResp struct { JSON503 *GetCLIMessageRawErrorResponseJSON503 } +type GetCLIMessageThreadResp struct { + HTTPResponse *http.Response + Body []byte + StatusCode int + JSON200 *GetCLIMessageThreadResponse + JSON400 *GetCLIMessageThreadErrorResponse + JSON404 *GetCLIMessageThreadErrorResponseJSON + JSON409 *GetCLIMessageThreadErrorResponseJSON409 + JSON503 *GetCLIMessageThreadErrorResponseJSON503 +} + type RebuildCLIFTSResp struct { HTTPResponse *http.Response Body []byte diff --git a/pkg/client/generated/types.go b/pkg/client/generated/types.go index 3970039af..8ac948fbb 100644 --- a/pkg/client/generated/types.go +++ b/pkg/client/generated/types.go @@ -2137,6 +2137,27 @@ func (c CliMessageResponse) Validate() error { return errors } +type CliOriginalMessageResponse struct { + Message MessageRecord `json:"message"` + Mime string `json:"mime" validate:"required"` +} + +func (c CliOriginalMessageResponse) Validate() error { + var errors runtime.ValidationErrors + if v, ok := any(c.Message).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Message", err) + } + } + if err := typesValidator.Var(c.Mime, "required"); err != nil { + errors = errors.Append("Mime", err) + } + if len(errors) == 0 { + return nil + } + return errors +} + type CliRebuildFTSEvent struct { Done *int64 `json:"done,omitempty"` ErrorData *string `json:"error,omitzero"` @@ -5665,6 +5686,21 @@ func (m MessageListResponse) Validate() error { return errors } +type MessageRecord struct { + Account string `json:"account" validate:"required"` + ConversationID int64 `json:"conversation_id"` + LastSyncAt *time.Time `json:"last_sync_at,omitempty"` + MessageID *int64 `json:"message_id,omitempty"` + SourceConversationID string `json:"source_conversation_id" validate:"required"` + SourceID int64 `json:"source_id"` + SourceMessageID *string `json:"source_message_id,omitzero"` + SourceType string `json:"source_type" validate:"required"` +} + +func (m MessageRecord) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(m)) +} + type MessageSummary struct { Bcc []string `json:"bcc,omitempty"` Cc []string `json:"cc,omitempty"` @@ -12391,6 +12427,93 @@ func (t TextSearchResponse) Validate() error { return errors } +type ThreadMessage struct { + AttachmentCount int64 `json:"attachment_count"` + Cc []Address `json:"cc" validate:"required"` + DeletedFromSourceAt *time.Time `json:"deleted_from_source_at,omitempty"` + From []Address `json:"from" validate:"required"` + HasRaw bool `json:"has_raw"` + ID int64 `json:"id"` + SentAt *time.Time `json:"sent_at,omitempty"` + SourceMessageID string `json:"source_message_id" validate:"required"` + Subject string `json:"subject" validate:"required"` + To []Address `json:"to" validate:"required"` +} + +func (t ThreadMessage) Validate() error { + var errors runtime.ValidationErrors + for i, item := range t.Cc { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("Cc[%d]", i), err) + } + } + } + for i, item := range t.From { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("From[%d]", i), err) + } + } + } + if err := typesValidator.Var(t.SourceMessageID, "required"); err != nil { + errors = errors.Append("SourceMessageID", err) + } + if err := typesValidator.Var(t.Subject, "required"); err != nil { + errors = errors.Append("Subject", err) + } + for i, item := range t.To { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("To[%d]", i), err) + } + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type ThreadPage struct { + Account string `json:"account" validate:"required"` + ConversationID int64 `json:"conversation_id"` + HasMore bool `json:"has_more"` + LastSyncAt *time.Time `json:"last_sync_at,omitempty"` + MessageID *int64 `json:"message_id,omitempty"` + Messages []ThreadMessage `json:"messages" validate:"required"` + Offset int64 `json:"offset"` + SourceConversationID string `json:"source_conversation_id" validate:"required"` + SourceID int64 `json:"source_id"` + SourceMessageID *string `json:"source_message_id,omitzero"` + SourceType string `json:"source_type" validate:"required"` + Total int64 `json:"total"` +} + +func (t ThreadPage) Validate() error { + var errors runtime.ValidationErrors + if err := typesValidator.Var(t.Account, "required"); err != nil { + errors = errors.Append("Account", err) + } + for i, item := range t.Messages { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("Messages[%d]", i), err) + } + } + } + if err := typesValidator.Var(t.SourceConversationID, "required"); err != nil { + errors = errors.Append("SourceConversationID", err) + } + if err := typesValidator.Var(t.SourceType, "required"); err != nil { + errors = errors.Append("SourceType", err) + } + if len(errors) == 0 { + return nil + } + return errors +} + type TimelineRow struct { AnchorMessageID *int64 `json:"anchor_message_id,omitempty"` ConversationID *int64 `json:"conversation_id,omitempty"` diff --git a/pkg/client/openapi.yaml b/pkg/client/openapi.yaml index 214d6b8e4..aff84b123 100644 --- a/pkg/client/openapi.yaml +++ b/pkg/client/openapi.yaml @@ -2302,6 +2302,17 @@ components: - body_text - body_html type: object + CliOriginalMessageResponse: + properties: + message: + $ref: "#/components/schemas/MessageRecord" + mime: + format: base64 + type: string + required: + - message + - mime + type: object CliRebuildFTSEvent: properties: done: @@ -6074,6 +6085,36 @@ components: - page_size - messages type: object + MessageRecord: + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + last_sync_at: + format: date-time + nullable: true + type: string + message_id: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + required: + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object MessageSummary: properties: bcc: @@ -12807,6 +12848,96 @@ components: - limit - messages type: object + ThreadMessage: + properties: + attachment_count: + format: int64 + type: integer + cc: + items: + $ref: "#/components/schemas/Address" + type: array + deleted_from_source_at: + format: date-time + nullable: true + type: string + from: + items: + $ref: "#/components/schemas/Address" + type: array + has_raw: + type: boolean + id: + format: int64 + type: integer + sent_at: + format: date-time + nullable: true + type: string + source_message_id: + type: string + subject: + type: string + to: + items: + $ref: "#/components/schemas/Address" + type: array + required: + - id + - source_message_id + - subject + - from + - to + - cc + - has_raw + - attachment_count + type: object + ThreadPage: + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + has_more: + type: boolean + last_sync_at: + format: date-time + nullable: true + type: string + message_id: + format: int64 + type: integer + messages: + items: + $ref: "#/components/schemas/ThreadMessage" + type: array + offset: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + total: + format: int64 + type: integer + required: + - total + - offset + - has_more + - messages + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object TimelineRow: properties: anchor_message_id: @@ -13198,7 +13329,7 @@ components: type: apiKey info: title: msgvault API - version: 2.32.0 + version: 2.33.0 openapi: 3.0.3 paths: /api/daemon/identity: @@ -16147,6 +16278,80 @@ paths: summary: Get one message for CLI output tags: - API + /api/v1/cli/message/original: + get: + operationId: getCLIMessageOriginal + parameters: + - description: Internal message ID + in: query + name: id + schema: + format: int64 + type: integer + - description: Maximum decoded MIME bytes; omit for an unrestricted export + in: query + name: max_bytes + schema: + format: int64 + type: integer + - description: Provider message ID + in: query + name: source_message_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CliOriginalMessageResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get one message's original MIME for export + tags: + - API /api/v1/cli/message/raw: get: operationId: getCLIMessageRaw @@ -16207,6 +16412,90 @@ paths: summary: Get one raw message for CLI export tags: - API + /api/v1/cli/message/thread: + get: + operationId: getCLIMessageThread + parameters: + - description: Internal ID of a message in the conversation + in: query + name: id + schema: + format: int64 + type: integer + - description: Provider ID of a message in the conversation + in: query + name: source_message_id + schema: + type: string + - description: Provider conversation ID + in: query + name: thread_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + - description: Return fixed membership for the entire conversation; cannot be combined with limit or offset + in: query + name: all + schema: + type: boolean + - description: Messages per page (default 100, max 500) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Messages to skip + in: query + name: offset + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadPage" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List one conversation in chronological order for export + tags: + - API /api/v1/cli/rebuild-fts: post: operationId: rebuildCLIFTS diff --git a/web/src/lib/api/generated/api/api.ts b/web/src/lib/api/generated/api/api.ts index 0c3b9817d..21b003ba4 100644 --- a/web/src/lib/api/generated/api/api.ts +++ b/web/src/lib/api/generated/api/api.ts @@ -71,6 +71,7 @@ import type { CliCollectionsResponse, CliInitDBResponse, CliMessageResponse, + CliOriginalMessageResponse, CliRebuildFTSEvent, CliSearchResponse, CliStatsResponse, @@ -135,8 +136,10 @@ import type { GetAttributeDefinitionPathParameters, GetCLIAttachmentParams, GetCLICollectionParams, + GetCLIMessageOriginalParams, GetCLIMessageParams, GetCLIMessageRawParams, + GetCLIMessageThreadParams, GetCLIStatsParams, GetCacheBuildStatusPathParameters, GetCardDAVConflictPathParameters, @@ -400,6 +403,7 @@ import type { TextConversationsResponse, TextMessagesResponse, TextSearchResponse, + ThreadPage, TokenUploadRequest, TotalStatsResponse, TriggerSyncParams, @@ -1131,6 +1135,18 @@ export const getCLIMessage = ( options, ); }; +/** + * @summary Get one message's original MIME for export + */ +export const getCLIMessageOriginal = ( + params?: GetCLIMessageOriginalParams, + options?: SecondParameter>, +) => { + return orvalFetch( + { url: `/api/v1/cli/message/original`, method: "GET", params }, + options, + ); +}; /** * @summary Get one raw message for CLI export */ @@ -1148,6 +1164,18 @@ export const getCLIMessageRaw = ( options, ); }; +/** + * @summary List one conversation in chronological order for export + */ +export const getCLIMessageThread = ( + params?: GetCLIMessageThreadParams, + options?: SecondParameter>, +) => { + return orvalFetch( + { url: `/api/v1/cli/message/thread`, method: "GET", params }, + options, + ); +}; /** * @summary Rebuild the CLI full-text search index */ diff --git a/web/src/lib/api/generated/models/cliOriginalMessageResponse.ts b/web/src/lib/api/generated/models/cliOriginalMessageResponse.ts new file mode 100644 index 000000000..2007604c2 --- /dev/null +++ b/web/src/lib/api/generated/models/cliOriginalMessageResponse.ts @@ -0,0 +1,10 @@ +/** + * Generated by Orval. Do not edit manually. + */ +import type { MessageRecord } from "./messageRecord"; + +export interface CliOriginalMessageResponse { + message: MessageRecord; + mime: string; + [key: string]: unknown; +} diff --git a/web/src/lib/api/generated/models/getCLIMessageOriginalParams.ts b/web/src/lib/api/generated/models/getCLIMessageOriginalParams.ts new file mode 100644 index 000000000..3b08575a4 --- /dev/null +++ b/web/src/lib/api/generated/models/getCLIMessageOriginalParams.ts @@ -0,0 +1,22 @@ +/** + * Generated by Orval. Do not edit manually. + */ + +export type GetCLIMessageOriginalParams = { + /** + * Internal message ID + */ + id?: number; + /** + * Maximum decoded MIME bytes; omit for an unrestricted export + */ + max_bytes?: number; + /** + * Provider message ID + */ + source_message_id?: string; + /** + * Source identifier that narrows the lookup + */ + account?: string; +}; diff --git a/web/src/lib/api/generated/models/getCLIMessageThreadParams.ts b/web/src/lib/api/generated/models/getCLIMessageThreadParams.ts new file mode 100644 index 000000000..e3b6dacb6 --- /dev/null +++ b/web/src/lib/api/generated/models/getCLIMessageThreadParams.ts @@ -0,0 +1,34 @@ +/** + * Generated by Orval. Do not edit manually. + */ + +export type GetCLIMessageThreadParams = { + /** + * Internal ID of a message in the conversation + */ + id?: number; + /** + * Provider ID of a message in the conversation + */ + source_message_id?: string; + /** + * Provider conversation ID + */ + thread_id?: string; + /** + * Source identifier that narrows the lookup + */ + account?: string; + /** + * Return fixed membership for the entire conversation; cannot be combined with limit or offset + */ + all?: boolean; + /** + * Messages per page (default 100, max 500) + */ + limit?: number; + /** + * Messages to skip + */ + offset?: number; +}; diff --git a/web/src/lib/api/generated/models/index.ts b/web/src/lib/api/generated/models/index.ts index c41ca3952..30b11089e 100644 --- a/web/src/lib/api/generated/models/index.ts +++ b/web/src/lib/api/generated/models/index.ts @@ -135,6 +135,7 @@ export * from "./cliInitDBResponse"; export * from "./cliMessageAddress"; export * from "./cliMessageAttachment"; export * from "./cliMessageResponse"; +export * from "./cliOriginalMessageResponse"; export * from "./cLIQueryMessageSummary"; export * from "./cliRebuildFTSEvent"; export * from "./cLIRepairEncodingEvent"; @@ -294,8 +295,10 @@ export * from "./getCardDAVConflictPathParameters"; export * from "./getCardDAVPublicationPathParameters"; export * from "./getCLIAttachmentParams"; export * from "./getCLICollectionParams"; +export * from "./getCLIMessageOriginalParams"; export * from "./getCLIMessageParams"; export * from "./getCLIMessageRawParams"; +export * from "./getCLIMessageThreadParams"; export * from "./getCLIStatsParams"; export * from "./getConversationParams"; export * from "./getConversationPathParameters"; @@ -453,6 +456,7 @@ export * from "./mergePersonRequest"; export * from "./mergePersonsPathParameters"; export * from "./messageDetail"; export * from "./messageListResponse"; +export * from "./messageRecord"; export * from "./messageSummary"; export * from "./metrics"; export * from "./monthTotals"; @@ -822,6 +826,8 @@ export * from "./textConversationRow"; export * from "./textConversationsResponse"; export * from "./textMessagesResponse"; export * from "./textSearchResponse"; +export * from "./threadMessage"; +export * from "./threadPage"; export * from "./timelineRow"; export * from "./tokenUploadRequest"; export * from "./tokenUsage"; diff --git a/web/src/lib/api/generated/models/messageRecord.ts b/web/src/lib/api/generated/models/messageRecord.ts new file mode 100644 index 000000000..1a4c4b064 --- /dev/null +++ b/web/src/lib/api/generated/models/messageRecord.ts @@ -0,0 +1,16 @@ +/** + * Generated by Orval. Do not edit manually. + */ + +export interface MessageRecord { + account: string; + conversation_id: number; + /** @nullable */ + last_sync_at?: string | null; + message_id?: number; + source_conversation_id: string; + source_id: number; + source_message_id?: string; + source_type: string; + [key: string]: unknown; +} diff --git a/web/src/lib/api/generated/models/threadMessage.ts b/web/src/lib/api/generated/models/threadMessage.ts new file mode 100644 index 000000000..55baeb6db --- /dev/null +++ b/web/src/lib/api/generated/models/threadMessage.ts @@ -0,0 +1,20 @@ +/** + * Generated by Orval. Do not edit manually. + */ +import type { Address } from "./address"; + +export interface ThreadMessage { + attachment_count: number; + cc: Address[]; + /** @nullable */ + deleted_from_source_at?: string | null; + from: Address[]; + has_raw: boolean; + id: number; + /** @nullable */ + sent_at?: string | null; + source_message_id: string; + subject: string; + to: Address[]; + [key: string]: unknown; +} diff --git a/web/src/lib/api/generated/models/threadPage.ts b/web/src/lib/api/generated/models/threadPage.ts new file mode 100644 index 000000000..ea090b757 --- /dev/null +++ b/web/src/lib/api/generated/models/threadPage.ts @@ -0,0 +1,21 @@ +/** + * Generated by Orval. Do not edit manually. + */ +import type { ThreadMessage } from "./threadMessage"; + +export interface ThreadPage { + account: string; + conversation_id: number; + has_more: boolean; + /** @nullable */ + last_sync_at?: string | null; + message_id?: number; + messages: ThreadMessage[]; + offset: number; + source_conversation_id: string; + source_id: number; + source_message_id?: string; + source_type: string; + total: number; + [key: string]: unknown; +} From 289f3e492a8a0ecafb9a80e700024d75a0ec40e7 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 29 Sep 2026 08:44:50 -0500 Subject: [PATCH 083/125] fix(people): stop repeating failed enrichment lookups (#988) Person enrichment now collapses duplicate values from Exa and Sixtyfour before committing claims. Repeated employment or attribute values previously aborted the claim generation and left work unfinished. Exa identity mismatches are reported as `identity_rejected` instead of `invalid_output`. The scheduler stops recreating work after terminal failures or exhausted retries. Expired claims count as consumed even when their lookup coalesces with manual or identity work. Later expirations, new manual requests, and identity changes remain eligible. Disabling or removing a profile retains its queued requests and refresh schedule. Workers claim only their exact configured profile, so a replacement with the same name cannot consume the retained work. Re-enabling resumes it; cleanup cancellations use `profile_unavailable` and a fresh operation without replaying actual failures. Work deleted by an earlier version is not reconstructed on re-enablement. The upgrade note documents recovery through a fresh manual lookup. No schema change or rewrite of historical attempts. Builds on #983. Fixes #985. Co-authored-by: Wes McKinney --- cmd/msgvault/cmd/person_enrichment.go | 6 +- .../cmd/person_enrichment_schedule_test.go | 21 +- cmd/msgvault/cmd/person_enrichment_test.go | 4 +- cmd/msgvault/cmd/serve.go | 7 +- cmd/msgvault/cmd/serve_test.go | 4 +- docs/changelog.md | 11 +- internal/personenrichment/claims.go | 37 +++ internal/personenrichment/errors.go | 17 +- internal/personenrichment/exa.go | 51 ++-- internal/personenrichment/exa_test.go | 46 ++- internal/personenrichment/sixtyfour.go | 33 +-- internal/personenrichment/work.go | 11 +- internal/personenrichment/worker.go | 21 +- internal/personenrichment/worker_test.go | 261 +++++++++++++++++- .../store/person_enrichment_budget_test.go | 2 +- .../store/person_enrichment_deletion_test.go | 2 +- .../person_enrichment_invalidation_test.go | 8 +- .../store/person_enrichment_results_test.go | 83 +++++- internal/store/person_enrichment_runs.go | 2 +- internal/store/person_enrichment_runs_test.go | 6 +- internal/store/person_enrichment_triggers.go | 18 ++ .../store/person_enrichment_triggers_test.go | 130 ++++++++- internal/store/person_enrichment_work.go | 29 +- internal/store/person_enrichment_work_test.go | 97 ++++++- 24 files changed, 784 insertions(+), 123 deletions(-) create mode 100644 internal/personenrichment/claims.go diff --git a/cmd/msgvault/cmd/person_enrichment.go b/cmd/msgvault/cmd/person_enrichment.go index eb5d1b76c..b653a5835 100644 --- a/cmd/msgvault/cmd/person_enrichment.go +++ b/cmd/msgvault/cmd/person_enrichment.go @@ -768,6 +768,7 @@ func newPersonEnrichmentCLIWorker( } factories := make(map[string]personenrichment.ProviderFactory) providerConfigs := make(map[string]personenrichment.ProviderConfig) + providerFingerprints := make(map[string]string) for _, configured := range config.Providers { provider := configured if !provider.Enabled { @@ -781,6 +782,7 @@ func newPersonEnrichmentCLIWorker( return nil, err } providerConfigs[provider.Name] = provider + providerFingerprints[provider.Name] = profile.Fingerprint switch provider.Kind { case personenrichment.ProviderExa: factories[provider.Name] = func(config personenrichment.ProviderConfig, credential string) (personenrichment.Provider, error) { @@ -799,6 +801,8 @@ func newPersonEnrichmentCLIWorker( return personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "daemon-person-enrichment-manual", LeaseDuration: config.LeaseDuration, RenewEvery: config.LeaseDuration / 4, Clock: time.Now, - Jitter: func(delay time.Duration) time.Duration { return delay }, ProviderConfigs: providerConfigs, + Jitter: func(delay time.Duration) time.Duration { return delay }, + ProviderConfigs: providerConfigs, + ProviderFingerprints: providerFingerprints, }) } diff --git a/cmd/msgvault/cmd/person_enrichment_schedule_test.go b/cmd/msgvault/cmd/person_enrichment_schedule_test.go index 87dee8d75..2ed9ede29 100644 --- a/cmd/msgvault/cmd/person_enrichment_schedule_test.go +++ b/cmd/msgvault/cmd/person_enrichment_schedule_test.go @@ -155,7 +155,7 @@ func TestPersonEnrichmentScheduleResumesRunningRunAfterCrash(t *testing.T) { worker := scheduleEnrichmentWorkerFunc(func(ctx context.Context, runID int64) (bool, error) { calls = append(calls, runID) lease, claimErr := f.Store.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "recovery-worker", ProviderName: profile.Name, + RunID: runID, Owner: "recovery-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) if claimErr != nil || lease == nil { @@ -481,12 +481,15 @@ func TestRegisterPersonEnrichmentJobCancelsWorkForUnavailableProfiles(t *testing requirements.NoError(err) checks.Equal("terminal", stored.State) requirements.NotNil(stored.FailureClass) - checks.Equal(string(personenrichment.FailurePolicy), *stored.FailureClass) + checks.Equal(string(personenrichment.FailureProfileUnavailable), *stored.FailureClass) work, err := f.Store.ListPersonEnrichmentWorkContext(t.Context(), store.PersonEnrichmentWorkFilter{ PersonID: person.ID, ProfileFingerprint: staleProfile.Fingerprint, Limit: 10, }) requirements.NoError(err) - checks.Empty(work) + requirements.Len(work, 1) + checks.Nil(work[0].RunID) + checks.Nil(work[0].ActiveAttemptID) + checks.Nil(work[0].LeaseOwner) requirements.NoError(f.Store.CompleteRun(t.Context(), run.ID, personenrichment.RunCompletion{ CompletedAt: now.Add(time.Second), })) @@ -556,10 +559,18 @@ func scheduleWorker( require.NoError(t, err) gate, err := personenrichment.NewEgressGate(st, st, hasher, credential) require.NoError(t, err) + catalog, err := st.BuildPersonFactCatalogContext(t.Context(), true) + require.NoError(t, err) + fingerprints := make(map[string]string, len(configs)) + for name, config := range configs { + profile, err := config.Profile(catalog) + require.NoError(t, err) + fingerprints[name] = profile.Fingerprint + } worker, err := personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "schedule-real-worker", LeaseDuration: time.Minute, RenewEvery: 10 * time.Second, Clock: func() time.Time { return now }, Jitter: func(delay time.Duration) time.Duration { return delay }, - ProviderConfigs: configs, + ProviderConfigs: configs, ProviderFingerprints: fingerprints, }) require.NoError(t, err) return worker @@ -661,7 +672,7 @@ func scheduleTestAttempt( ) *personenrichment.DurableAttempt { t.Helper() lease, err := st.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: runID, Owner: owner, ProviderName: profile.Name, + RunID: runID, Owner: owner, ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, err) diff --git a/cmd/msgvault/cmd/person_enrichment_test.go b/cmd/msgvault/cmd/person_enrichment_test.go index 29e331e17..93b9c6d8f 100644 --- a/cmd/msgvault/cmd/person_enrichment_test.go +++ b/cmd/msgvault/cmd/person_enrichment_test.go @@ -474,7 +474,7 @@ func TestPersonEnrichmentManualRunPersistsAndReusesRunIDBeforeWork(t *testing.T) require.NoError(t, workErr) require.NotEmpty(t, work) lease, claimErr := st.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "manual-test-worker", ProviderName: profile.Name, + RunID: runID, Owner: "manual-test-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, claimErr) @@ -596,7 +596,7 @@ func TestPersonEnrichmentManualRunKeepsRunIDOnLeaseAndAttemptAndReportsFinalCoun return false, nil } lease, claimErr := st.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "manual-final-worker", ProviderName: profile.Name, + RunID: runID, Owner: "manual-final-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, claimErr) diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index 1fedbd655..b7f9d960d 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -3595,6 +3595,7 @@ func registerPersonEnrichmentJob( } factories := make(map[string]personenrichment.ProviderFactory) providerConfigs := make(map[string]personenrichment.ProviderConfig) + providerFingerprints := make(map[string]string) activeFingerprints := make([]string, 0, len(enrichmentConfig.Providers)) for _, configured := range enrichmentConfig.Providers { provider := configured @@ -3610,6 +3611,7 @@ func registerPersonEnrichmentJob( } activeFingerprints = append(activeFingerprints, profile.Fingerprint) providerConfigs[provider.Name] = provider + providerFingerprints[provider.Name] = profile.Fingerprint switch provider.Kind { case personenrichment.ProviderExa: factories[provider.Name] = func(config personenrichment.ProviderConfig, credential string) (personenrichment.Provider, error) { @@ -3636,8 +3638,9 @@ func registerPersonEnrichmentJob( worker, err := personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "daemon-person-enrichment", LeaseDuration: enrichmentConfig.LeaseDuration, RenewEvery: enrichmentConfig.LeaseDuration / 4, Clock: time.Now, - Jitter: func(delay time.Duration) time.Duration { return delay }, - ProviderConfigs: providerConfigs, + Jitter: func(delay time.Duration) time.Duration { return delay }, + ProviderConfigs: providerConfigs, + ProviderFingerprints: providerFingerprints, }) if err != nil { return fmt.Errorf("configure person enrichment worker: %w", err) diff --git a/cmd/msgvault/cmd/serve_test.go b/cmd/msgvault/cmd/serve_test.go index ad0caa510..43cbb335c 100644 --- a/cmd/msgvault/cmd/serve_test.go +++ b/cmd/msgvault/cmd/serve_test.go @@ -79,7 +79,7 @@ func TestStoreAPIAdapterDeletePersonSuppressesCurrentIdentifiers(t *testing.T) { DueAt: now, })) lease, err := f.Store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "disabled-deletion-worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "disabled-deletion-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -218,7 +218,7 @@ func TestStoreAPIAdapterDeletePersonRejectsRecordedAttemptKeyMismatch(t *testing DueAt: now, })) lease, err := f.Store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "attempt-key-worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "attempt-key-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(err) diff --git a/docs/changelog.md b/docs/changelog.md index 9f8d2359d..cf124d505 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-27" +last_edited: "2026-09-29" title: Changelog description: Release history for msgvault --- @@ -8,6 +8,15 @@ All notable changes to msgvault, grouped by release. ## Unreleased +- Exa and Sixtyfour person enrichment collapse repeated values before committing + claims, so duplicate provider output no longer aborts the claim generation. +- Person enrichment retains queued lookups and refresh schedules when a provider + profile is disabled or removed. Work deleted by an earlier version is not + restored when that profile is re-enabled after upgrading. Request a + [manual lookup](cli-reference.md#person-enrichment) with a new idempotency key + to resume enrichment for an affected person; a later identity change or claim + expiry can also enqueue work. + - Beeper media sync stops retrying files the network has deleted, such as expired WhatsApp media. Each is requested once, recorded as unavailable, and reported in the sync summary. Other failed downloads get at most three attempts per run instead of eight, so one bad file no longer stalls a sync. - `msgvault search` keeps complete sender and subject/snippet text when piped or redirected. Terminal tables fit the available display width, with aligned Unicode and emoji and complete fixed fields. - `msgvault search` shows snippets for subjectless chat hits, keeps Unicode characters whole when truncating, and shows `-` when a message has no recorded size. diff --git a/internal/personenrichment/claims.go b/internal/personenrichment/claims.go new file mode 100644 index 000000000..466df8664 --- /dev/null +++ b/internal/personenrichment/claims.go @@ -0,0 +1,37 @@ +package personenrichment + +import ( + "encoding/json/jsontext" + "errors" + + "go.kenn.io/msgvault/internal/personfacts" +) + +func claimsForValues( + target personfacts.TargetDescriptor, + values []jsontext.Value, + score int, + evidence []personfacts.EvidenceInput, +) ([]personfacts.ProposedClaim, error) { + claims := make([]personfacts.ProposedClaim, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + normalized, failure, err := personfacts.NormalizeClaimValue(target, value) + if err != nil || failure != nil || normalized == nil { + return nil, errors.New("provider returned an unsupported target value") + } + // Values for one target share confidence and evidence. Keep the first + // occurrence of each canonical value. + if _, duplicate := seen[normalized.Fingerprint]; duplicate { + continue + } + seen[normalized.Fingerprint] = struct{}{} + claims = append(claims, personfacts.ProposedClaim{ + Target: target, Relation: personfacts.RelationSupport, + SubmittedValue: append(jsontext.Value(nil), value...), Evidence: evidence, + Origin: personfacts.OriginEnrichment, + Confidence: personfacts.ConfidenceInputs{ReportedScore: score}, + }) + } + return claims, nil +} diff --git a/internal/personenrichment/errors.go b/internal/personenrichment/errors.go index 4412a7ecb..2161bb2d5 100644 --- a/internal/personenrichment/errors.go +++ b/internal/personenrichment/errors.go @@ -9,14 +9,15 @@ import ( type FailureClass string const ( - FailurePolicy FailureClass = "policy" - FailureSuppressed FailureClass = "suppressed" - FailureRateLimited FailureClass = "rate_limited" - FailureTransient FailureClass = "transient" - FailureInvalidOutput FailureClass = "invalid_output" - FailureIdentityRejected FailureClass = "identity_rejected" - FailureTerminal FailureClass = "terminal" - FailureUncertainStart FailureClass = "uncertain_start" + FailurePolicy FailureClass = "policy" + FailureProfileUnavailable FailureClass = "profile_unavailable" + FailureSuppressed FailureClass = "suppressed" + FailureRateLimited FailureClass = "rate_limited" + FailureTransient FailureClass = "transient" + FailureInvalidOutput FailureClass = "invalid_output" + FailureIdentityRejected FailureClass = "identity_rejected" + FailureTerminal FailureClass = "terminal" + FailureUncertainStart FailureClass = "uncertain_start" ) var ( diff --git a/internal/personenrichment/exa.go b/internal/personenrichment/exa.go index bf1b7fbbb..41ab1ccaf 100644 --- a/internal/personenrichment/exa.go +++ b/internal/personenrichment/exa.go @@ -36,7 +36,10 @@ const ( exaMaxResponseBytes = 1 << 20 ) -var errExaSynchronous = errors.New("exa provider is synchronous and cannot be polled") +var ( + errExaSynchronous = errors.New("exa provider is synchronous and cannot be polled") + errExaIdentityMismatch = errors.New("missing Exa returned identity match") +) type exaProvider struct { config ProviderConfig @@ -168,7 +171,11 @@ func (p *exaProvider) Start(ctx context.Context, request Request) (Attempt, erro result, err = decodeExaPeopleResult(wire, request, now) } if err != nil { - return Attempt{}, exaFailure(response.StatusCode, FailureInvalidOutput, wire.RequestID, "") + class := FailureInvalidOutput + if errors.Is(err, errExaIdentityMismatch) { + class = FailureIdentityRejected + } + return Attempt{}, exaFailure(response.StatusCode, class, wire.RequestID, "") } result.AdapterVersion = ExaAdapterVersionV1 result.SchemaVersion = ExaSearchWireSchemaV1 @@ -432,17 +439,15 @@ func decodeExaPeopleResult(wire exaSearchResponse, request Request, now time.Tim if err != nil || len(values) == 0 { return Result{}, errors.New("exa typed target value is missing or malformed") } - for _, value := range values { - claim, err := exaClaim(target, value, ExaTypedUngroundedScore, []Citation{citation}) - if err != nil { - return Result{}, err - } - claims = append(claims, claim) + targetClaims, err := exaClaims(target, values, ExaTypedUngroundedScore, []Citation{citation}) + if err != nil { + return Result{}, err } + claims = append(claims, targetClaims...) } matches, identityConfidence := exaTypedIdentityMatches(request.Identity, selected.Properties, profileURL) if len(matches) == 0 || identityConfidence == 0 { - return Result{}, errors.New("missing Exa returned identity match") + return Result{}, errExaIdentityMismatch } cost, err := exaCost(wire.CostDollars) if err != nil { @@ -529,13 +534,11 @@ func decodeExaDeepResult(wire exaSearchResponse, request Request, now time.Time) if err != nil || len(values) == 0 { return Result{}, errors.New("unsupported Exa synthesized value") } - for _, value := range values { - claim, err := exaClaim(target, value, score, citations) - if err != nil { - return Result{}, err - } - claims = append(claims, claim) + targetClaims, err := exaClaims(target, values, score, citations) + if err != nil { + return Result{}, err } + claims = append(claims, targetClaims...) } citations := make([]Citation, 0, len(citationOrder)) for _, key := range citationOrder { @@ -658,15 +661,12 @@ func exaSubmittedValues(target personfacts.TargetDescriptor, raw jsontext.Value) return values, nil } -func exaClaim( +func exaClaims( target personfacts.TargetDescriptor, - value jsontext.Value, + values []jsontext.Value, score int, citations []Citation, -) (personfacts.ProposedClaim, error) { - if normalized, failure, err := personfacts.NormalizeClaimValue(target, value); err != nil || failure != nil || normalized == nil { - return personfacts.ProposedClaim{}, errors.New("exa returned an unsupported target value") - } +) ([]personfacts.ProposedClaim, error) { evidence := make([]personfacts.EvidenceInput, len(citations)) for i, citation := range citations { evidence[i] = personfacts.EvidenceInput{ @@ -675,12 +675,7 @@ func exaClaim( SourceURL: citation.URL, } } - return personfacts.ProposedClaim{ - Target: target, Relation: personfacts.RelationSupport, - SubmittedValue: append(jsontext.Value(nil), value...), Evidence: evidence, - Origin: personfacts.OriginEnrichment, - Confidence: personfacts.ConfidenceInputs{ReportedScore: score}, - }, nil + return claimsForValues(target, values, score, evidence) } func exaTypedIdentityMatches( @@ -739,7 +734,7 @@ func exaDeepResultIdentityMatch(identity Identity, results []exaSearchResult) ([ return []IdentityMatch{{Class: IdentifierPublicProfileURL, Value: canonical, Confidence: 1000}}, 1000, nil } } - return nil, 0, errors.New("missing Exa returned identity match") + return nil, 0, errExaIdentityMismatch } func exactExaIdentityMatch(class IdentifierClass, left, right string) bool { diff --git a/internal/personenrichment/exa_test.go b/internal/personenrichment/exa_test.go index d3a648aef..ec162cef4 100644 --- a/internal/personenrichment/exa_test.go +++ b/internal/personenrichment/exa_test.go @@ -264,7 +264,7 @@ func TestExaRejectsHistoricalEmploymentAsCurrentCompanyMatch(t *testing.T) { requirements.Error(err) var providerErr *personenrichment.ProviderError requirements.ErrorAs(err, &providerErr) - assert.Equal(t, personenrichment.FailureInvalidOutput, providerErr.Class) + assert.Equal(t, personenrichment.FailureIdentityRejected, providerErr.Class) } func TestExaDeepModesPreserveGroundingAndBindGeneratedSchema(t *testing.T) { @@ -496,7 +496,13 @@ func TestExaRejectsInvalidTypedEntitiesAndStructuredOutput(t *testing.T) { if test.mode != "people" { targets = exaDeepTargets(t) } - _, err = provider.Start(t.Context(), personenrichment.Request{Identity: personenrichment.Identity{Name: "Test User"}, Targets: targets}) + _, err = provider.Start(t.Context(), personenrichment.Request{ + Identity: personenrichment.Identity{ + Name: "Test User", CurrentCompany: "Example Labs", + PublicProfileURLs: []string{"https://sources.example.test/test-user"}, + }, + Targets: targets, + }) requirements.Error(err) checks.NotContains(err.Error(), "private") var providerErr *personenrichment.ProviderError @@ -506,6 +512,36 @@ func TestExaRejectsInvalidTypedEntitiesAndStructuredOutput(t *testing.T) { } } +func TestExaDeepDeduplicatesEmployment(t *testing.T) { + require := require.New(t) + var wire map[string]any + require.NoError(json.Unmarshal(exaFixture(t, "exa_deep_success.json"), &wire)) + output, ok := wire["output"].(map[string]any) + require.True(ok) + content, ok := output["content"].(map[string]any) + require.True(ok) + work, ok := content["system:employment"].([]any) + require.True(ok) + require.NotEmpty(work) + content["system:employment"] = []any{work[0], work[0]} + body, err := json.Marshal(wire) + require.NoError(err) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + provider, err := personenrichment.NewExaProvider(exaConfig(server.URL, "deep", 1), "test-key", server.Client()) + require.NoError(err) + attempt, err := provider.Start(t.Context(), personenrichment.Request{ + Identity: personenrichment.Identity{PublicProfileURLs: []string{"https://sources.example.test/test-user"}}, + Targets: exaDeepTargets(t), + }) + require.NoError(err) + require.NotNil(attempt.Result) + assert.Len(t, attempt.Result.Claims, 2, "one summary and one employment claim") +} + func TestExaRejectsDeepOutputWithoutReturnedIdentityMatch(t *testing.T) { requirements := require.New(t) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -526,7 +562,7 @@ func TestExaRejectsDeepOutputWithoutReturnedIdentityMatch(t *testing.T) { requirements.Error(err) var providerErr *personenrichment.ProviderError requirements.ErrorAs(err, &providerErr) - assert.Equal(t, personenrichment.FailureInvalidOutput, providerErr.Class) + assert.Equal(t, personenrichment.FailureIdentityRejected, providerErr.Class) } func TestExaRejectsDeepOutputAcrossAmbiguousResultRows(t *testing.T) { @@ -569,13 +605,13 @@ func TestExaRejectsTypedOutputWithoutReturnedIdentityMatch(t *testing.T) { ) requirements.NoError(err) _, err = provider.Start(t.Context(), personenrichment.Request{ - Identity: personenrichment.Identity{Name: "Different Synthetic Person"}, + Identity: personenrichment.Identity{Name: "Different Synthetic Person", CurrentCompany: "Example Labs"}, Targets: exaTypedTargets(t), }) requirements.Error(err) var providerErr *personenrichment.ProviderError requirements.ErrorAs(err, &providerErr) - assert.Equal(t, personenrichment.FailureInvalidOutput, providerErr.Class) + assert.Equal(t, personenrichment.FailureIdentityRejected, providerErr.Class) } func TestExaRejectsDuplicateDeepContentMembers(t *testing.T) { diff --git a/internal/personenrichment/sixtyfour.go b/internal/personenrichment/sixtyfour.go index 59cee829a..3bd4ede0e 100644 --- a/internal/personenrichment/sixtyfour.go +++ b/internal/personenrichment/sixtyfour.go @@ -412,13 +412,11 @@ func decodeSixtyfourCompleted( if valueErr != nil || len(values) == 0 { return Result{}, errors.New("unsupported Sixtyfour result value") } - for _, value := range values { - claim, claimErr := sixtyfourClaim(target, value, factConfidence) - if claimErr != nil { - return Result{}, claimErr - } - claims = append(claims, claim) + targetClaims, claimErr := sixtyfourClaims(target, values, factConfidence) + if claimErr != nil { + return Result{}, claimErr } + claims = append(claims, targetClaims...) } return Result{ @@ -468,24 +466,15 @@ func sixtyfourTargetByKey(attempt Attempt, key string) (personfacts.TargetDescri return personfacts.TargetDescriptor{}, false } -func sixtyfourClaim( +func sixtyfourClaims( target personfacts.TargetDescriptor, - value jsontext.Value, + values []jsontext.Value, score int, -) (personfacts.ProposedClaim, error) { - if normalized, failure, err := personfacts.NormalizeClaimValue(target, value); err != nil || failure != nil || normalized == nil { - return personfacts.ProposedClaim{}, errors.New("sixtyfour returned an unsupported target value") - } - return personfacts.ProposedClaim{ - Target: target, Relation: personfacts.RelationSupport, - SubmittedValue: append(jsontext.Value(nil), value...), - Evidence: []personfacts.EvidenceInput{{ - SourceClass: personfacts.EvidenceProviderAssertion, - Directness: personfacts.Indirect, Authority: personfacts.AuthorityAggregator, - }}, - Origin: personfacts.OriginEnrichment, - Confidence: personfacts.ConfidenceInputs{ReportedScore: score}, - }, nil +) ([]personfacts.ProposedClaim, error) { + return claimsForValues(target, values, score, []personfacts.EvidenceInput{{ + SourceClass: personfacts.EvidenceProviderAssertion, + Directness: personfacts.Indirect, Authority: personfacts.AuthorityAggregator, + }}) } func parseSixtyfourConfidence(value jsontext.Value) (int, error) { diff --git a/internal/personenrichment/work.go b/internal/personenrichment/work.go index a6b4fe0cd..241327f65 100644 --- a/internal/personenrichment/work.go +++ b/internal/personenrichment/work.go @@ -53,11 +53,12 @@ type WorkStore interface { } type ClaimOptions struct { - RunID int64 - Owner string - ProviderName string - Now time.Time - LeaseDuration time.Duration + ProfileFingerprint string + RunID int64 + Owner string + ProviderName string + Now time.Time + LeaseDuration time.Duration } type LeaseToken struct { diff --git a/internal/personenrichment/worker.go b/internal/personenrichment/worker.go index 82c508915..903d087ad 100644 --- a/internal/personenrichment/worker.go +++ b/internal/personenrichment/worker.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "reflect" "slices" "sort" @@ -16,12 +17,13 @@ import ( ) type WorkerOptions struct { - Owner string - LeaseDuration time.Duration - RenewEvery time.Duration - Clock func() time.Time - Jitter func(time.Duration) time.Duration - ProviderConfigs map[string]ProviderConfig + Owner string + LeaseDuration time.Duration + RenewEvery time.Duration + Clock func() time.Time + Jitter func(time.Duration) time.Duration + ProviderConfigs map[string]ProviderConfig + ProviderFingerprints map[string]string } type Worker struct { @@ -130,6 +132,9 @@ func NewWorker( if name == "" || name != rawName || factory == nil { return nil, errors.New("person enrichment provider factory map is invalid") } + if !isSHA256Hex(options.ProviderFingerprints[name]) { + return nil, fmt.Errorf("person enrichment provider %q requires a configured profile fingerprint", name) + } factoryCopy[name] = factory names = append(names, name) } @@ -139,6 +144,7 @@ func NewWorker( configCopy[name] = cloneWorkerProviderConfig(config) } options.ProviderConfigs = configCopy + options.ProviderFingerprints = maps.Clone(options.ProviderFingerprints) return &Worker{ work: work, sink: sink, gate: gate, providers: factoryCopy, providerNames: names, options: options, @@ -166,7 +172,8 @@ func (w *Worker) RunOnce(ctx context.Context, runID int64) (processed bool, err } lease, claimErr := w.work.ClaimWork(ctx, ClaimOptions{ RunID: runID, Owner: w.options.Owner, ProviderName: providerName, - Now: now, LeaseDuration: w.options.LeaseDuration, + ProfileFingerprint: w.options.ProviderFingerprints[providerName], + Now: now, LeaseDuration: w.options.LeaseDuration, }) if claimErr != nil { return false, claimErr diff --git a/internal/personenrichment/worker_test.go b/internal/personenrichment/worker_test.go index ca68ea7dc..1da4d3f5a 100644 --- a/internal/personenrichment/worker_test.go +++ b/internal/personenrichment/worker_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + "net/http" + "net/http/httptest" "strings" "sync" "sync/atomic" @@ -111,7 +113,7 @@ func (f *workerFixture) options(configs map[string]personenrichment.ProviderConf return personenrichment.WorkerOptions{ Owner: "worker-test", LeaseDuration: time.Minute, RenewEvery: 20 * time.Second, Clock: time.Now, Jitter: func(time.Duration) time.Duration { return 0 }, - ProviderConfigs: configs, + ProviderConfigs: configs, ProviderFingerprints: map[string]string{f.profile.Name: f.profile.Fingerprint}, } } @@ -335,7 +337,11 @@ func TestWorkerCompletesSynchronousResultAndIsolatesProviderFailure(t *testing.T configs := map[string]personenrichment.ProviderConfig{ failed.config.Name: failed.config, goodConfig.Name: goodConfig, } - worker := failed.newWorker(t, factories, configs, func(string) (string, bool) { return "test-key", true }) + options := failed.options(configs) + options.ProviderFingerprints[goodConfig.Name] = goodProfile.Fingerprint + worker, err := personenrichment.NewWorker(failed.store, failed.store, + failed.gate(t, func(string) (string, bool) { return "test-key", true }), factories, options) + requirements.NoError(err) processed, err := worker.RunOnce(t.Context(), failed.run.ID) requirements.NoError(err) @@ -353,6 +359,50 @@ func TestWorkerCompletesSynchronousResultAndIsolatesProviderFailure(t *testing.T checks.ElementsMatch([]string{"succeeded", "terminal"}, states) } +func TestWorkerPreservesQueuedWorkForUnavailableProfile(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newWorkerFixture(t, "profile-selection", nil) + f.enqueue(t) + replacementConfig := f.config + replacementConfig.Endpoint = "https://replacement.example.test/search" + replacement, err := replacementConfig.Profile(personfacts.Catalog{Targets: f.profile.Targets}) + require.NoError(err) + _, err = f.store.EnsurePersonEnrichmentProfile(t.Context(), replacement) + require.NoError(err) + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), []string{replacement.Fingerprint})) + var starts atomic.Int64 + factories := map[string]personenrichment.ProviderFactory{ + f.config.Name: func(personenrichment.ProviderConfig, string) (personenrichment.Provider, error) { + return &functionProvider{start: func(context.Context, personenrichment.Request) (personenrichment.Attempt, error) { + starts.Add(1) + return personenrichment.Attempt{}, &personenrichment.ProviderError{Class: personenrichment.FailureInvalidOutput} + }}, nil + }, + } + options := f.options(map[string]personenrichment.ProviderConfig{replacement.Name: replacementConfig}) + options.ProviderFingerprints[replacement.Name] = replacement.Fingerprint + worker, err := personenrichment.NewWorker(f.store, f.store, + f.gate(t, func(string) (string, bool) { return "test-key", true }), factories, options) + require.NoError(err) + processed, err := worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + assert.False(processed, "a replacement with the same name must leave the old profile's work queued") + assert.Zero(starts.Load()) + + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), []string{f.profile.Fingerprint})) + worker = f.newWorker(t, factories, map[string]personenrichment.ProviderConfig{f.config.Name: f.config}, + func(string) (string, bool) { return "test-key", true }) + processed, err = worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + assert.True(processed, "re-enabling must resume the queued operation") + assert.Equal(int64(1), starts.Load()) + processed, err = worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + assert.False(processed, "a terminal failure must not be retried") + assert.Equal(int64(1), starts.Load()) +} + func TestWorkerConcurrentRunOnceStartsProviderOnlyOnce(t *testing.T) { checks := assert.New(t) requirements := require.New(t) @@ -1750,6 +1800,117 @@ func TestWorkerStopsRetryingActiveAttemptAtLimit(t *testing.T) { checks.Equal(int64(2), attempts[0].AttemptCount) } +func TestWorkerExaPeopleResults(t *testing.T) { + for _, test := range []struct { + name string + state string + failure string + claims int + }{ + {"duplicate employment", "succeeded", "", 2}, + {"identity mismatch", "identity_rejected", "identity_rejected", 0}, + {"invalid output", "terminal", "invalid_output", 0}, + } { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var wire map[string]any + require.NoError(json.Unmarshal(exaFixture(t, "exa_people_success.json"), &wire)) + results, ok := wire["results"].([]any) + require.True(ok) + require.NotEmpty(results) + row, ok := results[0].(map[string]any) + require.True(ok) + row["url"] = "https://profiles.example.test/worker-person" + entities, ok := row["entities"].([]any) + require.True(ok) + require.NotEmpty(entities) + entity, ok := entities[0].(map[string]any) + require.True(ok) + properties, ok := entity["properties"].(map[string]any) + require.True(ok) + switch test.name { + case "duplicate employment": + history, ok := properties["workHistory"].([]any) + require.True(ok) + require.NotEmpty(history) + work, ok := history[0].(map[string]any) + require.True(ok) + properties["workHistory"] = []any{work, work, map[string]any{ + "title": " Engineer ", "location": "Test City", "dates": work["dates"], + "company": map[string]any{"name": " Example Labs "}, + }, map[string]any{ + "title": "Senior Engineer", "location": "Test City", "dates": work["dates"], + "company": work["company"], + }} + case "identity mismatch": + row["url"] = "https://profiles.example.test/different-person" + case "invalid output": + properties["workHistory"] = nil + } + body, err := json.Marshal(wire) + require.NoError(err) + var calls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + f := newWorkerFixture(t, "exa-people", func(cfg *personenrichment.ProviderConfig) { + cfg.Mode = "people" + cfg.Endpoint = server.URL + cfg.TargetKeys = []string{"system:employment"} + }) + worker := f.newWorker(t, + map[string]personenrichment.ProviderFactory{f.config.Name: func(cfg personenrichment.ProviderConfig, key string) (personenrichment.Provider, error) { + return personenrichment.NewExaProvider(cfg, key, server.Client()) + }}, map[string]personenrichment.ProviderConfig{f.config.Name: f.config}, + func(string) (string, bool) { return "test-key", true }, + ) + processed, err := worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + require.True(processed) + attempts, err := f.store.ListPersonEnrichmentAttemptsContext(t.Context(), store.PersonEnrichmentAttemptFilter{ + PersonID: f.person.ID, RunID: f.run.ID, Limit: 10, + }) + require.NoError(err) + require.Len(attempts, 1) + assert.Equal(test.state, attempts[0].State) + if test.failure == "" { + assert.Nil(attempts[0].FailureClass) + } else { + require.NotNil(attempts[0].FailureClass) + assert.Equal(test.failure, *attempts[0].FailureClass) + } + var claims int + require.NoError(f.store.DB().QueryRowContext(t.Context(), "SELECT COUNT(*) FROM person_fact_claims").Scan(&claims)) + assert.Equal(test.claims, claims) + for range 2 { + queued, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), time.Now().UTC(), 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Zero(queued) + processed, err = worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + assert.False(processed) + } + assert.Equal(int32(1), calls.Load()) + require.NoError(f.store.CompleteRun(t.Context(), f.run.ID, personenrichment.RunCompletion{})) + run, err := f.store.GetPersonEnrichmentRunContext(t.Context(), f.run.ID) + require.NoError(err) + switch test.state { + case "identity_rejected": + assert.Equal(int64(1), run.IdentityRejectedCount) + assert.Zero(run.FailedCount) + case "terminal": + assert.Equal(int64(1), run.FailedCount) + default: + assert.Equal(int64(1), run.SucceededCount) + } + }) + } +} + func TestWorkerInvalidOutputWritesNoResultState(t *testing.T) { checks := assert.New(t) requirements := require.New(t) @@ -1889,7 +2050,7 @@ func TestWorkerRejectsStaleCommitAfterLeaseReclaim(t *testing.T) { ProgramFingerprint: workerProgramFingerprint(t, false, ""), Result: &result, } reclaimed, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: f.run.ID, Owner: "replacement-worker", ProviderName: f.config.Name, + RunID: f.run.ID, Owner: "replacement-worker", ProviderName: f.config.Name, ProfileFingerprint: f.profile.Fingerprint, Now: claimTime.Add(80 * time.Millisecond), LeaseDuration: time.Minute, }) requirements.NoError(err) @@ -2327,3 +2488,97 @@ func TestWorkerAsyncPollFencesConsentAndRechecksReturnedSuppression(t *testing.T }) } } + +func TestWorkerSixtyfourDuplicateValues(t *testing.T) { + for _, test := range []struct { + name string + values []string + }{ + {"unique", []string{"Example value", "Different value"}}, + {"duplicate", []string{"Example value", "Different value", "Example value", " Example value "}}, + } { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newWorkerFixture(t, "sixtyfour-values", nil) + organization, err := f.store.CreateOrganizationContext(t.Context(), store.OrganizationInput{Name: "Example Labs"}) + require.NoError(err) + _, err = f.store.AddEmploymentContext(t.Context(), store.EmploymentInput{ + PersonID: f.person.ID, OrganizationID: organization.ID, + IsCurrent: new(true), Source: store.ProvenanceUser, + }) + require.NoError(err) + catalog, err := f.store.BuildPersonFactCatalogContext(t.Context(), true) + require.NoError(err) + var target personfacts.TargetDescriptor + for _, candidate := range catalog.Targets { + if candidate.Kind == personfacts.TargetAttribute && candidate.ValueType == personfacts.ValueText && candidate.Cardinality == personfacts.CardinalityMulti && !candidate.Sensitive { + target = candidate + break + } + } + require.NotEmpty(target.Key) + startBody := sixtyfourFixture(t, "sixtyfour_start.json") + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost { + _, _ = w.Write(startBody) + return + } + assert.NoError(json.NewEncoder(w).Encode(map[string]any{ + "task_id": "opaque-job-42", "status": "completed", "charge_amount": 12, + "result": map[string]any{ + "structured_data": map[string]any{target.Key: test.values, "name": "Worker Person", "company": "Example Labs"}, + "confidence_score": 9, "findings": []any{}, + }, + })) + })) + defer server.Close() + f.config = sixtyfourConfig(server.URL+"/start", server.URL+"/job-status") + f.config.TargetKeys = []string{target.Key} + f.config.RequestTimeout = 30 * time.Second + f.profile, err = f.config.Profile(catalog) + require.NoError(err) + _, err = f.store.EnsurePersonEnrichmentProfile(t.Context(), f.profile) + require.NoError(err) + _, _, err = f.store.GrantPersonEnrichmentConsent(t.Context(), f.profile.Fingerprint, "test") + require.NoError(err) + options := f.options(map[string]personenrichment.ProviderConfig{f.config.Name: f.config}) + now := time.Now().UTC() + options.Clock = func() time.Time { return now } + worker, err := personenrichment.NewWorker(f.store, f.store, + f.gate(t, func(string) (string, bool) { return "test-key", true }), + map[string]personenrichment.ProviderFactory{f.config.Name: func(cfg personenrichment.ProviderConfig, key string) (personenrichment.Provider, error) { + return personenrichment.NewSixtyfourProvider(cfg, key, server.Client()) + }}, options) + require.NoError(err) + processed, err := worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + require.True(processed) + work, err := f.store.ListPersonEnrichmentWorkContext(t.Context(), store.PersonEnrichmentWorkFilter{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, Limit: 10, + }) + require.NoError(err) + require.Len(work, 1) + now = work[0].DueAt.Add(time.Second) + processed, err = worker.RunOnce(t.Context(), f.run.ID) + assert.True(processed) + require.NoError(err) + attempts, err := f.store.ListPersonEnrichmentAttemptsContext(t.Context(), store.PersonEnrichmentAttemptFilter{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, Limit: 10, + }) + require.NoError(err) + require.Len(attempts, 1) + assert.Equal("succeeded", attempts[0].State) + require.NotNil(attempts[0].FactGenerationKey) + var claims int + require.NoError(f.store.DB().QueryRowContext(t.Context(), f.store.Rebind(` + SELECT COUNT(*) FROM person_fact_claims + WHERE generation_id = (SELECT id FROM person_fact_generations WHERE generation_key = ?)`), *attempts[0].FactGenerationKey).Scan(&claims)) + assert.Equal(2, claims) + processed, err = worker.RunOnce(t.Context(), f.run.ID) + require.NoError(err) + assert.False(processed, "the completed result must not be polled again") + }) + } +} diff --git a/internal/store/person_enrichment_budget_test.go b/internal/store/person_enrichment_budget_test.go index d9a80b5e4..ae1fb7acb 100644 --- a/internal/store/person_enrichment_budget_test.go +++ b/internal/store/person_enrichment_budget_test.go @@ -67,7 +67,7 @@ func newBudgetClaims(t *testing.T, profile personenrichment.ProviderProfile) (*s Trigger: personenrichment.Trigger{Kind: personenrichment.TriggerManual, Generation: generation}, DueAt: now, })) lease, claimErr := fixture.Store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "worker-" + generation, ProviderName: profile.Name, + RunID: run.ID, Owner: "worker-" + generation, ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, claimErr) diff --git a/internal/store/person_enrichment_deletion_test.go b/internal/store/person_enrichment_deletion_test.go index 2c026890f..2f75dc963 100644 --- a/internal/store/person_enrichment_deletion_test.go +++ b/internal/store/person_enrichment_deletion_test.go @@ -157,7 +157,7 @@ func newEnrichmentDeletionFixture(t *testing.T) enrichmentDeletionFixture { DueAt: f.now.Add(time.Minute), })) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "deletion-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "deletion-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now.Add(time.Minute), LeaseDuration: time.Minute, }) require.NoError(t, err) diff --git a/internal/store/person_enrichment_invalidation_test.go b/internal/store/person_enrichment_invalidation_test.go index ca944b951..4c6a1ab9b 100644 --- a/internal/store/person_enrichment_invalidation_test.go +++ b/internal/store/person_enrichment_invalidation_test.go @@ -175,7 +175,7 @@ func TestPersonEnrichmentEmploymentAdvancesGenerationAndPreservesFreshCompanyWor }) require.NoError(err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "employment-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "employment-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -273,7 +273,7 @@ func TestPersonEnrichmentClaimedPublicationFencesBeginAndPreservesExactWork(t *t }) require.NoError(err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "claimed-publication-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "claimed-publication-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -417,7 +417,7 @@ func claimUnboundInvalidationWork( }) require.NoError(t, err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "unbound-" + suffix, ProviderName: f.profile.Name, + RunID: run.ID, Owner: "unbound-" + suffix, ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(t, err) @@ -478,7 +478,7 @@ func prepareCurrentEnrichmentResult( }) require.NoError(t, err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "invalidation-worker-" + suffix, ProviderName: f.profile.Name, + RunID: run.ID, Owner: "invalidation-worker-" + suffix, ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(t, err) diff --git a/internal/store/person_enrichment_results_test.go b/internal/store/person_enrichment_results_test.go index 49ad4d585..405f2c458 100644 --- a/internal/store/person_enrichment_results_test.go +++ b/internal/store/person_enrichment_results_test.go @@ -81,7 +81,7 @@ func newEnrichmentResultFixture(t *testing.T) *enrichmentResultFixture { DueAt: now, })) lease, err := st.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "result-worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "result-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: 5 * time.Minute, }) require.NoError(t, err) @@ -153,6 +153,85 @@ func newEnrichmentResultFixture(t *testing.T) *enrichmentResultFixture { } } +func TestPersonEnrichmentCatchUpDoesNotConsumeProducingAttempt(t *testing.T) { + require := require.New(t) + f := newEnrichmentResultFixture(t) + f.result.Claims[0].ValidUntil = new(f.now.Add(-time.Minute)) + f.reseal(t) + outcome, err := f.store.CommitEnrichmentClaims(t.Context(), f.commit) + require.NoError(err) + require.Equal(personenrichment.ClaimApplied, outcome.Status) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Equal(t, 1, count, "a result already expired on arrival still needs an expiry lookup") +} + +func TestPersonEnrichmentProfileCleanupPreservesRefresh(t *testing.T) { + for _, laterFailure := range []bool{false, true} { + t.Run("later_failure="+strconv.FormatBool(laterFailure), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newEnrichmentResultFixture(t) + outcome, err := f.store.CommitEnrichmentClaims(t.Context(), f.commit) + require.NoError(err) + require.Equal(personenrichment.ClaimApplied, outcome.Status) + refreshAt := f.now.Add(f.profile.RefreshInterval) + now := f.now.Add(time.Hour) + SetPersonEnrichmentClockForTest(f.store, func() time.Time { return now }) + if laterFailure { + now = refreshAt + lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ + RunID: f.attempt.RunID, Owner: "refresh-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, + Now: now, LeaseDuration: time.Minute, + }) + require.NoError(err) + require.NotNil(lease) + person, err := f.store.GetPersonContext(t.Context(), f.person.ID) + require.NoError(err) + attempt, _, err := f.store.BeginAttempt(t.Context(), lease.Token, personenrichment.AttemptStart{ + RunID: lease.RunID, PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, + PersonRevision: person.Revision, Trigger: lease.Trigger, + PayloadHash: strings.Repeat("3", 64), RequestHash: strings.Repeat("4", 64), + }) + require.NoError(err) + require.NoError(f.store.MarkTerminal(t.Context(), attempt.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + } + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), nil)) + _, err = f.store.EnsurePersonEnrichmentProfile(t.Context(), f.profile) + require.NoError(err) + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), []string{f.profile.Fingerprint})) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + work, err := f.store.ListPersonEnrichmentWorkContext(t.Context(), PersonEnrichmentWorkFilter{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, Limit: 10, + }) + require.NoError(err) + if laterFailure { + assert.Zero(count) + assert.Empty(work, "re-enabling must not replay a failed refresh") + return + } + require.Zero(count) + require.Len(work, 1) + assert.Equal(int64(8), work[0].TriggerMask) + assert.Equal("refresh:"+outcome.Generation.GenerationKey, work[0].TriggerGeneration) + assert.True(refreshAt.Equal(work[0].DueAt), "keep the original refresh due time") + count, err = f.store.EnqueueDuePersonEnrichmentContext(t.Context(), refreshAt, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Zero(count, "repeated catch-up must not republish the refresh") + lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ + RunID: f.attempt.RunID, Owner: "reenabled-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, + Now: refreshAt, LeaseDuration: time.Minute, + }) + require.NoError(err) + require.NotNil(lease) + assert.Equal(personenrichment.TriggerRefresh, lease.Trigger.Kind) + }) + } +} + func TestPersonEnrichmentSynchronousResultCommitsFromStartingAttempt(t *testing.T) { requirements := require.New(t) checks := assert.New(t) @@ -635,7 +714,7 @@ func TestCommitEnrichmentClaimsReusesCitationAcrossAttempts(t *testing.T) { }) requirements.NoError(err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "result-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "result-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: now, LeaseDuration: 5 * time.Minute, }) requirements.NoError(err) diff --git a/internal/store/person_enrichment_runs.go b/internal/store/person_enrichment_runs.go index 05808f47f..0ddb31a38 100644 --- a/internal/store/person_enrichment_runs.go +++ b/internal/store/person_enrichment_runs.go @@ -63,7 +63,7 @@ func derivePersonEnrichmentRunOutcomeTx( SELECT COUNT(*), COUNT(*), COALESCE(SUM(CASE WHEN state = 'succeeded' THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN state IN ('terminal','uncertain_start') - AND COALESCE(failure_class, '') <> 'policy' THEN 1 ELSE 0 END), 0), + AND COALESCE(failure_class, '') NOT IN ('policy', 'profile_unavailable') THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN state = 'suppressed' THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN state = 'identity_rejected' THEN 1 ELSE 0 END), 0) FROM person_enrichment_attempts WHERE run_id = ?`, runID).Scan( diff --git a/internal/store/person_enrichment_runs_test.go b/internal/store/person_enrichment_runs_test.go index 9f3c37f88..24d857a98 100644 --- a/internal/store/person_enrichment_runs_test.go +++ b/internal/store/person_enrichment_runs_test.go @@ -38,7 +38,7 @@ func TestPersonEnrichmentClaimLocksRunBeforeBindingWork(t *testing.T) { claimErr := make(chan error, 1) go func() { lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "claim-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "claim-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) claimResult <- lease @@ -89,7 +89,7 @@ func TestPersonEnrichmentClaimRetriesSQLiteSnapshotContention(t *testing.T) { result := make(chan claimOutcome, 1) go func() { lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "claim-worker", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "claim-worker", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) result <- claimOutcome{lease: lease, err: err} @@ -349,7 +349,7 @@ func TestPersonEnrichmentRecoveryReconcilesUncertainStartCostOnce(t *testing.T) }) require.NoError(err) lease, err := st.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: nextRun.ID, Owner: "post-recovery-worker", ProviderName: profile.Name, + RunID: nextRun.ID, Owner: "post-recovery-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: recoveredAt, LeaseDuration: time.Minute, }) require.NoError(err) diff --git a/internal/store/person_enrichment_triggers.go b/internal/store/person_enrichment_triggers.go index 9963f0184..ddee9d921 100644 --- a/internal/store/person_enrichment_triggers.go +++ b/internal/store/person_enrichment_triggers.go @@ -209,6 +209,9 @@ func (s *Store) EnqueueDuePersonEnrichmentContext( } expiredArgs := append([]any(nil), profileArgs...) expiredArgs = append(expiredArgs, now, expiryMask, now, limit) + // A lookup started after a claim expired consumes that expiry even when + // manual or identity work won trigger coalescing. The attempt that + // produced the claim cannot consume its own expiry. expiredRows, err := tx.QueryContext(ctx, ` SELECT c.person_id, g.provider_policy_fingerprint, MAX(c.id) FROM person_fact_claims c @@ -224,6 +227,15 @@ func (s *Store) EnqueueDuePersonEnrichmentContext( AND g.provider_policy_fingerprint IN (`+profilePlaceholders+`) AND c.valid_until IS NOT NULL AND c.valid_until <= ? + AND NOT EXISTS ( + SELECT 1 FROM person_enrichment_attempts attempted + WHERE attempted.person_id = c.person_id + AND attempted.profile_fingerprint = g.provider_policy_fingerprint + AND attempted.created_at >= c.valid_until + AND attempted.created_at >= g.resolved_at + AND (attempted.fact_generation_key IS NULL + OR attempted.fact_generation_key <> g.generation_key) + ) AND NOT EXISTS ( SELECT 1 FROM person_fact_generations newer WHERE newer.person_id = g.person_id @@ -284,6 +296,12 @@ func (s *Store) EnqueueDuePersonEnrichmentContext( WHERE c.revoked_at IS NULL AND c.profile_fingerprint IN (`+profilePlaceholders+`) AND w.person_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM person_enrichment_attempts attempted + WHERE attempted.person_id = pt.person_id + AND attempted.profile_fingerprint = c.profile_fingerprint + AND attempted.person_revision = p.revision + ) ORDER BY pt.person_id, c.profile_fingerprint LIMIT ?`, missingArgs...) if err != nil { diff --git a/internal/store/person_enrichment_triggers_test.go b/internal/store/person_enrichment_triggers_test.go index 1739665f5..d0801184c 100644 --- a/internal/store/person_enrichment_triggers_test.go +++ b/internal/store/person_enrichment_triggers_test.go @@ -173,7 +173,7 @@ func TestPersonEnrichmentManualRunCompletedIdempotencyRejectsDifferentPerson(t * requirements.NoError(err) requirements.True(created) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "manual-target-worker", ProviderName: f.profiles[0].Name, + RunID: run.ID, Owner: "manual-target-worker", ProviderName: f.profiles[0].Name, ProfileFingerprint: f.profiles[0].Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) requirements.NoError(err) @@ -507,7 +507,7 @@ func TestPersonEnrichmentTriggerCoalescingKeepsSelectedKindAndGenerationPaired(t }) require.NoError(err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "pair-worker", ProviderName: f.profiles[0].Name, + RunID: run.ID, Owner: "pair-worker", ProviderName: f.profiles[0].Name, ProfileFingerprint: f.profiles[0].Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -645,6 +645,124 @@ func TestPersonEnrichmentCatchUpRepairsMissingTrackedWorkAndExcludesUntracked(t assert.Zero(count) } +func TestPersonEnrichmentCatchUpDoesNotRepeatTerminalAttempts(t *testing.T) { + for _, kind := range []personenrichment.TriggerKind{ + personenrichment.TriggerTracked, personenrichment.TriggerManual, personenrichment.TriggerExpiry, + } { + t.Run(string(kind), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newEnrichmentWorkFixture(t) + run := f.startRun(t, "terminal-catch-up") + trigger := personenrichment.Trigger{Kind: kind, Generation: "initial"} + if kind == personenrichment.TriggerExpiry { + claimID := insertProviderClaim(t, f.store, f.person.ID, f.profile.Fingerprint, "expired", f.now.Add(-time.Minute)) + trigger.Generation = "claim:" + strconv.FormatInt(claimID, 10) + } + require.NoError(f.store.PutPersonEnrichmentWorkContext(t.Context(), store.PersonEnrichmentWorkInput{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, Trigger: trigger, DueAt: f.now, + })) + lease := f.claim(t, run.ID, "worker") + start := testAttemptStart(&f, run.ID, "a") + start.Trigger = trigger + attempt, _, err := f.store.BeginAttempt(t.Context(), lease.Token, start) + require.NoError(err) + require.NoError(f.store.MarkTerminal(t.Context(), attempt.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + for range 2 { + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Zero(count) + assert.Empty(f.work(t)) + } + + // A deliberate new operation must still be possible after failure. + require.NoError(f.store.EnqueuePersonEnrichmentContext(t.Context(), store.EnrichmentTriggerInput{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, + Kind: personenrichment.TriggerManual, Generation: "manual:retry", DueAt: f.now, + })) + work := f.work(t) + require.Len(work, 1) + assert.Equal("manual:retry", work[0].TriggerGeneration) + require.NoError(clearEnrichmentWork(t, f.store, f.person.ID)) + + // Catch-up can repair work for a later person revision. + _, err = f.store.AddPersonContactPointContext(t.Context(), f.person.ID, store.PersonContactPointInput{ + AddressKind: store.ContactAddressURL, OriginalValue: "https://profiles.example.test/updated-person", + Envelope: store.ValueEnvelopeInput{Source: store.ProvenanceUser}, + }) + require.NoError(err) + require.NoError(clearEnrichmentWork(t, f.store, f.person.ID)) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Equal(1, count) + }) + } +} + +func TestPersonEnrichmentCatchUpConsumesExpiryWithHigherPriorityWork(t *testing.T) { + for _, kind := range []personenrichment.TriggerKind{personenrichment.TriggerManual, personenrichment.TriggerIdentity} { + t.Run(string(kind), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newEnrichmentWorkFixture(t) + insertProviderClaim(t, f.store, f.person.ID, f.profile.Fingerprint, "coalesced-expiry", f.now.Add(-time.Minute)) + run := f.startRun(t, "coalesced-expiry") + trigger := personenrichment.Trigger{Kind: kind, Generation: "higher-priority"} + require.NoError(f.store.EnqueuePersonEnrichmentContext(t.Context(), store.EnrichmentTriggerInput{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, + Kind: kind, Generation: trigger.Generation, DueAt: f.now, + })) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + require.Equal(1, count) + work := f.work(t) + require.Len(work, 1) + require.NotZero(work[0].TriggerMask & 4) + lease := f.claim(t, run.ID, "worker") + require.Equal(trigger, lease.Trigger) + start := testAttemptStart(&f, run.ID, "a") + start.Trigger = lease.Trigger + attempt, _, err := f.store.BeginAttempt(t.Context(), lease.Token, start) + require.NoError(err) + require.NoError(f.store.MarkTerminal(t.Context(), attempt.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + require.Empty(f.work(t)) + for range 2 { + count, err = f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Zero(count) + assert.Empty(f.work(t)) + } + }) + } +} + +func TestPersonEnrichmentCatchUpDoesNotConsumeFutureExpiry(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newEnrichmentWorkFixture(t) + expires := f.now.Add(time.Minute) + claimID := insertProviderClaim(t, f.store, f.person.ID, f.profile.Fingerprint, "future-expiry", expires) + run := f.startRun(t, "before-expiry") + f.enqueue(t) + lease := f.claim(t, run.ID, "worker") + attempt, _, err := f.store.BeginAttempt(t.Context(), lease.Token, testAttemptStart(&f, run.ID, "a")) + require.NoError(err) + require.NoError(f.store.MarkTerminal(t.Context(), attempt.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + f.setNow(expires.Add(time.Minute)) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Equal(1, count) + work := f.work(t) + require.Len(work, 1) + assert.Equal("claim:"+strconv.FormatInt(claimID, 10), work[0].TriggerGeneration) +} + func TestPersonEnrichmentCatchUpSerializesWithConsentRevocation(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -698,7 +816,7 @@ func TestPersonEnrichmentCatchUpSerializesWithConsentRevocation(t *testing.T) { assert.Empty(f.work(t, 0)) } -func TestPersonEnrichmentCatchUpDoesNotRecreateUnavailableProfileWork(t *testing.T) { +func TestPersonEnrichmentCatchUpLeavesUnavailableProfileWorkQueued(t *testing.T) { for _, expiredClaim := range []bool{false, true} { t.Run("expired_claim="+strconv.FormatBool(expiredClaim), func(t *testing.T) { requirements := require.New(t) @@ -714,13 +832,13 @@ func TestPersonEnrichmentCatchUpDoesNotRecreateUnavailableProfileWork(t *testing } requirements.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext( t.Context(), []string{f.profiles[0].Fingerprint})) - checks.Empty(f.work(t, 1)) + checks.Len(f.work(t, 1), 1) count, err := f.store.EnqueueDuePersonEnrichmentContext( t.Context(), f.now, 200, []string{f.profiles[0].Fingerprint}) requirements.NoError(err) checks.Zero(count) - checks.Empty(f.work(t, 1)) + checks.Len(f.work(t, 1), 1) }) } } @@ -799,7 +917,7 @@ func TestPersonEnrichmentTriggerConsentGrantAndRevocationCancelPendingWork(t *te }) require.NoError(err) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "test-worker", ProviderName: f.profiles[0].Name, + RunID: run.ID, Owner: "test-worker", ProviderName: f.profiles[0].Name, ProfileFingerprint: f.profiles[0].Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(err) diff --git a/internal/store/person_enrichment_work.go b/internal/store/person_enrichment_work.go index af1f0f95c..7e8e849c6 100644 --- a/internal/store/person_enrichment_work.go +++ b/internal/store/person_enrichment_work.go @@ -206,7 +206,8 @@ func (s *Store) PutPersonEnrichmentWorkContext( } // CancelPersonEnrichmentWorkOutsideProfilesContext terminalizes active -// attempts and removes work whose immutable profile is no longer configured. +// attempts for unavailable profiles, retaining queued work until that exact +// profile is configured again. Claims select only configured fingerprints. func (s *Store) CancelPersonEnrichmentWorkOutsideProfilesContext( ctx context.Context, activeFingerprints []string, ) error { @@ -273,7 +274,7 @@ func (s *Store) CancelPersonEnrichmentWorkOutsideProfilesContext( SET state = 'terminal', failure_class = ?, completed_at = ?, next_action_at = NULL, lease_owner = NULL, lease_until = NULL WHERE id = ? AND state IN ('queued','starting','pending','retry_wait','uncertain_start')`, - personenrichment.FailurePolicy, completedAt, item.activeAttempt.Int64) + personenrichment.FailureProfileUnavailable, completedAt, item.activeAttempt.Int64) if err != nil { return fmt.Errorf("terminalize unavailable person enrichment attempt: %w", err) } @@ -281,10 +282,18 @@ func (s *Store) CancelPersonEnrichmentWorkOutsideProfilesContext( return err } } - result, err := tx.ExecContext(ctx, `DELETE FROM person_enrichment_work - WHERE person_id = ? AND profile_fingerprint = ?`, item.personID, item.fingerprint) + result, err := tx.ExecContext(ctx, `UPDATE person_enrichment_work + SET trigger_mask = CASE WHEN active_attempt_id IS NOT NULL AND NOT has_fresh_trigger + THEN 1 ELSE trigger_mask END, + trigger_generation = CASE WHEN active_attempt_id IS NOT NULL AND NOT has_fresh_trigger + THEN 'profile:' || CAST(active_attempt_id AS TEXT) ELSE trigger_generation END, + due_at = CASE WHEN active_attempt_id IS NOT NULL AND NOT has_fresh_trigger + THEN ? ELSE due_at END, + lease_owner = NULL, lease_until = NULL, run_id = NULL, + active_attempt_id = NULL, has_fresh_trigger = FALSE + WHERE person_id = ? AND profile_fingerprint = ?`, completedAt, item.personID, item.fingerprint) if err != nil { - return fmt.Errorf("delete unavailable person enrichment work: %w", err) + return fmt.Errorf("retain unavailable person enrichment work: %w", err) } if err := requireOneLeaseRow(result); err != nil { return err @@ -317,7 +326,7 @@ func (s *Store) ClaimWork( options.Owner = strings.TrimSpace(options.Owner) options.ProviderName = strings.TrimSpace(options.ProviderName) if options.RunID <= 0 || options.Owner == "" || options.ProviderName == "" || - options.Now.IsZero() || options.LeaseDuration <= 0 { + !validLowerSHA256(options.ProfileFingerprint) || options.Now.IsZero() || options.LeaseDuration <= 0 { return nil, errors.New("person enrichment claim options are invalid") } options.Now = options.Now.UTC() @@ -351,7 +360,7 @@ func (s *Store) claimWorkOnce( SELECT w.person_id, w.profile_fingerprint FROM person_enrichment_work w JOIN person_enrichment_profiles p ON p.fingerprint = w.profile_fingerprint - WHERE p.provider_name = ? + WHERE p.provider_name = ? AND w.profile_fingerprint = ? AND (w.run_id = ? OR w.run_id IS NULL) AND w.due_at <= ? AND (w.lease_owner IS NULL OR w.lease_until <= ?) @@ -374,7 +383,7 @@ func (s *Store) claimWorkOnce( until nullableTimestamp ) err := tx.QueryRowContext(ctx, query, - options.ProviderName, options.RunID, options.Now, options.Now, options.RunID, + options.ProviderName, options.ProfileFingerprint, options.RunID, options.Now, options.Now, options.RunID, options.RunID, options.Owner, leaseUntil).Scan( &personID, &fingerprint, &triggerMask, &generation, &runID, &activeID, &fence, &until) if errors.Is(err, sql.ErrNoRows) { @@ -1298,7 +1307,7 @@ func (s *Store) MarkTerminal( state = "identity_rejected" case personenrichment.FailureSuppressed: state = "suppressed" - case personenrichment.FailurePolicy, personenrichment.FailureRateLimited, + case personenrichment.FailurePolicy, personenrichment.FailureProfileUnavailable, personenrichment.FailureRateLimited, personenrichment.FailureTransient, personenrichment.FailureInvalidOutput, personenrichment.FailureTerminal, personenrichment.FailureUncertainStart: // The default terminal state is correct for every other safe failure. @@ -1913,7 +1922,7 @@ func validateSafeFailure(failure personenrichment.SafeFailure) error { func validPersonEnrichmentFailureClass(class personenrichment.FailureClass) bool { switch class { - case personenrichment.FailurePolicy, personenrichment.FailureSuppressed, + case personenrichment.FailurePolicy, personenrichment.FailureProfileUnavailable, personenrichment.FailureSuppressed, personenrichment.FailureRateLimited, personenrichment.FailureTransient, personenrichment.FailureInvalidOutput, personenrichment.FailureIdentityRejected, personenrichment.FailureTerminal, personenrichment.FailureUncertainStart: diff --git a/internal/store/person_enrichment_work_test.go b/internal/store/person_enrichment_work_test.go index 2fc1eb7f2..969db36b7 100644 --- a/internal/store/person_enrichment_work_test.go +++ b/internal/store/person_enrichment_work_test.go @@ -73,7 +73,7 @@ func (f *enrichmentWorkFixture) enqueue(t *testing.T) { func (f *enrichmentWorkFixture) claim(t *testing.T, runID int64, owner string) *personenrichment.WorkLease { t.Helper() lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: runID, Owner: owner, ProviderName: f.profile.Name, + RunID: runID, Owner: owner, ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: 5 * time.Minute, }) require.NoError(t, err) @@ -291,7 +291,96 @@ func TestPersonEnrichmentProfileCleanupRejectsAuthorizedDispatch(t *testing.T) { stored, err = f.store.GetPersonEnrichmentAttemptContext(t.Context(), attempt.ID) requirements.NoError(err) checks.Equal("terminal", stored.State) - checks.Empty(f.work(t)) + requirements.Len(f.work(t), 1) + + // Re-enable the same profile after cleanup canceled the unfinished lookup. + requirements.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext( + t.Context(), []string{f.profile.Fingerprint})) + queued, err := f.store.EnqueueDuePersonEnrichmentContext( + t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + requirements.NoError(err) + requirements.Zero(queued) + resumed := f.claim(t, run.ID, "reenabled-worker") + checks.NotEqual(lease.Trigger.Generation, resumed.Trigger.Generation) + start := testAttemptStart(&f, run.ID, "e") + start.Trigger = resumed.Trigger + restarted, created, err := f.store.BeginAttempt(t.Context(), resumed.Token, start) + requirements.NoError(err) + requirements.True(created) + checks.NotEqual(attempt.ID, restarted.ID) + requirements.NoError(f.store.MarkTerminal(t.Context(), restarted.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + queued, err = f.store.EnqueueDuePersonEnrichmentContext( + t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + requirements.NoError(err) + checks.Zero(queued, "a later provider failure must stay terminal") +} + +func TestPersonEnrichmentProfileCleanupPreservesNewWork(t *testing.T) { + for _, kind := range []personenrichment.TriggerKind{personenrichment.TriggerManual, personenrichment.TriggerIdentity} { + for _, active := range []bool{false, true} { + t.Run(string(kind)+"/active="+strconv.FormatBool(active), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newEnrichmentWorkFixture(t) + run := f.startRun(t, "before-disable") + f.enqueue(t) + lease := f.claim(t, run.ID, "before-disable-worker") + attempt, _, err := f.store.BeginAttempt(t.Context(), lease.Token, testAttemptStart(&f, run.ID, "a")) + require.NoError(err) + if !active { + require.NoError(f.store.MarkTerminal(t.Context(), attempt.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + } + trigger := personenrichment.Trigger{Kind: kind, Generation: "later-operation"} + require.NoError(f.store.EnqueuePersonEnrichmentContext(t.Context(), store.EnrichmentTriggerInput{ + PersonID: f.person.ID, ProfileFingerprint: f.profile.Fingerprint, + Kind: trigger.Kind, Generation: trigger.Generation, DueAt: f.now.Add(time.Hour), + })) + queued := f.work(t) + require.Len(queued, 1) + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), nil)) + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), nil)) + preserved := f.work(t) + require.Len(preserved, 1, "cleanup must retain the new operation") + assert.Equal(queued[0].TriggerMask, preserved[0].TriggerMask) + assert.Equal(trigger.Generation, preserved[0].TriggerGeneration) + assert.True(queued[0].DueAt.Equal(preserved[0].DueAt)) + assert.Nil(preserved[0].ActiveAttemptID) + assert.Nil(preserved[0].RunID) + assert.Nil(preserved[0].LeaseOwner) + require.NoError(f.store.CompleteRun(t.Context(), run.ID, personenrichment.RunCompletion{CompletedAt: f.now})) + + require.NoError(f.store.CancelPersonEnrichmentWorkOutsideProfilesContext(t.Context(), []string{f.profile.Fingerprint})) + f.setNow(f.now.Add(time.Hour)) + nextRun := f.startRun(t, "after-enable") + resumed := f.claim(t, nextRun.ID, "after-enable-worker") + assert.Equal(trigger, resumed.Trigger) + start := testAttemptStart(&f, nextRun.ID, "b") + start.Trigger = trigger + next, created, err := f.store.BeginAttempt(t.Context(), resumed.Token, start) + require.NoError(err) + require.True(created) + assert.NotEqual(attempt.ID, next.ID) + require.NoError(f.store.MarkTerminal(t.Context(), next.Token, personenrichment.SafeFailure{ + Class: personenrichment.FailureInvalidOutput, Message: "invalid provider output", + })) + count, err := f.store.EnqueueDuePersonEnrichmentContext(t.Context(), f.now, 200, []string{f.profile.Fingerprint}) + require.NoError(err) + assert.Zero(count) + assert.Empty(f.work(t), "failure of the new operation must stay stopped") + if !active { + original, err := f.store.GetPersonEnrichmentAttemptContext(t.Context(), attempt.ID) + require.NoError(err) + assert.Equal("terminal", original.State) + require.NotNil(original.FailureClass) + assert.Equal(string(personenrichment.FailureInvalidOutput), *original.FailureClass) + } + }) + } + } } func TestPersonEnrichmentProfileIdentityMutationInvalidatesProviderBindingAndAttempt(t *testing.T) { @@ -823,7 +912,7 @@ func TestPersonEnrichmentClaimRejectsCorruptDurableAttemptTargets(t *testing.T) f.setNow(lease.LeaseUntil.Add(time.Nanosecond)) _, err = f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "worker-b", ProviderName: f.profile.Name, + RunID: run.ID, Owner: "worker-b", ProviderName: f.profile.Name, ProfileFingerprint: f.profile.Fingerprint, Now: f.now, LeaseDuration: 5 * time.Minute, }) assert.ErrorContains(t, err, "durable attempt targets") @@ -1297,7 +1386,7 @@ func TestPersonEnrichmentWorkLoadsCurrentMinimumRequestInput(t *testing.T) { DueAt: f.now, })) lease, err := f.store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: f.now, LeaseDuration: time.Minute, }) require.NoError(err) From 0b48fa82623d9888da187414da9e4cbe8b536d54 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 29 Sep 2026 16:40:42 -0500 Subject: [PATCH 084/125] feat(web): replace the tab bar with a grouped sidebar and global search (#995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Web UI now has a sidebar, a global search box, and one page layout shared by every workspace. Before this change, ten equal tabs sat in one centered bar, each workspace drew its own header (some with orange "Archive workspace" labels), and search existed only inside Everything. This is the first of four pull requests from the [Web UI overhaul design](https://github.com/kenn-io/msgvault/blob/ui-ux-improvements/docs/internal/web-ui-overhaul-design.md). It changes the shell and the visual language; each workspace's own controls change in the next three. No capability is removed. | Before | After | | --- | --- | | ![Everything before](https://github.com/user-attachments/assets/7429f1cd-e3ec-410f-9d30-09c70dec42e3) | ![Everything after](https://github.com/user-attachments/assets/d8eaed16-d95f-4c56-981a-b61ba8d26fbe) | | ![Sources before](https://github.com/user-attachments/assets/2fbd2656-9331-418f-b624-3997c7f5a4fe) | ![Sources after](https://github.com/user-attachments/assets/1338a1dc-438e-49a5-ad69-5c3820f55bdc) | ## What changes for people using the web app - **Navigation:** a left sidebar groups workspaces into People (Relationships, Directory, Reviews), Archive (Everything, Files, Saved views), and Manage (Sources, Operations, Deletions, Settings). It collapses to an icon rail with tooltips and remembers that choice per browser. Below 900px it becomes a slide-out menu; the scrim, Escape, or choosing an item closes it, and focus returns to the menu button. - **Search:** one search box in the top bar on every page. On Everything and Files it updates results as you type, as before. On other pages, Enter opens Everything with the query in a single history entry. `/` focuses it from anywhere. - **Display menu:** the per-tab density override and "Use daemon theme" move from the top bar into a Display menu next to the theme toggle. Their behavior is unchanged. - **Page headers:** every workspace has the same title, optional description, and action area. The browser tab title names the workspace, for example "Sources · msgvault". - **Look:** platform fonts, sentence case, one blue accent, and monospace only for identifiers and cron text. Muted text measures at least 4.6:1 against every surface, including selected rows. Everything's selected row uses the shared blue instead of teal. - **Keyboard hints:** the hint strip under Everything is gone. The Keyboard shortcuts dialog (`?`, or the sidebar entry) lists every shortcut, now including Shift+Space and plain `a`.
Phone width, dark mode, and the icon rail | Before (420px) | After (420px) | Navigation menu | | --- | --- | --- | | ![Phone before](https://github.com/user-attachments/assets/f7f87692-2243-436f-9572-65abc9a0ebd6) | ![Phone after](https://github.com/user-attachments/assets/4182d6b7-2973-4e18-b49d-b43671ce5b69) | ![Phone navigation menu](https://github.com/user-attachments/assets/c79d92fc-3fdc-4390-80a6-266ce5bd2525) | ![Everything in dark mode](https://github.com/user-attachments/assets/f9fb9c6e-fd1d-4a94-b785-8427161ddc5e) ![Collapsed sidebar rail with a tooltip](https://github.com/user-attachments/assets/e8d3db7e-5930-40b5-96aa-71557b1f6680)
## Bugs fixed along the way - Shortcuts relayed to the results grid from outside it did nothing. The shell dispatched a keydown that did not bubble, and Svelte 5 handles `onkeydown` by delegation at the root, so j/k, Space, A, x, Enter, and the same commands run from the command palette only moved focus. The relay now bubbles. `relayGridKey` still returns early for events already aimed at the grid, so nothing fires twice. A Playwright test presses `j` with focus outside the grid. - Escape inside the Display menu also closed the reading pane or a grouping level. The menu now stops the key after closing itself. ## Things to check in review - **Accessible names:** moved controls keep their names: nav "Primary" with `aria-current="page"`, form "Search Everything", searchbox "Search everything", radiogroup "Search mode", and button "Search". The density control changed from the combobox "Temporary density" to a radio group of the same name inside the Display menu. The Playwright helpers `selectWorkspace` and `setTemporaryDensity` replace the old top-bar helpers. - **Focus management** concentrates in `AppShell.svelte`. The drawer refocuses its opener after `inert` clears; jsdom ignores `inert`, so this was verified in Chromium. The shortcuts dialog returns focus to whatever opened it, and Back or Forward on narrow screens falls back to the menu button. - **Palette location:** `web/palette.css` sits outside `src/` because `kit-ui-check` guards component styles, not palette files. The font stack omits `system-ui`: on Linux, fontconfig can resolve it to a font without a bold face, which flattened every heading during testing. - **kit-ui coupling:** kit's Tooltip opens only above or below, so the rail uses a rail-only CSS offset that names its removal condition. The expanded sidebar hides kit tooltip popovers by class. - **Accessibility checks:** `tests/e2e/accessibility.spec.ts` now also checks the collapsed rail with a tooltip open, the narrow menu, and the Display menu. Every violation the new shell introduced was fixed in product code; `main` passed the same spec before this branch. ## Left for the next pull requests - Everything, Files, and Saved views: one toolbar row, the Save view dialog, and a single Files view. - Relationships, Directory, and Reviews structure. - Sources, Operations, Deletions, and Settings, including readable labels for raw codes such as `source_not_schedulable` in the screenshot above. - The remaining purple and green solid buttons (Saved views, Directory promotion, CardDAV, Settings) become blue as each screen is redone. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Wes McKinney --- docs/internal/README.md | 1 + docs/internal/web-ui-overhaul-design.md | 600 +++++ docs/internal/web-ui-overhaul-pr1-plan.md | 2111 +++++++++++++++++ web/palette.css | 98 + web/src/App.svelte | 43 +- web/src/App.test.ts | 6 + web/src/app.css | 12 +- web/src/lib/commands/registry.ts | 3 +- web/src/lib/components/auth/Login.svelte | 50 +- .../deletions/DeletionsWorkspace.svelte | 31 +- .../directory/DirectoryReviewCentre.svelte | 39 +- .../directory/DirectoryWorkspace.svelte | 21 +- .../directory/IdentityCandidateCard.svelte | 2 +- .../directory/PersonMergeHistory.svelte | 15 +- .../directory/PersonMergeHistory.test.ts | 3 + .../components/directory/PersonNetwork.svelte | 5 +- .../directory/PersonNetwork.test.ts | 1 + .../directory/RelationshipReviewCard.svelte | 2 +- .../components/explore/EverythingTable.svelte | 6 +- .../explore/FilesPresentation.svelte | 2 +- .../lib/components/explore/GroupTable.svelte | 2 - .../components/explore/SelectionBar.svelte | 17 +- .../components/files/FilesWorkspace.svelte | 27 +- .../operations/OperationRelatedStatus.svelte | 10 +- .../operations/OperationRunDetail.svelte | 2 +- .../operations/OperationsWorkspace.svelte | 30 +- .../operations/OperationsWorkspace.test.ts | 16 + .../components/people/PersonTimeline.svelte | 5 +- .../relationships/RelationshipTimeline.svelte | 2 +- .../RelationshipsWorkspace.svelte | 98 +- .../saved-views/SavedViewsWorkspace.svelte | 45 +- .../saved-views/SavedViewsWorkspace.test.ts | 10 +- .../lib/components/search/SearchBar.svelte | 109 +- .../lib/components/search/SearchBar.test.ts | 100 +- .../search/SearchModeControl.svelte | 5 +- .../search/SearchModeControl.test.ts | 13 - .../settings/CardDAVConflicts.svelte | 2 +- .../settings/SettingsWorkspace.svelte | 21 +- .../settings/SettingsWorkspace.test.ts | 5 + web/src/lib/components/shell/AppShell.svelte | 820 ++++--- web/src/lib/components/shell/AppShell.test.ts | 391 ++- .../lib/components/shell/AppSidebar.svelte | 260 ++ .../lib/components/shell/AppSidebar.test.ts | 107 + .../lib/components/shell/DisplayMenu.svelte | 63 + .../lib/components/shell/DisplayMenu.test.ts | 77 + .../shell/EverythingWorkspace.svelte | 115 +- .../shell/EverythingWorkspace.test.ts | 12 + .../lib/components/shell/KeyboardHelp.svelte | 2 - .../lib/components/shell/KeyboardHelp.test.ts | 9 + .../components/shell/NavigationDrawer.svelte | 65 + .../components/shell/NavigationDrawer.test.ts | 56 + .../lib/components/shell/PageHeader.svelte | 64 + .../lib/components/shell/PageHeader.test.ts | 25 + web/src/lib/components/shell/navigation.ts | 62 + .../sources/SourcesWorkspace.svelte | 46 +- web/src/lib/explore/state.svelte.ts | 17 + web/src/lib/explore/state.test.ts | 21 + web/src/main.ts | 2 + web/src/styles/tokens.css | 17 +- web/tests/density-restoration.spec.ts | 8 +- web/tests/docs-fixture-screenshots.spec.ts | 6 +- web/tests/e2e/accessibility.spec.ts | 34 +- web/tests/e2e/directory.spec.ts | 2 +- web/tests/everything-virtualization.spec.ts | 2 +- web/tests/kit-ui.ts | 26 +- web/tests/presentations.spec.ts | 2 +- web/tests/shell-navigation.spec.ts | 94 + web/tests/theme-keyboard.spec.ts | 24 +- 68 files changed, 5140 insertions(+), 859 deletions(-) create mode 100644 docs/internal/web-ui-overhaul-design.md create mode 100644 docs/internal/web-ui-overhaul-pr1-plan.md create mode 100644 web/palette.css create mode 100644 web/src/lib/components/shell/AppSidebar.svelte create mode 100644 web/src/lib/components/shell/AppSidebar.test.ts create mode 100644 web/src/lib/components/shell/DisplayMenu.svelte create mode 100644 web/src/lib/components/shell/DisplayMenu.test.ts create mode 100644 web/src/lib/components/shell/NavigationDrawer.svelte create mode 100644 web/src/lib/components/shell/NavigationDrawer.test.ts create mode 100644 web/src/lib/components/shell/PageHeader.svelte create mode 100644 web/src/lib/components/shell/PageHeader.test.ts create mode 100644 web/src/lib/components/shell/navigation.ts create mode 100644 web/tests/shell-navigation.spec.ts diff --git a/docs/internal/README.md b/docs/internal/README.md index 6381d0b06..a496e24f4 100644 --- a/docs/internal/README.md +++ b/docs/internal/README.md @@ -17,6 +17,7 @@ For maintenance rules, see the [documentation contributor guide](../README.md). | Slack ingestion and reply discovery | [Ingestion](slack-ingestion-design.md), [reply sweep](slack-reply-sweep-design.md) | [Slack](../usage/slack.md) | | Message exports | [Design](message-export-design.md) and [plan](message-export-plan.md) | [Exporting](../usage/exporting.md) | | People and relationships | [Relationship index](relationship-list-index-design.md), [merge reversal](person-merge-reversal.md), [conversation brief](last-time-we-talked-design.md) | [People and profiles](../usage/people.md) | +| Web UI navigation and visual language | [Design](web-ui-overhaul-design.md) and [foundation plan](web-ui-overhaul-pr1-plan.md) | [Web UI](../web-ui.md) | | Daemon command routing | [CLI audit](daemon-cli-request-audit.md) | [Daemon guide](../guides/daemon-migration.md) | | PostgreSQL | [Original implementation tracker](PG_STATUS.md) | [PostgreSQL backend](../architecture/postgresql.md) | | Recovery | [Recovery notes](recovery.md) | [Backup](../usage/backup.md) and [troubleshooting](../troubleshooting.md) | diff --git a/docs/internal/web-ui-overhaul-design.md b/docs/internal/web-ui-overhaul-design.md new file mode 100644 index 000000000..49ad33e9f --- /dev/null +++ b/docs/internal/web-ui-overhaul-design.md @@ -0,0 +1,600 @@ +# Web UI overhaul + +Status: in review. Design sections were approved 2026-09-28; this revision +(2026-09-29) addresses review findings. PR 1 (foundation and shell) is +implemented on branch `ui-ux-improvements` and not yet merged; PRs 2–4 are not +started. Delivery is four stacked pull requests, described under +[Delivery](#delivery). This record describes the intended end state; the +current source remains authoritative until each pull request lands. + +## Summary + +The Web UI exposes a large, capable archive, but its screens grew one feature +at a time. Ten peer tabs share one centered top bar, each workspace invents its +own header and toolbar, several controls do nothing, and many labels are raw +API codes. People who use msgvault every day have to learn each screen +separately. + +This overhaul gives every workspace the same shell, page structure, visual +language, and status vocabulary. It keeps every existing capability. Controls +move to predictable places; none are removed. The visual language follows the +docbank web restyle (kenn-io/docbank#722) so the kenn tools feel like one +family. + +## Problems observed + +Observed on `main` at 5d1ce392 with the Enron docs fixture at 1440×900 and +420×860, light and dark. + +- **Navigation has no hierarchy.** Relationships, Directory, Reviews, + Everything, Files, Saved Views, Sources, Operations, Deletions, and Settings + are equal-weight tabs. On a phone they collapse into a dropdown beside the + density selector. +- **Each page invents its header.** Sources, Deletions, and Saved Views show an + orange "ARCHIVE WORKSPACE" eyebrow; Operations shows "ARCHIVE OPERATIONS"; + Everything and Directory show none. Saved Views uses a centered column while + other pages fill the width. In Files the page title sits below its toolbar. +- **Everything stacks four control strips.** Search, the context bar, an + always-visible "No items selected" bar, and a Columns strip sit above the + results, with a hard-coded keyboard-hint footer below. The result count + appears twice ("20 items" and "20 results"). +- **Files is two different views.** Everything's "Show as: Files" renders + `explore/FilesPresentation.svelte` (grid "Files in current context"). The + Files workspace renders `FilesWorkspace` (grid "Files results") with other + columns and filters. In the Files workspace, "Show as: Table" silently moves + to Everything. +- **Directory shows seven inline filters.** The date filters are free-text + `YYYY-MM-DD` fields whose placeholder is truncated; an invalid date is + silently ignored. +- **Accent colors compete.** Search is blue, Save is purple, Save settings is + green, and eyebrows are orange. Timestamps and counts use monospace as + decoration. +- **Status is alarming or raw.** Operations shows a red dot for every feature + that is simply not configured. Sources shows `source_not_schedulable` in red + as its "Action". Deletions shows "No deletion manifests yet" in an orange + warning box. Raw codes also appear in Files (MIME families), Operations + (error codes), Deletions (reasons, ISO timestamps), Tasks, Meetings, and the + context-bar crumbs (`full_text`). +- **Some controls do nothing.** "Newest first" and the `s`/`r` shortcuts only + announce that sort is fixed; in Files the button has no handler. + "Open selection in source" never receives a handler. Directory → Media & + Files mounts `FilesWorkspace` without change handlers, so its filename + filter, type checkboxes, and sort headers are inert. Saved Views declares an + unused `selection` prop. `search/SearchBar.svelte` is imported only by its + test. +- **Settings contradicts itself.** Appearance says "Changes apply right away" + above a Save settings bar. The phrase means "no daemon restart after saving". + The shell reads `web.theme` and `web.density` only at sign-in, so saving them + may not update the open tab. +- **Useful actions are hidden.** Staging a deletion is reachable only with `d` + or `D` in Everything. Tasks are a collapsed disclosure in the reading pane. + "Save this view" lives on the Saved Views page, away from the view it saves. +- **Cross-links are missing.** Directory cannot open a person's Relationships + view. Nothing links to Reviews → Facts for a person, and Facts with no person + selected is a dead end. +- **The browser tab title is always "Everything · msgvault".** + +## Goals + +- One navigation model, one page structure, and one toolbar pattern across + every workspace. +- One accent color. Status colors mean status only. +- Human-readable labels for every code the UI displays, with the raw code kept + in a tooltip or detail view for diagnosis. +- Every existing capability reachable, with a visible entry point for actions + that are keyboard-only today. +- Existing URLs, URL state keys, API calls, and accessible names preserved + unless this document names the change. +- Light and dark themes, both densities, keyboard use, and phone widths work on + every screen. + +## Non-goals + +- New archive capabilities. The follow-ups listed below need backend work and + ship separately. +- Changes to the TUI, CLI, API, or MCP server. +- Replacing kit-ui components with local ones. Where kit-ui lacks a token, + `app.css` overrides the kit class, as docbank does. + +## Visual language + +These rules match docbank#722. + +- **Palette.** A new `web/palette.css` retunes kit-ui tokens: neutral surfaces, + one blue accent, and green, amber, and red for status only. It lives outside + `src/` because `kit-ui-check` guards component styles, not palette files. + `src/styles/tokens.css` keeps the semantic aliases it defines today. +- **Buttons.** One solid primary action per screen, in blue. Purple and green + are no longer button colors. Destructive confirmation buttons are red; the + button that opens a destructive review is not. +- **Type.** Platform system fonts. Sentence case for every label, heading, + table header, and chip ("Saved views", not "Saved Views"; no uppercase + letter-spaced section labels). `[data-section-label]` becomes sentence case. +- **Monospace** is for identifiers, hashes, code, keys, and cron text only. + `[data-mono]` becomes tabular-figure sans, so timestamps and counts stop + switching typeface. Identifiers that use `[data-mono]` today move to + `` or `[data-metadata]`, which stay monospace. +- **Status vocabulary.** Every status dot and chip uses one mapping: + + | Meaning | Tone | Examples | + |---|---|---| + | Healthy or finished | green | Succeeded, Completed, Ready | + | In progress | blue | Running, Syncing, Queued | + | Needs attention | amber | Partial, Stale, Conflict | + | Failed | red | Failed | + | Off or not set up | gray | Not configured, Disabled, Cancelled | + +- **Empty states** use kit `EmptyState` in a neutral tone and name the next + step. A missing item is never a warning. +- **Code labels.** Each enum the UI displays gets one label map next to the + component that renders it. Unknown codes fall back to a sentence-cased form of + the code. The raw code stays available in a tooltip or detail panel. + +## Information architecture + +### Sidebar + +A labeled left sidebar replaces the top-bar tabs. It is a `nav` still named +"Primary", and the active item keeps `aria-current="page"`, which history focus +restoration depends on. + +| Group | Items (icon) | Workspace id | +|---|---|---| +| People | Relationships (users), Directory (contact), Reviews (check-check) | `relationships`, `directory`, `directory_review` | +| Archive | Everything (inbox), Files (paperclip), Saved views (bookmark) | `everything`, `files`, `saved_views` | +| Manage | Sources (plug), Operations (activity), Deletions (trash-2), Settings (settings) | `sources`, `operations`, `deletions`, `settings` | + +People comes first because Relationships is the default landing workspace. +Icons come from `@lucide/svelte`, which is already a dependency; final icon +choices may change during implementation. + +- **Layout.** PR 1 adds a local `AppSidebar` component and owns all of the + behavior below. kit `CollapsibleSidebar` is not used: its collapsed branch + renders only an expand toggle, and its narrow-screen overlay positions the + sidebar without a scrim or focus management. The sidebar has a fixed width + and no resize handle. +- **Expanded and rail modes.** Expanded, the sidebar is about 232px with group + headings. Collapsed, it is an icon rail about 56px wide. Each rail item shows + a kit `Tooltip` with its label and keeps its full accessible name. A toggle at + the bottom of the sidebar switches modes. The mode is stored per browser in + localStorage `msgvault.sidebar.collapsed`. Group headings are hidden in the + rail; a divider separates the groups. +- **Narrow screens.** Below 900px the sidebar is hidden and a menu button + labeled "Open navigation" appears at the start of the top bar. It opens the + expanded sidebar as a modal slide-out menu: + - A scrim covers the page. Selecting the scrim, pressing Escape, or choosing + a navigation item closes the menu. + - Focus moves to the current navigation item when the menu opens. kit + `trapFocus` keeps focus inside the menu while it is open, and the page + behind it is `inert`. + - Closing the menu returns focus to the menu button, except after choosing an + item, when focus follows the existing workspace-change focus rules. + - The narrow-screen menu ignores the rail setting and always shows labels. + - Escape closes the menu before any other Escape handler runs, so it never + also closes the reading pane or a grouping level. +- **Footer.** The archive status indicator (dot plus "Local archive", + "Searching", or "Attention") and a "Keyboard shortcuts" entry showing `?` + move to the sidebar footer. +- **Tab title.** The browser tab title follows the workspace, for example + "Directory · msgvault". + +### Top bar + +The top bar holds only global search and appearance controls. + +- **Global search.** The search form moves from Everything into the top bar and + reuses `search/SearchBar.svelte` (today imported only by its test). It keeps + the names form "Search Everything", searchbox "Search everything", radiogroup + "Search mode" with "Full text", "Semantic", and "Hybrid", and button + "Search". Query and mode state are unchanged (`query` in the `explore` URL + state, top-level `mode`, localStorage `msgvault-search-mode`). + - On Everything and Files, typing updates the current view as it does today. + - On any other workspace, typing is a local draft; Enter or Search commits the + query and opens Everything. + - `/` focuses the global search from any workspace. AppShell's `searchInput` + reference binds to this input, so "Refine search" and the saved-view focus + fallback keep working. + - Below 900px the mode control becomes a compact select inside the search + field; the radio semantics are kept at wider widths. +- **Theme.** kit `ThemeToggle` stays, matching the other kenn tools. +- **Display menu.** A new "Display" menu button holds the per-tab density + override (Auto, Compact, Comfortable) and "Use daemon theme". Both keep their + sessionStorage behavior (`msgvault.appearance.override`). The combobox + "Temporary density" becomes a radio group named "Temporary density" inside + this menu. + +### Page structure + +A shared local `PageHeader` component renders the title (h1), an optional +one-line description, right-aligned actions, and an optional row below for view +tabs. Every workspace uses it. Eyebrows are removed. Horizontal padding is the +same on every page. Data workspaces fill the width; form pages (Settings, +Saved views editing) use the same left edge with a maximum content width. + +- **View switches** (different views of the same data, such as Messages or + Files, and review type) use kit `SegmentedControl`. +- **Record sections** (sections of one record, such as person detail) use the + ARIA tablist pattern that `PersonDetail` already implements. + +### Toolbars + +Data workspaces use one toolbar row: search or filter input first, then +Filters, view options, and sort, with the result count at the right edge. When +filters, groupings, or a query are active, a second line shows them as +removable chips with readable labels. Notices (semantic coverage, "More results +may match") appear between the toolbar and the results only when they apply. + +### Keyboard + +The hard-coded footer in Everything and the key badges in the selection bar are +removed. The existing "Keyboard shortcuts" dialog, generated from +`lib/commands/registry.ts`, becomes the single reference. Before removing the +footer, the registry gains the entries only the footer documents today: +Shift+Space (extend selection) and plain `a` (select visible rows). + +## Workspaces + +### Everything + +``` +Everything [Save view…] +[Filters 2] [Show as: Table] [Group by] [Sort: Newest first] [Columns] [▭|▯] 20 items +Full text: "network" × Source: example@example.com × Grouped by Year × +┌ results ────────────────────────────────────────────────────────────────┐ +├ reading pane ───────────────────────── [Open relationship] [Tasks 2] [×] ┤ +└─────────────────────────────────────────────────────────────────────────┘ + ┌ 3 selected · Select all 20 matching · Export · Review for deletion… · Clear ┐ +``` + +- **Toolbar.** Filters, Show as, Group by, Sort, and Columns sit in one row. + - Columns moves from a `
` strip into a menu with the same seven + checkboxes and the same `columns` URL state. + - Preview position becomes a two-icon segmented control, keeping radiogroup + "Preview position" with radios "Below" and "Right". It still appears only + when the results are at least 960px wide. + - The count appears once, at the right edge. +- **Sort.** "Newest first" becomes a menu listing the one supported order, + marked "Other orders aren't available yet". It keeps the accessible name + "Sort: newest first", so `s` opens it. `r` keeps announcing that reversing + isn't supported. +- **Context chips.** The query, filters, and groupings appear as removable chips + with readable labels ("Full text", "Source"). Removing a grouping keeps the + name "Remove {label} grouping". +- **Selection bar.** The existing `SelectionBar` moves below the results as a + compact, sticky action strip. It appears only while at least one row is + selected. kit `BottomDock` is not used, because it is a resizable panel that + opens at half the viewport height. The bar keeps "Select all N matching + items", "Export selection", meeting-context export, and "Clear selection". + - It adds **Review for deletion…**, which starts the same flow as `d` (or `D` + in all-matching mode): `openDeletionReview` switches to Deletions and runs + the preflight. + - "Open selection in source" moves to an overflow menu. It is shown disabled + with its reason as a sentence, which matches today's behavior because no + handler exists. +- **Reading pane.** "Tasks for this message" becomes a header button labeled + "Tasks" with the linked-task count, opening the same `TaskLinks` sheet. + "Close" becomes an icon button that keeps the name "Close reading pane". The + meta strip uses readable labels. + +### Files + +Files becomes the single file view. + +- Everything's "Show as: Files" opens the Files workspace with the same query, + filters, and groupings. Files shows the same "Show as" control; choosing + Table or Timeline opens Everything. Switching is symmetric and visible. +- `explore/FilesPresentation.svelte` is deleted. Its per-row "Open containing + item" action moves into the Files grid as a row action. +- **Existing links and history.** Explore state can encode + `workspace: 'everything'` with `presentation: 'files'` in shared links, + browser history entries, and saved views. `normalize()` in + `src/lib/explore/state.svelte.ts` maps that combination to + `workspace: 'files'` and keeps the query, search mode, filters, grouping + chain, and columns. URL parsing, every commit, popstate restoration, and + saved-view navigation all pass through `normalize()`, so one rule covers + every entry path: + - Opening an old link lands in Files with the same context. The address bar + is rewritten with a history replace, so no extra history entry appears. + - Back and Forward into an old history entry restore Files with the same + context and the entry's keyboard focus and scroll state. + - An attachment selection (`selectedRow` of the form `attachment:`) + reopens the same file viewer in Files. + - In the Files workspace, `presentation` is always `files`. Choosing Table or + Timeline sets `workspace: 'everything'` with that presentation. +- Saved views with `presentation: 'files'` open in the Files workspace through + the same rule. +- **Toolbar.** Filename search, a **Type** menu, Filters, Show as, Group by, a + **Sort** menu (Date, Filename, Size), and a **Visual search** toggle. + - The Type menu replaces the eight raw MIME-family checkboxes with Images, + PDFs, Audio, Video, Text, Documents, Archives, and Other. URL state + `fileMIMEFamilies` is unchanged. + - Column-header sorting still works and stays in sync with the Sort menu + (`fileSort`). + - Turning on Visual search expands a second row with the visual query, + query image, and the unchanged provider disclosure. +- **Columns.** Type shows a readable name ("PDF", "PNG image"); the raw MIME + type moves to the cell tooltip and the file viewer. Source shows the account + display name when one is available. +- **Fixes.** The Sort control works (it has no handler today). The count comes + from the Files query instead of "Count pending". Directory → Media & files + passes change handlers so its filters and sorting work. + +### Saved views + +- **Save view…** is a header action in Everything and Files. It opens a dialog + with Name and Description and saves the current view through the existing + saved-views API. It keeps the field names "Name" and "Description" and the + submit button "Save". In Files, the dialog states: "Filename, type, and file + sort aren't saved with the view." This limitation goes away only with the + backend follow-up below. +- The Saved views page becomes a library. Each view shows its name, + description, a readable summary of its query, filters, grouping, and layout, + and the actions "Open {name}", "Edit {name}", and "Delete {name}". The empty + state points to Save view… in Everything and Files. +- The unused `selection` prop is removed. + +### Relationships + +- The list search is relabeled "Filter people and domains" (placeholder) to + separate it from global search; its accessible name "Search people and + domains" is unchanged. +- The person header groups "Open in Directory" and "Same person…" as one action + set beside the Messages | Files view switch. +- On narrow screens the list-drawer button reads "People" instead of + "Contacts". + +### Directory + +- **Toolbar.** Search, a **Filters** popover, and a **Sort** menu. + - The popover holds contact state, category, organization, primary channel, + and last contacted. The URL keys (`directoryContactState`, + `directoryCategory`, `directoryOrganization`, `directoryPrimaryChannel`, + `directoryLastContactAfter`, `directoryLastContactBefore`) and API + parameters are unchanged. + - "Last contacted after" and "Last contacted before" stay two independent + fields, because either boundary can be set alone today. Each becomes a + native `` with a clear button, keeping its accessible + name and URL key. The native input produces only valid `YYYY-MM-DD` + values, so invalid dates can no longer be silently ignored. kit + `DateRangePicker` is not used: it commits only completed custom ranges. + - A one-sided filter shows as one chip, such as "Last contacted after + Jan 5, 2024". Removing it clears only that boundary. + - Sort keeps the three orders and the `directorySort` URL key. + - Active filters show as removable chips. +- **Person detail sections** (tablist "Person detail sections"): + + | Section | Contents | + |---|---| + | Overview | "Last time we talked" brief, agenda, contact state, activity, meeting activity | + | Profile | Structured profile (names, contact points, addresses, dates, categories, media), attributes, profile history | + | Organizations | Unchanged | + | Connections | Curated person-to-person relationships; renamed from "Relationships" to avoid confusion with the Relationships workspace | + | Network | Unchanged | + | Media & files | Person-scoped files, now with working filters and sort | + | Maintenance | Profile-maintenance tracking, CardDAV publication, merge history and split | + +- **Header actions.** + - **Open relationship** opens the person's Relationships view through the + shell's existing `openRelationship(participantID)`, using the person's + `participant_ids` from the person response. It appears only when that list + is not empty. PR 3 confirms which entry opens the person's relationship + cluster when a person has several participants. + - **Review facts** opens Reviews → Facts with `directoryPersonID` set. + - An overflow menu holds "Rename person", "View profile history", and "Delete + person". Rename and delete keep their existing confirmation steps and + accessible names. +- **Partial dates in editors** stay text fields because a date picker cannot + express `YYYY` or `YYYY-MM`. They get inline validation messages. + +### Reviews + +- One view switch under the header: Identity matches, Facts, Imported + relationships. It keeps the radiogroup "Review type" and the `reviewKind` URL + key. +- The second header ("Identity matches" with its own segmented control) is + removed. Each queue's status filter becomes a **Show** menu in its list + toolbar, keeping the names "Identity review state" and "Imported + relationship review state" and the URL keys `identityState` and + `relationshipReviewState`. +- Facts with no person selected shows a person picker that uses the existing + directory search, instead of only linking to Directory. + +### Sources + +- **Header.** Title "Sources", description "Accounts and imports in your + archive, and when they last synced." The action "View source operations" is + renamed **Sync history** and still opens Operations filtered to source sync. +- **Table.** + - Source: display name and a readable source type ("Mbox import", "Gmail"). + - Schedule: a sentence from CronField's summary logic, with the cron text in + a tooltip. + - Status: a compact chip. + - Last successful sync: unchanged. + - Action: "Sync now {name}", or a muted reason: + + | Code | Label | + |---|---| + | `source_not_schedulable` | Imported file — nothing to sync | + | `sync_already_running` | Sync in progress | + | `scheduler_unavailable` | Scheduler unavailable | + | `sync_not_configured` | Sync not set up | + | `sync_unavailable` | Sync unavailable | + +- Error messages, item errors, and scheduler errors move to an expandable row + detail. `sync_start_not_observed` and `stale_last_result` get sentences. + +### Operations + +- **Lane summary.** The five lane cards become one compact status list: one row + per operation kind, grouped by lane, showing the status tone, the latest + run's time, and the row's actions. + - "Not configured" shows as gray **Off** with a **Set up** link to the related + setting. + - "History available" is no longer printed; "History unavailable" still is. + - Related-status buttons ("Open Sources status" and the others) and actions + ("Start CardDAV sync", "Build visual index", "Resume visual index") keep + their names. +- **Toolbar.** Lane, Kind, State, and the date range stay in one row with their + URL keys. "Refresh operations" becomes kit `RefreshControl` in the header. +- **Runs.** Counters read "20 messages processed · 20 added". An unknown + trigger shows "—" instead of "Unspecified". Each `OperationPublicErrorCode` + maps to a sentence; the detail panel keeps the raw code. + +### Deletions + +- **Header.** Title "Deletions", description "Deletions you've staged. Nothing + is deleted until you run `msgvault delete-staged`." +- **Review.** The review panel appears when a selection arrives from + Everything. Without one, an empty state explains: "Select items in + Everything, then choose Review for deletion…". The summary shows expiry as + relative time and each unavailable reason as a sentence. Only "Confirm stage + deletion" is red. +- **Manifests.** A table of ID (monospace), description, items, status chip + (Pending, In progress, Completed, Failed, Cancelled), and the row actions + "Inspect {id}" and "Cancel {id}". The detail opens beside the table. + +### Settings + +- The category list stays as the page's second-level navigation under a normal + page header. The selected category is stored in a new `settingsCategory` + field of the `explore` URL state, so reload and Back keep the category. + `settingsAuthority` deep links keep working. +- **Save model.** + - Catalog categories show the save bar only when drafts exist, with + "Discard" and "Save changes" in blue. The status text "N unsaved changes" + stays. + - Each category states when saved changes apply: "After the daemon restarts", + or for Appearance "Right away — no restart needed". + - Controls that save through their own endpoints (provider credentials, + enrichment providers, CardDAV account, People sweep) say "Saves + immediately" beside their buttons. +- **Appearance.** After a successful save, SettingsWorkspace passes the saved + `web.*` values to App through a callback, and App updates its defaults. + - **Theme and density** apply to the open tab right away, unless a Display + menu override is active in this tab. The override still wins, as it does + today; the note under Appearance explains that "Use daemon theme" and + "Density: Auto" return to the saved default. + - **Default search mode** changes only future searches, not the current one. + Search mode is resolved in this order: explicit URL mode, then this + browser's remembered mode (localStorage `msgvault-search-mode`), then the + daemon default. Saving does not change the open view's mode or its URL, + so results on screen and shared links stay stable. Saving does replace + this browser's remembered mode with the saved value, so the next tab + opened here without a mode in its link uses the new default. Other + browsers keep their remembered mode until someone changes the mode there. + The setting's description says: "Used when a tab opens without a search + mode in its link. Your current search keeps its mode." +- The plain-HTTP warning stays as a compact kit `Notice`. + +### Sign-in and boot screens + +The login, connecting, connection-error, and OAuth-callback screens use the +same palette and type. + +## Control relocation + +Every control not listed here keeps its location, label, and accessible name. + +| Control | Today | After | +|---|---|---| +| Workspace tabs | Top bar center, nav "Primary" | Sidebar, nav "Primary"; icon rail when collapsed; modal menu below 900px | +| Archive status | Top bar right | Sidebar footer | +| Temporary density | Top bar combobox | Display menu radio group | +| Use daemon theme | Top bar button | Display menu item | +| Search form, mode, Search button | Everything search row | Top bar | +| Preview position | Everything header, text plus segmented control | Everything toolbar, icon segmented control | +| Result count | Everything header and context bar | Toolbar right edge, once | +| Columns | `
` strip above the table | Toolbar menu | +| Newest first | Button that only announces | Sort menu with the one supported order | +| Keyboard hints | Everything footer, selection bar badges | Keyboard shortcuts dialog; sidebar footer entry | +| Selection bar | Always visible above results | Sticky strip below results while a selection exists | +| Stage deletion entry | `d` / `D` only | Also "Review for deletion…" in the selection bar | +| Open selection in source | Selection bar, reason text | Selection bar overflow, disabled with sentence reason | +| Tasks for this message | Collapsed disclosure | Reading-pane header button with count | +| Close reading pane | Text button | Icon button, same name | +| Show as: Files | Everything-only files grid | Opens the Files workspace; old links and history entries normalize there | +| File type filter | Eight raw checkboxes | Type menu with readable names | +| File sort | Column headers only | Sort menu and column headers | +| Save this view | Saved Views page form | "Save view…" in Everything and Files headers | +| Directory filters | Seven inline controls | Filters popover and Sort menu | +| Directory date filters | Two `YYYY-MM-DD` text fields | Two independent native date fields in the Filters popover | +| Person detail Overview sections | One long Overview | Overview, Profile, Maintenance sections | +| Person "Relationships" tab | Record tab | Renamed "Connections" | +| Rename, profile history, delete person | Structured profile section | Person header overflow menu | +| Review state filters | Second header segmented control | Show menu in the queue toolbar | +| Sources header action | "View source operations" | "Sync history" | +| Refresh operations | Header button | Header `RefreshControl` | +| Settings category | Transient | `settingsCategory` URL state | + +New entry points: Directory → Relationships ("Open relationship"), Directory → +Reviews Facts ("Review facts"), and a person picker in Facts. + +## Tests and accessible names + +- Accessible names stay the same wherever the control survives in the same + form. The kit test helper `selectKitTopBarTab` in `web/tests/kit-ui.ts` + changes to click the sidebar item in nav "Primary". +- Expected test edits, listed in each pull request: + - The combobox "Temporary density" becomes a menu radio group + (`density-restoration`, `theme-keyboard`, `docs-fixture-screenshots`, + `e2e/accessibility`). + - The grid "Files in current context" and region "Files presentation" are + replaced by the Files grid (`presentations`). + - Visible labels change to sentence case, such as "Saved views". + - The person tab "Relationships" becomes "Connections", and Overview + sections move (`e2e/directory`, `directory-network`, `directory-review`, + `e2e/accessibility`). + - "View source operations" becomes "Sync history" (`operations`). +- Each pull request adds tests for the behavior it changes: the Save view + dialog and its Files note, the deletion entry in the selection bar, and the + Type and Sort menus. They also cover: + - Files round trips, including an old `everything` + `files` link, Back and + Forward into an old history entry, and an old saved view, each landing in + Files with the query, filters, grouping, and attachment selection intact. + - The narrow-screen menu: scrim, Escape, and item selection close it; focus + starts on the current item, stays inside while open, and returns to the + menu button; Escape does not also close the reading pane. + - The icon rail: each item keeps its accessible name, and the mode survives + a reload. + - Directory dates: setting and clearing each boundary alone. + - The Settings category URL. + - Appearance saves: theme and density update the open tab unless a Display + override is active; saving the default search mode leaves the open view's + mode and URL unchanged and changes the mode a new tab opens with. +- `e2e/accessibility` (axe) runs on every workspace in both themes. +- Each pull request includes before and after screenshots from the real daemon + with the Enron docs fixture, at desktop and phone widths, in light and dark. + The docs screenshots (`docs/screenshots/generate-web-fixture-screenshots.sh`) + are regenerated in the final pull request. + +## Delivery + +Four stacked pull requests. Each leaves the application shippable. + +1. **Foundation and shell.** Palette, type, and status vocabulary; the local + `AppSidebar` with its icon rail, narrow-screen modal menu, scrim and Escape + dismissal, focus trap, and focus restoration; top bar, global search, + Display menu; `PageHeader` applied to every workspace; empty-state + conventions; keyboard registry additions and footer removal; tab titles; + sign-in and boot screens. +2. **Everything, Files, and Saved views.** Toolbar, context chips, Sort and + Columns menus, selection bar, reading-pane header, Files unification with + the `normalize()` rule for old links, history, and saved views, Files + fixes, Save view dialog, Saved views library. +3. **People.** Relationships header, Directory filters and person sections, + cross-links, Reviews structure and fact person picker. +4. **Manage.** Sources, Operations, Deletions, Settings, and the code-label + maps they need; docs screenshots and the [Web UI guide](../web-ui.md). + +## Follow-ups + +These need backend changes and are not part of this overhaul. + +- **Reviews count.** No API returns a pending review count, so the sidebar shows + no badge. A count endpoint would enable one. +- **Files fields in saved views.** The saved-view schema has no fields for the + filename filter, type filter, or file sort, so saving a Files view keeps only + the shared context. +- **Sort orders.** Everything supports only newest first. +- **Open in source.** The daemon always reports + `open_in_source:trusted_source_link_unavailable`. diff --git a/docs/internal/web-ui-overhaul-pr1-plan.md b/docs/internal/web-ui-overhaul-pr1-plan.md new file mode 100644 index 000000000..0d1458ae3 --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr1-plan.md @@ -0,0 +1,2111 @@ +# Web UI overhaul PR 1: foundation and shell — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task by task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the ten-tab top bar with a grouped, collapsible sidebar and a +slim top bar that holds global search, and give every workspace the same page +header, palette, and type rules, without removing any capability. + +**Architecture:** New focused shell components (`AppSidebar`, +`NavigationDrawer`, `DisplayMenu`, `PageHeader`, and the reworked `SearchBar`) +live in `web/src/lib/components/shell/` and `web/src/lib/components/search/`. +`AppShell.svelte` composes them and keeps ownership of navigation state, +shortcuts, and focus. A palette file outside `src/` retunes kit-ui tokens. + +**Tech stack:** Svelte 5 (runes), `@kenn-io/kit-ui` at pin `e43c820f`, +`@lucide/svelte` 1.26.0, Vitest with Testing Library (jsdom), Playwright. + +**Spec:** [Web UI overhaul](web-ui-overhaul-design.md). Read its "Visual +language", "Information architecture", and "Control relocation" sections before +starting. This plan covers delivery item 1 only. + +## Global constraints + +- Keep every capability. A control that moves keeps its accessible name unless + the spec names the change. +- Keep URL state keys, the `explore` JSON format, and API calls unchanged. +- No new npm dependencies. Icons come from `@lucide/svelte` + (`@lucide/svelte/icons/`). +- One solid primary button per screen: `tone="info" surface="solid"`. +- Sentence case for every visible label, heading, table header, and chip. +- Monospace only for identifiers, hashes, code, keys, and cron text. +- Raw colors appear only in `web/palette.css`. Components use tokens; + `bun run check:kit-ui` enforces this for `src/`. +- Tests use Vitest `expect` and Testing Library for unit tests, and Playwright + for browser tests. Prefer role and accessible-name queries. +- Run commands from `web/` unless a step says otherwise. +- Commit after each task with the `kenn-io-commit` skill. The repository is + public: run the private-data scan before each commit. + +## File structure + +| File | Responsibility | +|---|---| +| `web/palette.css` (new) | kit-ui token values: neutral surfaces, one blue accent, status colors, system fonts, type scale, light and dark | +| `web/src/main.ts` | Imports the palette after kit-ui's theme and before `app.css` | +| `web/src/styles/tokens.css` | Semantic aliases; `[data-mono]` and `[data-section-label]` rules | +| `web/src/app.css` | Base element rules and the few kit-class overrides kit has no token for | +| `web/src/lib/commands/registry.ts` | Adds the Shift+Space command so the dialog documents it | +| `web/src/lib/explore/state.svelte.ts` | Adds `commitSearchIn(workspace, query, mode)` | +| `web/src/lib/components/search/SearchBar.svelte` | Global search form: live or draft mode, narrow layout | +| `web/src/lib/components/shell/navigation.ts` (new) | Sidebar groups, labels, icons, and workspace ids | +| `web/src/lib/components/shell/AppSidebar.svelte` (new) | Grouped nav, icon rail, footer (status and shortcuts), collapse toggle | +| `web/src/lib/components/shell/NavigationDrawer.svelte` (new) | Narrow-screen modal: scrim, Escape, focus trap, focus restoration | +| `web/src/lib/components/shell/DisplayMenu.svelte` (new) | Per-tab density override and "Use daemon theme" | +| `web/src/lib/components/shell/PageHeader.svelte` (new) | Title, description, actions, and optional view row | +| `web/src/lib/components/shell/AppShell.svelte` | Composes the shell; wires search, sidebar, drawer, title | +| `web/src/lib/components/shell/EverythingWorkspace.svelte` | Loses its search form and keyboard footer; uses `PageHeader` | +| `web/src/lib/components/explore/SelectionBar.svelte` | Loses its key badges | +| Each workspace component | Uses `PageHeader`; loses eyebrows; sentence case | +| `web/src/App.svelte`, `web/src/lib/components/auth/Login.svelte` | Boot and sign-in screens use the palette and lose eyebrows | +| `web/tests/kit-ui.ts` | Navigation and display-menu helpers for Playwright | + +--- + +### Task 1: Palette and type rules + +**Files:** +- Create: `web/palette.css` +- Modify: `web/src/main.ts`, `web/src/styles/tokens.css`, `web/src/app.css` +- Modify (uppercase removal): `DeletionsWorkspace.svelte:462`, + `OperationRunDetail.svelte:152`, `OperationsWorkspace.svelte:377`, + `OperationRelatedStatus.svelte:167`, `CardDAVConflicts.svelte:308`, + `SourcesWorkspace.svelte:473`, `FilesPresentation.svelte:357`, + `GroupTable.svelte:437`, `IdentityCandidateCard.svelte:89`, + `EverythingTable.svelte:693`, `KeyboardHelp.svelte:78`, + `RelationshipReviewCard.svelte:61`, `FilesWorkspace.svelte:1107`, + `PersonTimeline.svelte:54`, `SavedViewsWorkspace.svelte:361` + +**Interfaces:** +- Produces: kit tokens with new values; `--nav-width: 232px`, + `--nav-rail-width: 56px`, `--nav-active-bg`, `--header-height: 48px`. + Later tasks read these tokens. + +CSS has no meaningful unit test. This task is verified by `check:kit-ui`, +the build, and screenshot review in Task 13. + +- [ ] **Step 1: Capture before screenshots** + +The fixture daemon from the design review runs at `http://127.0.0.1:18733`. If +it is not running, start it as described in +`docs/screenshots/generate-web-fixture-screenshots.sh`, with an isolated +`--home`. Save captures of all ten workspaces at 1440×900 light, 1440×900 dark, +and 420×860 light into the scratchpad `before/` directory, using the script +from the design review (`shoot.mjs`). + +- [ ] **Step 2: Create `web/palette.css`** + +```css +/* + * msgvault retunes kit-ui's tokens for a quiet archive workspace: platform + * system fonts, neutral surfaces, one blue accent, and green, amber, and red + * for status only. kit-ui components read only these tokens, so declaring + * them after kit-ui/theme.css restyles every component consistently. + * + * This is the palette definition file, so raw colors belong here. It lives + * outside src/ because kit-ui-check guards component styles, not palettes. + */ +:root { + --bg-primary: #f7f8fa; + --bg-surface: #ffffff; + --bg-surface-hover: #f2f4f7; + --bg-inset: #f4f5f7; + --border-default: #dde1e6; + --border-muted: #eceef1; + + --text-primary: #1b1f24; + --text-secondary: #464e58; + --text-muted: #666e79; + + --accent-blue: #0061d5; + --accent-amber: #b45309; + --accent-purple: #6d4ad6; + --accent-green: #047857; + --accent-red: #c62828; + --accent-teal: #0e7490; + + --nav-active-bg: #e6eefb; + + --shadow-sm: 0 1px 2px rgba(16, 24, 40, 0.05); + --shadow-md: 0 4px 12px rgba(16, 24, 40, 0.08); + --shadow-lg: 0 16px 40px rgba(16, 24, 40, 0.16); + --overlay-bg: rgba(16, 24, 40, 0.32); + + --radius-sm: 4px; + --radius-md: 6px; + --radius-lg: 8px; + + --font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, Roboto, + "Helvetica Neue", Arial, sans-serif; + --font-mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, + "Liberation Mono", monospace; + --letter-spacing-label: 0; + + --font-size-2xs: 0.6875rem; + --font-size-xs: 0.75rem; + --font-size-sm: 0.8125rem; + --font-size-md: 0.875rem; + --font-size-lg: 0.9375rem; + --font-size-xl: 1.125rem; + --font-size-2xl: 1.5rem; + + --header-height: 48px; + --nav-width: 232px; + --nav-rail-width: 56px; +} + +:root.dark { + --bg-primary: #151618; + --bg-surface: #1c1d20; + --bg-surface-hover: #26282b; + --bg-inset: #222427; + --border-default: #34373c; + --border-muted: #2a2c30; + + --text-primary: #ececee; + --text-secondary: #b8bcc3; + --text-muted: #8f949c; + + --accent-blue: #4d9cff; + --accent-amber: #f0a64a; + --accent-purple: #a995f5; + --accent-green: #4cc38a; + --accent-red: #f27474; + --accent-teal: #3fc1d6; + + --nav-active-bg: #243650; + + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.35); + --shadow-lg: 0 16px 40px rgba(0, 0, 0, 0.5); + --overlay-bg: rgba(0, 0, 0, 0.55); +} + +/* Repeats kit-ui's handheld type scale, which the base block above would + * otherwise override by source order. */ +@media (hover: none) and (pointer: coarse) { + :root { + --font-size-2xs: 0.75rem; + --font-size-xs: 0.875rem; + --font-size-sm: 0.9375rem; + --font-size-md: 1rem; + --font-size-lg: 1.0625rem; + --font-size-xl: 1.25rem; + --font-size-2xl: 1.75rem; + } +} +``` + +- [ ] **Step 3: Import the palette** + +In `web/src/main.ts`, change the import block to: + +```ts +import '@kenn-io/kit-ui/theme.css'; +import '../palette.css'; +import './app.css'; +``` + +In `web/src/app.css`, delete the line `@import '@kenn-io/kit-ui/theme.css';`, +because `main.ts` now imports it first. + +- [ ] **Step 4: Update the type rules in `web/src/styles/tokens.css`** + +1. Delete the `--text-muted: color-mix(...)` override and its comment. The + palette's `--text-muted` meets WCAG AA on both surfaces. +2. Replace the `[data-mono]` block and its comment with: + +```css +/* Data values (timestamps, counts, sizes) keep the UI typeface with tabular + * figures so columns align. Identifiers use or [data-metadata]. */ +:where([data-mono]) { + font-variant-numeric: tabular-nums; +} +``` + +3. Replace the `[data-section-label]` block and its comment with: + +```css +/* Pane headers, month markers, and group headings: small sentence-case + * labels, never uppercase. */ +:where([data-section-label]) { + color: var(--text-muted); + font-family: var(--font-sans); + font-size: var(--font-size-xs); + font-weight: 600; +} +``` + +- [ ] **Step 5: Move identifiers off `[data-mono]`** + +Run `rg -n 'data-mono' src/lib --glob '*.svelte'`. For each hit, decide: +- The value is an ID, hash, raw code, email address in a code-like position, + or cron text: replace `data-mono` with `data-metadata`. +- The value is a timestamp, count, duration, or size: keep `data-mono`. + +List the files you changed in the commit body. + +- [ ] **Step 6: Remove uppercase styling** + +In each file listed under **Files**, delete `text-transform: uppercase;` and +any `letter-spacing` on the same rule. Where the rule also sets +`font-weight: 800`, change it to `600`. In `OperationsWorkspace.svelte:377` and +`OperationRelatedStatus.svelte:167` also change +`color: var(--status-warning-ink)` to `color: var(--text-muted)`, because +eyebrows are not warnings. Task 10 removes those eyebrows entirely. + +- [ ] **Step 7: Add kit overrides for treatments kit has no token for** + +Append to `web/src/app.css`: + +```css +/* kit-ui has no tokens for these treatments. If kit renames the classes, + * these overrides stop applying; check them when bumping the kit pin. */ +.kit-chip { + text-transform: none; + letter-spacing: 0; +} + +.kit-table th, +.kit-table-header-cell { + text-transform: none; + letter-spacing: 0; + color: var(--text-muted); + font-weight: 600; +} +``` + +Before committing, confirm each selector exists in kit with +`rg -n 'kit-chip\b|kit-table th|kit-table-header-cell' node_modules/@kenn-io/kit-ui/src/lib/components`. +Delete any rule whose selector kit does not use. + +- [ ] **Step 8: Verify** + +Run: `bun run check && bun run check:kit-ui && bun run build` +Expected: all pass with no warnings. + +Capture the ten workspaces again into `after-task1/` and compare with +`before/`. Nothing should be uppercase, and no text should fall below AA +contrast. + +- [ ] **Step 9: Commit** + +Subject: `feat(web): adopt the quiet kenn palette and sentence-case type`. + +--- + +### Task 2: Document Shift+Space in the shortcut registry + +The Everything footer is the only place that documents Shift+Space +(extending a selection). Task 9 removes the footer, so the registry must +document it first. + +**Files:** +- Modify: `web/src/lib/commands/registry.ts` +- Modify: `web/src/lib/components/shell/AppShell.svelte` (`relay`, + `relayGridKey`, `commandHandlers`) +- Test: `web/src/lib/components/shell/KeyboardHelp.test.ts`, + `web/src/lib/components/shell/AppShell.test.ts` + +**Interfaces:** +- Produces: command id `extend-selection`, label + `Extend selection to focused row`, keys `['Shift', 'Space']`, combo + `shift+space`, section `Selection`. + +- [ ] **Step 1: Write the failing tests** + +Add to `KeyboardHelp.test.ts`: + +```ts +it('documents extending a selection with Shift+Space', () => { + const handlers = Object.fromEntries( + COMMAND_DEFINITIONS.map(({ id }) => [id, () => undefined]) + ) as CommandHandlers; + render(KeyboardHelp, { commands: createCommandRegistry(handlers), onclose: () => undefined }); + + expect(screen.getByText('Extend selection to focused row')).toBeTruthy(); +}); +``` + +Import `COMMAND_DEFINITIONS`, `createCommandRegistry`, and `CommandHandlers` +from `../../commands/registry` if the file does not already. + +Add to `AppShell.test.ts`, next to the existing selection-shortcut tests: + +```ts +it('extends a selection with Shift+Space pressed outside the grid', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rows = [0, 1, 2].map((index) => entry(index)); + render(AppShell, { + client: createAPIClient(vi.fn(async () => Response.json(exploreResponse({ rows, total_count: 3 })))), + state + }); + const grid = await screen.findByRole('grid', { name: 'Everything results' }); + grid.focus(); + await fireEvent.keyDown(grid, { key: ' ' }); + await fireEvent.keyDown(grid, { key: 'j' }); + await fireEvent.keyDown(grid, { key: 'j' }); + document.body.focus(); + + await fireEvent.keyDown(window, { key: ' ', shiftKey: true }); + + await waitFor(() => expect(screen.getByText('3 selected')).toBeTruthy()); + state.destroy(); +}); +``` + +`entry(index)` is the existing row fixture in `AppShell.test.ts`. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/KeyboardHelp.test.ts src/lib/components/shell/AppShell.test.ts -t "Shift"` +Expected: FAIL. The dialog lacks the label, and the selection stays at 1. + +- [ ] **Step 3: Add the command** + +In `registry.ts`, after the `toggle-selection` entry, add: + +```ts + command('extend-selection', 'Extend selection to focused row', ['Shift', 'Space'], ['shift+space'], 'Selection'), +``` + +In `AppShell.svelte`, let `relay` and `relayGridKey` carry modifier keys: + +```ts +function relayGridKey(event: KeyboardEvent, key: string, init: KeyboardEventInit = {}): void { + if (event.target instanceof Element && event.target.closest('button, a, summary, [role="button"]')) return; + const grid = currentGrid(); + if (!grid || event.target === grid) return; + grid.focus(); + grid.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: false, cancelable: true, ...init })); +} +``` + +```ts +function relay(event: KeyboardEvent | undefined, key: string | undefined = undefined, init: KeyboardEventInit = {}): void { + const resolvedKey = key ?? event?.key; + if (!resolvedKey) return; + if (event) { + relayGridKey(event, resolvedKey, init); + return; + } + queueMicrotask(() => { + const grid = currentGrid(); + if (!grid) return; + grid.focus(); + grid.dispatchEvent(new KeyboardEvent('keydown', { key: resolvedKey, bubbles: false, cancelable: true, ...init })); + }); +} +``` + +Add the handler to `commandHandlers`: + +```ts +'extend-selection': (event) => relay(event, ' ', { shiftKey: true }), +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. +Expected: PASS. Then run `bunx vitest run src/lib/commands src/lib/components/shell` +and confirm nothing else regressed. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): list Shift+Space in the keyboard shortcuts dialog`. + +--- + +### Task 3: One history entry for a search that changes workspace + +**Files:** +- Modify: `web/src/lib/explore/state.svelte.ts` +- Test: `web/src/lib/explore/state.test.ts` + +**Interfaces:** +- Produces: + `ExploreState.commitSearchIn(workspace: ExploreWorkspace, query: string, searchMode: ExploreSearchMode): void`. + It pushes one history entry that sets the workspace, query, and mode. It + clears the same transient fields as `commitWorkspace`, and it remembers the + mode like `commitSearch`. + +- [ ] **Step 1: Write the failing test** + +```ts +it('commits a search into another workspace as one history entry', () => { + window.history.replaceState(null, '', '/?workspace=sources&mode=full_text'); + const state = new ExploreState(window); + const before = window.history.length; + + state.commitSearchIn('everything', 'quarterly report', 'hybrid'); + + expect(window.history.length).toBe(before + 1); + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe('quarterly report'); + expect(state.current.searchMode).toBe('hybrid'); + expect(state.current.selectedRow).toBeNull(); + state.destroy(); +}); +``` + +The existing state tests drive the jsdom `window` directly, as this test does. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/lib/explore -t "one history entry"` +Expected: FAIL with `state.commitSearchIn is not a function`. + +- [ ] **Step 3: Implement** + +Add after `commitWorkspace` in `state.svelte.ts`: + +```ts + commitSearchIn(workspace: ExploreWorkspace, query: string, searchMode: ExploreSearchMode): void { + rememberSearchMode(searchMode, this.preferenceStorage); + this.navigate({ + workspace, + query, + searchMode, + analysisTarget: null, + selectedIdentifier: null, + activeRow: null, + selectedRow: null, + conversationAnchor: null, + scrollAnchor: null, + operationStatus: '', + settingsAuthority: '' + }, 'push'); + } +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): commit a search into another workspace in one step`. + +--- + +### Task 4: Global search form + +`search/SearchBar.svelte` is imported only by its own test. It becomes the +top-bar search. It keeps today's Everything names: form "Search Everything", +searchbox "Search everything", radiogroup "Search mode", button "Search". + +**Files:** +- Modify: `web/src/lib/components/search/SearchBar.svelte` +- Test: `web/src/lib/components/search/SearchBar.test.ts` (replace its + contents) + +**Interfaces:** +- Consumes: `SearchModeControl` (unchanged), `ExploreSearchMode`. +- Produces: `SearchBar` props: + - `query: string` — committed query from explore state + - `mode: ExploreSearchMode` + - `live: boolean` — true on Everything and Files + - `compact: boolean` — true below 900px + - `onDraft: (query: string, mode: ExploreSearchMode) => void` — called while + typing or changing mode, only when `live` + - `onSubmit: (query: string, mode: ExploreSearchMode) => void` + - `inputEl?: HTMLInputElement` — `$bindable`, for `/` and "Refine search" + +- [ ] **Step 1: Write the failing tests** + +Replace `SearchBar.test.ts` with: + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import SearchBar from './SearchBar.svelte'; + +function setup(live: boolean) { + const onDraft = vi.fn(); + const onSubmit = vi.fn(); + render(SearchBar, { query: 'budget', mode: 'full_text', live, compact: false, onDraft, onSubmit }); + return { onDraft, onSubmit, input: screen.getByRole('searchbox', { name: 'Search everything' }) }; +} + +describe('SearchBar', () => { + it('reports each keystroke when live', async () => { + const { onDraft, input } = setup(true); + await fireEvent.input(input, { target: { value: 'budget q3' } }); + expect(onDraft).toHaveBeenLastCalledWith('budget q3', 'full_text'); + }); + + it('keeps typing local when not live and submits the draft', async () => { + const { onDraft, onSubmit, input } = setup(false); + await fireEvent.input(input, { target: { value: ' merger ' } }); + await fireEvent.click(screen.getByRole('radio', { name: 'Hybrid' })); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + expect(onDraft).not.toHaveBeenCalled(); + expect(onSubmit).toHaveBeenCalledWith('merger', 'hybrid'); + }); + + it('shows the committed query again when it changes', async () => { + const { input } = setup(false); + expect((input as HTMLInputElement).value).toBe('budget'); + }); + + it('offers the search mode as a select when compact', () => { + render(SearchBar, { query: '', mode: 'semantic', live: true, compact: true, onDraft: vi.fn(), onSubmit: vi.fn() }); + expect(screen.getByRole('combobox', { name: /^Search mode:/ })).toBeTruthy(); + expect(screen.queryByRole('radiogroup', { name: 'Search mode' })).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/search/SearchBar.test.ts` +Expected: FAIL, because the current component uses different names and props. + +- [ ] **Step 3: Implement** + +Replace `SearchBar.svelte` with: + +```svelte + + +
+ + +``` + +Check the `KbdBadge` class name with +`rg -n 'class="kit-kbd' node_modules/@kenn-io/kit-ui/src/lib/components/KbdBadge.svelte` +and adjust the `:global()` selector to match. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 6: Commit** + +Subject: `feat(web): add a grouped sidebar with an icon rail`. + +--- + +### Task 6: Narrow-screen navigation drawer + +**Files:** +- Create: `web/src/lib/components/shell/NavigationDrawer.svelte` +- Test: `web/src/lib/components/shell/NavigationDrawer.test.ts` + +**Interfaces:** +- Produces the `NavigationDrawer` props: `onclose: () => void` and + `children: Snippet`. The parent mounts the drawer only while it is open. +- Behavior contract: + - Mounting pushes the shortcut scope `navigation-drawer`. That suspends all + root shortcuts, including `close-layer`, so Escape cannot also close the + reading pane. + - Escape, registered in that scope, calls `onclose`. + - Selecting the scrim calls `onclose`. + - On mount, focus moves to the drawer's `[aria-current="page"]` button. kit + `trapFocus` keeps Tab inside and, on unmount, restores focus to the element + that had it before, which is the "Open navigation" button. + - Choosing an item also restores focus to "Open navigation". The spec's + "workspace-change focus rules" leave focus on the navigation control that + was used; in the drawer that control is the menu button. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { appShortcuts } from '@kenn-io/kit-ui'; +import { createRawSnippet } from 'svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import NavigationDrawer from './NavigationDrawer.svelte'; + +const items = createRawSnippet(() => ({ + render: () => '' +})); + +describe('NavigationDrawer', () => { + it('focuses the current item and closes on Escape without reaching root shortcuts', async () => { + const root = vi.fn(); + const unregister = appShortcuts.register('escape', root); + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + + expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Everything' })); + await fireEvent.keyDown(document.activeElement!, { key: 'Escape' }); + + expect(onclose).toHaveBeenCalledOnce(); + expect(root).not.toHaveBeenCalled(); + unregister(); + }); + + it('closes when the scrim is selected', async () => { + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + await fireEvent.click(screen.getByRole('button', { name: 'Close navigation' })); + expect(onclose).toHaveBeenCalledOnce(); + }); + + it('returns focus to the opener when unmounted', () => { + const opener = document.createElement('button'); + document.body.append(opener); + opener.focus(); + const rendered = render(NavigationDrawer, { onclose: vi.fn(), children: items }); + rendered.unmount(); + expect(document.activeElement).toBe(opener); + opener.remove(); + }); +}); +``` + +The first test relies on `initShortcuts` having attached kit's keydown +listener. If it is not attached in unit tests, call `initShortcuts()` in a +`beforeAll` and its returned cleanup in `afterAll`, as `AppShell.svelte` does. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/NavigationDrawer.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte + + +
+ + +
+ + +``` + +`trapFocus` runs its initial focus before `tick()` resolves. The `tick()` +callback then moves focus to the current item, which is inside the trap. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): add the narrow-screen navigation drawer`. + +--- + +### Task 7: Display menu + +**Files:** +- Create: `web/src/lib/components/shell/DisplayMenu.svelte` +- Test: `web/src/lib/components/shell/DisplayMenu.test.ts` + +**Interfaces:** +- Produces the `DisplayMenu` props: + - `density: 'daemon' | 'compact' | 'comfortable'` — `'daemon'` means no + override + - `themeOverridden: boolean` + - `onDensityChange: (value: 'daemon' | 'compact' | 'comfortable') => void` + - `onUseDaemonTheme: () => void` +- Accessible names: trigger button "Display"; radio group "Temporary + density" with radios "Auto", "Compact", and "Comfortable"; menu item "Use + daemon theme", shown only when `themeOverridden`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import DisplayMenu from './DisplayMenu.svelte'; + +describe('DisplayMenu', () => { + it('sets and clears the per-tab density override', async () => { + const onDensityChange = vi.fn(); + render(DisplayMenu, { density: 'daemon', themeOverridden: false, onDensityChange, onUseDaemonTheme: vi.fn() }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + + const group = screen.getByRole('group', { name: 'Temporary density' }); + expect(group).toBeTruthy(); + await fireEvent.click(screen.getByRole('menuitemradio', { name: 'Comfortable' })); + expect(onDensityChange).toHaveBeenCalledWith('comfortable'); + expect(screen.queryByRole('menuitem', { name: 'Use daemon theme' })).toBeNull(); + }); + + it('offers the daemon theme only while a theme override is active', async () => { + const onUseDaemonTheme = vi.fn(); + render(DisplayMenu, { density: 'compact', themeOverridden: true, onDensityChange: vi.fn(), onUseDaemonTheme }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Use daemon theme' })); + expect(onUseDaemonTheme).toHaveBeenCalledOnce(); + }); +}); +``` + +Check the roles kit renders with +`rg -n 'role=' node_modules/@kenn-io/kit-ui/src/lib/components/MenuRadio*.svelte node_modules/@kenn-io/kit-ui/src/lib/components/MenuItem.svelte`. +If the group role is `radiogroup`, use that in the test. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/DisplayMenu.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte + + + + + + + onDensityChange(value as Density)}> + Auto + Compact + Comfortable + + {#if themeOverridden} + + Use daemon theme + {/if} + + +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): move per-tab display overrides into a menu`. + +--- + +### Task 8: Page header + +**Files:** +- Create: `web/src/lib/components/shell/PageHeader.svelte` +- Test: `web/src/lib/components/shell/PageHeader.test.ts` + +**Interfaces:** +- Produces the `PageHeader` props: + - `title: string` + - `description?: string` + - `actions?: Snippet` + - `view?: Snippet` — a row below the title for view switches + - `visuallyHiddenTitle?: boolean` — defaults to false. It exists only for + the narrow Operations detail view, which renders its own heading. + +- [ ] **Step 1: Write the failing test** + +```ts +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { describe, expect, it } from 'vitest'; + +import PageHeader from './PageHeader.svelte'; + +describe('PageHeader', () => { + it('renders one level-one heading, the description, and actions', () => { + const actions = createRawSnippet(() => ({ render: () => '' })); + render(PageHeader, { title: 'Operations', description: 'Background work and its history.', actions }); + + expect(screen.getByRole('heading', { level: 1, name: 'Operations' })).toBeTruthy(); + expect(screen.getByText('Background work and its history.')).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Refresh operations' })).toBeTruthy(); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/lib/components/shell/PageHeader.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte + + +
+
+
+

{title}

+ {#if description}

{description}

{/if} +
+ {#if actions}
{@render actions()}
{/if} +
+ {#if view}
{@render view()}
{/if} +
+ + +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): add a shared page header`. + +--- + +### Task 9: Compose the shell + +This task replaces kit `TopBar` in `AppShell.svelte` with the new layout. It +moves the search form out of Everything and removes the keyboard footer and +the selection-bar key badges. + +**Files:** +- Modify: `web/src/lib/components/shell/AppShell.svelte` +- Modify: `web/src/lib/components/shell/EverythingWorkspace.svelte` +- Modify: `web/src/lib/components/explore/SelectionBar.svelte` +- Modify: `web/src/App.svelte` (remove the static ``) +- Test: `web/src/lib/components/shell/AppShell.test.ts`, + `web/src/lib/components/shell/EverythingWorkspace.test.ts` + +**Interfaces:** +- Consumes: `SearchBar` (Task 4), `AppSidebar`, `NAVIGATION_GROUPS`, + `workspaceLabel`, `SIDEBAR_COLLAPSED_KEY` (Task 5), `NavigationDrawer` + (Task 6), `DisplayMenu` (Task 7), `ExploreState.commitSearchIn` (Task 3). +- `EverythingWorkspace` changes from `bind:searchInput` to a plain prop + `searchInput: HTMLInputElement | undefined`. "Refine search" still calls + `searchInput?.focus()`. The `commitSearch` and `SearchCoverage` props are + unchanged. + +- [ ] **Step 1: Write the failing tests** + +Add to `AppShell.test.ts`. Replace the existing test "presents the primary +navigation tabs…" with the first test below; it checks the new order and +labels. + +```ts +it('groups workspaces in the sidebar with Relationships first', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rendered = render(AppShell, { + client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false + }); + const nav = screen.getByRole('navigation', { name: 'Primary' }); + expect(within(nav).getAllByRole('button').map((button) => button.textContent?.trim())).toEqual([ + 'Relationships', 'Directory', 'Reviews', 'Everything', 'Files', 'Saved views', 'Sources', 'Operations', 'Deletions', 'Settings' + ]); + expect(within(nav).getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe('page'); + rendered.unmount(); + state.destroy(); +}); + +it('names the browser tab after the workspace', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'sources' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json({ sources: [] }))), state, enabled: false }); + await waitFor(() => expect(document.title).toBe('Sources · msgvault')); + await fireEvent.click(screen.getByRole('button', { name: 'Directory' })); + await waitFor(() => expect(document.title).toBe('Directory · msgvault')); + state.destroy(); +}); + +it('opens Everything with the query when searching from another workspace', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'sources' }))}`); + const state = new ExploreState(window); + const length = window.history.length; + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + + const search = screen.getByRole('searchbox', { name: 'Search everything' }); + await fireEvent.input(search, { target: { value: 'pipeline' } }); + expect(state.current.workspace).toBe('sources'); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe('pipeline'); + expect(window.history.length).toBe(length + 1); + state.destroy(); +}); + +it('updates Everything results as the global search is typed', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + await fireEvent.input(screen.getByRole('searchbox', { name: 'Search everything' }), { target: { value: 'gas' } }); + expect(state.current.query).toBe('gas'); + state.destroy(); +}); + +it('remembers the collapsed sidebar across reloads', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const first = render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + await fireEvent.click(screen.getByRole('button', { name: 'Collapse sidebar' })); + first.unmount(); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + expect(screen.getByRole('button', { name: 'Expand sidebar' })).toBeTruthy(); + state.destroy(); +}); + +it('opens a modal navigation menu on narrow screens and closes it on Escape', async () => { + window.innerWidth = 480; + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + + expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull(); + const opener = screen.getByRole('button', { name: 'Open navigation' }); + await fireEvent.click(opener); + const current = screen.getByRole('button', { name: 'Everything' }); + expect(document.activeElement).toBe(current); + + await fireEvent.keyDown(current, { key: 'Escape' }); + await waitFor(() => expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull()); + expect(document.activeElement).toBe(opener); + window.innerWidth = 1024; + state.destroy(); +}); +``` + +Also update the test "focuses search with slash…". It must still pass with +the searchbox in the top bar and without the Everything search form. + +In `EverythingWorkspace.test.ts`, delete the assertions that query the +Everything search form or the keyboard footer. Add: + +```ts +it('leaves searching to the global search box', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + const main = screen.getByRole('main', { name: 'Everything' }); + expect(within(main).queryByRole('search')).toBeNull(); + expect(screen.getAllByRole('search')).toHaveLength(1); + expect(screen.queryByRole('contentinfo', { name: 'Keyboard shortcuts' })).toBeNull(); + state.destroy(); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell` +Expected: the new tests FAIL. + +- [ ] **Step 3: Restructure the AppShell markup** + +Replace the `<TopBar …>…</TopBar>` block and the `app-shell` wrapper with this +structure. Keep every `{#if exploreState.current.workspace === …}` branch +inside `<div class="app-main">` unchanged, and keep the elements that follow +the wrapper (palette, keyboard help, viewers) as they are. + +```svelte +<svelte:window bind:innerWidth={viewportWidth} /> + +<div class="app-shell" class:app-shell--narrow={narrow}> + <span class="kit-sr-only" role="status" aria-label="Operation status" aria-live="polite"> + {#key operationAnnouncement.key}<span>{operationAnnouncement.message}</span>{/key} + </span> + {#if !narrow} + <AppSidebar active={exploreState.current.workspace} collapsed={sidebarCollapsed} showCollapseToggle + status={archiveStatus} onNavigate={openWorkspaceTab} onToggleCollapsed={toggleSidebar} + onOpenShortcuts={() => { keyboardHelpOpen = true; }} /> + {/if} + <div class="app-column" inert={drawerOpen}> + <header class="app-top-bar"> + {#if narrow} + <IconButton label="Open navigation" onclick={() => { drawerOpen = true; }} bind:element={drawerOpener}> + <Menu size={18} aria-hidden="true" /> + </IconButton> + {/if} + <SearchBar query={exploreState.current.query} mode={exploreState.current.searchMode} + live={exploreState.current.workspace === 'everything' || exploreState.current.workspace === 'files'} + compact={narrow} bind:inputEl={searchInput} onDraft={(query, mode) => exploreState.replaceSearchDraft(query, mode)} + onSubmit={submitGlobalSearch} /> + <div class="app-top-bar__end" aria-label="Appearance controls"> + <ThemeToggle /> + <DisplayMenu density={appearance.temporary.density ?? 'daemon'} + themeOverridden={appearance.temporary.theme !== undefined} + onDensityChange={applyTemporaryDensity} onUseDaemonTheme={() => appearance.clearTemporary('theme')} /> + </div> + </header> + <div class="app-main"> + <!-- existing workspace branches, unchanged --> + </div> + </div> + {#if narrow && drawerOpen} + <NavigationDrawer onclose={() => { drawerOpen = false; }}> + <AppSidebar active={exploreState.current.workspace} collapsed={false} showCollapseToggle={false} + status={archiveStatus} onNavigate={(id) => { drawerOpen = false; openWorkspaceTab(id); }} + onToggleCollapsed={() => undefined} onOpenShortcuts={() => { drawerOpen = false; keyboardHelpOpen = true; }} /> + </NavigationDrawer> + {/if} +</div> +``` + +Check the `IconButton` prop names with +`sed -n '/interface Props/,/}: Props/p' node_modules/@kenn-io/kit-ui/src/lib/components/IconButton.svelte`. +If it has no element binding, wrap it in a `<span bind:this>` and focus the +`button` inside. `trapFocus` restores focus to the opener either way, so the +reference is needed only if restoration fails in the browser test. + +- [ ] **Step 4: Add the script state and functions** + +In `AppShell.svelte`'s script: + +```ts +import AppSidebar from './AppSidebar.svelte'; +import DisplayMenu from './DisplayMenu.svelte'; +import NavigationDrawer from './NavigationDrawer.svelte'; +import SearchBar from '../search/SearchBar.svelte'; +import { SIDEBAR_COLLAPSED_KEY, workspaceLabel } from './navigation'; +import Menu from '@lucide/svelte/icons/menu'; +``` + +Add `IconButton` to the kit-ui import. Remove `TopBar`, `SelectDropdown`, +`StatusDot`, `Button` (if no longer used), and the `tabs` and +`densityOptions` constants. + +```ts +const NARROW_WIDTH = 900; +let viewportWidth = $state(typeof window === 'undefined' ? 1280 : window.innerWidth); +const narrow = $derived(viewportWidth < NARROW_WIDTH); +let drawerOpen = $state(false); +let drawerOpener = $state<HTMLElement>(); +let sidebarCollapsed = $state(readSidebarCollapsed()); + +function readSidebarCollapsed(): boolean { + try { + return localStorage.getItem(SIDEBAR_COLLAPSED_KEY) === 'true'; + } catch { + return false; + } +} + +function toggleSidebar(): void { + sidebarCollapsed = !sidebarCollapsed; + try { + localStorage.setItem(SIDEBAR_COLLAPSED_KEY, String(sidebarCollapsed)); + } catch { + // Storage may be disabled; the rail still toggles for this page view. + } +} + +$effect(() => { + if (!narrow) drawerOpen = false; +}); + +$effect(() => { + document.title = `${workspaceLabel(exploreState.current.workspace)} · msgvault`; +}); + +const archiveStatus = $derived( + loader.loading + ? { tone: 'working' as const, label: 'Searching', text: 'Searching' } + : loader.error || loader.unavailable + ? { tone: 'unclean' as const, label: 'Archive needs attention', text: 'Attention' } + : { tone: 'idle' as const, label: 'Local archive ready', text: 'Local archive' } +); + +function submitGlobalSearch(query: string, mode: ExploreSearchMode): void { + const workspace = exploreState.current.workspace; + if (workspace === 'everything' || workspace === 'files') { + commitSearch(query, mode); + focusGrid(); + return; + } + beforeCommit(); + exploreState.commitSearchIn('everything', query, mode); +} +``` + +`applyTemporaryDensity` keeps its body. It now receives +`'daemon' | 'compact' | 'comfortable'` from `DisplayMenu`. + +- [ ] **Step 5: Update EverythingWorkspace** + +In `EverythingWorkspace.svelte`: +1. Delete the `<form class="search-bar" …>…</form>` block, the + `submitSearch` function, and the `.search-bar` and `.query-control` styles + and their media-query rules. +2. Change the `searchInput` prop from `$bindable()` to a plain prop, and + update the `Props` interface. +3. Delete the `<footer class="keyboard-help" …>` block, its styles, and the + `KbdBadge` import if nothing else uses it. +4. Keep "Refine search", `SearchCoverage`, and every other block. + +In `AppShell.svelte`, change `bind:searchInput` on `<EverythingWorkspace>` to +`{searchInput}`. + +In `SelectionBar.svelte`, delete the "Space toggle" and "A visible" key-badge +markup at lines 65–66 and any styles only they use. + +In `App.svelte`, delete `<svelte:head><title>Everything · msgvault`. +AppShell now owns the title. The boot and login screens set theirs in Task 11. + +- [ ] **Step 6: Add the layout styles** + +Replace the `.app-shell`, `.brand`, `.app-shell :global(.kit-top-bar…)`, +`.archive-state…`, and `.appearance-controls` rules with: + +```css +.app-shell { + display: flex; + height: 100vh; + min-height: 100vh; + overflow: hidden; + background: var(--bg-primary); + color: var(--text-primary); +} + +.app-column { + display: flex; + min-width: 0; + flex: 1; + flex-direction: column; +} + +.app-top-bar { + display: flex; + min-height: var(--header-height); + align-items: center; + gap: var(--space-3); + padding: 0 var(--space-5); + background: var(--bg-surface); + border-bottom: 1px solid var(--border-default); +} + +.app-shell--narrow .app-top-bar { + padding: 0 var(--space-3); +} + +.app-top-bar__end { + display: flex; + align-items: center; + gap: var(--space-2); + margin-left: auto; +} + +.app-main { + display: flex; + min-height: 0; + flex: 1; + flex-direction: column; + overflow: hidden; +} +``` + +Keep `.files-shell` and `.archive-navigation-status`. + +- [ ] **Step 7: Run the tests to verify they pass** + +Run: `bunx vitest run src/lib/components/shell src/lib/components/search src/lib/components/explore` +Expected: PASS. Fix any older test that looked for the Everything search form +inside `main[aria-label="Everything"]`; the searchbox is now in the top bar. + +- [ ] **Step 8: Check types and build** + +Run: `bun run check && bun run check:kit-ui && bun run build` +Expected: no errors or warnings. + +- [ ] **Step 9: Commit** + +Subject: `feat(web): move navigation to a sidebar and search to the top bar`. + +--- + +### Task 10: Page headers and sentence case in every workspace + +**Files:** +- Modify: `EverythingWorkspace.svelte`, `FilesWorkspace.svelte`, + `SavedViewsWorkspace.svelte`, `SourcesWorkspace.svelte`, + `OperationsWorkspace.svelte`, `OperationRelatedStatus.svelte`, + `DeletionsWorkspace.svelte`, `DirectoryWorkspace.svelte`, + `DirectoryReviewCentre.svelte`, `RelationshipsWorkspace.svelte`, + `SettingsWorkspace.svelte`, `PersonTimeline.svelte` (eyebrow only), + and `AppShell.svelte` (`.files-shell`) +- Test: `web/src/lib/components/shell/AppShell.test.ts` + +**Interfaces:** +- Consumes: `PageHeader` (Task 8). + +Use these titles and descriptions. Existing descriptions are kept where they +already say what the page is for. + +| Workspace | Title | Description | Actions and view row | +|---|---|---|---| +| Relationships | Relationships | People and domains you've exchanged messages with. | none | +| Directory | Directory | People you've saved, with profiles and contact details. | Promote to person (when present) | +| Reviews | Reviews | Decide which identities and facts belong together. | view row: the existing "Review type" control | +| Everything | Everything | none | `actions`: the existing Preview position control and result count | +| Files | Files (Attachments when person-scoped) | none | file count in `actions` | +| Saved views | Saved views | Searches and layouts you've saved to reuse. | none | +| Sources | Sources | Accounts and imports in your archive, and when they last synced. | View source operations | +| Operations | Operations | Background work and its history. | Refresh operations | +| Deletions | Deletions | Deletions you've staged. Nothing is deleted until you run `msgvault delete-staged`. | none | +| Settings | Settings | How this daemon and the web app behave. | none | + +The "Live authority" related-status view in `OperationRelatedStatus.svelte` +keeps its own `h1`, because it replaces the Operations page body. Remove only +its eyebrow paragraph. + +- [ ] **Step 1: Write the failing test** + +```ts +it.each([ + ['relationships', 'Relationships'], ['directory', 'Directory'], ['directory_review', 'Reviews'], + ['everything', 'Everything'], ['files', 'Files'], ['saved_views', 'Saved views'], + ['sources', 'Sources'], ['operations', 'Operations'], ['deletions', 'Deletions'] +])('shows one visible page title in %s', async (workspace, title) => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn(async () => Response.json(exploreResponse()))), state, enabled: false }); + const headings = await screen.findAllByRole('heading', { level: 1 }); + expect(headings.map((heading) => heading.textContent?.trim())).toEqual([title]); + expect(headings[0]!.closest('.kit-sr-only')).toBeNull(); + expect(screen.queryByText(/archive workspace|archive operations/i)).toBeNull(); + state.destroy(); +}); +``` + +Settings is covered separately, because it renders through the `settings` +snippet. Add a test in `SettingsWorkspace.test.ts` that finds the visible +level-one heading "Settings". + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/AppShell.test.ts -t "page title"` +Expected: FAIL for Relationships (screen-reader-only title), Saved views +(title case), and the workspaces with eyebrows. + +- [ ] **Step 3: Apply `PageHeader`** + +For each workspace, replace its header markup with `` using the +table above. Move existing action buttons into the `actions` snippet without +changing their labels or handlers. Delete eyebrow paragraphs and their CSS. +Delete each workspace's local `h1` styles. + +Use the same outer padding everywhere: +`padding: var(--space-5) var(--space-6) var(--space-4);`, with +`padding-inline: var(--space-4)` below 760px. Data workspaces fill the width: +remove `max-width` and `margin-inline: auto` from `.everything-workspace`, +`.files-shell`, and `.saved-views`. Saved views keeps a readable width on its +form content only: `max-width: 960px` on the form and list, not on the page. + +For Relationships, render `PageHeader` above the list and detail split, and +remove the screen-reader-only `h1`. For Settings, replace the +screen-reader-only `h1` with `PageHeader` above `SettingsLayout`. + +For Files, move the Everything `ContextBar` in `AppShell.svelte`'s `files-shell` +below the Files `PageHeader`. Do this by moving `FilesWorkspace`'s header into +`AppShell`'s `files-shell` above `ContextBar`. `FilesWorkspace` keeps rendering +its own header when `personScoped`, because Relationships and Directory embed +it without the shell. + +- [ ] **Step 4: Apply sentence case** + +Change these visible strings, keeping the accessible names the spec names: +- `main aria-label="Saved Views"` → `"Saved views"`; heading and empty state + "No Saved Views yet" → "No saved views yet"; "Loading Saved Views…" → + "Loading saved views…"; modal "Delete Saved View?" → "Delete saved view?" +- The command palette entry "Open Reviews" keeps its label; it names a + workspace. + +Then run `rg -n "Saved Views" src --glob '*.svelte'` and change each remaining +visible occurrence the same way. Playwright's default name match is +case-insensitive, so specs that use `'Saved Views'` without `exact: true` keep +working. Task 12 fixes any spec that uses `exact: true`. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `bunx vitest run src/lib/components` +Expected: PASS. + +- [ ] **Step 6: Check and commit** + +Run: `bun run check && bun run check:kit-ui` +Subject: `feat(web): give every workspace the same page header`. + +--- + +### Task 11: Sign-in and boot screens + +**Files:** +- Modify: `web/src/App.svelte` (boot markup and styles) +- Modify: `web/src/lib/components/auth/Login.svelte` +- Test: `web/src/App.test.ts` + +- [ ] **Step 1: Write the failing test** + +Add to `App.test.ts`, following its existing session fixtures: + +```ts +it('titles the connecting screen msgvault', () => { + const session = createSessionController(() => new Promise(() => undefined)); + render(App, { session }); + expect(screen.getByRole('main', { name: 'Connecting' })).toBeDefined(); + expect(document.title).toBe('msgvault'); +}); +``` + +The eyebrow removal is visual and is checked in the Task 13 screenshots. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/App.test.ts -t "connecting screen msgvault"` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +In `App.svelte`, add +`{#if !shellMounted || messageID !== undefined}msgvault{/if}`. +Replace each `

msgvault

` with a brand line matching the +sidebar brand: `

msgvault

`. Use sentence case and +palette tokens in the `.boot` styles: 14px base, `--text-muted` for secondary +text, one solid "Retry" button. + +In `Login.svelte`, make the same eyebrow change. Keep the heading "Log in" +and every field name. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `bunx vitest run src/App.test.ts src/lib/components/auth` +Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): match the sign-in and boot screens to the new palette`. + +--- + +### Task 12: Browser tests + +**Files:** +- Modify: `web/tests/kit-ui.ts` +- Modify: `web/tests/density-restoration.spec.ts`, + `web/tests/theme-keyboard.spec.ts`, + `web/tests/docs-fixture-screenshots.spec.ts`, + `web/tests/e2e/accessibility.spec.ts`, and any spec found by the searches in + Step 1 +- Create: `web/tests/shell-navigation.spec.ts` + +**Interfaces:** +- Produces in `tests/kit-ui.ts`: + - `selectWorkspace(page, label)` — replaces `selectKitTopBarTab` + - `setTemporaryDensity(page, 'Auto' | 'Compact' | 'Comfortable')` + +- [ ] **Step 1: Find the affected specs** + +Run: +```bash +rg -ln "selectKitTopBarTab|Temporary density|Density: |name: 'Primary'|keyboard-help|Preview position" tests +rg -n "exact: true" tests | rg -i "saved views" +``` + +- [ ] **Step 2: Update the helpers** + +Replace `selectKitTopBarTab` in `tests/kit-ui.ts` with: + +```ts +export async function selectWorkspace(page: Page, label: string): Promise { + const opener = page.getByRole('button', { name: 'Open navigation' }); + if (await opener.isVisible()) await opener.click(); + await page.getByRole('navigation', { name: 'Primary' }).getByRole('button', { name: label, exact: true }).click(); +} + +export async function setTemporaryDensity( + page: Page, + density: 'Auto' | 'Compact' | 'Comfortable' +): Promise { + await page.getByRole('button', { name: 'Display' }).click(); + await page.getByRole('menuitemradio', { name: density }).click(); + await page.keyboard.press('Escape'); +} +``` + +Rename every `selectKitTopBarTab` call to `selectWorkspace`, and use sentence +case labels ("Saved views"). Replace every +`selectKitOption(page, 'Temporary density', 'Density: X')` with +`setTemporaryDensity(page, 'X')`. Replace assertions on the combobox +"Temporary density: …" with assertions on the checked radio: +`page.getByRole('menuitemradio', { name: 'Compact', checked: true })` after +opening the Display menu. + +- [ ] **Step 3: Write the new spec** + +`tests/shell-navigation.spec.ts` uses the same archive fixture routing as +`session-navigation.spec.ts`; copy its `beforeEach` setup. + +```ts +import { expect, test } from '@playwright/test'; + +import { selectWorkspace } from './kit-ui'; + +test('sidebar rail keeps names and survives reload', async ({ page }) => { + await page.goto('/?workspace=everything'); + await page.getByRole('button', { name: 'Collapse sidebar' }).click(); + await expect(page.getByRole('navigation', { name: 'Primary' }).getByRole('button', { name: 'Saved views' })).toBeVisible(); + await page.reload(); + await expect(page.getByRole('button', { name: 'Expand sidebar' })).toBeVisible(); +}); + +test('narrow navigation menu traps focus and closes three ways', async ({ page }) => { + await page.setViewportSize({ width: 420, height: 860 }); + await page.goto('/?workspace=everything'); + const opener = page.getByRole('button', { name: 'Open navigation' }); + + await opener.click(); + await expect(page.getByRole('button', { name: 'Everything' })).toBeFocused(); + for (let index = 0; index < 20; index += 1) await page.keyboard.press('Tab'); + await expect(page.getByRole('dialog', { name: 'Navigation' }).locator(':focus')).toHaveCount(1); + await page.keyboard.press('Escape'); + await expect(opener).toBeFocused(); + + await opener.click(); + await page.getByRole('button', { name: 'Close navigation' }).click({ position: { x: 400, y: 400 } }); + await expect(opener).toBeFocused(); + + await selectWorkspace(page, 'Sources'); + await expect(page.getByRole('heading', { level: 1, name: 'Sources' })).toBeVisible(); + await expect(page.getByRole('dialog', { name: 'Navigation' })).toHaveCount(0); +}); + +test('Escape in the narrow menu leaves an open reading pane open', async ({ page }) => { + await page.setViewportSize({ width: 420, height: 860 }); + await page.goto('/?workspace=everything'); + await page.getByRole('grid', { name: 'Everything results' }).getByRole('row').nth(1).click(); + await expect(page.getByRole('complementary', { name: /^Reading pane/ })).toBeVisible(); + await page.getByRole('button', { name: 'Open navigation' }).click(); + await page.keyboard.press('Escape'); + await expect(page.getByRole('complementary', { name: /^Reading pane/ })).toBeVisible(); +}); + +test('global search from another workspace opens Everything', async ({ page }) => { + await page.goto('/?workspace=sources'); + await page.getByRole('searchbox', { name: 'Search everything' }).fill('fixture'); + await page.keyboard.press('Enter'); + await expect(page.getByRole('main', { name: 'Everything' })).toBeVisible(); + await page.goBack(); + await expect(page.getByRole('main', { name: 'Sources' })).toBeVisible(); +}); +``` + +- [ ] **Step 4: Run the browser tests** + +From the repository root, run: `make web-test-browser` +Expected: all specs pass, including `e2e/accessibility` (axe) in both themes. + +- [ ] **Step 5: Commit** + +Subject: `test(web): cover the sidebar, navigation menu, and global search`. +In the body, list every existing spec edit and why it was needed, as the spec's +"Tests and accessible names" section requires. + +--- + +### Task 13: Verify and prepare the pull request + +- [ ] **Step 1: Run the full web checks** + +From the repository root: +`make web-check && make web-test && make web-test-browser && make lint-ci` +Expected: all pass with no warnings. + +- [ ] **Step 2: Capture after screenshots** + +Rebuild the branch binary with `make build` and restart the isolated fixture +daemon with the new binary. Capture the same set as Task 1 Step 1, plus the +collapsed rail at 1440×900 and the open narrow menu at 420×860, into `after/`. +View every image. Check that: +- no page shows an eyebrow, uppercase label, or duplicate title; +- one blue primary button at most per screen; +- the top bar never wraps at 420px; +- dark mode has no unreadable text. + +- [ ] **Step 3: Review the diff** + +Run `git diff main...HEAD --stat` and read the full diff. Remove unused +imports, styles, and props. Confirm nothing in `web/src/lib/api/generated` +changed. + +- [ ] **Step 4: Open the pull request** + +Use the `kenn-io-commit-push-pr` skill, and the `kenn-io-pr-desc` skill for +the body. Run the private-data scrub on the body and on every screenshot. The +Enron fixture captures are the documented exception in `AGENTS.md`. + +The body describes the result for people using the Web UI, what reviewers +should check (accessible-name changes, kit overrides in `app.css`, the palette +outside `src/`), and before and after screenshots. It has no validation or +test-plan section. + +## Self-review notes + +- **Spec coverage for delivery item 1:** + - Palette, type, and status tones: Task 1. + - Sidebar with rail, footer, and collapse storage: Task 5 and Task 9. + - Narrow menu with scrim, Escape, focus trap, focus restoration, and Escape + precedence: Task 6, Task 9, and Task 12. + - Top bar, global search, and `/`: Task 3, Task 4, and Task 9. + - Display menu: Task 7 and Task 9. + - `PageHeader` on every workspace: Task 8 and Task 10. + - Keyboard registry and footer removal: Task 2 and Task 9. + - Tab titles: Task 9 and Task 11. + - Sign-in and boot screens: Task 11. +- **Deferred to later pull requests, per the spec:** empty-state copy changes + in Deletions and Saved views, the code-label maps, the selection-bar + redesign, and everything else in delivery items 2 through 4. +- **Focus after choosing a drawer item** returns to "Open navigation". This is + the drawer's equivalent of today's rule, where focus stays on the tab you + clicked. diff --git a/web/palette.css b/web/palette.css new file mode 100644 index 000000000..1c70bcbde --- /dev/null +++ b/web/palette.css @@ -0,0 +1,98 @@ +/* + * msgvault retunes kit-ui's tokens for a quiet archive workspace: platform + * system fonts, neutral surfaces, one blue accent, and green, amber, and red + * for status only. kit-ui components read only these tokens, so declaring + * them after kit-ui/theme.css restyles every component consistently. + * + * This is the palette definition file, so raw colors belong here. It lives + * outside src/ because kit-ui-check guards component styles, not palettes. + */ +:root { + --bg-primary: #f7f8fa; + --bg-surface: #ffffff; + --bg-surface-hover: #f2f4f7; + --bg-inset: #f4f5f7; + --border-default: #dde1e6; + --border-muted: #eceef1; + + --text-primary: #1b1f24; + --text-secondary: #464e58; + --text-muted: #5f6772; + + --accent-blue: #0061d5; + --accent-amber: #b45309; + --accent-purple: #6d4ad6; + --accent-green: #047857; + --accent-red: #c62828; + --accent-teal: #0e7490; + + --nav-active-bg: #e6eefb; + + --shadow-sm: 0 1px 2px rgba(16, 24, 40, 0.05); + --shadow-md: 0 4px 12px rgba(16, 24, 40, 0.08); + --shadow-lg: 0 16px 40px rgba(16, 24, 40, 0.16); + --overlay-bg: rgba(16, 24, 40, 0.32); + + --radius-sm: 4px; + --radius-md: 6px; + --radius-lg: 8px; + + --font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Helvetica Neue", Helvetica, Arial, sans-serif; + --font-mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, + "Liberation Mono", monospace; + --letter-spacing-label: 0; + + --font-size-2xs: 0.6875rem; + --font-size-xs: 0.75rem; + --font-size-sm: 0.8125rem; + --font-size-md: 0.875rem; + --font-size-lg: 0.9375rem; + --font-size-xl: 1.125rem; + --font-size-2xl: 1.5rem; + + --header-height: 48px; + --nav-width: 232px; + --nav-rail-width: 56px; +} + +:root.dark { + --bg-primary: #151618; + --bg-surface: #1c1d20; + --bg-surface-hover: #26282b; + --bg-inset: #222427; + --border-default: #34373c; + --border-muted: #2a2c30; + + --text-primary: #ececee; + --text-secondary: #b8bcc3; + --text-muted: #92979f; + + --accent-blue: #4d9cff; + --accent-amber: #f0a64a; + --accent-purple: #a995f5; + --accent-green: #4cc38a; + --accent-red: #f27474; + --accent-teal: #3fc1d6; + + --nav-active-bg: #243650; + + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.35); + --shadow-lg: 0 16px 40px rgba(0, 0, 0, 0.5); + --overlay-bg: rgba(0, 0, 0, 0.55); +} + +/* Repeats kit-ui's handheld type scale, which the base block above would + * otherwise override by source order. */ +@media (hover: none) and (pointer: coarse) { + :root { + --font-size-2xs: 0.75rem; + --font-size-xs: 0.875rem; + --font-size-sm: 0.9375rem; + --font-size-md: 1rem; + --font-size-lg: 1.0625rem; + --font-size-xl: 1.25rem; + --font-size-2xl: 1.75rem; + } +} diff --git a/web/src/App.svelte b/web/src/App.svelte index 72035dd03..b925c03fd 100644 --- a/web/src/App.svelte +++ b/web/src/App.svelte @@ -81,7 +81,7 @@ pathname = window.location.pathname} /> - Everything · msgvault + {#if !shellMounted || messageID !== undefined}msgvault{/if} {#if oauthCallback} @@ -106,14 +106,51 @@ {/if} {:else if session.error !== undefined}
-

msgvault

+

msgvault

Can't reach the msgvault daemon

{session.error}

{:else}
-

msgvault

+

msgvault

Connecting…

{/if} + + diff --git a/web/src/App.test.ts b/web/src/App.test.ts index 8799c1e92..5c1c27b7b 100644 --- a/web/src/App.test.ts +++ b/web/src/App.test.ts @@ -26,6 +26,12 @@ describe('application foundation', () => { expect(await screen.findByRole('form', { name: 'Log in' })).toBeDefined(); expect(screen.queryByRole('main', { name: 'Relationships' })).toBeNull(); }); + it('titles the connecting screen msgvault', () => { + const session = createSessionController(() => new Promise(() => undefined)); + render(App, { session }); + expect(screen.getByRole('main', { name: 'Connecting' })).toBeDefined(); + expect(document.title).toBe('msgvault'); + }); it('shows a bootstrap error with retry instead of the shell, and recovers on retry', async () => { let sessionCalls = 0; const fetchFn = vi.fn(async (input) => { diff --git a/web/src/app.css b/web/src/app.css index 3b36747f5..d2a354e6c 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -1,4 +1,3 @@ -@import '@kenn-io/kit-ui/theme.css'; @import './styles/tokens.css'; @import './styles/density.css'; @@ -24,3 +23,14 @@ select, textarea { font: inherit; } + +/* Keep unstyled section headings below the page title (h1, --font-size-xl). */ +h2 { + font-size: var(--font-size-lg); + font-weight: 600; +} + +h3 { + font-size: var(--font-size-md); + font-weight: 600; +} diff --git a/web/src/lib/commands/registry.ts b/web/src/lib/commands/registry.ts index 4028838e9..df9224cf0 100644 --- a/web/src/lib/commands/registry.ts +++ b/web/src/lib/commands/registry.ts @@ -13,7 +13,8 @@ export const COMMAND_DEFINITIONS = [ command('close-layer', 'Close current layer or restore context', ['Esc'], ['escape'], 'Navigate'), command('focus-search', 'Focus search', ['/'], ['/'], 'Navigate'), command('toggle-selection', 'Toggle focused row selection', ['Space'], ['space'], 'Selection'), - command('select-visible', 'Select all visible rows', ['A'], ['shift+a'], 'Selection'), + command('extend-selection', 'Extend selection to focused row', ['Shift', 'Space'], ['shift+space'], 'Selection'), + command('select-visible', 'Select all visible rows', ['A'], ['a', 'shift+a'], 'Selection'), command('clear-selection', 'Clear selection', ['x'], ['x'], 'Selection'), command('review-delete-selected', 'Review selected messages for deletion', ['d'], ['d'], 'Safety', true), command('review-delete-matching', 'Review all matching messages for deletion', ['D'], ['shift+d'], 'Safety', true), diff --git a/web/src/lib/components/auth/Login.svelte b/web/src/lib/components/auth/Login.svelte index 45871d2e2..51bd23a5a 100644 --- a/web/src/lib/components/auth/Login.svelte +++ b/web/src/lib/components/auth/Login.svelte @@ -14,7 +14,7 @@
-

msgvault

+

Log in

Enter the API key configured for this daemon.

@@ -42,3 +42,51 @@ />
+ + diff --git a/web/src/lib/components/deletions/DeletionsWorkspace.svelte b/web/src/lib/components/deletions/DeletionsWorkspace.svelte index 1cd296812..fc7598637 100644 --- a/web/src/lib/components/deletions/DeletionsWorkspace.svelte +++ b/web/src/lib/components/deletions/DeletionsWorkspace.svelte @@ -9,6 +9,7 @@ import { Button, Card, KbdBadge, Modal, appShortcuts } from '@kenn-io/kit-ui'; import { onDestroy, onMount } from 'svelte'; import type { APIClient } from '../../api/client'; + import PageHeader from '../shell/PageHeader.svelte'; import type { DeletionManifestDetail as GeneratedDeletionManifestDetail, DeletionManifestSummary as GeneratedDeletionManifestSummary, @@ -266,13 +267,10 @@
-
-
-

Archive workspace

-

Deletions

-
- Staged manifest lifecycle -
+ {#if error}{/if} @@ -435,9 +433,8 @@ flex: 1; flex-direction: column; gap: var(--space-4); - padding: var(--space-5) var(--space-6); + padding: var(--space-5) var(--space-6) var(--space-4); } - header, article, .actions, .staging { @@ -445,23 +442,10 @@ align-items: center; gap: var(--space-3); } - header { - justify-content: space-between; - } - header p, - h1, h2, .staging p { margin: 0; } - header p { - color: var(--status-warning-ink); - font-size: var(--font-size-2xs); - font-weight: 800; - letter-spacing: 0.1em; - text-transform: uppercase; - } - header span, article span, .staging p, .actions span, @@ -506,6 +490,9 @@ border-color: var(--accent-red); } @media (max-width: 760px) { + .deletions { + padding-inline: var(--space-4); + } article, .staging { align-items: stretch; diff --git a/web/src/lib/components/directory/DirectoryReviewCentre.svelte b/web/src/lib/components/directory/DirectoryReviewCentre.svelte index 78d6b051f..af9cbdd68 100644 --- a/web/src/lib/components/directory/DirectoryReviewCentre.svelte +++ b/web/src/lib/components/directory/DirectoryReviewCentre.svelte @@ -13,6 +13,7 @@ import IdentityDecisionModal from './IdentityDecisionModal.svelte'; import FactReviewPanel from './FactReviewPanel.svelte'; import RelationshipReviewQueue from './RelationshipReviewQueue.svelte'; + import PageHeader from '../shell/PageHeader.svelte'; import type { RelationshipReviewController } from '../../directory/relationship-review-controller.svelte'; import PersonBindingConflictModal from './PersonBindingConflictModal.svelte'; import type { PersonMergeSuccess, ValidatedPersonMergeRequired } from '../../directory/person-merge'; @@ -127,19 +128,17 @@
-
-
-

Reviews

-

Inspect identity evidence and imported relationship review records.

-
- -
+ + {#snippet view()} + + {/snippet} + {#if controller.reviewKind === 'identity'}
@@ -252,11 +251,11 @@ {/if} diff --git a/web/src/lib/components/directory/DirectoryWorkspace.svelte b/web/src/lib/components/directory/DirectoryWorkspace.svelte index 74a98f794..d625dd330 100644 --- a/web/src/lib/components/directory/DirectoryWorkspace.svelte +++ b/web/src/lib/components/directory/DirectoryWorkspace.svelte @@ -8,6 +8,7 @@ import { DirectoryController } from '../../directory/controller.svelte'; import { bufferedCallback } from '../../util/buffered-callback'; import DirectoryList from './DirectoryList.svelte'; + import PageHeader from '../shell/PageHeader.svelte'; import PersonDetail from './PersonDetail.svelte'; interface Props { @@ -129,12 +130,13 @@
-
-

Directory

Durable people and their recorded contact context.

- {#if promotionParticipantID !== undefined} -
+ + {#snippet actions()} + {#if promotionParticipantID !== undefined} +
diff --git a/web/src/lib/components/operations/OperationRunDetail.svelte b/web/src/lib/components/operations/OperationRunDetail.svelte index 634aae95e..03d803814 100644 --- a/web/src/lib/components/operations/OperationRunDetail.svelte +++ b/web/src/lib/components/operations/OperationRunDetail.svelte @@ -149,7 +149,7 @@ .detail { display: grid; align-content: start; gap: var(--space-4); min-width: 0; padding: var(--space-4); } header { display: flex; align-items: start; justify-content: space-between; gap: var(--space-3); } h2, h3, p, dl, dd { margin: 0; } - header p { color: var(--text-muted); font-size: var(--font-size-2xs); font-weight: 800; letter-spacing: .08em; text-transform: uppercase; } + header p { color: var(--text-muted); font-size: var(--font-size-2xs); font-weight: 600; } h2 { font-size: var(--font-size-lg); } h3 { margin-bottom: var(--space-2); font-size: var(--font-size-sm); } .facts, .counters { display: grid; gap: var(--space-2); } diff --git a/web/src/lib/components/operations/OperationsWorkspace.svelte b/web/src/lib/components/operations/OperationsWorkspace.svelte index 459258578..9238b64b2 100644 --- a/web/src/lib/components/operations/OperationsWorkspace.svelte +++ b/web/src/lib/components/operations/OperationsWorkspace.svelte @@ -26,6 +26,7 @@ import OperationRelatedStatus from './OperationRelatedStatus.svelte'; import OperationRunDetail from './OperationRunDetail.svelte'; import OperationRunTable from './OperationRunTable.svelte'; + import PageHeader from '../shell/PageHeader.svelte'; type RelatedStatus = NonNullable; type Controller = Pick; @@ -280,10 +281,11 @@ {/key} {:else if narrow && urlState.operationRunID !== null}
-
-

Operation detail

-
+ + {#snippet actions()} +
{:else} -
-

Archive operations

Operations

-
+ + {#snippet actions()} +
+
+ {#if layout === 'narrow'} +
diff --git a/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte b/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte index 2baacba0d..660a7c4f0 100644 --- a/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte +++ b/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte @@ -7,6 +7,7 @@ } from '../../api/generated/api/api'; import { Button, Card, EmptyState, Modal, TextInput } from '@kenn-io/kit-ui'; import { onMount } from 'svelte'; + import PageHeader from '../shell/PageHeader.svelte'; import type { APIClient } from '../../api/client'; import type { SavedView as GeneratedSavedView, @@ -43,10 +44,10 @@ error = ''; try { const { data, error: responseError } = await generatedListSavedViews(client); - if (!data) throw new Error(messageFor(responseError, 'Unable to load Saved Views.')); + if (!data) throw new Error(messageFor(responseError, 'Unable to load saved views.')); views = data.saved_views ?? []; } catch (cause) { - error = cause instanceof Error ? cause.message : 'Unable to load Saved Views.'; + error = cause instanceof Error ? cause.message : 'Unable to load saved views.'; } finally { loading = false; } @@ -118,7 +119,7 @@ ); if (!data) { if (response.status === 409) - throw new Error('This Saved View changed in another session. Reload and review the latest revision.'); + throw new Error('This saved view changed in another session. Reload and review the latest revision.'); throw new Error(messageFor(responseError, 'Unable to update this view.')); } views = views.map((view) => (view.id === data.id ? data : view)); @@ -193,13 +194,8 @@ } -
-
-
-

Archive workspace

-

Saved Views

-
-
+
+ {#if error}{/if} @@ -224,11 +220,11 @@ {#if loading} -

Loading Saved Views…

+

Loading saved views…

{:else if views.length === 0} - + {:else} -
+
{#each views as view (view.id)} {@const incompatibility = incompatibilityFor(view)}
@@ -302,7 +298,7 @@ {#if deleting} { deleting = undefined; @@ -332,34 +328,26 @@ .saved-views { display: flex; width: 100%; - max-width: 1080px; min-height: 0; flex: 1; flex-direction: column; gap: var(--space-4); - margin-inline: auto; - padding: var(--space-5) var(--space-6); + padding: var(--space-5) var(--space-6) var(--space-4); + } + .saved-views > :global(:not(header)) { + width: 100%; + max-width: 960px; } - header, article, .actions { display: flex; align-items: center; gap: var(--space-3); } - header p, - h1, h2, article p { margin: 0; } - header p { - color: var(--status-warning-ink); - font-size: var(--font-size-2xs); - font-weight: 800; - letter-spacing: 0.1em; - text-transform: uppercase; - } article p { color: var(--text-muted); font-size: var(--font-size-xs); @@ -401,6 +389,9 @@ color: var(--text-danger); } @media (max-width: 760px) { + .saved-views { + padding-inline: var(--space-4); + } .create { grid-template-columns: 1fr; } diff --git a/web/src/lib/components/saved-views/SavedViewsWorkspace.test.ts b/web/src/lib/components/saved-views/SavedViewsWorkspace.test.ts index 6615b76b4..05b8170b8 100644 --- a/web/src/lib/components/saved-views/SavedViewsWorkspace.test.ts +++ b/web/src/lib/components/saved-views/SavedViewsWorkspace.test.ts @@ -58,7 +58,7 @@ describe('SavedViewsWorkspace', () => { const component = render(SavedViewsWorkspace, { client: createAPIClient(fetchFn), currentState: { ...currentState, query, searchMode } }); - await screen.findByText('No Saved Views yet'); + await screen.findByText('No saved views yet'); await fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'Invoices' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save' })); await screen.findByRole('heading', { name: 'Invoices' }); @@ -98,7 +98,7 @@ describe('SavedViewsWorkspace', () => { selection: { mode: 'all_matching', operationToken: 'session-secret' } }); - await screen.findByText('No Saved Views yet'); + await screen.findByText('No saved views yet'); await fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'Invoices' } }); await fireEvent.input(screen.getByLabelText('Description'), { target: { value: 'Quarterly review' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save' })); @@ -213,12 +213,12 @@ describe('SavedViewsWorkspace', () => { expect(requests[1]!.headers.get('If-Match')).toBe('"saved-view-7-r3"'); await fireEvent.click(screen.getByRole('button', { name: 'Delete Invoices 2026' })); - expect(screen.getByRole('dialog', { name: 'Delete Saved View?' })).toBeDefined(); + expect(screen.getByRole('dialog', { name: 'Delete saved view?' })).toBeDefined(); expect(requests).toHaveLength(2); await fireEvent.click(screen.getByRole('button', { name: 'Confirm delete' })); await waitFor(() => expect(requests).toHaveLength(3)); expect(requests[2]!.headers.get('If-Match')).toBe('"saved-view-7-r4"'); - expect(await screen.findByText('No Saved Views yet')).toBeDefined(); + expect(await screen.findByText('No saved views yet')).toBeDefined(); }); it('keeps incompatible schema records visible and offers confirmed removal, not migration', async () => { @@ -238,7 +238,7 @@ describe('SavedViewsWorkspace', () => { expect(screen.getByRole('alert').textContent).toContain('Automatic migration is not supported'); expect((screen.getByRole('button', { name: 'Open Invoices' }) as HTMLButtonElement).disabled).toBe(true); await fireEvent.click(screen.getByRole('button', { name: 'Remove incompatible Invoices' })); - expect(screen.getByRole('dialog', { name: 'Delete Saved View?' })).toBeDefined(); + expect(screen.getByRole('dialog', { name: 'Delete saved view?' })).toBeDefined(); expect(requests).toHaveLength(1); await fireEvent.click(screen.getByRole('button', { name: 'Confirm delete' })); await waitFor(() => expect(requests).toHaveLength(2)); diff --git a/web/src/lib/components/search/SearchBar.svelte b/web/src/lib/components/search/SearchBar.svelte index 8a0b9b24e..ee6817a0f 100644 --- a/web/src/lib/components/search/SearchBar.svelte +++ b/web/src/lib/components/search/SearchBar.svelte @@ -1,63 +1,112 @@ -
-
+ +
- { - mode = next === 'full_text' ? 'fts' : next === 'semantic' ? 'vector' : 'hybrid'; - }} - /> -
{#if hint && !switchedOff}{hint}{/if} @@ -606,14 +607,16 @@ {/snippet}
-

Settings

+
+ +
{#if loading}

Loading settings…

{:else} {#snippet panel(activeId)} @@ -633,6 +636,7 @@ {#if activeId === 'carddav'} +

CardDAV settings

{ expect(await screen.findByRole('heading', { name: 'Appearance' })).toBeDefined(); expect(screen.getByRole('main', { name: 'Settings' })).toBeDefined(); + const title = screen.getByRole('heading', { level: 1, name: 'Settings' }); + expect(title.closest('.kit-sr-only')).toBeNull(); + expect(screen.getAllByText('Settings')).toHaveLength(1); expect(screen.getByText('Changes apply right away.')).toBeDefined(); expect(screen.queryByText(/Restart required/)).toBeNull(); await openSettingsCategory('Daemon'); @@ -857,6 +860,8 @@ describe('SettingsWorkspace', () => { await openSettingsCategory('CardDAV account'); expect(await screen.findByRole('heading', { name: 'CardDAV account' })).toBeDefined(); + const levels = screen.getAllByRole('heading').map((heading) => Number(heading.tagName.slice(1))); + expect(levels.every((level, index) => index === 0 || level <= levels[index - 1]! + 1)).toBe(true); expect(screen.getByLabelText('Base URL')).toBeDefined(); expect(screen.getByLabelText('Username')).toBeDefined(); expect(screen.getByLabelText('Password')).toBeDefined(); diff --git a/web/src/lib/components/shell/AppShell.svelte b/web/src/lib/components/shell/AppShell.svelte index 3cc4cc6a2..a2ed17ade 100644 --- a/web/src/lib/components/shell/AppShell.svelte +++ b/web/src/lib/components/shell/AppShell.svelte @@ -2,13 +2,10 @@ import { getCLIMessageRaw as generatedGetCLIMessageRaw } from '../../api/generated/api/api'; import { preflightExploreSelection as generatedPreflightExploreSelection } from '../../api/generated/exploration/exploration'; import { - Button, CommandPalette, getThemeMode, - SelectDropdown, - StatusDot, + IconButton, ThemeToggle, - TopBar, appShortcuts, initShortcuts, type PaletteCommand, @@ -22,7 +19,6 @@ } from '../../api/generated/models'; import type { EntryRow, - ExploreColumn, ExploreGroupDimension, ExploreGroupRow, ExploreFileFact, @@ -34,7 +30,7 @@ FileSearchSort, } from '../../explore/models'; import { attachmentSelection, parseAttachmentSelection } from '../../explore/attachment-authority'; - import { filtersForGroup, parseGroupSelection } from '../../explore/group-context'; + import { filtersForGroup } from '../../explore/group-context'; import { ExploreLoader } from '../../explore/loader.svelte'; import { GROUPING_CATALOG, groupingByDimension } from '../../grouping/catalog'; import { canonicalFingerprint, predicateFingerprint } from '../../explore/selection'; @@ -71,10 +67,17 @@ import DirectoryWorkspace from '../directory/DirectoryWorkspace.svelte'; import DirectoryReviewWorkspace from '../directory/DirectoryReviewWorkspace.svelte'; import KeyboardHelp from './KeyboardHelp.svelte'; + import PageHeader from './PageHeader.svelte'; import ArchivedMeetingReader from '../meetings/ArchivedMeetingReader.svelte'; import { ArchiveMeetingNavigation, archiveMeetingSelection, parseArchiveMeetingSelection } from '../../meetings/archive-navigation.svelte'; import { ARCHIVE_MEETING_HISTORY_KEY, parseArchiveMeetingHistory } from '../../meetings/archive-selection'; import EverythingWorkspace from './EverythingWorkspace.svelte'; + import AppSidebar from './AppSidebar.svelte'; + import DisplayMenu from './DisplayMenu.svelte'; + import NavigationDrawer from './NavigationDrawer.svelte'; + import SearchBar from '../search/SearchBar.svelte'; + import { SIDEBAR_COLLAPSED_KEY, workspaceLabel } from './navigation'; + import Menu from '@lucide/svelte/icons/menu'; import { EverythingSessionState } from './EverythingSessionState.svelte'; import { bufferedCallback } from '../../util/buffered-callback'; interface Props { @@ -105,6 +108,7 @@ const archivedMeeting = new ArchiveMeetingNavigation(untrack(() => client)); let archiveReturnFocus: HTMLElement | undefined; let archiveReturnSelection = $state(null); + let fileCount = $state(null); let archiveWasOpen = false; const archiveMeetingID = $derived(parseArchiveMeetingSelection(exploreState.current.selectedRow)); const archiveNavigationFingerprint = $derived(canonicalFingerprint(exploreState.current)); @@ -328,23 +332,57 @@ let operationAnnouncement = $state({ key: 0, message: '' }); type APIExploreSelection = GeneratedExploreSelection; type ExplorePreflight = GeneratedExplorePreflightResponse; - const tabs = [ - { id: 'relationships', label: 'Relationships' }, - { id: 'directory', label: 'Directory' }, - { id: 'directory_review', label: 'Reviews' }, - { id: 'everything', label: 'Everything' }, - { id: 'files', label: 'Files' }, - { id: 'saved_views', label: 'Saved Views' }, - { id: 'sources', label: 'Sources' }, - { id: 'operations', label: 'Operations' }, - { id: 'deletions', label: 'Deletions' }, - { id: 'settings', label: 'Settings' }, - ]; - const densityOptions = [ - { value: 'daemon', label: 'Density: Auto' }, - { value: 'compact', label: 'Density: Compact' }, - { value: 'comfortable', label: 'Density: Comfortable' }, - ]; + const NARROW_WIDTH = 900; + let viewportWidth = $state(typeof window === 'undefined' ? 1280 : window.innerWidth); + const narrow = $derived(viewportWidth < NARROW_WIDTH); + let drawerOpen = $state(false); + let drawerOpenerHost = $state(); + let shellRoot = $state(); + let sidebarCollapsed = $state(readSidebarCollapsed()); + + function readSidebarCollapsed(): boolean { + try { + return localStorage.getItem(SIDEBAR_COLLAPSED_KEY) === 'true'; + } catch { + return false; + } + } + + function toggleSidebar(): void { + sidebarCollapsed = !sidebarCollapsed; + try { + localStorage.setItem(SIDEBAR_COLLAPSED_KEY, String(sidebarCollapsed)); + } catch { + // Storage may be disabled; the rail still toggles for this page view. + } + } + + function drawerOpener(): HTMLElement | null { + return drawerOpenerHost?.querySelector('button') ?? null; + } + + // Below NARROW_WIDTH the sidebar is not rendered, so the drawer opener + // stands in for the current navigation item. + function navigationFocusTarget(): HTMLElement | null { + return document.querySelector('button[aria-current="page"]') ?? drawerOpener(); + } + + // trapFocus restores focus while the page column may still be inert, so + // focus the opener again once the drawer has unmounted. + async function closeDrawer(): Promise { + drawerOpen = false; + await tick(); + drawerOpener()?.focus(); + } + + $effect(() => { + if (!narrow) drawerOpen = false; + }); + + $effect(() => { + document.title = `${workspaceLabel(exploreState.current.workspace)} · msgvault`; + }); + $effect(() => { if (exploreState.current.workspace !== 'settings') cardDAVSettingsRequest = undefined; }); @@ -414,8 +452,28 @@ // behind an {#if}, so it is destroyed and recreated on every switch away // from and back to 'everything'. const everythingSession = new EverythingSessionState(); + const archiveStatus = $derived( + loader.loading + ? { tone: 'working' as const, label: 'Searching', text: 'Searching' } + : loader.error || loader.unavailable + ? { tone: 'unclean' as const, label: 'Archive needs attention', text: 'Attention' } + : { tone: 'idle' as const, label: 'Local archive ready', text: 'Local archive' } + ); + function submitGlobalSearch(query: string, mode: ExploreSearchMode): void { + const workspace = exploreState.current.workspace; + if (workspace === 'everything' || workspace === 'files') { + commitSearch(query, mode); + // The ungrouped Files grid stays out of currentGrid() so shortcut relays keep ignoring it. + (currentGrid() ?? document.querySelector('[role="grid"][aria-label="Files results"]'))?.focus(); + return; + } + beforeCommit(); + directoryPromotionParticipantID = undefined; + exploreState.commitSearchIn('everything', query, mode); + } let paletteOpen = $state(false); let keyboardHelpOpen = $state(false); + let keyboardHelpReturnFocus: HTMLElement | undefined; let keyboardHelpScopeCleanup: (() => void) | undefined; let sortNotice = $state(DEFAULT_SORT_NOTICE); let editableScopeCleanup: (() => void) | undefined; @@ -667,7 +725,7 @@ if (exploreState.current.selectedRow === null) focusGrid(); return; } - document.querySelector('button[aria-current="page"]')?.focus(); + navigationFocusTarget()?.focus(); } async function focusGridAfterUpdate(): Promise { await tick(); @@ -678,12 +736,12 @@ '[role="grid"][aria-label="Everything results"], [role="grid"][aria-label^="Everything grouped by"], [role="grid"][aria-label="Files in current context"]', ); } - function relayGridKey(event: KeyboardEvent, key: string): void { + function relayGridKey(event: KeyboardEvent, key: string, init: KeyboardEventInit = {}): void { if (event.target instanceof Element && event.target.closest('button, a, summary, [role="button"]')) return; const grid = currentGrid(); if (!grid || event.target === grid) return; grid.focus(); - grid.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: false, cancelable: true })); + grid.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true, cancelable: true, ...init })); } async function closeReadingPane(): Promise { commitNavigation({ selectedRow: null }); @@ -709,7 +767,7 @@ // Kit releases its focus trap during teardown; focus the surviving source // link after that cleanup (or the current workspace's own control). await new Promise((resolve) => setTimeout(resolve, 0)); - const target = archiveReturnFocus?.isConnected ? archiveReturnFocus : currentGrid() ?? document.querySelector('button[aria-current="page"]'); + const target = archiveReturnFocus?.isConnected ? archiveReturnFocus : currentGrid() ?? navigationFocusTarget(); target?.focus(); } @@ -836,18 +894,18 @@ const next = loader.rows[Math.max(0, Math.min(loader.rows.length - 1, index + delta))]; if (next && next.key !== exploreState.current.selectedRow) openRow(next); } - function relay(event: KeyboardEvent | undefined, key: string | undefined = undefined): void { + function relay(event: KeyboardEvent | undefined, key: string | undefined = undefined, init: KeyboardEventInit = {}): void { const resolvedKey = key ?? event?.key; if (!resolvedKey) return; if (event) { - relayGridKey(event, resolvedKey); + relayGridKey(event, resolvedKey, init); return; } queueMicrotask(() => { const grid = currentGrid(); if (!grid) return; grid.focus(); - grid.dispatchEvent(new KeyboardEvent('keydown', { key: resolvedKey, bubbles: false, cancelable: true })); + grid.dispatchEvent(new KeyboardEvent('keydown', { key: resolvedKey, bubbles: true, cancelable: true, ...init })); }); } const commandHandlers: CommandHandlers = { @@ -865,6 +923,7 @@ if (!editableTarget(event?.target ?? null)) searchInput?.focus(); }, 'toggle-selection': (event) => relay(event, ' '), + 'extend-selection': (event) => relay(event, ' ', { shiftKey: true }), 'select-visible': (event) => relay(event, 'A'), 'clear-selection': (event) => { if (event) relay(event, 'x'); @@ -879,9 +938,7 @@ 'open-grouping': () => openContextControl('grouping'), 'change-sort': () => openContextControl('sort'), 'reverse-sort': fixedSortNotice, - 'open-keyboard-help': () => { - keyboardHelpOpen = true; - }, + 'open-keyboard-help': openKeyboardHelp, 'open-command-palette': (event) => { if (!editableTarget(event?.target ?? null)) paletteOpen = true; }, @@ -951,9 +1008,25 @@ function runPalette(command: PaletteCommand): void { commandRegistry.find(({ id }) => id === command.id)?.run(); } - function applyTemporaryDensity(value: string): void { + // KeyboardHelp's search input takes focus before the Modal's trap records + // where focus came from, so the shell remembers the return target itself. + function openKeyboardHelp(): void { + const active = document.activeElement; + keyboardHelpReturnFocus = active instanceof HTMLElement && active !== document.body ? active : undefined; + keyboardHelpOpen = true; + } + async function closeKeyboardHelp(): Promise { + keyboardHelpOpen = false; + await tick(); + const target = keyboardHelpReturnFocus?.isConnected + ? keyboardHelpReturnFocus + : (shellRoot?.querySelector('button[aria-label="Keyboard shortcuts"]') ?? drawerOpener()); + keyboardHelpReturnFocus = undefined; + target?.focus(); + } + function applyTemporaryDensity(value: 'daemon' | DensityPreference): void { if (value === 'daemon') appearance.clearTemporary('density'); - else appearance.setTemporary({ density: value as DensityPreference }); + else appearance.setTemporary({ density: value }); } function openRow(row: EntryRow): void { // Single-click selects AND opens; re-opening the already-open row must @@ -1121,302 +1194,327 @@ }); -
+ + +
{#key operationAnnouncement.key}{operationAnnouncement.message}{/key} - openWorkspaceTab(workspace as ExploreWorkspace)} - > - {#snippet left()} -
msgvault
- {/snippet} - {#snippet right()} -
- - {#if appearance.temporary.theme !== undefined} -
- - - - - {loader.loading ? 'Searching' : loader.error || loader.unavailable ? 'Attention' : 'Local archive'} - - - {/snippet} - - - {#if exploreState.current.workspace === 'settings'} - {#if settings}{@render settings(cardDAVSettingsRequest, consumeCardDAVSettingsRequest, settingsNavigationTarget)}{/if} - {:else if exploreState.current.workspace === 'saved_views'} - { - void openSavedView(state); - }} - /> - {:else if exploreState.current.workspace === 'sources'} - openOperations('messages', 'source_sync')} /> - {:else if exploreState.current.workspace === 'operations'} - commitNavigation(patch)} - onNavigate={openOperationAuthority} - onConfigure={openOperationConfiguration} - onAnnounce={announceOperation} - /> - {:else if exploreState.current.workspace === 'deletions'} - { - pendingDeletionReview = undefined; - }} + {#if !narrow} + - {:else if exploreState.current.workspace === 'relationships'} - commitNavigation({ relationshipFacet })} - onTargetChange={(relationshipTarget) => commitNavigation({ relationshipTarget, relationshipFiles: false })} - onShowAllChange={(relationshipShowAll) => commitNavigation({ relationshipShowAll })} - onFilesToggle={(relationshipFiles) => commitNavigation({ relationshipFiles })} - onPersonFilePresentationChange={(personFilePresentation) => - commitNavigation({ - personFilePresentation, - activeRow: null, - selectedRow: null, - scrollAnchor: null, - })} - onPersonFileDirectionsChange={(personFileDirections) => - commitNavigation({ - personFileDirections, - activeRow: null, - selectedRow: null, - scrollAnchor: null, - })} - onOpenEverything={() => commitWorkspace('everything')} - onOpenDirectory={openDirectoryFromRelationship} - onOpenDirectoryPerson={openDirectoryPerson} - onAnnounce={announceOperation} - onOpenFileItem={openFileItem} - onOpenFileConversation={openFileConversation} - onOpenMeeting={(meeting) => void openArchivedMeeting(meeting)} - /> - {:else if exploreState.current.workspace === 'directory'} - void openArchivedMeeting(meeting)} - promotionParticipantID={directoryPromotionParticipantID} - state={{ - directoryQuery: exploreState.current.directoryQuery, - directoryContactState: exploreState.current.directoryContactState, - directoryCategory: exploreState.current.directoryCategory, - directoryOrganization: exploreState.current.directoryOrganization, - directoryPrimaryChannel: exploreState.current.directoryPrimaryChannel, - directoryLastContactAfter: exploreState.current.directoryLastContactAfter, - directoryLastContactBefore: exploreState.current.directoryLastContactBefore, - directorySort: exploreState.current.directorySort, - directoryPersonID: exploreState.current.directoryPersonID, - }} - onOpenCardDAVConflict={openCardDAVConflict} - onOpenCardDAVSettings={openCardDAVSettings} - onAnnounce={announceOperation} - /> - {:else if exploreState.current.workspace === 'directory_review'} - commitWorkspace('directory')} - onOpenPerson={openDirectoryPerson} - onAnnounce={announceOperation} - /> - {:else if exploreState.current.workspace === 'files'} -
- +
+ {#if narrow} + + { drawerOpen = true; }}> +
+
+ {#if exploreState.current.workspace === 'settings'} + {#if settings}{@render settings(cardDAVSettingsRequest, consumeCardDAVSettingsRequest, settingsNavigationTarget)}{/if} + {:else if exploreState.current.workspace === 'saved_views'} + { + void openSavedView(state); + }} + /> + {:else if exploreState.current.workspace === 'sources'} + openOperations('messages', 'source_sync')} /> + {:else if exploreState.current.workspace === 'operations'} + commitNavigation(patch)} + onNavigate={openOperationAuthority} + onConfigure={openOperationConfiguration} + onAnnounce={announceOperation} + /> + {:else if exploreState.current.workspace === 'deletions'} + { - exploreState.acknowledgeRestoration(epoch); + selection={apiSelection} + reviewOnMount={pendingDeletionReview === apiSelection?.mode} + onReviewStarted={() => { + pendingDeletionReview = undefined; }} - onSortChange={(fileSort: FileSearchSort) => + /> + {:else if exploreState.current.workspace === 'relationships'} + commitNavigation({ relationshipFacet })} + onTargetChange={(relationshipTarget) => commitNavigation({ relationshipTarget, relationshipFiles: false })} + onShowAllChange={(relationshipShowAll) => commitNavigation({ relationshipShowAll })} + onFilesToggle={(relationshipFiles) => commitNavigation({ relationshipFiles })} + onPersonFilePresentationChange={(personFilePresentation) => commitNavigation({ - fileSort, - activeRow: null, - scrollAnchor: null, - })} - onFilenameQueryChange={(fileFilenameQuery) => - debouncedSearchPatch({ - fileFilenameQuery, + personFilePresentation, activeRow: null, selectedRow: null, scrollAnchor: null, })} - onMIMEFamiliesChange={(fileMIMEFamilies) => + onPersonFileDirectionsChange={(personFileDirections) => commitNavigation({ - fileMIMEFamilies, + personFileDirections, activeRow: null, selectedRow: null, scrollAnchor: null, })} - onActiveKey={(activeRow) => replaceTransient({ activeRow })} - onSelectedKey={(selectedRow) => - selectedRow ? commitNavigation({ selectedRow }) : replaceCommittedNavigation({ selectedRow: null })} - onOpenItem={openFileItem} - onOpenConversation={openFileConversation} + onOpenEverything={() => commitWorkspace('everything')} + onOpenDirectory={openDirectoryFromRelationship} + onOpenDirectoryPerson={openDirectoryPerson} + onAnnounce={announceOperation} + onOpenFileItem={openFileItem} + onOpenFileConversation={openFileConversation} + onOpenMeeting={(meeting) => void openArchivedMeeting(meeting)} + /> + {:else if exploreState.current.workspace === 'directory'} + void openArchivedMeeting(meeting)} + promotionParticipantID={directoryPromotionParticipantID} + state={{ + directoryQuery: exploreState.current.directoryQuery, + directoryContactState: exploreState.current.directoryContactState, + directoryCategory: exploreState.current.directoryCategory, + directoryOrganization: exploreState.current.directoryOrganization, + directoryPrimaryChannel: exploreState.current.directoryPrimaryChannel, + directoryLastContactAfter: exploreState.current.directoryLastContactAfter, + directoryLastContactBefore: exploreState.current.directoryLastContactBefore, + directorySort: exploreState.current.directorySort, + directoryPersonID: exploreState.current.directoryPersonID, + }} + onOpenCardDAVConflict={openCardDAVConflict} + onOpenCardDAVSettings={openCardDAVSettings} + onAnnounce={announceOperation} + /> + {:else if exploreState.current.workspace === 'directory_review'} + commitWorkspace('directory')} + onOpenPerson={openDirectoryPerson} + onAnnounce={announceOperation} + /> + {:else if exploreState.current.workspace === 'files'} +
+ + {#snippet actions()} + {#if fileCount !== null && exploreState.current.groupingChain.length === 0} + {fileCount.toLocaleString()} files + {/if} + {/snippet} + + 0 ? loader.result?.totalCount : undefined} + presentation="files" + onPresentationChange={(presentation) => { + if (presentation === 'files') return; + commitNavigation({ + workspace: 'everything', + presentation, + analysisTarget: null, + selectedIdentifier: null, + activeRow: null, + selectedRow: null, + conversationAnchor: null, + scrollAnchor: null, + }); + }} + onAddGroup={(dimension) => commitGrouping(dimension)} + onRemoveGroup={(index) => + commitNavigation({ + groupingChain: exploreState.current.groupingChain.filter((_, position) => position !== index), + activeRow: null, + scrollAnchor: null, + })} + onClearFilters={() => commitNavigation({ filters: [], activeRow: null, scrollAnchor: null })} + onFiltersChange={(filters) => + commitNavigation({ + filters, + activeRow: null, + selectedRow: null, + scrollAnchor: null, + })} + /> + {sortNotice} + {#if exploreState.current.groupingChain.length > 0} + replaceTransient({ activeRow })} + onScrollAnchor={(key, offset) => replaceTransient({ scrollAnchor: { key, offset } })} + onRetry={loader.retry} + /> + {:else} + { + exploreState.acknowledgeRestoration(epoch); + }} + onSortChange={(fileSort: FileSearchSort) => + commitNavigation({ + fileSort, + activeRow: null, + scrollAnchor: null, + })} + onFilenameQueryChange={(fileFilenameQuery) => + debouncedSearchPatch({ + fileFilenameQuery, + activeRow: null, + selectedRow: null, + scrollAnchor: null, + })} + onMIMEFamiliesChange={(fileMIMEFamilies) => + commitNavigation({ + fileMIMEFamilies, + activeRow: null, + selectedRow: null, + scrollAnchor: null, + })} + onActiveKey={(activeRow) => replaceTransient({ activeRow })} + onSelectedKey={(selectedRow) => + selectedRow ? commitNavigation({ selectedRow }) : replaceCommittedNavigation({ selectedRow: null })} + onOpenItem={openFileItem} + onOpenConversation={openFileConversation} + /> + {/if} +
+ {:else} + void exportSelection()} + {commitNavigation} + {commitWorkspace} + {commitGrouping} + {fixedSortNotice} + {focusGrid} + {openRow} + {drillGroup} + {openFileItem} + {openContextualFile} + closeReadingPane={() => void closeReadingPane()} + {openRelationship} + {changeConversationAnchor} + onOpenMeeting={(meeting) => void openArchivedMeeting(meeting)} /> {/if}
- {:else} - void exportSelection()} - {commitNavigation} - {commitWorkspace} - {commitGrouping} - {commitSearch} - {fixedSortNotice} - {focusGrid} - {openRow} - {drillGroup} - {openFileItem} - {openContextualFile} - closeReadingPane={() => void closeReadingPane()} - {openRelationship} - {changeConversationAnchor} - onOpenMeeting={(meeting) => void openArchivedMeeting(meeting)} - /> +
+ {#if narrow && drawerOpen} + void closeDrawer()}> + { + openWorkspaceTab(id); + void closeDrawer(); + }} + onToggleCollapsed={() => undefined} + onOpenShortcuts={() => { + void closeDrawer().then(openKeyboardHelp); + }} + /> + {/if}
@@ -1437,9 +1535,7 @@ {#if keyboardHelpOpen} { - keyboardHelpOpen = false; - }} + onclose={() => void closeKeyboardHelp()} /> {/if} @@ -1467,93 +1563,71 @@ .app-shell { display: flex; - min-width: 0; - min-height: 100vh; height: 100vh; - flex-direction: column; + min-height: 100vh; overflow: hidden; background: var(--bg-primary); color: var(--text-primary); } - .brand { - display: inline-flex; - align-items: center; - gap: var(--space-2); - color: var(--text-primary); - font-family: var(--font-sans); - font-size: var(--font-size-md); - font-weight: 650; - letter-spacing: 0.01em; - } - - .brand span { - color: var(--artifact-ink); - font-size: var(--font-size-sm); - } - - /* Machined app-bar boundary: darker hairline plus a faint sheen line. */ - .app-shell :global(.kit-top-bar) { - box-shadow: 0 1px 0 var(--hairline-sheen); + .app-column { + display: flex; + min-width: 0; + flex: 1; + flex-direction: column; } - /* Integrated app-bar tabs: quiet text buttons with a soft active pill - * instead of kit-ui's detached inset track. */ - .app-shell :global(.kit-top-bar__tabs) { - gap: var(--space-1); - padding: 0; - background: transparent; - border-radius: 0; + .app-top-bar { + display: flex; + min-height: var(--header-height); + align-items: center; + gap: var(--space-3); + padding: 0 var(--space-5); + background: var(--bg-surface); + border-bottom: 1px solid var(--border-default); } - .app-shell :global(.kit-top-bar__tab) { - padding: 5px 12px; - border-radius: var(--radius-md); - font-size: var(--font-size-md); + .app-shell--narrow .app-top-bar { + padding: 0 var(--space-3); } - .app-shell :global(.kit-top-bar__tab.active) { - background: var(--bg-subtle); - box-shadow: none; + .app-top-bar__menu { + display: contents; } - .archive-state { - display: inline-flex; + .app-top-bar__end { + display: flex; align-items: center; gap: var(--space-2); - margin-left: var(--space-3); - color: var(--text-muted); - font-size: var(--font-size-xs); - white-space: nowrap; - } - - .appearance-controls { - display: inline-flex; - gap: var(--space-2); - } - - /* Keep the longest status label's width while searches are in flight. */ - .archive-state__label { - display: inline-grid; + margin-left: auto; } - .archive-state__label > span { - grid-area: 1 / 1; - } - - .archive-state__reserve { - visibility: hidden; + .app-main { + display: flex; + min-height: 0; + flex: 1; + flex-direction: column; + overflow: hidden; } .files-shell { display: flex; width: 100%; - max-width: 1760px; min-height: 0; flex: 1; flex-direction: column; gap: var(--space-4); - margin-inline: auto; - padding: var(--space-6) var(--space-7) var(--space-4); + padding: var(--space-5) var(--space-6) var(--space-4); + } + + .files-count { + color: var(--text-muted); + font-size: var(--font-size-xs); + } + + @media (max-width: 760px) { + .files-shell { + padding-inline: var(--space-4); + } } diff --git a/web/src/lib/components/shell/AppShell.test.ts b/web/src/lib/components/shell/AppShell.test.ts index 722eb8f7d..a135ebf4f 100644 --- a/web/src/lib/components/shell/AppShell.test.ts +++ b/web/src/lib/components/shell/AppShell.test.ts @@ -1,14 +1,15 @@ -import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; import { appShortcuts } from '@kenn-io/kit-ui'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createRawSnippet } from 'svelte'; import { meetingFixtureResponse } from '../../meetings/fixtures.test-support'; import { createAPIClient } from '../../api/client'; import { LOAD_THROUGH_END_MAX_PAGES } from '../../explore/paging'; -import { ExploreState, parseExploreURLState, serializeExploreURLState } from '../../explore/state.svelte'; +import { ExploreState, serializeExploreURLState } from '../../explore/state.svelte'; import { chooseSelectOption } from '../../../test/kit-ui'; import AppShell from './AppShell.svelte'; +import { SIDEBAR_COLLAPSED_KEY } from './navigation'; function exploreResponse(overrides: Record = {}) { return { @@ -287,10 +288,12 @@ describe('AppShell', () => { }); await fireEvent.click(screen.getByRole('button', { name: 'Change theme (current: Dark)' })); - await fireEvent.click(screen.getByRole('button', { name: 'Use daemon theme' })); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Use daemon theme' })); expect(screen.getByRole('button', { name: 'Change theme (current: Dark)' })).toBeDefined(); - expect(screen.queryByRole('button', { name: 'Use daemon theme' })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + expect(screen.queryByRole('menuitem', { name: 'Use daemon theme' })).toBeNull(); expect(sessionStorage.getItem('msgvault.appearance.override')).toBeNull(); rendered.unmount(); @@ -313,7 +316,7 @@ describe('AppShell', () => { const rendered = render(AppShell, { client: createAPIClient(fetchFn), state, enabled: false }); for (const [tab, label, workspace] of [ - ['Saved Views', 'Saved Views', 'saved_views'], + ['Saved views', 'Saved views', 'saved_views'], ['Sources', 'Sources', 'sources'], ['Operations', 'Operations', 'operations'], ['Deletions', 'Deletions', 'deletions'] @@ -328,23 +331,254 @@ describe('AppShell', () => { }); - it('presents the primary navigation tabs with Relationships first and People/Domains retired', async () => { - window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); - const state = new ExploreState(window); - const rendered = render(AppShell, { - client: createAPIClient(vi.fn(async () => Response.json(exploreResponse()))), - state, enabled: false + describe('shell chrome', () => { + const states: ExploreState[] = []; + + function shellState(workspace: string): ExploreState { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace }))}`); + const state = new ExploreState(window); + states.push(state); + return state; + } + + function exploreClient() { + return createAPIClient(vi.fn(async () => Response.json(exploreResponse()))); + } + + afterEach(() => { + cleanup(); + for (const state of states.splice(0)) state.destroy(); + localStorage.removeItem(SIDEBAR_COLLAPSED_KEY); + window.innerWidth = 1024; }); - const nav = screen.getByRole('navigation', { name: 'Primary' }); - expect(within(nav).getAllByRole('button').map((button) => button.textContent?.trim())).toEqual([ - 'Relationships', 'Directory', 'Reviews', 'Everything', 'Files', 'Saved Views', 'Sources', 'Operations', 'Deletions', 'Settings' - ]); - expect(screen.queryByRole('button', { name: 'People' })).toBeNull(); - expect(screen.queryByRole('button', { name: 'Domains' })).toBeNull(); + it('groups workspaces in the sidebar with Relationships first', () => { + render(AppShell, { client: exploreClient(), state: shellState('everything'), enabled: false }); + const nav = screen.getByRole('navigation', { name: 'Primary' }); + expect(within(nav).getAllByRole('button').map((button) => button.textContent?.trim())).toEqual([ + 'Relationships', 'Directory', 'Reviews', 'Everything', 'Files', 'Saved views', 'Sources', 'Operations', 'Deletions', 'Settings' + ]); + expect(within(nav).getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe('page'); + expect(screen.queryByRole('button', { name: 'People' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Domains' })).toBeNull(); + }); - rendered.unmount(); - state.destroy(); + it('names the browser tab after the workspace', async () => { + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/sources/status')) return Response.json({ sources: [] }); + return Response.json(exploreResponse()); + }); + render(AppShell, { client: createAPIClient(fetchFn), state: shellState('sources'), enabled: false }); + expect(await screen.findByRole('main', { name: 'Sources' })).toBeDefined(); + await waitFor(() => expect(document.title).toBe('Sources · msgvault')); + await fireEvent.click(screen.getByRole('button', { name: 'Directory' })); + await waitFor(() => expect(document.title).toBe('Directory · msgvault')); + }); + + it.each([ + ['relationships', 'Relationships'], ['directory', 'Directory'], ['directory_review', 'Reviews'], + ['everything', 'Everything'], ['files', 'Files'], ['saved_views', 'Saved views'], + ['sources', 'Sources'], ['operations', 'Operations'], ['deletions', 'Deletions'] + ])('shows one visible page title in %s', async (workspace, title) => { + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/saved-views')) return Response.json({ saved_views: [] }); + if (path.endsWith('/sources/status')) return Response.json({ sources: [] }); + if (path.endsWith('/operations/status')) return Response.json({ lanes: [] }); + if (path.endsWith('/operations/runs')) return Response.json({ runs: [], unavailable_kinds: [], membership_revision: 1 }); + if (path.endsWith('/deletions')) return Response.json({ manifests: [] }); + return Response.json(exploreResponse()); + }); + render(AppShell, { client: createAPIClient(fetchFn), state: shellState(workspace), enabled: false }); + const headings = await screen.findAllByRole('heading', { level: 1 }); + expect(headings.map((heading) => heading.textContent?.trim())).toEqual([title]); + expect(headings[0]!.closest('.kit-sr-only')).toBeNull(); + expect(screen.queryByText(/archive workspace|archive operations/i)).toBeNull(); + }); + + it('puts the Files title above the Everything context bar', async () => { + render(AppShell, { client: exploreClient(), state: shellState('files'), enabled: false }); + const heading = await screen.findByRole('heading', { level: 1, name: 'Files' }); + const bar = screen.getByRole('region', { name: 'Active analytical context' }); + expect(heading.compareDocumentPosition(bar) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + + it.each([ + ['ungrouped', []], + ['grouped', ['source']] + ])('keeps the %s Files title inside one Files main landmark', async (_case, groupingChain) => { + window.history.replaceState( + null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'files', groupingChain }))}` + ); + const state = new ExploreState(window); + states.push(state); + render(AppShell, { client: exploreClient(), state, enabled: false }); + const heading = await screen.findByRole('heading', { level: 1, name: 'Files' }); + const mains = screen.getAllByRole('main', { name: 'Files' }); + expect(mains).toHaveLength(1); + expect(mains[0]!.contains(heading)).toBe(true); + expect(screen.getAllByRole('main')).toHaveLength(1); + // testing-library maps every
to banner; browsers only do so outside sectioning content. + const banners = [...document.querySelectorAll('header')].filter((header) => !header.closest('main, section, article, aside, nav')); + expect(banners).toHaveLength(1); + }); + + it('shows the file count beside the Files title without a second heading', async () => { + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/files/search')) { + return Response.json({ files: [], total_count: 7, cache_revision: 'cache-1', search_provenance: {} }); + } + return Response.json(exploreResponse()); + }); + render(AppShell, { client: createAPIClient(fetchFn), state: shellState('files'), enabled: false }); + expect(await screen.findByText('7 files')).toBeDefined(); + expect(screen.getAllByRole('heading', { level: 1 })).toHaveLength(1); + }); + + it('opens Everything with the query when searching from another workspace', async () => { + const state = shellState('sources'); + const length = window.history.length; + render(AppShell, { client: exploreClient(), state, enabled: false }); + + const search = screen.getByRole('searchbox', { name: 'Search everything' }); + await fireEvent.input(search, { target: { value: 'pipeline' } }); + expect(state.current.workspace).toBe('sources'); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe('pipeline'); + expect(window.history.length).toBe(length + 1); + }); + + it('moves focus to the Files results after submitting search on Files', async () => { + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/files/search')) { + return Response.json({ files: [], total_count: 0, cache_revision: 'cache-1', search_provenance: {} }); + } + return Response.json(exploreResponse()); + }); + const state = shellState('files'); + render(AppShell, { client: createAPIClient(fetchFn), state, enabled: false }); + const grid = await screen.findByRole('grid', { name: 'Files results' }); + + const search = screen.getByRole('searchbox', { name: 'Search everything' }); + search.focus(); + await fireEvent.input(search, { target: { value: 'invoice' } }); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + + expect(state.current.query).toBe('invoice'); + expect(document.activeElement).toBe(grid); + }); + + it('drops an unsubmitted search draft when the sidebar opens another workspace', async () => { + const state = shellState('sources'); + render(AppShell, { client: exploreClient(), state, enabled: false }); + + const search = screen.getByRole('searchbox', { name: 'Search everything' }) as HTMLInputElement; + await fireEvent.input(search, { target: { value: 'pipeline' } }); + await fireEvent.click(within(screen.getByRole('navigation', { name: 'Primary' })).getByRole('button', { name: 'Everything' })); + + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe(''); + await waitFor(() => expect(search.value).toBe('')); + }); + + it('updates Everything results as the global search is typed', async () => { + const state = shellState('everything'); + render(AppShell, { client: exploreClient(), state, enabled: false }); + await fireEvent.input(screen.getByRole('searchbox', { name: 'Search everything' }), { target: { value: 'gas' } }); + expect(state.current.query).toBe('gas'); + }); + + it('remembers the collapsed sidebar across reloads', async () => { + const state = shellState('everything'); + const first = render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + await fireEvent.click(screen.getByRole('button', { name: 'Collapse sidebar' })); + first.unmount(); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + expect(screen.getByRole('button', { name: 'Expand sidebar' })).toBeTruthy(); + }); + + it('opens a modal navigation menu on narrow screens and closes it on Escape', async () => { + window.innerWidth = 480; + render(AppShell, { client: createAPIClient(vi.fn()), state: shellState('everything'), enabled: false }); + + expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull(); + const opener = screen.getByRole('button', { name: 'Open navigation' }); + await fireEvent.click(opener); + const current = await screen.findByRole('button', { name: 'Everything' }); + await waitFor(() => expect(document.activeElement).toBe(current)); + + await fireEvent.keyDown(current, { key: 'Escape' }); + await waitFor(() => expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull()); + expect(document.activeElement).toBe(opener); + }); + + it('closes the narrow navigation menu after choosing a workspace and returns focus to its opener', async () => { + window.innerWidth = 480; + const state = shellState('everything'); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + + const opener = screen.getByRole('button', { name: 'Open navigation' }); + await fireEvent.click(opener); + await fireEvent.click(await screen.findByRole('button', { name: 'Settings' })); + + expect(state.current.workspace).toBe('settings'); + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Navigation' })).toBeNull()); + expect(document.activeElement).toBe(opener); + }); + + it('returns focus to the sidebar entry when the shortcuts dialog closes', async () => { + render(AppShell, { client: createAPIClient(vi.fn()), state: shellState('everything'), enabled: false }); + const entry = screen.getByRole('button', { name: 'Keyboard shortcuts' }); + await fireEvent.click(entry); + const search = await screen.findByRole('searchbox', { name: 'Search keyboard shortcuts' }); + + await fireEvent.keyDown(search, { key: 'Escape' }); + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Keyboard shortcuts' })).toBeNull()); + await waitFor(() => expect(document.activeElement).toBe(entry)); + }); + + it('returns focus to the prior control when the shortcuts dialog was opened with ?', async () => { + render(AppShell, { client: createAPIClient(vi.fn()), state: shellState('settings'), enabled: false }); + const theme = screen.getByRole('button', { name: /^Change theme/ }); + theme.focus(); + await fireEvent.keyDown(theme, { key: '?', shiftKey: true }); + await fireEvent.click(await screen.findByRole('button', { name: 'Close' })); + + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Keyboard shortcuts' })).toBeNull()); + await waitFor(() => expect(document.activeElement).toBe(theme)); + }); + + it('returns focus to the navigation opener when shortcuts opened from the narrow menu close', async () => { + window.innerWidth = 480; + render(AppShell, { client: createAPIClient(vi.fn()), state: shellState('everything'), enabled: false }); + const opener = screen.getByRole('button', { name: 'Open navigation' }); + await fireEvent.click(opener); + await fireEvent.click(await screen.findByRole('button', { name: 'Keyboard shortcuts' })); + const search = await screen.findByRole('searchbox', { name: 'Search keyboard shortcuts' }); + + await fireEvent.keyDown(search, { key: 'Escape' }); + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Keyboard shortcuts' })).toBeNull()); + await waitFor(() => expect(document.activeElement).toBe(opener)); + }); + + it('focuses the navigation opener after Back on narrow screens', async () => { + window.innerWidth = 480; + const state = shellState('sources'); + render(AppShell, { client: exploreClient(), state, enabled: false }); + await fireEvent.click(screen.getByRole('button', { name: 'Open navigation' })); + await fireEvent.click(await screen.findByRole('button', { name: 'Settings' })); + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Navigation' })).toBeNull()); + (document.activeElement as HTMLElement).blur(); + + window.history.back(); + await waitFor(() => expect(state.current.workspace).toBe('sources')); + await waitFor(() => expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Open navigation' }))); + }); }); it('restores Operations filters and detail through popstate and routes related authority through shell state', async () => { @@ -1464,6 +1698,58 @@ describe('AppShell', () => { state.destroy(); }); + it.each(['sidebar', 'global search'] as const)( + 'drops the relationship promotion when leaving Directory by %s and going back', + async (exit) => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ + workspace: 'relationships', relationshipTarget: 'cluster:11' + }))}`); + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + const meetingResponse = meetingFixtureResponse(path); + if (meetingResponse) return meetingResponse; + if (path === '/api/v1/relationships') return Response.json({ rows: [] }); + if (path === '/api/v1/participants/11') return Response.json({ + id: 11, display_label: 'Synthetic Candidate', partial_label: false, identifiers: [], + activity_count: 1, file_count: 0, source_counts: [], first_at: '2026-07-19T10:00:00Z', + last_at: '2026-07-19T10:00:00Z', cache_revision: 'cache-rel' + }); + if (path === '/api/v1/relationships/11/timeline') return Response.json({ + canonical_id: 11, identity_revision: 1, cache_revision: 'cache-rel', rows: [], total_count: 0 + }); + if (path === '/api/v1/people/directory') return Response.json({ people: [] }); + return Response.json(exploreResponse()); + }); + const state = new ExploreState(window); + const rendered = render(AppShell, { client: createAPIClient(fetchFn), state }); + + expect(await screen.findByRole('heading', { name: 'Synthetic Candidate' })).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Open in Directory' })); + expect(await screen.findByRole('button', { name: 'Promote to person' })).toBeDefined(); + + if (exit === 'sidebar') { + await fireEvent.click(within(screen.getByRole('navigation', { name: 'Primary' })) + .getByRole('button', { name: 'Everything' })); + } else { + await fireEvent.input(screen.getByRole('searchbox', { name: 'Search everything' }), { + target: { value: 'pipeline' } + }); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + } + expect(await screen.findByRole('main', { name: 'Everything' })).toBeDefined(); + + const restored = new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); + window.history.back(); + await restored; + expect(await screen.findByRole('main', { name: 'Directory' })).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Promote to person' })).toBeNull(); + + rendered.unmount(); + state.destroy(); + } + ); + it('renders actionable Directory guidance for a relationship promotion conflict', async () => { window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'relationships', relationshipTarget: 'cluster:11' @@ -1987,4 +2273,69 @@ describe('AppShell', () => { rendered.unmount(); state.destroy(); }); + + it('selects three rows when Shift+Space is pressed outside the grid', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rows = [0, 1, 2].map((index) => entry(index)); + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/explore/preflight')) { + return Response.json({ + count: 3, deletable_count: 3, estimated_bytes: 30, cache_revision: 'cache-1', + search_provenance: {}, unavailable_actions: [], action_targets: [] + }); + } + return Response.json(exploreResponse({ rows, total_count: 3 })); + }); + const rendered = render(AppShell, { client: createAPIClient(fetchFn), state }); + const grid = await screen.findByRole('grid', { name: 'Everything results' }); + await screen.findByText('Synthetic subject 0'); + grid.focus(); + await fireEvent.keyDown(grid, { key: ' ' }); + await fireEvent.keyDown(grid, { key: 'j' }); + await fireEvent.keyDown(grid, { key: 'j' }); + await waitFor(() => expect(screen.getByText('1 selected')).toBeTruthy()); + grid.blur(); + expect(document.activeElement).toBe(document.body); + + await fireEvent.keyDown(document.body, { key: ' ', shiftKey: true }); + + await waitFor(() => expect(screen.getByText('3 selected')).toBeTruthy()); + rendered.unmount(); + state.destroy(); + }); + + it('selects the visible rows once when plain a is pressed outside the grid', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rows = [0, 1, 2].map((index) => entry(index)); + const fetchFn = vi.fn(async (input) => { + const path = new URL(input instanceof Request ? input.url : String(input)).pathname; + if (path.endsWith('/explore/preflight')) { + return Response.json({ + count: 3, deletable_count: 3, estimated_bytes: 30, cache_revision: 'cache-1', + search_provenance: {}, unavailable_actions: [], action_targets: [] + }); + } + return Response.json(exploreResponse({ rows, total_count: 3 })); + }); + const rendered = render(AppShell, { client: createAPIClient(fetchFn), state }); + const grid = await screen.findByRole('grid', { name: 'Everything results' }); + await screen.findByText('Synthetic subject 0'); + const gridKeys: string[] = []; + grid.addEventListener('keydown', (event) => gridKeys.push(event.key)); + + await fireEvent.keyDown(document.body, { key: 'a' }); + + await waitFor(() => expect(screen.getByText('3 selected')).toBeTruthy()); + expect(gridKeys).toEqual(['A']); + + gridKeys.length = 0; + grid.focus(); + await fireEvent.keyDown(grid, { key: 'a' }); + expect(gridKeys).toEqual(['a']); + rendered.unmount(); + state.destroy(); + }); }); diff --git a/web/src/lib/components/shell/AppSidebar.svelte b/web/src/lib/components/shell/AppSidebar.svelte new file mode 100644 index 000000000..fc75c5532 --- /dev/null +++ b/web/src/lib/components/shell/AppSidebar.svelte @@ -0,0 +1,260 @@ + + + + + diff --git a/web/src/lib/components/shell/AppSidebar.test.ts b/web/src/lib/components/shell/AppSidebar.test.ts new file mode 100644 index 000000000..7166d9615 --- /dev/null +++ b/web/src/lib/components/shell/AppSidebar.test.ts @@ -0,0 +1,107 @@ +import { fireEvent, render, screen, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import AppSidebar from './AppSidebar.svelte'; + +const status = { tone: 'idle' as const, label: 'Local archive ready', text: 'Local archive' }; + +function renderSidebar(overrides: Record = {}) { + const props = { + active: 'everything' as const, + collapsed: false, + showCollapseToggle: true, + status, + onNavigate: vi.fn(), + onToggleCollapsed: vi.fn(), + onOpenShortcuts: vi.fn(), + ...overrides + }; + const view = render(AppSidebar, props); + return { ...props, rerender: view.rerender, container: view.container }; +} + +describe('AppSidebar', () => { + it('lists workspaces in People, Archive, Manage order with the active one current', () => { + renderSidebar(); + const nav = screen.getByRole('navigation', { name: 'Primary' }); + expect( + within(nav) + .getAllByRole('button') + .map((b) => b.getAttribute('aria-label') ?? b.textContent?.trim()) + ).toEqual([ + 'Relationships', + 'Directory', + 'Reviews', + 'Everything', + 'Files', + 'Saved views', + 'Sources', + 'Operations', + 'Deletions', + 'Settings' + ]); + expect(within(nav).getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe( + 'page' + ); + expect(within(nav).getByRole('button', { name: 'Files' }).hasAttribute('aria-current')).toBe( + false + ); + }); + + it('navigates when an item is chosen', async () => { + const props = renderSidebar(); + await fireEvent.click(screen.getByRole('button', { name: 'Deletions' })); + expect(props.onNavigate).toHaveBeenCalledWith('deletions'); + }); + + it('keeps full accessible names in the icon rail and hides group headings', () => { + renderSidebar({ collapsed: true }); + expect(screen.getByRole('button', { name: 'Saved views' })).toBeTruthy(); + expect(screen.queryByText('People')).toBeNull(); + expect(screen.getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe( + 'page' + ); + }); + + it('keeps every sidebar item inside its landmark and hides the decorative status dot', () => { + const { container } = renderSidebar(); + const sidebar = screen.getByRole('complementary', { name: 'Sidebar' }); + expect(sidebar.textContent).toBe(container.textContent); + expect(within(sidebar).getByText('msgvault')).toBeTruthy(); + expect(within(sidebar).getByRole('navigation', { name: 'Primary' })).toBeTruthy(); + const dot = container.querySelector('[aria-label="Local archive ready"]'); + expect(dot?.closest('[aria-hidden="true"]')).toBeTruthy(); + }); + + it('gives assistive tech the full archive status in the rail', () => { + renderSidebar({ collapsed: true }); + expect(screen.getByText('Local archive ready').classList.contains('kit-sr-only')).toBe(true); + expect(screen.queryByText('Local archive')).toBeNull(); + }); + + it('lets keyboard users open the archive status tooltip in the rail', async () => { + renderSidebar({ collapsed: true }); + const trigger = screen.getByText('Local archive ready').closest('.kit-tooltip-trigger') as HTMLElement; + expect(trigger.tabIndex).toBe(0); + trigger.focus(); + expect((await screen.findByRole('tooltip')).textContent?.trim()).toBe('Local archive ready'); + }); + + it('toggles the rail and opens shortcuts from the footer', async () => { + const props = renderSidebar(); + await fireEvent.click(screen.getByRole('button', { name: 'Collapse sidebar' })); + await fireEvent.click(screen.getByRole('button', { name: /Keyboard shortcuts/ })); + expect(props.onToggleCollapsed).toHaveBeenCalled(); + expect(props.onOpenShortcuts).toHaveBeenCalled(); + expect(screen.getByText('Local archive')).toBeTruthy(); + }); + + it('keeps focus on the toggle when the parent switches the sidebar to the rail', async () => { + const { rerender } = renderSidebar(); + const toggle = screen.getByRole('button', { name: 'Collapse sidebar' }); + toggle.focus(); + await fireEvent.click(toggle); + await rerender({ collapsed: true }); + expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Expand sidebar' })); + }); +}); diff --git a/web/src/lib/components/shell/DisplayMenu.svelte b/web/src/lib/components/shell/DisplayMenu.svelte new file mode 100644 index 000000000..3444f2bbb --- /dev/null +++ b/web/src/lib/components/shell/DisplayMenu.svelte @@ -0,0 +1,63 @@ + + + + + + + + onDensityChange(value as Density)} + > + Auto + Compact + Comfortable + + {#if themeOverridden} + + Use daemon theme + {/if} + + + + + diff --git a/web/src/lib/components/shell/DisplayMenu.test.ts b/web/src/lib/components/shell/DisplayMenu.test.ts new file mode 100644 index 000000000..7f9eb3ea3 --- /dev/null +++ b/web/src/lib/components/shell/DisplayMenu.test.ts @@ -0,0 +1,77 @@ +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import DisplayMenu from './DisplayMenu.svelte'; + +describe('DisplayMenu', () => { + it('sets and clears the per-tab density override', async () => { + const onDensityChange = vi.fn(); + render(DisplayMenu, { + density: 'daemon', + themeOverridden: false, + onDensityChange, + onUseDaemonTheme: vi.fn(), + }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + + expect(screen.getByRole('group', { name: 'Temporary density' })).toBeTruthy(); + await fireEvent.click(screen.getByRole('menuitemradio', { name: 'Comfortable' })); + expect(onDensityChange).toHaveBeenCalledWith('comfortable'); + await fireEvent.click(screen.getByRole('menuitemradio', { name: 'Auto' })); + expect(onDensityChange).toHaveBeenCalledWith('daemon'); + expect(screen.queryByRole('menuitem', { name: 'Use daemon theme' })).toBeNull(); + }); + + it('shows the active density override as checked', async () => { + render(DisplayMenu, { + density: 'compact', + themeOverridden: false, + onDensityChange: vi.fn(), + onUseDaemonTheme: vi.fn(), + }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + + const checked = (name: string) => + screen.getByRole('menuitemradio', { name }).getAttribute('aria-checked'); + expect(checked('Compact')).toBe('true'); + expect(checked('Auto')).toBe('false'); + expect(checked('Comfortable')).toBe('false'); + }); + + it('offers the daemon theme only while a theme override is active', async () => { + const onUseDaemonTheme = vi.fn(); + render(DisplayMenu, { + density: 'compact', + themeOverridden: true, + onDensityChange: vi.fn(), + onUseDaemonTheme, + }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Use daemon theme' })); + expect(onUseDaemonTheme).toHaveBeenCalledOnce(); + }); + + it('closes on Escape without passing the key to window shortcuts', async () => { + render(DisplayMenu, { + density: 'compact', + themeOverridden: false, + onDensityChange: vi.fn(), + onUseDaemonTheme: vi.fn(), + }); + const trigger = screen.getByRole('button', { name: 'Display' }); + await fireEvent.click(trigger); + const windowKeydown = vi.fn(); + window.addEventListener('keydown', windowKeydown); + try { + await fireEvent.keyDown(screen.getByRole('menuitemradio', { name: 'Comfortable' }), { + key: 'Escape', + }); + } finally { + window.removeEventListener('keydown', windowKeydown); + } + + expect(screen.queryByRole('menu', { name: 'Display' })).toBeNull(); + expect(document.activeElement).toBe(trigger); + expect(windowKeydown).not.toHaveBeenCalled(); + }); +}); diff --git a/web/src/lib/components/shell/EverythingWorkspace.svelte b/web/src/lib/components/shell/EverythingWorkspace.svelte index 3bde697e7..62e8b43fe 100644 --- a/web/src/lib/components/shell/EverythingWorkspace.svelte +++ b/web/src/lib/components/shell/EverythingWorkspace.svelte @@ -1,5 +1,5 @@
-
-
-

Everything

-
-
+ + {#snippet actions()} {#if canPreviewRight}
Preview position @@ -610,29 +598,8 @@ Modality-neutral archive {/if}

-
-
- -
diff --git a/web/src/lib/components/shell/EverythingWorkspace.test.ts b/web/src/lib/components/shell/EverythingWorkspace.test.ts index d71d590ee..a46634dea 100644 --- a/web/src/lib/components/shell/EverythingWorkspace.test.ts +++ b/web/src/lib/components/shell/EverythingWorkspace.test.ts @@ -59,6 +59,18 @@ describe('EverythingWorkspace', () => { }; } + it('leaves searching to the global search box', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn(async () => Response.json(exploreResponse()))), state, enabled: false }); + const main = screen.getByRole('main', { name: 'Everything' }); + expect(within(main).queryByRole('search')).toBeNull(); + expect(screen.getAllByRole('search')).toHaveLength(1); + expect(screen.queryByRole('contentinfo', { name: 'Keyboard shortcuts' })).toBeNull(); + expect(within(main).queryByText('Keyboard shortcuts')).toBeNull(); + state.destroy(); + }); + it('explains how to refine a semantic search when the candidate pool is capped', async () => { window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); const fetchFn = vi.fn(async (input) => { diff --git a/web/src/lib/components/shell/KeyboardHelp.svelte b/web/src/lib/components/shell/KeyboardHelp.svelte index a8690fc97..5062dfde8 100644 --- a/web/src/lib/components/shell/KeyboardHelp.svelte +++ b/web/src/lib/components/shell/KeyboardHelp.svelte @@ -74,8 +74,6 @@ margin: 0 0 var(--space-2); color: var(--text-muted); font-size: var(--font-size-xs); - letter-spacing: 0.08em; - text-transform: uppercase; } dl { diff --git a/web/src/lib/components/shell/KeyboardHelp.test.ts b/web/src/lib/components/shell/KeyboardHelp.test.ts index 5ff6b1095..b16b0b17a 100644 --- a/web/src/lib/components/shell/KeyboardHelp.test.ts +++ b/web/src/lib/components/shell/KeyboardHelp.test.ts @@ -56,6 +56,15 @@ describe('keyboard command registry', () => { expect(command?.querySelector('[aria-label="Mod K"]')).not.toBeNull(); expect(command?.textContent).not.toContain('or'); }); + + it('documents extending a selection with Shift+Space', () => { + render(KeyboardHelp, { + commands: createCommandRegistry(handlersFor(COMMAND_DEFINITIONS.map(({ id }) => id))), + onclose: vi.fn() + }); + + expect(screen.getByText('Extend selection to focused row')).toBeTruthy(); + }); }); function handlersFor(ids: CommandID[]): CommandHandlers { diff --git a/web/src/lib/components/shell/NavigationDrawer.svelte b/web/src/lib/components/shell/NavigationDrawer.svelte new file mode 100644 index 000000000..f465662ce --- /dev/null +++ b/web/src/lib/components/shell/NavigationDrawer.svelte @@ -0,0 +1,65 @@ + + + +
+ + +
+ + diff --git a/web/src/lib/components/shell/NavigationDrawer.test.ts b/web/src/lib/components/shell/NavigationDrawer.test.ts new file mode 100644 index 000000000..a959d9da7 --- /dev/null +++ b/web/src/lib/components/shell/NavigationDrawer.test.ts @@ -0,0 +1,56 @@ +import { appShortcuts, initShortcuts } from '@kenn-io/kit-ui'; +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import NavigationDrawer from './NavigationDrawer.svelte'; + +const items = createRawSnippet(() => ({ + render: () => + '' +})); + +let detachShortcuts: () => void; + +beforeAll(() => { + detachShortcuts = initShortcuts(); +}); + +afterAll(() => { + detachShortcuts(); +}); + +describe('NavigationDrawer', () => { + it('focuses the current item and closes on Escape without reaching root shortcuts', async () => { + const root = vi.fn(); + const unregister = appShortcuts.register('escape', root); + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + + await vi.waitFor(() => + expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Everything' })) + ); + await fireEvent.keyDown(document.activeElement as Element, { key: 'Escape' }); + + expect(onclose).toHaveBeenCalledOnce(); + expect(root).not.toHaveBeenCalled(); + unregister(); + }); + + it('closes when the scrim is selected', async () => { + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + await fireEvent.click(screen.getByRole('button', { name: 'Close navigation' })); + expect(onclose).toHaveBeenCalledOnce(); + }); + + it('returns focus to the opener when unmounted', () => { + const opener = document.createElement('button'); + document.body.append(opener); + opener.focus(); + const rendered = render(NavigationDrawer, { onclose: vi.fn(), children: items }); + rendered.unmount(); + expect(document.activeElement).toBe(opener); + opener.remove(); + }); +}); diff --git a/web/src/lib/components/shell/PageHeader.svelte b/web/src/lib/components/shell/PageHeader.svelte new file mode 100644 index 000000000..8286bd487 --- /dev/null +++ b/web/src/lib/components/shell/PageHeader.svelte @@ -0,0 +1,64 @@ + + +
+
+
+

{title}

+ {#if description}

{description}

{/if} +
+ {#if actions}
{@render actions()}
{/if} +
+ {#if view}
{@render view()}
{/if} +
+ + diff --git a/web/src/lib/components/shell/PageHeader.test.ts b/web/src/lib/components/shell/PageHeader.test.ts new file mode 100644 index 000000000..1d03dae80 --- /dev/null +++ b/web/src/lib/components/shell/PageHeader.test.ts @@ -0,0 +1,25 @@ +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { describe, expect, it } from 'vitest'; + +import PageHeader from './PageHeader.svelte'; + +describe('PageHeader', () => { + it('renders one level-one heading, the description, and actions', () => { + const actions = createRawSnippet(() => ({ render: () => '' })); + render(PageHeader, { title: 'Operations', description: 'Background work and its history.', actions }); + + expect(screen.getByRole('heading', { level: 1, name: 'Operations' })).toBeTruthy(); + expect(screen.getByText('Background work and its history.')).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Refresh operations' })).toBeTruthy(); + }); + + it('renders the view row below the title', () => { + const view = createRawSnippet(() => ({ render: () => '
Views
' })); + render(PageHeader, { title: 'Operations', view }); + + const heading = screen.getByRole('heading', { level: 1, name: 'Operations' }); + const row = screen.getByRole('tablist'); + expect(heading.compareDocumentPosition(row) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/shell/navigation.ts b/web/src/lib/components/shell/navigation.ts new file mode 100644 index 000000000..3c3bfe9f3 --- /dev/null +++ b/web/src/lib/components/shell/navigation.ts @@ -0,0 +1,62 @@ +import type { Component } from 'svelte'; +import Activity from '@lucide/svelte/icons/activity'; +import Bookmark from '@lucide/svelte/icons/bookmark'; +import CheckCheck from '@lucide/svelte/icons/check-check'; +import Contact from '@lucide/svelte/icons/contact'; +import Inbox from '@lucide/svelte/icons/inbox'; +import Paperclip from '@lucide/svelte/icons/paperclip'; +import Plug from '@lucide/svelte/icons/plug'; +import Settings from '@lucide/svelte/icons/settings'; +import Trash2 from '@lucide/svelte/icons/trash-2'; +import Users from '@lucide/svelte/icons/users'; + +import type { ExploreWorkspace } from '../../explore/models'; + +export interface NavigationItem { + id: ExploreWorkspace; + label: string; + icon: Component; +} + +export interface NavigationGroup { + label: string; + items: NavigationItem[]; +} + +export const SIDEBAR_COLLAPSED_KEY = 'msgvault.sidebar.collapsed'; + +export const NAVIGATION_GROUPS: NavigationGroup[] = [ + { + label: 'People', + items: [ + { id: 'relationships', label: 'Relationships', icon: Users }, + { id: 'directory', label: 'Directory', icon: Contact }, + { id: 'directory_review', label: 'Reviews', icon: CheckCheck } + ] + }, + { + label: 'Archive', + items: [ + { id: 'everything', label: 'Everything', icon: Inbox }, + { id: 'files', label: 'Files', icon: Paperclip }, + { id: 'saved_views', label: 'Saved views', icon: Bookmark } + ] + }, + { + label: 'Manage', + items: [ + { id: 'sources', label: 'Sources', icon: Plug }, + { id: 'operations', label: 'Operations', icon: Activity }, + { id: 'deletions', label: 'Deletions', icon: Trash2 }, + { id: 'settings', label: 'Settings', icon: Settings } + ] + } +]; + +export function workspaceLabel(id: ExploreWorkspace): string { + for (const group of NAVIGATION_GROUPS) { + const item = group.items.find((candidate) => candidate.id === id); + if (item) return item.label; + } + return 'msgvault'; +} diff --git a/web/src/lib/components/sources/SourcesWorkspace.svelte b/web/src/lib/components/sources/SourcesWorkspace.svelte index 0844e97f9..71b1039f9 100644 --- a/web/src/lib/components/sources/SourcesWorkspace.svelte +++ b/web/src/lib/components/sources/SourcesWorkspace.svelte @@ -7,6 +7,7 @@ import { onDestroy, onMount } from 'svelte'; import type { APIClient } from '../../api/client'; import { scheduleSummary } from '../../settings/cron'; + import PageHeader from '../shell/PageHeader.svelte'; import type { SourceStatus as GeneratedSourceStatus, SyncRunStatus as GeneratedSyncRunStatus, @@ -301,16 +302,11 @@
-
-
-

Archive workspace

-

Sources

-
-
- Status and incremental sync + + {#snippet actions()}
-
+ {/snippet} + {#if statusError}
diff --git a/web/src/lib/components/directory/PersonDetail.svelte b/web/src/lib/components/directory/PersonDetail.svelte index a1fa83594..5627cb1e0 100644 --- a/web/src/lib/components/directory/PersonDetail.svelte +++ b/web/src/lib/components/directory/PersonDetail.svelte @@ -1,4 +1,5 @@ + + diff --git a/web/src/lib/components/explore/ColumnsMenu.test.ts b/web/src/lib/components/explore/ColumnsMenu.test.ts new file mode 100644 index 000000000..05efcfb49 --- /dev/null +++ b/web/src/lib/components/explore/ColumnsMenu.test.ts @@ -0,0 +1,51 @@ +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import type { ExploreColumn } from '../../explore/models'; +import ColumnsMenu from './ColumnsMenu.svelte'; + +describe('ColumnsMenu', () => { + it('toggles a column off and announces whether each column is shown', async () => { + const onchange = vi.fn(); + const rendered = render(ColumnsMenu, { + columns: ['kind', 'title', 'excerpt', 'time'], onchange + }); + + await fireEvent.click(screen.getByRole('button', { name: 'Columns' })); + const excerpt = screen.getByRole('button', { name: 'Excerpt' }); + expect(describedBy(excerpt)).toBe('Shown'); + await fireEvent.click(excerpt); + + const next: ExploreColumn[] = ['kind', 'title', 'time']; + expect(onchange).toHaveBeenCalledWith(next); + await rendered.rerender({ columns: next }); + expect(describedBy(screen.getByRole('button', { name: 'Excerpt' }))).toBe('Hidden'); + }); + + it('adds a hidden column in canonical position', async () => { + const onchange = vi.fn(); + render(ColumnsMenu, { columns: ['title', 'time'], onchange }); + + await fireEvent.click(screen.getByRole('button', { name: 'Columns' })); + await fireEvent.click(screen.getByRole('button', { name: 'Kind' })); + + expect(onchange).toHaveBeenCalledWith(['kind', 'title', 'time']); + }); + + it('disables the last visible column', async () => { + const onchange = vi.fn(); + render(ColumnsMenu, { columns: ['title'], onchange }); + + await fireEvent.click(screen.getByRole('button', { name: 'Columns' })); + const title = screen.getByRole('button', { name: 'Subject / title' }) as HTMLButtonElement; + expect(title.disabled).toBe(true); + await fireEvent.click(title); + + expect(onchange).not.toHaveBeenCalled(); + }); +}); + +function describedBy(element: HTMLElement): string | undefined { + const id = element.getAttribute('aria-describedby'); + return id ? document.getElementById(id)?.textContent?.trim() : undefined; +} diff --git a/web/src/lib/components/explore/ContextBar.svelte b/web/src/lib/components/explore/ContextBar.svelte index 8ce0ab5e2..c6b8cc56e 100644 --- a/web/src/lib/components/explore/ContextBar.svelte +++ b/web/src/lib/components/explore/ContextBar.svelte @@ -1,29 +1,45 @@
-
-
- Columns -
- {#each ALL_COLUMNS as column (column.id)} - toggleColumn(column.id)} - /> - {/each} -
-
-
-
- {column === 'attachments' ? '⌕' : ALL_COLUMNS.find((entry) => entry.id === column)?.label} + {column === 'attachments' ? '⌕' : EXPLORE_COLUMNS.find((entry) => entry.id === column)?.label} {/each}
@@ -617,44 +578,6 @@ box-shadow: var(--shadow-sm); } - .table-tools { - display: flex; - min-height: 30px; - align-items: center; - justify-content: flex-end; - padding: 0 var(--space-4); - border-bottom: 1px solid var(--border-muted); - background: var(--bg-subtle); - } - - details { - position: relative; - color: var(--text-secondary); - font-size: var(--font-size-xs); - } - - summary { - cursor: pointer; - } - - .column-picker { - position: absolute; - z-index: var(--z-popover); - top: 24px; - right: 0; - display: grid; - width: 176px; - gap: var(--space-3); - padding: var(--space-4); - } - - .column-picker :global(.kit-checkbox) { - display: flex; - align-items: center; - gap: var(--space-3); - white-space: nowrap; - } - .table-header, .data-row { display: grid; diff --git a/web/src/lib/components/explore/EverythingTable.test.ts b/web/src/lib/components/explore/EverythingTable.test.ts index 59ecf9b66..b26514694 100644 --- a/web/src/lib/components/explore/EverythingTable.test.ts +++ b/web/src/lib/components/explore/EverythingTable.test.ts @@ -143,20 +143,18 @@ describe('EverythingTable', () => { expect(rendered.textContent).not.toContain(', Charlie'); }); - it('exposes size through the column picker without showing it initially', async () => { - const onColumnsChange = vi.fn(); - render(EverythingTable, { + it('renders exactly the columns it is given', async () => { + const rendered = render(EverythingTable, { rows: [row(1, { attachment_size: 2048 })], - selection: new ExploreSelectionState(), - onColumnsChange + selection: new ExploreSelectionState() }); expect(screen.queryByRole('columnheader', { name: 'Size' })).toBeNull(); - await fireEvent.click(screen.getByText('Columns')); - await fireEvent.click(screen.getByRole('checkbox', { name: 'Size' })); + expect(screen.queryByText('Columns')).toBeNull(); + await rendered.rerender({ columns: ['title', 'size'] }); - expect(screen.getByRole('columnheader', { name: 'Size' })).toBeDefined(); - expect(onColumnsChange).toHaveBeenCalled(); + expect(screen.getAllByRole('columnheader').map((header) => header.textContent?.trim())) + .toEqual(['Subject / title', 'Size']); }); it('keeps keyboard focus on the grid while j/k move a stable keyed cursor', async () => { diff --git a/web/src/lib/components/explore/FilesPresentation.svelte b/web/src/lib/components/explore/FilesPresentation.svelte deleted file mode 100644 index 42c4d1a50..000000000 --- a/web/src/lib/components/explore/FilesPresentation.svelte +++ /dev/null @@ -1,375 +0,0 @@ - - -
-
Files in context{totalCount?.toLocaleString() ?? '—'} files
-
-
- ReceivedFilename - Containing itemSource - Size -
-
- {#if unavailable} -
- {:else if error && files.length === 0} -
- {:else if loading && files.length === 0} -
Loading files in this context…
- {:else if files.length === 0} -
No files match this view.
- {:else if !slice || rowHeight === undefined} -
Preparing files layout…
- {:else} -
-
- {#each renderedFiles as file, offset (file.key)} - {@const index = slice.start + offset} - -
{ activeKey = file.key; onActiveKey?.(file.key); grid?.focus(); }} - onclick={(event) => { - if (!(event.target as Element).closest('button')) onOpenFile?.(file); - }} - > - {formatTime(file.occurred_at)} - {file.filename || 'Unnamed file'} - - {file.source_identifier} - {formatBytes(file.size)} -
- {/each} -
-
- {#if pageError} - -
- {:else if error} -
- {:else if loadingMore} -
Loading more… {files.length.toLocaleString()} loaded
- {:else if hasMore} -
- {/if} - {/if} -
-
-
- - diff --git a/web/src/lib/components/explore/FilesPresentation.test.ts b/web/src/lib/components/explore/FilesPresentation.test.ts deleted file mode 100644 index 73e74ae64..000000000 --- a/web/src/lib/components/explore/FilesPresentation.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; -import { describe, expect, it, vi } from 'vitest'; - -import type { ExploreFileFact } from '../../explore/models'; -import FilesPresentation from './FilesPresentation.svelte'; - -function file(index: number): ExploreFileFact { - return { - id: index, - key: `message:${index}:file:${index}`, - entry_key: `message:${index}`, - message_id: index, - conversation_id: index + 1000, - occurred_at: '2026-07-18T12:00:00Z', - source_id: 1, - source_identifier: 'archive@example.com', - title: `Containing item ${index}`, - filename: `file-${index}.pdf`, - mime_type: 'application/pdf', - size: 2048 - }; -} - -describe('FilesPresentation', () => { - it('reports cache initialization and waits for automatic recovery', () => { - render(FilesPresentation, { - files: [], - unavailable: { - error: 'analytical_cache_unavailable', - message: 'The analytical cache is being prepared', - readiness: 'building', - recovery_action: '' - } - }); - - const alert = screen.getByRole('alert'); - expect(alert.textContent).toContain('Preparing analytical cache'); - expect(alert.textContent).toContain('This view will refresh automatically.'); - expect(screen.queryByRole('button', { name: 'Retry cache check' })).toBeNull(); - }); - - it('virtualizes a large loaded slice and implements bounded table navigation', async () => { - const files = Array.from({ length: 1_000 }, (_, index) => file(index + 1)); - const onActiveKey = vi.fn(); - const onLoadMore = vi.fn(); - render(FilesPresentation, { - files, hasMore: true, totalCount: 100_000, onActiveKey, onLoadMore - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await waitFor(() => expect(screen.getAllByRole('row').length).toBeLessThan(80)); - await fireEvent.keyDown(grid, { key: 'PageDown' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:10:file:10'); - await fireEvent.keyDown(grid, { key: 'PageUp' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:1:file:1'); - await fireEvent.keyDown(grid, { key: 'End' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:1000:file:1000'); - expect(onLoadMore).not.toHaveBeenCalled(); - }); - - it('loads at most one next page when keyboard movement crosses the loaded boundary', async () => { - let files = Array.from({ length: 600 }, (_, index) => file(index + 1)); - const onActiveKey = vi.fn(); - let rendered: ReturnType; - const onLoadMore = vi.fn(async () => { - files = Array.from({ length: 700 }, (_, index) => file(index + 1)); - await rendered.rerender({ - files, hasMore: true, totalCount: 10_000, - focusedKey: 'message:600:file:600', onActiveKey, onLoadMore - }); - return { status: 'advanced' }; - }); - rendered = render(FilesPresentation, { - files, hasMore: true, totalCount: 10_000, - focusedKey: 'message:600:file:600', onActiveKey, onLoadMore - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await fireEvent.keyDown(grid, { key: 'ArrowDown' }); - - expect(onLoadMore).toHaveBeenCalledTimes(1); - await waitFor(() => expect(onActiveKey).toHaveBeenLastCalledWith('message:601:file:601')); - expect(screen.getAllByRole('row').length).toBeLessThan(80); - }); - - it('keeps loaded files visible and exposes an accessible paging retry', async () => { - const files = Array.from({ length: 600 }, (_, index) => file(index + 1)); - const onLoadMore = vi.fn(async () => ({ status: 'advanced' })); - render(FilesPresentation, { - files, hasMore: true, totalCount: 10_000, - pageError: 'The next file page could not be loaded.', onLoadMore - }); - - expect(screen.getByText('file-1.pdf')).toBeDefined(); - expect(screen.getByRole('alert').textContent).toContain('The next file page could not be loaded.'); - await fireEvent.click(screen.getByRole('button', { name: 'Retry loading more files' })); - expect(onLoadMore).toHaveBeenCalledTimes(1); - }); - - it('restores a keyed scroll anchor and reports user scroll authority', async () => { - const onScrollAnchor = vi.fn(); - const files = Array.from({ length: 200 }, (_, index) => file(index + 1)); - const rendered = render(FilesPresentation, { - files, - focusedKey: 'message:150:file:150', - scrollAnchor: { key: 'message:140:file:140', offset: 5 }, - restoring: true, - onScrollAnchor - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }) as HTMLDivElement; - await waitFor(() => expect(grid.scrollTop).toBe(139 * 36 + 5)); - expect(grid.getAttribute('aria-activedescendant')).toContain('message-3a-150'); - - await rendered.rerender({ - files, focusedKey: 'message:150:file:150', - scrollAnchor: { key: 'message:140:file:140', offset: 5 }, - restoring: false, onScrollAnchor - }); - grid.scrollTop = 72; - await fireEvent.scroll(grid); - expect(onScrollAnchor).toHaveBeenLastCalledWith('message:3:file:3', 0); - }); - - it('opens the attachment while keeping the containing item a separate action', async () => { - const onOpenFile = vi.fn(); - const onOpenItem = vi.fn(); - const selected = file(7); - render(FilesPresentation, { files: [selected], onOpenFile, onOpenItem }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await fireEvent.keyDown(grid, { key: 'Enter' }); - expect(onOpenFile).toHaveBeenCalledWith(selected); - expect(onOpenItem).not.toHaveBeenCalled(); - - await fireEvent.click(screen.getByRole('button', { name: 'Open containing item 7' })); - expect(onOpenItem).toHaveBeenCalledWith('message:7'); - }); -}); diff --git a/web/src/lib/components/explore/RowKind.svelte b/web/src/lib/components/explore/RowKind.svelte index 26d0c9eaa..7639983e5 100644 --- a/web/src/lib/components/explore/RowKind.svelte +++ b/web/src/lib/components/explore/RowKind.svelte @@ -1,27 +1,14 @@ - {presentation.label.replace(' item', '').replace('Archive', 'Item')} + {presentation.name} diff --git a/web/src/lib/components/operations/OperationsWorkspace.svelte b/web/src/lib/components/operations/OperationsWorkspace.svelte index 9238b64b2..1bac1b065 100644 --- a/web/src/lib/components/operations/OperationsWorkspace.svelte +++ b/web/src/lib/components/operations/OperationsWorkspace.svelte @@ -373,7 +373,7 @@
diff --git a/web/src/lib/components/saved-views/SaveViewDialog.svelte b/web/src/lib/components/saved-views/SaveViewDialog.svelte new file mode 100644 index 000000000..2f81f7a35 --- /dev/null +++ b/web/src/lib/components/saved-views/SaveViewDialog.svelte @@ -0,0 +1,135 @@ + + + +
{ + event.preventDefault(); + void save(); + }} + > + + + {#if exploreState.workspace === 'files'} +

Filename, type, and file sort aren’t saved with the view.

+ {/if} + {#if error}{/if} +
+
+
+ + + diff --git a/web/src/lib/components/saved-views/SaveViewDialog.test.ts b/web/src/lib/components/saved-views/SaveViewDialog.test.ts new file mode 100644 index 000000000..a56115d78 --- /dev/null +++ b/web/src/lib/components/saved-views/SaveViewDialog.test.ts @@ -0,0 +1,145 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import { defaultExploreURLState } from '../../explore/state.svelte'; +import type { ExploreURLState } from '../../explore/models'; +import SaveViewDialog from './SaveViewDialog.svelte'; + +const state: ExploreURLState = { + ...defaultExploreURLState, + workspace: 'everything', + query: ' invoice ', + searchMode: 'full_text', + filters: [{ dimension: 'source', values: ['1'] }], + groupingChain: ['domain'], + presentation: 'table', + sort: [{ field: 'occurred_at', direction: 'desc' }], + columns: ['kind', 'title'], + inspectorPinned: true +}; + +function savedView() { + return { + id: 7, name: 'Invoices', description: 'Quarterly review', canonical_state: {}, + schema_version: 1, revision: 1, + created_at: '2026-07-19T10:00:00Z', updated_at: '2026-07-19T10:00:00Z' + }; +} + +function renderDialog(fetchFn: typeof fetch, overrides: Partial = {}) { + const onSaved = vi.fn(); + const onclose = vi.fn(); + render(SaveViewDialog, { + client: createAPIClient(fetchFn), state: { ...state, ...overrides }, onSaved, onclose + }); + return { onSaved, onclose }; +} + +describe('SaveViewDialog', () => { + it('puts focus in Name when it opens', () => { + renderDialog(vi.fn()); + expect(document.activeElement).toBe(screen.getByLabelText('Name')); + }); + + it('submits once when Enter is pressed in a named Name field', async () => { + const fetchFn = vi.fn(async () => Response.json(savedView(), { status: 201 })); + const { onSaved } = renderDialog(fetchFn); + const name = screen.getByLabelText('Name') as HTMLInputElement; + await fireEvent.input(name, { target: { value: 'Invoices' } }); + // jsdom has no implicit Enter submission; submitting the Name field's form is its effect. + await fireEvent.submit(name.form!); + await waitFor(() => expect(onSaved).toHaveBeenCalledOnce()); + expect(fetchFn).toHaveBeenCalledOnce(); + }); + + it('keeps Save disabled until a name is typed', async () => { + renderDialog(vi.fn()); + expect(screen.getByRole('dialog', { name: 'Save view' })).toBeDefined(); + const save = screen.getByRole('button', { name: 'Save' }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + await fireEvent.input(screen.getByLabelText('Name'), { target: { value: ' ' } }); + expect(save.disabled).toBe(true); + await fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'Invoices' } }); + expect(save.disabled).toBe(false); + }); + + it.each(['full_text', 'semantic', 'hybrid'] as const)( + 'posts the canonical %s state without session details and reports the saved view', + async (searchMode) => { + const requests: Request[] = []; + const fetchFn = vi.fn(async (input) => { + requests.push(input instanceof Request ? input : new Request(input)); + return Response.json(savedView(), { status: 201 }); + }); + const { onSaved } = renderDialog(fetchFn, { searchMode, activeRow: 'message:9' }); + + await fireEvent.input(screen.getByLabelText('Name'), { target: { value: ' Invoices ' } }); + await fireEvent.input( + screen.getByLabelText('Description'), { target: { value: 'Quarterly review' } } + ); + await fireEvent.click(screen.getByRole('button', { name: 'Save' })); + + await waitFor(() => + expect(onSaved).toHaveBeenCalledWith(expect.objectContaining({ id: 7, name: 'Invoices' })) + ); + expect(requests[0]!.method).toBe('POST'); + const body = await requests[0]!.clone().json(); + expect(body).toEqual({ + name: 'Invoices', description: 'Quarterly review', schema_version: 1, + canonical_state: { + query: 'invoice', search_mode: searchMode, + filters: [{ field: 'source', operator: 'in', values: ['1'] }], + grouping: ['domain'], presentation: 'table', + sort: [{ field: 'occurred_at', direction: 'desc' }], + columns: ['kind', 'title'] + } + }); + expect(JSON.stringify(body)).not.toContain('inspector_pinned'); + expect(JSON.stringify(body)).not.toContain('message:9'); + } + ); + + it.each([ + ['semantic', ''], ['semantic', ' \t\n '], ['hybrid', ''], ['hybrid', ' \t\n '] + ] as const)( + 'saves a filter-only %s view with query %j without a query or mode', + async (searchMode, query) => { + const requests: Request[] = []; + const fetchFn = vi.fn(async (input) => { + requests.push(input instanceof Request ? input : new Request(input)); + return Response.json(savedView(), { status: 201 }); + }); + const { onSaved } = renderDialog(fetchFn, { query, searchMode }); + await fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'Invoices' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Save' })); + await waitFor(() => expect(onSaved).toHaveBeenCalledOnce()); + const { canonical_state: saved } = await requests[0]!.clone().json(); + expect(saved).not.toHaveProperty('query'); + expect(saved).not.toHaveProperty('search_mode'); + expect(saved.filters).toEqual([{ field: 'source', operator: 'in', values: ['1'] }]); + } + ); + + it('shows an API error in the dialog and keeps it open', async () => { + const fetchFn = vi.fn(async () => + Response.json({ message: 'Name already used' }, { status: 409 }) + ); + const { onSaved, onclose } = renderDialog(fetchFn); + await fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'Invoices' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Save' })); + expect((await screen.findByRole('alert')).textContent).toContain('Name already used'); + expect(onSaved).not.toHaveBeenCalled(); + expect(onclose).not.toHaveBeenCalled(); + expect(screen.getByRole('dialog', { name: 'Save view' })).toBeDefined(); + }); + + it('notes what a Files view leaves out only in Files', () => { + const note = 'Filename, type, and file sort aren’t saved with the view.'; + renderDialog(vi.fn()); + expect(screen.queryByText(note)).toBeNull(); + cleanup(); + renderDialog(vi.fn(), { workspace: 'files', presentation: 'files' }); + expect(screen.getByText(note)).toBeDefined(); + }); +}); diff --git a/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte b/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte index 660a7c4f0..51406d94e 100644 --- a/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte +++ b/web/src/lib/components/saved-views/SavedViewsWorkspace.svelte @@ -1,39 +1,32 @@ - - - - - - onDensityChange(value as Density)} - > - Auto - Compact - Comfortable - - {#if themeOverridden} - - Use daemon theme - {/if} - - - - - + + + + + onDensityChange(value as Density)} + > + Auto + Compact + Comfortable + + {#if themeOverridden} + + Use daemon theme + {/if} + + diff --git a/web/src/lib/components/shell/DisplayMenu.test.ts b/web/src/lib/components/shell/DisplayMenu.test.ts index 7f9eb3ea3..b190e7e04 100644 --- a/web/src/lib/components/shell/DisplayMenu.test.ts +++ b/web/src/lib/components/shell/DisplayMenu.test.ts @@ -1,4 +1,4 @@ -import { fireEvent, render, screen } from '@testing-library/svelte'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { describe, expect, it, vi } from 'vitest'; import DisplayMenu from './DisplayMenu.svelte'; @@ -51,7 +51,7 @@ describe('DisplayMenu', () => { expect(onUseDaemonTheme).toHaveBeenCalledOnce(); }); - it('closes on Escape without passing the key to window shortcuts', async () => { + it('closes on Escape and returns focus to its trigger', async () => { render(DisplayMenu, { density: 'compact', themeOverridden: false, @@ -60,18 +60,12 @@ describe('DisplayMenu', () => { }); const trigger = screen.getByRole('button', { name: 'Display' }); await fireEvent.click(trigger); - const windowKeydown = vi.fn(); - window.addEventListener('keydown', windowKeydown); - try { - await fireEvent.keyDown(screen.getByRole('menuitemradio', { name: 'Comfortable' }), { - key: 'Escape', - }); - } finally { - window.removeEventListener('keydown', windowKeydown); - } + const item = await screen.findByRole('menuitemradio', { name: 'Comfortable' }); + await waitFor(() => expect(document.activeElement).not.toBe(document.body)); - expect(screen.queryByRole('menu', { name: 'Display' })).toBeNull(); + await fireEvent.keyDown(item, { key: 'Escape' }); + + await waitFor(() => expect(screen.queryByRole('menu', { name: 'Display' })).toBeNull()); expect(document.activeElement).toBe(trigger); - expect(windowKeydown).not.toHaveBeenCalled(); }); }); diff --git a/web/src/lib/components/shell/EverythingWorkspace.svelte b/web/src/lib/components/shell/EverythingWorkspace.svelte index 62e8b43fe..452a79bf8 100644 --- a/web/src/lib/components/shell/EverythingWorkspace.svelte +++ b/web/src/lib/components/shell/EverythingWorkspace.svelte @@ -12,10 +12,9 @@ EntryRow, AllMatchingExploreSelection, ExploreCacheUnavailable, - ExploreColumn, - ExploreFileFact, ExploreGroupDimension, ExploreGroupRow, + ExploreSearchMode, ExploreURLState, ExploreWorkspace, } from '../../explore/models'; @@ -26,9 +25,9 @@ import { groupingByDimension } from '../../grouping/catalog'; import { canonicalFingerprint, createAllMatchingSelection, predicateFingerprint } from '../../explore/selection'; import type { ExploreSelectionState, ExploreState } from '../../explore/state.svelte'; + import ColumnsMenu from '../explore/ColumnsMenu.svelte'; import ContextBar from '../explore/ContextBar.svelte'; import EverythingTable from '../explore/EverythingTable.svelte'; - import FilesPresentation from '../explore/FilesPresentation.svelte'; import GroupTable from '../explore/GroupTable.svelte'; import SelectionBar from '../explore/SelectionBar.svelte'; import SplitPane from '../layout/SplitPane.svelte'; @@ -38,7 +37,12 @@ import type { SearchCoverageAction } from '../../search/modes'; import MeetingPanel from '../meetings/MeetingPanel.svelte'; import PageHeader from './PageHeader.svelte'; - import { exploreMeetingScope } from '../../meetings/scopes'; + import { + exploreMeetingScope, + filtersToMeetingTranscripts, + isMeetingTranscript, + restrictsToMeetingTranscripts, + } from '../../meetings/scopes'; import type { EverythingSessionState } from './EverythingSessionState.svelte'; type ExplorePreflight = GeneratedExplorePreflightResponse; @@ -57,19 +61,20 @@ selectionPreflight: ExplorePreflight | undefined; meetingSelection: GeneratedExploreSelection | undefined; exportSelection: () => void; + onReviewDeletion: (mode: 'explicit' | 'all_matching') => void; commitNavigation: (patch: Partial) => void; + commitSearch: (query: string, mode: ExploreSearchMode) => void; commitWorkspace: (workspace: ExploreWorkspace) => void; commitGrouping: (dimension: ExploreGroupDimension) => void; fixedSortNotice: () => void; focusGrid: () => void; openRow: (row: EntryRow) => void; drillGroup: (row: ExploreGroupRow) => void; - openFileItem: (entryKey: string) => void; - openContextualFile: (file: ExploreFileFact) => void; closeReadingPane: () => void; openRelationship: (participantID: number) => void; changeConversationAnchor: (anchorId: number) => void; onOpenMeeting?: (meeting: MeetingRef) => void; + onSaveView: () => void; } let { @@ -86,23 +91,38 @@ selectionPreflight, meetingSelection, exportSelection, + onReviewDeletion, commitNavigation, + commitSearch, commitWorkspace, commitGrouping, fixedSortNotice, focusGrid, openRow, drillGroup, - openFileItem, - openContextualFile, closeReadingPane, openRelationship, changeConversationAnchor, onOpenMeeting = undefined, + onSaveView, }: Props = $props(); const api = createExploreAPI(untrack(() => client)); + const countLabel = $derived.by(() => { + const result = loader.result; + if (loader.loading) return 'Counting…'; + if (!result || loader.error || loader.unavailable) return ''; + if (result.candidatePoolSaturated) { + const shown = loader.rows.length; + return `${shown.toLocaleString()} ${shown === 1 ? 'result' : 'results'} shown`; + } + if (result.totalCount === undefined) return ''; + const count = result.totalCount; + const [one, many] = exploreState.current.groupingChain.length > 0 ? ['group', 'groups'] : ['item', 'items']; + return `${count.toLocaleString()} ${count === 1 ? one : many}`; + }); + function storedPreviewPosition(): 'below' | 'right' { try { return localStorage.getItem('msgvault.reading-pane.position') === 'right' ? 'right' : 'below'; @@ -330,6 +350,13 @@ }); }); + // The meeting context API rejects any selection that contains a non-meeting row. + const canExportMeetings = $derived.by(() => { + if (selection.mode === 'all_matching') return restrictsToMeetingTranscripts(exploreState.predicate()); + const selected = loader.rows.filter((row) => selection.isSelected(row.key)); + return selected.length > 0 && selected.every(isMeetingTranscript); + }); + let meetingReloadRequestedAt: number | undefined; const meetingPredicateFingerprint = $derived(predicateFingerprint(exploreState.predicate())); const meetingScope = $derived(session.meetingOverview?.fingerprint === meetingPredicateFingerprint @@ -342,8 +369,7 @@ const loading = loader.loading; const generation = loader.resultGeneration; const resultFingerprint = loader.resultFingerprint; - const isMeetingView = exploreState.current.workspace === 'everything' && - predicate.filters?.some((filter) => filter.dimension === 'message_type' && filter.values.includes('meeting_transcript')); + const isMeetingView = exploreState.current.workspace === 'everything' && filtersToMeetingTranscripts(predicate); untrack(() => { if (!isMeetingView || session.meetingOverview?.fingerprint !== fingerprint) session.meetingOverview = undefined; if (!isMeetingView || !result || loading || resultFingerprint !== fingerprint) return; @@ -433,7 +459,6 @@ } if ( !currentResult || - exploreState.current.presentation === 'files' || !predicate.query || (predicate.search_mode !== 'full_text' && predicate.search_mode !== 'hybrid') || loader.resultFingerprint !== predicateFingerprint(predicate) || @@ -577,58 +602,26 @@
- {#snippet actions()} - {#if canPreviewRight} -
- Preview position - -
- {/if} -

- {#if loader.result?.candidatePoolSaturated} - {loader.rows.length.toLocaleString()} {loader.rows.length === 1 ? 'result' : 'results'} shown - {:else if loader.result?.totalCount !== undefined} - {loader.result.totalCount.toLocaleString()} items - {:else} - Modality-neutral archive - {/if} -

- {/snippet} + {#snippet actions()}
+``` + +Styles: lanes stack vertically (`display: grid; gap: var(--space-4)`); each +`li` is a four-column grid `minmax(10rem, 1fr) auto minmax(10rem, 1fr) auto` +with a `border-top: 1px solid var(--border-muted)`; under +`@media (max-width: 760px)` it becomes one column. `.history-note` uses +`color: var(--status-warning-ink)`. Do not give one rule background, border, +and radius together (`hand-rolled-card`). "Last succeeded" appears when the +row's shown run did not succeed and a successful run exists (state comparison; +run IDs are not stable to compare). + +`OperationsWorkspace.svelte`: import `OperationStatusList` instead of +`OperationLaneCards`; pass `onSetUp`; change `onConfigure` to `() => void` +and pass it through to `OperationRelatedStatus`. + +`OperationRelatedStatus.svelte`: + +- Delete `settingsLabels`. Props: `onConfigure?: () => void`. +- Configured-false branch: + +```svelte +{:else if configured === false} +
+

Off {configurationLabels[authority]}

+
+ {#if authority === 'getVisualAttachmentStatus'} +
+ +{/if} +``` + + Arriving from Everything keeps `reviewOnMount` and the immediate + confirmation (spec decision 5); the summary renders behind the modal. +- Manifests (replaces :329-370): + +```svelte +{#if loading}

Loading deletion manifests…

+{:else if manifests.length === 0} + +{:else} +
+ + {#snippet header()} + + + {/snippet} + {#each manifests as manifest (manifest.id)} + {@const chip = manifestStatusChip(manifest.status)} + + + + + + + + + {/each} +
{manifest.id}{manifest.description}{manifest.message_count.toLocaleString()} {manifest.message_count === 1 ? 'item' : 'items'}{chip.label} +
+ {#if detail} + {@const detailChip = manifestStatusChip(detail.status)} + + + + {/if} +
+{/if} +``` + + "Confirm stage deletion" and "Confirm cancel manifest" keep + `tone="danger" surface="solid"`. +- Styles: `.manifests.has-detail { display: grid; grid-template-columns: minmax(0, 2fr) minmax(16rem, 1fr); gap: var(--space-4); align-items: start; }` + and `@media (max-width: 900px) { .manifests.has-detail { grid-template-columns: 1fr; } }`. + `.result` is neutral (`border: 1px solid var(--border-default); background: var(--bg-subtle)`, + no radius on the same rule as a background if `kit-ui-check` flags it). + `.warning` keeps the amber border. `.reason` uses `var(--text-secondary)`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it('explains how to start when nothing is selected', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async () => Response.json({ manifests: [] }))) }); + expect(screen.getByText('Nothing selected for deletion')).toBeDefined(); + expect(screen.getByText('Select items in Everything, then choose Review for deletion…')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Review selection' })).toBeNull(); + expect(await screen.findByText('No staged deletions')).toBeDefined(); + expect(screen.getByText('msgvault delete-staged').tagName).toBe('CODE'); +}); + +it('summarizes a review with size, relative expiry, and only the staging reason', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/explore/preflight')) return Response.json(preflight({ + count: 2, deletable_count: 2, estimated_bytes: 3 * 1024, expires_at: '2026-07-19T12:00:00Z', + unavailable_actions: [ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + { action: 'export', reason: 'browser_export_requires_single_message' }, + { action: 'open_in_source', reason: 'trusted_source_link_unavailable' } + ] + })); + return Response.json({ manifests: [] }); + }); + render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit, now: () => new Date('2026-07-19T10:00:00Z') }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + + expect(await screen.findByText('2 items · 3 KB')).toBeDefined(); + expect(screen.getByText('in 2 hours').closest('span')?.textContent).toBe('Review expires in 2 hours'); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + expect(screen.queryByText(/Export works for one message/)).toBeNull(); + expect(screen.queryByText(/provide links to open/)).toBeNull(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); +}); + +it('makes only the confirmation button red', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + return new URL(request.url).pathname.endsWith('/explore/preflight') ? Response.json(preflight()) : Response.json({ manifests: [] }); + })), selection: explicit }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + const stage = await screen.findByRole('button', { name: 'Stage deletion…' }); + expect(stage.className).toContain('kit-button--info'); + expect(stage.className).toContain('kit-button--solid'); + expect(screen.getByRole('button', { name: 'Dry run' }).className).toContain('kit-button--outline'); + await fireEvent.click(stage); + expect(screen.getByRole('button', { name: 'Confirm stage deletion' }).className).toContain('kit-button--danger'); +}); + +it('lists manifests with status chips and closes the detail', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/batch-1')) return Response.json({ ...listResponse().manifests[0], account: 'archive@example.com' }); + return Response.json(listResponse()); + })) }); + const table = await screen.findByRole('table', { name: 'Deletion manifests' }); + expect(within(table).getAllByRole('columnheader').map((cell) => cell.textContent?.trim())) + .toEqual(['ID', 'Description', 'Items', 'Status', 'Created', 'Actions']); + expect(within(table).getByText('Pending')).toBeDefined(); + expect(within(table).getByRole('button', { name: 'Cancel batch-1' }).className).not.toContain('kit-button--danger'); + await fireEvent.click(within(table).getByRole('button', { name: 'Inspect batch-1' })); + expect(await screen.findByText('archive@example.com')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Close manifest detail' })); + expect(screen.queryByText('archive@example.com')).toBeNull(); +}); +``` + +Existing tests: rename "Stage deletion" to "Stage deletion…" (:70, :102, :148, +:253, :399, :428); "1 item · 120 bytes" (:374) → "1 item · 120 B"; :398 +`/selection_contains_items_that_cannot_be_deleted_from_source/` → the +sentence; :359 `findAllByText('cancelled')` → `findAllByText('Cancelled')`. +The partial-staging assertions (:146, :156) still find `role="alert"`. Import +`within` from Testing Library. + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/deletions src/lib/components/shell`. + Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): give Deletions empty states, a review panel, and a manifests table`. + +--- + +### Task 7: Selection bar staging reason + +**Files:** +- Modify: `web/src/lib/components/explore/SelectionBar.svelte:44-46,111-118` +- Test: `web/src/lib/components/explore/SelectionBar.test.ts` + +**Interfaces:** +- Consumes: `preflightReasonLabel` (already imported). + +Implementation: + +```ts +const stageReason = $derived(preflight?.unavailable_actions.find((item) => item.action === 'stage_deletion')?.reason); +const stageReasonID = $props.id(); +``` + +```svelte +{#if onReviewDeletion} +
{:else} -
-

msgvault

+
+

msgvault

Connecting…

{/if} - - diff --git a/web/src/App.test.ts b/web/src/App.test.ts index 7607eac9e..225510240 100644 --- a/web/src/App.test.ts +++ b/web/src/App.test.ts @@ -1,12 +1,18 @@ import { getHealth as generatedGetHealth } from './lib/api/generated/api/api'; -import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; -import { describe, expect, it, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import App from './App.svelte'; import { createAPIClient } from './lib/api/client'; import { createSessionController } from './lib/api/session.svelte'; -import { SEARCH_MODE_PREFERENCE_KEY } from './lib/search/modes'; +import { resolveInitialSearchMode, SEARCH_MODE_PREFERENCE_KEY } from './lib/search/modes'; import { chooseSelectOption } from './test/kit-ui'; describe('application foundation', () => { + afterEach(() => { + localStorage.removeItem(SEARCH_MODE_PREFERENCE_KEY); + sessionStorage.removeItem('msgvault.appearance.override'); + document.documentElement.classList.remove('dark'); + window.history.replaceState(null, '', '/'); + }); it('mounts the Relationships landmark once bootstrap succeeds', async () => { const session = createSessionController(async () => Response.json({ auth_mode: 'loopback', https: false, plain_http_warning: true }), @@ -32,6 +38,30 @@ describe('application foundation', () => { expect(screen.getByRole('main', { name: 'Connecting' })).toBeDefined(); expect(document.title).toBe('msgvault'); }); + it('shows the OAuth callback in the same layout as connecting', () => { + vi.stubGlobal( + 'BroadcastChannel', + class { + postMessage(): void {} + close(): void {} + }, + ); + const close = vi.spyOn(window, 'close').mockImplementation(() => undefined); + window.history.replaceState(null, '', '/?state=msgvault-carddav-synthetic&code=synthetic'); + try { + render(App, { session: createSessionController(vi.fn()) }); + const main = screen.getByRole('main'); + expect(main.className).toContain('boot-screen'); + expect(within(main).getByText('msgvault').className).toContain('boot-screen__brand'); + expect( + within(main).getByText('Return to CardDAV settings to finish connecting. You can close this window.'), + ).toBeDefined(); + } finally { + close.mockRestore(); + vi.unstubAllGlobals(); + window.history.replaceState(null, '', '/'); + } + }); it('shows a bootstrap error with retry instead of the shell, and recovers on retry', async () => { let sessionCalls = 0; const fetchFn = vi.fn(async (input) => { @@ -112,13 +142,14 @@ describe('application foundation', () => { await session.bootstrap(); await fireEvent.click(await screen.findByRole('button', { name: 'Settings' })); await chooseSelectOption(await screen.findByLabelText('Theme'), 'Dark'); - await fireEvent.click(screen.getByRole('button', { name: 'Save settings' })); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); await waitFor(() => expect(requests.some((request) => request.method === 'PATCH')).toBe(true)); const patch = requests.find((request) => request.method === 'PATCH'); expect(patch?.headers.get('X-CSRF-Token')).toBe('csrf-token'); expect(patch?.headers.get('If-Match')).toBe('"etag-a"'); }); it('returns to login when a settings mutation is unauthorized', async () => { + window.history.replaceState(null, '', '/?workspace=settings'); const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; @@ -138,7 +169,7 @@ describe('application foundation', () => { await session.bootstrap(); await fireEvent.click(await screen.findByRole('button', { name: 'Settings' })); await chooseSelectOption(await screen.findByLabelText('Theme'), 'Dark'); - await fireEvent.click(screen.getByRole('button', { name: 'Save settings' })); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); expect(await screen.findByRole('form', { name: 'Log in' })).toBeDefined(); }); it('loads appearance once after interactive login while a session override wins', async () => { @@ -230,6 +261,93 @@ describe('application foundation', () => { window.history.replaceState(null, '', '/'); }, ); + it('applies a saved theme and density to the open tab', async () => { + sessionStorage.removeItem('msgvault.appearance.override'); + await openAppearance(appearanceDaemon()); + await chooseSelectOption(screen.getByLabelText('Theme'), 'Dark'); + await chooseSelectOption(screen.getByLabelText('Density'), 'Comfortable'); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await waitFor(() => expect(document.documentElement.classList.contains('dark')).toBe(true)); + expect(document.documentElement.dataset.density).toBe('comfortable'); + }); + it('keeps a saved theme when the browser-defaults load finishes after the save', async () => { + sessionStorage.removeItem('msgvault.appearance.override'); + const daemon = appearanceDaemon(); + let releaseFirstLoad: (() => void) | undefined; + let settingsReads = 0; + const fetchFn = vi.fn(async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + const path = new URL(request.url).pathname; + if (path === '/api/v1/settings' && request.method === 'GET' && ++settingsReads === 2) { + // The browser-defaults load reads the daemon before the save, then stalls. + const stale = await daemon(request.clone()); + const body = await stale.json(); + body.settings.find((setting: { key: string }) => setting.key === 'web.density').value = { string: 'comfortable' }; + await new Promise((resolve) => { releaseFirstLoad = resolve; }); + return Response.json(body); + } + return daemon(request); + }); + await openAppearance(fetchFn); + await chooseSelectOption(screen.getByLabelText('Theme'), 'Dark'); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await waitFor(() => expect(document.documentElement.classList.contains('dark')).toBe(true)); + await waitFor(() => expect(releaseFirstLoad).toBeDefined()); + releaseFirstLoad?.(); + await waitFor(() => expect(document.documentElement.dataset.density).toBe('comfortable')); + expect(document.documentElement.classList.contains('dark')).toBe(true); + }); + it('keeps the open view mode when the browser-defaults load finishes after a saved search mode', async () => { + localStorage.removeItem(SEARCH_MODE_PREFERENCE_KEY); + const daemon = appearanceDaemon(); + let releaseDefaultsLoad: (() => void) | undefined; + let settingsReads = 0; + const fetchFn = vi.fn(async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init); + const path = new URL(request.url).pathname; + if (path === '/api/v1/settings' && request.method === 'GET' && ++settingsReads === 2) { + // The browser-defaults load reads an older daemon default, then stalls past the save. + const stale = await daemon(request.clone()); + const body = await stale.json(); + body.settings.find((setting: { key: string }) => setting.key === 'web.default_search_mode').value = { string: 'semantic' }; + // An unsaved density change shows when the late load has been applied. + body.settings.find((setting: { key: string }) => setting.key === 'web.density').value = { string: 'comfortable' }; + await new Promise((resolve) => { releaseDefaultsLoad = resolve; }); + return Response.json(body); + } + return daemon(request); + }); + await openAppearance(fetchFn); + const before = window.location.search; + await chooseSelectOption(screen.getByLabelText('Default search mode'), 'Hybrid'); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await waitFor(() => expect(localStorage.getItem(SEARCH_MODE_PREFERENCE_KEY)).toBe('hybrid')); + await waitFor(() => expect(releaseDefaultsLoad).toBeDefined()); + releaseDefaultsLoad?.(); + await waitFor(() => expect(document.documentElement.dataset.density).toBe('comfortable')); + expect(screen.getByRole('radio', { name: 'Full text' }).getAttribute('aria-checked')).toBe('true'); + expect(window.location.search).toBe(before); + expect(localStorage.getItem(SEARCH_MODE_PREFERENCE_KEY)).toBe('hybrid'); + }); + it('keeps a Display menu theme override ahead of a saved theme', async () => { + sessionStorage.setItem('msgvault.appearance.override', JSON.stringify({ theme: 'light' })); + await openAppearance(appearanceDaemon()); + await chooseSelectOption(screen.getByLabelText('Theme'), 'Dark'); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await waitFor(() => expect(screen.queryByRole('button', { name: 'Save changes' })).toBeNull()); + expect(document.documentElement.classList.contains('dark')).toBe(false); + }); + it('remembers a saved default search mode without changing the open view', async () => { + localStorage.removeItem(SEARCH_MODE_PREFERENCE_KEY); + await openAppearance(appearanceDaemon()); + const before = window.location.search; + await chooseSelectOption(screen.getByLabelText('Default search mode'), 'Hybrid'); + await fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await waitFor(() => expect(localStorage.getItem(SEARCH_MODE_PREFERENCE_KEY)).toBe('hybrid')); + expect(window.location.search).toBe(before); + expect(screen.getByRole('radio', { name: 'Full text' }).getAttribute('aria-checked')).toBe('true'); + expect(resolveInitialSearchMode(undefined, localStorage, 'full_text')).toBe('hybrid'); + }); it('threads repeated same-conflict handoffs through AppShell as distinct exactly-once live events', async () => { window.history.replaceState( null, @@ -392,3 +510,71 @@ function settingsResponse(theme: string, etag: string, pendingRestart = false): { headers: { ETag: etag } }, ); } +function appearanceDaemon() { + const current = { theme: 'system', density: 'compact', mode: 'full_text' }; + const document = () => + Response.json( + { + groups: [{ id: 'browser', label: 'Appearance', description: 'How the web app looks.' }], + settings: [ + { + key: 'web.theme', + group: 'browser', + label: 'Theme', + kind: 'string', + value: { string: current.theme }, + options: ['system', 'light', 'dark'], + restart_required: false, + }, + { + key: 'web.density', + group: 'browser', + label: 'Density', + kind: 'string', + value: { string: current.density }, + options: ['compact', 'comfortable'], + restart_required: false, + }, + { + key: 'web.default_search_mode', + group: 'browser', + label: 'Default search mode', + kind: 'string', + value: { string: current.mode }, + options: ['full_text', 'semantic', 'hybrid'], + restart_required: false, + }, + ], + pending_restart: false, + }, + { headers: { ETag: '"appearance"' } }, + ); + return vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + if (path === '/api/session') { + return Response.json({ auth_mode: 'loopback', https: false, plain_http_warning: false }); + } + if (path === '/api/v1/settings' && request.method === 'PATCH') { + const body = (await request.json()) as { updates: Array<{ key: string; value: { string: string } }> }; + for (const { key, value } of body.updates) { + if (key === 'web.theme') current.theme = value.string; + if (key === 'web.density') current.density = value.string; + if (key === 'web.default_search_mode') current.mode = value.string; + } + return document(); + } + if (path === '/api/v1/settings') return document(); + if (path === '/api/v1/explore') { + return Response.json({ rows: [], total_count: 0, cache_revision: 'appearance', search_provenance: {} }); + } + return Response.json({}, { status: 404 }); + }); +} +async function openAppearance(fetchFn: ReturnType) { + window.history.replaceState(null, '', '/?workspace=settings'); + const session = createSessionController(fetchFn); + render(App, { session }); + await session.bootstrap(); + await screen.findByRole('heading', { level: 2, name: 'Appearance' }); +} diff --git a/web/src/app.css b/web/src/app.css index d2a354e6c..65968cf1c 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -34,3 +34,42 @@ h3 { font-size: var(--font-size-md); font-weight: 600; } + +/* Sign-in and boot screens share one narrow column with a brand line. */ +.boot-screen, +.boot-screen > form { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: var(--space-5); +} + +.boot-screen { + max-width: 28rem; + margin: 0 auto; + padding: var(--space-8) var(--space-6); + font-size: var(--font-size-md); +} + +.boot-screen p, +.boot-screen h1 { + margin: 0; +} + +.boot-screen h1 { + font-size: var(--font-size-xl); + font-weight: 650; +} + +.boot-screen__brand { + color: var(--text-primary); + font-weight: 650; +} + +.boot-screen p:not(.boot-screen__brand) { + color: var(--text-muted); +} + +.boot-screen p[role='alert'] { + color: var(--text-danger); +} diff --git a/web/src/lib/carddav/navigation.ts b/web/src/lib/carddav/navigation.ts index 8cde49da0..94078b0c8 100644 --- a/web/src/lib/carddav/navigation.ts +++ b/web/src/lib/carddav/navigation.ts @@ -1,9 +1,9 @@ const SETTINGS_NAVIGATION_TARGETS = { - document_index: { - authority: 'document_index', categoryID: 'archive', settingKey: 'analytics.auto_build_cache' + semantic_search: { + authority: 'semantic_search', categoryID: 'search', settingKey: 'vector.enabled' }, - document_vector: { - authority: 'document_vector', categoryID: 'search', settingKey: 'vector.enabled' + person_embeddings: { + authority: 'person_embeddings', categoryID: 'search', settingKey: 'vector.people.enabled' }, visual_attachments: { authority: 'visual_attachments', categoryID: 'search', settingKey: 'vector.multimodal.enabled' diff --git a/web/src/lib/commands/registry.ts b/web/src/lib/commands/registry.ts index df9224cf0..97eb0f717 100644 --- a/web/src/lib/commands/registry.ts +++ b/web/src/lib/commands/registry.ts @@ -21,7 +21,7 @@ export const COMMAND_DEFINITIONS = [ command('open-filters', 'Open filters', ['F'], ['f'], 'Analyze'), command('open-grouping', 'Open grouping controls', ['G'], ['g'], 'Analyze'), command('change-sort', 'Change sort', ['S'], ['s'], 'Analyze'), - command('reverse-sort', 'Reverse sort direction', ['R'], ['r'], 'Analyze'), + command('reverse-sort', 'Show sort order', ['R'], ['r'], 'Analyze'), command('open-keyboard-help', 'Open searchable keyboard help', ['?'], ['shift+/'], 'Help'), command('open-command-palette', 'Open command palette', ['Mod', 'K'], ['mod+k'], 'Help') ] as const; diff --git a/web/src/lib/components/auth/Login.svelte b/web/src/lib/components/auth/Login.svelte index 51bd23a5a..f6c478ee9 100644 --- a/web/src/lib/components/auth/Login.svelte +++ b/web/src/lib/components/auth/Login.svelte @@ -12,9 +12,9 @@ } -
+
- +

msgvault

Log in

Enter the API key configured for this daemon.

@@ -44,18 +44,8 @@
diff --git a/web/src/lib/components/auth/Login.test.ts b/web/src/lib/components/auth/Login.test.ts index 26981a63e..47003c5be 100644 --- a/web/src/lib/components/auth/Login.test.ts +++ b/web/src/lib/components/auth/Login.test.ts @@ -17,6 +17,13 @@ function response(status: number) { } describe('Login', () => { + it('uses the shared boot-screen layout', () => { + render(Login, { session: createSessionController(vi.fn()) }); + + expect(screen.getByRole('main', { name: 'Authentication' }).className).toContain('boot-screen'); + expect(screen.getByText('msgvault').className).toContain('boot-screen__brand'); + }); + it('exchanges the API key and leaves required mode on success', async () => { const fetchFn = vi.fn(async () => response(200)); const session = createSessionController(fetchFn); diff --git a/web/src/lib/components/deletions/DeletionsWorkspace.svelte b/web/src/lib/components/deletions/DeletionsWorkspace.svelte index 5c54c6b61..f5b87b8b6 100644 --- a/web/src/lib/components/deletions/DeletionsWorkspace.svelte +++ b/web/src/lib/components/deletions/DeletionsWorkspace.svelte @@ -1,3 +1,22 @@ + +
- + + {#snippet descriptionContent()}Deletions you've staged. Nothing is deleted until you run msgvault delete-staged.{/snippet} + {#if error}{/if} - -
-
-

Reviewed staging

-

Preflight the current session selection before creating a manifest.

-
-
-
- {#if reviewed} -
- {reviewed.count.toLocaleString()} - {reviewed.count === 1 ? 'item' : 'items'} · {reviewed.estimated_bytes.toLocaleString()} bytes - - {reviewed.deletable_count.toLocaleString()} can be staged · {(reviewed.count - reviewed.deletable_count).toLocaleString()} will be skipped.{selectionExclusions()} - - Authority expires {reviewed.expires_at} - {#if reviewed.search_deletion_scope === 'active'} - Semantic search covers active messages only. - {/if} - {#each reviewed.unavailable_actions as unavailable (`${unavailable.action}:${unavailable.reason}`)} - {unavailable.action}: {unavailable.reason} - {/each} -
-
-
- {/if} - {#if preview} -

{resultSummary(preview)}

- {#if stageCounts(preview).skipped > 0}{/if} - {/if} -
- + {#if reviewed} + {@const stageReason = unavailableReason('stage_deletion')} +
+ {reviewed.count.toLocaleString()} + {reviewed.count === 1 ? 'item' : 'items'} · {formatBytes(reviewed.estimated_bytes)} + + {reviewed.deletable_count.toLocaleString()} can be staged · {(reviewed.count - reviewed.deletable_count).toLocaleString()} will be skipped.{selectionExclusions()} + + Review expires + {#if reviewed.search_deletion_scope === 'active'} + Semantic search covers active messages only. + {/if} + {#if stageReason} + {preflightReasonLabel('stage_deletion', stageReason)} + {/if} +
+
+
+ {/if} + {#if preview} +

{resultSummary(preview)}

+ {#if stageCounts(preview).skipped > 0}{/if} + {/if} + + + {/if} {#if loading}

Loading deletion manifests…

- {:else if manifests.length === 0}

No deletion manifests yet.

+ {:else if manifests.length === 0} + {:else} -
- {#each manifests as manifest (manifest.id)} -
-
{manifest.id}{manifest.description}
- {manifest.message_count.toLocaleString()} {manifest.message_count === 1 ? 'item' : 'items'} - {manifest.status} -
-
+ {detailChip.label} + {detail.account || 'Account unavailable'} + {detail.message_count.toLocaleString()} items · {detail.description} + {#if detail.execution} + {detail.execution.succeeded} succeeded · {detail.execution.failed} failed + {#each detail.execution.failed_ids ?? [] as id}{id}{/each} {/if} - -
- {/each} -
- {/if} - - {#if detail} - - - + + + {/if} + {/if}
@@ -435,63 +513,74 @@ gap: var(--space-4); padding: var(--space-5) var(--page-gutter) var(--space-4); } - article, .actions, - .staging { + .staging-header, + .detail-header, + .row-actions { display: flex; align-items: center; gap: var(--space-3); } + .staging { + display: grid; + gap: var(--space-3); + } + .staging-header, + .detail-header { + justify-content: space-between; + } h2, .staging p { margin: 0; } - article span, - .staging p, - .actions span, - aside span { + .detail span { color: var(--text-muted); font-size: var(--font-size-xs); } - .staging { - flex-wrap: wrap; - justify-content: space-between; - } .review { display: grid; gap: var(--space-1); } - .reason, + .reason { + color: var(--text-secondary); + } .notice--error { color: var(--text-danger); } - .manifest-list { + .manifests.has-detail { display: grid; - border-top: 1px solid var(--border-muted); - } - article { - justify-content: space-between; - padding: var(--space-3); - border-bottom: 1px solid var(--border-muted); + grid-template-columns: minmax(0, 2fr) minmax(16rem, 1fr); + gap: var(--space-4); + align-items: start; } - article > div:first-child, - aside { + .detail { display: grid; - gap: var(--space-1); + gap: var(--space-2); } .notice, - .preview { + .result, + .warning { padding: var(--space-3); border: 1px solid var(--accent-amber); - border-radius: var(--radius-md); background: var(--bg-subtle); } + .notice, + .warning { + border-radius: var(--radius-md); + } + .result { + border-color: var(--border-default); + } .notice--error { border-color: var(--accent-red); } + @media (max-width: 900px) { + .manifests.has-detail { + grid-template-columns: 1fr; + } + } @media (max-width: 760px) { - article, - .staging { + .staging-header { align-items: stretch; flex-direction: column; } diff --git a/web/src/lib/components/deletions/DeletionsWorkspace.test.ts b/web/src/lib/components/deletions/DeletionsWorkspace.test.ts index 1c95d70a9..fdee9da3b 100644 --- a/web/src/lib/components/deletions/DeletionsWorkspace.test.ts +++ b/web/src/lib/components/deletions/DeletionsWorkspace.test.ts @@ -1,8 +1,9 @@ -import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { appShortcuts, initShortcuts } from '@kenn-io/kit-ui'; import { createAPIClient } from '../../api/client'; +import { formatDateTime } from '../../util/format'; import type { ExploreSelection as GeneratedExploreSelection } from '../../api/generated/models'; import DeletionsWorkspace from './DeletionsWorkspace.svelte'; @@ -56,6 +57,106 @@ function listResponse() { afterEach(() => document.body.replaceChildren()); describe('DeletionsWorkspace', () => { + it('explains how to start when nothing is selected', async () => { + render(DeletionsWorkspace, { + client: createAPIClient(vi.fn(async () => Response.json({ manifests: [] }))), + }); + expect(screen.getByText('Nothing selected for deletion')).toBeDefined(); + expect(screen.getByText('Select items in Everything, then choose Review for deletion…')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Review selection' })).toBeNull(); + expect(await screen.findByText('No staged deletions')).toBeDefined(); + expect(screen.getByText('msgvault delete-staged').tagName).toBe('CODE'); + }); + + it('summarizes a review with size, relative expiry, and only the staging reason', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/explore/preflight')) + return Response.json( + preflight({ + count: 2, + deletable_count: 2, + estimated_bytes: 3 * 1024, + expires_at: '2026-07-19T12:00:00Z', + unavailable_actions: [ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + { action: 'export', reason: 'browser_export_requires_single_message' }, + { action: 'open_in_source', reason: 'trusted_source_link_unavailable' }, + ], + }), + ); + return Response.json({ manifests: [] }); + }); + render(DeletionsWorkspace, { + client: createAPIClient(fetchFn), + selection: explicit, + now: () => new Date('2026-07-19T10:00:00Z'), + }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + + expect(await screen.findByText('2 items · 3 KB')).toBeDefined(); + expect(screen.getByText('in 2 hours').closest('span')?.textContent).toBe('Review expires in 2 hours'); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + expect(screen.queryByText(/Export works for one message/)).toBeNull(); + expect(screen.queryByText(/provide links to open/)).toBeNull(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('makes only the confirmation button red', async () => { + render(DeletionsWorkspace, { + client: createAPIClient( + vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + return new URL(request.url).pathname.endsWith('/explore/preflight') + ? Response.json(preflight()) + : Response.json({ manifests: [] }); + }), + ), + selection: explicit, + }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + const stage = await screen.findByRole('button', { name: 'Stage deletion…' }); + expect(stage.className).toContain('kit-button--info'); + expect(stage.className).toContain('kit-button--solid'); + expect(screen.getByRole('button', { name: 'Dry run' }).className).toContain('kit-button--outline'); + await fireEvent.click(stage); + expect(screen.getByRole('button', { name: 'Confirm stage deletion' }).className).toContain('kit-button--danger'); + }); + + it('lists manifests with status chips and closes the detail back to its Inspect button', async () => { + render(DeletionsWorkspace, { + client: createAPIClient( + vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/batch-1')) + return Response.json({ ...listResponse().manifests[0], account: 'archive@example.com' }); + return Response.json(listResponse()); + }), + ), + }); + const table = await screen.findByRole('table', { name: 'Deletion manifests' }); + expect(within(table).getAllByRole('columnheader').map((cell) => cell.textContent?.trim())).toEqual([ + 'ID', + 'Description', + 'Items', + 'Status', + 'Created', + 'Actions', + ]); + expect(within(table).getByText('Pending')).toBeDefined(); + expect(within(table).getByTitle(formatDateTime('2026-07-19T10:00:00Z', 'long'))).toBeDefined(); + expect(within(table).queryByTitle('2026-07-19T10:00:00Z')).toBeNull(); + expect(within(table).getByRole('button', { name: 'Cancel batch-1' }).className).not.toContain('kit-button--danger'); + await fireEvent.click(within(table).getByRole('button', { name: 'Inspect batch-1' })); + expect(await screen.findByText('archive@example.com')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Close manifest detail' })); + expect(screen.queryByText('archive@example.com')).toBeNull(); + await waitFor(() => + expect(document.activeElement).toBe(within(table).getByRole('button', { name: 'Inspect batch-1' })), + ); + }); + it('requires the deletable-count contract before offering staging', async () => { const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); @@ -67,7 +168,7 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit, reviewOnMount: true }); expect((await screen.findByRole('alert')).textContent).toContain('Upgrade the daemon and review again.'); - expect(screen.queryByRole('button', { name: 'Stage deletion' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Stage deletion…' })).toBeNull(); expect(screen.queryByRole('dialog')).toBeNull(); }); @@ -94,12 +195,12 @@ describe('DeletionsWorkspace', () => { await screen.findByText('batch-1'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - expect(await screen.findByText('1 item · 120 bytes')).toBeDefined(); + expect(await screen.findByText('1 item · 120 B')).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1 · Staged: 1 · Skipped: 0 in archive@example.com/)).toBeDefined(); expect(screen.queryByRole('alert')).toBeNull(); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); expect(screen.getByRole('dialog', { name: 'Confirm selected deletion' })).toBeDefined(); expect(deletionPosts).toBe(1); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); @@ -138,14 +239,14 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('3 items · 120 bytes'); + await screen.findByText('3 items · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1 in archive@example.com/)).toBeDefined(); expect(screen.getByRole('alert').textContent).toMatch(/Partial staging.*deletable Gmail and Microsoft Graph mail subset.*unsupported match will be skipped/); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); const dialog = screen.getByRole('dialog', { name: 'Confirm selected deletion' }); expect(dialog.textContent).toMatch(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1/); expect(dialog.textContent).toMatch(/Only deletable Gmail and Microsoft Graph mail messages will be staged/); @@ -173,9 +274,9 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('3 items · 120 bytes'); + await screen.findByText('3 items · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1/)).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); @@ -201,9 +302,9 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); await waitFor(() => expect(posts).toBe(1)); await rendered.rerender({ client: createAPIClient(fetchFn), selection: matching }); @@ -247,10 +348,10 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await screen.findByText('1 item · 120 B'); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); await waitFor(() => expect(stagePosts).toBe(1)); await rendered.rerender({ client: createAPIClient(fetchFn), selection: matching }); @@ -291,7 +392,7 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: matching }); try { - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.keyDown(window, { key: 'D', shiftKey: true }); expect(await screen.findByRole('dialog', { name: 'Confirm matching deletion' })).toBeDefined(); expect(screen.getByText(/Matched: 8 · Will stage: 6 · Will skip: 2.*After 1 exclusion/)).toBeDefined(); @@ -316,7 +417,7 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); try { - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.keyDown(window, { key: 'd' }); expect(await screen.findByRole('dialog', { name: 'Confirm selected deletion' })).toBeDefined(); expect(shellHandler).not.toHaveBeenCalled(); @@ -356,7 +457,7 @@ describe('DeletionsWorkspace', () => { expect(requests.some((request) => request.method === 'DELETE')).toBe(false); await fireEvent.click(screen.getByRole('button', { name: 'Confirm cancel manifest' })); await waitFor(() => expect(requests.some((request) => request.method === 'DELETE')).toBe(true)); - expect((await screen.findAllByText('cancelled')).length).toBeGreaterThan(0); + expect((await screen.findAllByText('Cancelled')).length).toBeGreaterThan(0); }); it('discloses the active-only deletion scope reported by the preflight review', async () => { @@ -371,7 +472,7 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); expect(screen.queryByText(/active messages only/)).toBeNull(); scoped = true; @@ -395,8 +496,8 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); - expect(await screen.findByText(/selection_contains_items_that_cannot_be_deleted_from_source/)).toBeDefined(); - expect((screen.getByRole('button', { name: 'Stage deletion' }) as HTMLButtonElement).disabled).toBe(true); + expect(await screen.findByText('None of the selected items can be deleted from their source.')).toBeDefined(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); }); it('clears stale result counts when dry-run and create requests fail', async () => { @@ -414,9 +515,9 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1/)).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); @@ -425,7 +526,7 @@ describe('DeletionsWorkspace', () => { await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1/)).toBeDefined(); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); expect(await screen.findByText('create failed')).toBeDefined(); expect(screen.queryByText(/Dry run: Matched: 1/)).toBeNull(); diff --git a/web/src/lib/components/explore/EverythingTable.svelte b/web/src/lib/components/explore/EverythingTable.svelte index 1995bd446..4a9fd137c 100644 --- a/web/src/lib/components/explore/EverythingTable.svelte +++ b/web/src/lib/components/explore/EverythingTable.svelte @@ -12,6 +12,7 @@ import { DEFAULT_EXPLORE_COLUMNS, isEmailMessageType } from '../../explore/models'; import type { ExploreSelectionState } from '../../explore/state.svelte'; import { rebaseVirtualScroll, RowGeometry, tableViewportHeight } from '../../theme/preferences.svelte'; + import { formatBytes } from '../../util/format'; import IdentityBadge from './IdentityBadge.svelte'; import RowKind from './RowKind.svelte'; @@ -239,12 +240,6 @@ }).format(date); } - function formatBytes(value: number): string { - if (value < 1024) return `${value} B`; - if (value < 1024 * 1024) return `${Math.round(value / 1024)} KB`; - return `${(value / (1024 * 1024)).toFixed(1)} MB`; - } - function scrollActiveIntoView(index: number): void { const height = rowHeight; if (!gridElement || height === undefined) return; diff --git a/web/src/lib/components/explore/SelectionBar.svelte b/web/src/lib/components/explore/SelectionBar.svelte index 6981766b9..3721eb288 100644 --- a/web/src/lib/components/explore/SelectionBar.svelte +++ b/web/src/lib/components/explore/SelectionBar.svelte @@ -43,6 +43,10 @@ const exportReason = $derived(preflight?.unavailable_actions.find((item) => item.action === 'export')?.reason); const openReason = $derived(preflight?.unavailable_actions.find((item) => item.action === 'open_in_source')?.reason); + const stageReason = $derived( + preflight?.unavailable_actions.find((item) => item.action === 'stage_deletion')?.reason, + ); + const stageReasonID = $props.id(); const visible = $derived(selection.mode === 'all_matching' || selection.count > 0); const openMenuVisible = $derived(Boolean(openReason) || Boolean(preflight && onOpenInSource)); const exportTarget = $derived(preflight?.action_targets?.find((item) => item.action === 'export')); @@ -113,8 +117,15 @@ size="sm" surface="soft" label="Review for deletion…" + disabled={Boolean(stageReason)} + ariaDescribedby={stageReason ? stageReasonID : undefined} onclick={() => onReviewDeletion(selection.mode === 'all_matching' ? 'all_matching' : 'explicit')} /> + {#if stageReason} + + {preflightReasonLabel('stage_deletion', stageReason)} + + {/if} {/if} {#if openMenuVisible} diff --git a/web/src/lib/components/explore/SelectionBar.test.ts b/web/src/lib/components/explore/SelectionBar.test.ts index b1448be75..73f2355ec 100644 --- a/web/src/lib/components/explore/SelectionBar.test.ts +++ b/web/src/lib/components/explore/SelectionBar.test.ts @@ -192,6 +192,35 @@ describe('SelectionBar', () => { expect(onReviewDeletion).toHaveBeenLastCalledWith('all_matching'); }); + it('disables Review for deletion with the staging reason as a sentence', () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { + selection, + totalCount: 2, + onReviewDeletion: vi.fn(), + preflight: preflight([ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + ]), + }); + + const review = screen.getByRole('button', { name: 'Review for deletion…' }) as HTMLButtonElement; + expect(review.disabled).toBe(true); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(review.getAttribute('aria-describedby')).toBe(reason.id); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + }); + + it('keeps Review for deletion enabled when staging is available', () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { selection, totalCount: 2, onReviewDeletion: vi.fn(), preflight: preflight() }); + + const review = screen.getByRole('button', { name: 'Review for deletion…' }) as HTMLButtonElement; + expect(review.disabled).toBe(false); + expect(review.getAttribute('aria-describedby')).toBeNull(); + }); + it('keeps meeting context independent from raw-export preflight eligibility', async () => { const selection = new ExploreSelectionState(); selection.selectVisible(['message:7', 'message:91']); diff --git a/web/src/lib/components/files/FileViewer.svelte b/web/src/lib/components/files/FileViewer.svelte index 0063128ae..433490393 100644 --- a/web/src/lib/components/files/FileViewer.svelte +++ b/web/src/lib/components/files/FileViewer.svelte @@ -1,11 +1,3 @@ - - + +

+ {setup.text} + {setup.guideLabel} +

+ + diff --git a/web/src/lib/components/operations/OperationLaneCards.svelte b/web/src/lib/components/operations/OperationLaneCards.svelte deleted file mode 100644 index 4daa01efb..000000000 --- a/web/src/lib/components/operations/OperationLaneCards.svelte +++ /dev/null @@ -1,203 +0,0 @@ - - -
- {#each lanes as lane (lane.lane)} -
-

{laneLabels[lane.lane]}

- {#if lane.kinds.length === 0} -

Status unavailable

- {:else} -
- {#each lane.kinds as kind (kind.kind)} -
-

{kindLabels[kind.kind]}

-
- - - {kind.configured ? 'Configured' : 'Not configured'} - - - - {kind.history_availability === 'available' ? 'History available' : 'History unavailable'} - -
-
- {#if kind.active} -
-
Active
-
- - {stateLabel(kind.active)} - -
-
- {/if} - {#if kind.latest} -
-
Latest
-
- - {stateLabel(kind.latest)} - -
-
- {/if} - {#if kind.latest_successful} -
-
Last successful
-
- - Succeeded - -
-
- {/if} -
- {#if kind.history_availability === 'available' && !kind.active && !kind.latest && !kind.latest_successful} -

No recorded runs

- {/if} - {#if kind.related_status || kind.supported_actions.length > 0} -
- {#if kind.related_status} -
- {/if} -
- {/each} -
- {/if} -
- {/each} -
- - diff --git a/web/src/lib/components/operations/OperationRelatedStatus.svelte b/web/src/lib/components/operations/OperationRelatedStatus.svelte index 606cd53af..abc6ca60e 100644 --- a/web/src/lib/components/operations/OperationRelatedStatus.svelte +++ b/web/src/lib/components/operations/OperationRelatedStatus.svelte @@ -4,7 +4,7 @@ getDocumentVectorStatus as generatedGetDocumentVectorStatus, getVisualAttachmentStatus as generatedGetVisualAttachmentStatus, } from '../../api/generated/api/api'; - import { Button, StatusDot } from '@kenn-io/kit-ui'; + import { Button, Chip, StatusDot } from '@kenn-io/kit-ui'; import { onMount } from 'svelte'; import type { APIClient } from '../../api/client'; @@ -14,6 +14,8 @@ Status as GeneratedVisualStatus, } from '../../api/generated/models'; import type { OperationStatusAuthority } from '../../explore/models'; + import { DOCUMENT_INDEX_SETUP, DOCUMENT_SEARCH_SETUP } from '../../operations/labels'; + import OperationHostSetup from './OperationHostSetup.svelte'; type DocumentStatus = GeneratedDocumentIndexStatusResponse; type DocumentVectorStatus = GeneratedDocumentVectorOperationsResponse; @@ -30,7 +32,7 @@ authority: OperationStatusAuthority; configured?: boolean; onClose?: () => void; - onConfigure?: (authority: OperationStatusAuthority) => void; + onConfigure?: () => void; } = $props(); let loading = $state(true); @@ -51,12 +53,6 @@ getVisualAttachmentStatus: 'Visual attachments need configuration' }; - const settingsLabels: Record = { - getDocumentIndexStatus: 'Open document index settings', - getDocumentVectorStatus: 'Open document vector settings', - getVisualAttachmentStatus: 'Open visual attachment settings' - }; - onMount(() => { if (configured === false) { loading = false; @@ -107,9 +103,15 @@

Loading {labels[authority].toLowerCase()}…

{:else if configured === false}
-

{configurationLabels[authority]}

+

Off {configurationLabels[authority]}

- {/each} @@ -107,15 +73,15 @@ class="run-link" type="button" data-run-id={run.id} - aria-label={`Open ${kindLabels[run.kind]} run`} + aria-label={`Open ${OPERATION_KIND_LABELS[run.kind]} run`} onclick={(event) => onSelect(run.id, event.currentTarget)} - >{kindLabels[run.kind]} + >{OPERATION_KIND_LABELS[run.kind]} - {titleCase(run.trigger)} - {titleCase(run.state)} - - {duration(run)} - {counters(run)} + {triggerLabel(run.trigger)} + {@render runState(run)} + + {operationDuration(run)} + {counterSummary(run.counters)} {/each} @@ -138,7 +104,8 @@ cursor: pointer; } .run-link:focus-visible { outline: var(--focus-ring); outline-offset: 2px; } - .state { display: inline-flex; align-items: center; gap: var(--space-2); white-space: nowrap; } + .state { display: grid; justify-items: start; gap: var(--space-1); } + .state .error { color: var(--status-error-ink); } .run-list { display: grid; gap: var(--space-2); } .run-list [role="listitem"] { border: 1px solid var(--border-default); border-radius: var(--radius-md); background: var(--bg-surface); } .run-list [role="listitem"].selected { border-color: var(--accent-blue); } diff --git a/web/src/lib/components/operations/OperationStatusList.svelte b/web/src/lib/components/operations/OperationStatusList.svelte new file mode 100644 index 000000000..f60fed1d7 --- /dev/null +++ b/web/src/lib/components/operations/OperationStatusList.svelte @@ -0,0 +1,139 @@ + + +
+ {#each lanes as lane (lane.lane)} + {@const headingID = `operation-lane-${lane.lane}`} +
+

{OPERATION_LANE_LABELS[lane.lane]}

+ {#if lane.kinds.length === 0} + Status unavailable + {:else} +
    + {#each lane.kinds as kind (kind.kind)} + {@const nameID = `operation-kind-${kind.kind}`} + {@const chip = statusChip(kind)} + {@const run = kind.active ?? kind.latest} + {@const setup = kind.configured ? undefined : OPERATION_SETUP[kind.kind]} +
  • + {OPERATION_KIND_LABELS[kind.kind]} + + {#if chip}{chip.label}{/if} + {#if kind.history_availability !== 'available'} + History unavailable + {/if} + + + {#if run}{/if} + {#if kind.latest_successful && run?.state !== 'succeeded'} + + Last succeeded + + + {/if} + + + {#if kind.related_status} +
    • + {/each} +
+ {/if} +
+ {/each} +
+ + diff --git a/web/src/lib/components/operations/OperationsRefresh.test.ts b/web/src/lib/components/operations/OperationsRefresh.test.ts new file mode 100644 index 000000000..4d5afd7bf --- /dev/null +++ b/web/src/lib/components/operations/OperationsRefresh.test.ts @@ -0,0 +1,120 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import { OperationsController } from '../../operations/controller.svelte'; +import type { OperationRunSummary, OperationsURLState } from '../../operations/models'; +import OperationsWorkspace from './OperationsWorkspace.svelte'; + +// The daemon encrypts run IDs with a fresh nonce per response +// (internal/api/operation_tokens.go), so the same run never carries the same +// ID twice. These fixtures do the same; a test cannot pass by matching IDs. +let encodings = 0; +const encode = (run: number) => `op2.${String(++encodings).padStart(32, '0')}.run${run}`; + +function summary(run: number): OperationRunSummary { + return { + id: encode(run), kind: 'source_sync', lane: 'messages', trigger: 'manual', state: 'succeeded', + started_at: `2026-08-30T1${run}:00:00Z`, finished_at: `2026-08-30T1${run}:01:00Z`, + counters: [{ name: 'processed', unit: 'messages', value: run }] + }; +} + +const state = (overrides: Partial = {}): OperationsURLState => ({ + operationLane: '', operationKind: '', operationState: '', operationStartedFrom: '', + operationStartedBefore: '', operationRunID: null, operationStatus: '', ...overrides +}); + +function daemon(pageTwo: () => Response = () => Response.json({ + runs: [summary(3), summary(4)], membership_revision: 7, unavailable_kinds: [] +})) { + const requests: string[] = []; + const fetchFn = vi.fn(async (input) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + requests.push(`${url.pathname}${url.search}`); + if (url.pathname === '/api/v1/operations/status') { + return Response.json({ lanes: [{ lane: 'messages', kind: 'source_sync', configured: true, + history_availability: 'available', supported_actions: [], latest: summary(1) }] }); + } + if (url.pathname === '/api/v1/operations/runs') { + if (url.searchParams.get('cursor') === 'page-two') return pageTwo(); + return Response.json({ + runs: [summary(1), summary(2)], membership_revision: 7, unavailable_kinds: [], next_cursor: 'page-two' + }); + } + return Response.json({ ...summary(3), related_status: 'listSourceStatus', supported_actions: [] }); + }); + return { requests, client: createAPIClient(fetchFn) }; +} + +const runButtons = () => screen.getAllByRole('button', { name: 'Open Source sync run' }); + +afterEach(() => vi.useRealTimers()); + +describe('Operations refresh', () => { + it('refreshes only status on its timer and on click, keeping paged rows, detail, and focus', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const { requests, client } = daemon(); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + await controller.loadMore(); + const selected = controller.snapshot.rows[2]!.id; + await controller.applyURLState(state({ operationRunID: selected })); + render(OperationsWorkspace, { controller, state: state({ operationRunID: selected }) }); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + runButtons()[2]!.focus(); + const focused = document.activeElement; + + requests.length = 0; + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await waitFor(() => expect(requests).toEqual(['/api/v1/operations/status'])); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + expect(document.activeElement).toBe(focused); + + requests.length = 0; + await fireEvent.click(screen.getByRole('button', { name: 'Refresh operation status' })); + await waitFor(() => expect(requests).toEqual(['/api/v1/operations/status'])); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + } finally { + controller.destroy(); + } + }); + + it('reloads status and page one of runs from Reload run history', async () => { + const { requests, client } = daemon(); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + await controller.loadMore(); + const before = controller.snapshot.rows.map((row) => row.id); + render(OperationsWorkspace, { controller, state: state() }); + requests.length = 0; + await fireEvent.click(screen.getByRole('button', { name: 'Reload run history' })); + await waitFor(() => expect(runButtons()).toHaveLength(2)); + expect(requests.sort()).toEqual(['/api/v1/operations/runs?limit=25', '/api/v1/operations/status']); + expect(controller.snapshot.rows.some((row) => before.includes(row.id))).toBe(false); + } finally { + controller.destroy(); + } + }); + + it('keeps the conflict notice when Load more meets a changed history', async () => { + const { client } = daemon(() => Response.json( + { error: 'operation_history_conflict', message: 'Operation history changed.' }, { status: 409 })); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + render(OperationsWorkspace, { controller, state: state() }); + await fireEvent.click(screen.getByRole('button', { name: 'Load more operation history' })); + const conflict = await screen.findByRole('alert', { name: 'Operation history conflict' }); + expect(conflict.textContent).toContain('Operation history changed. Restart from the first page.'); + expect(screen.getByRole('button', { name: 'Restart operation history' })).toBeDefined(); + } finally { + controller.destroy(); + } + }); +}); diff --git a/web/src/lib/components/operations/OperationsWorkspace.svelte b/web/src/lib/components/operations/OperationsWorkspace.svelte index 1bac1b065..c96541964 100644 --- a/web/src/lib/components/operations/OperationsWorkspace.svelte +++ b/web/src/lib/components/operations/OperationsWorkspace.svelte @@ -2,6 +2,7 @@ import { Button, DateRangePicker, + RefreshControl, SelectDropdown, resolveRange, type RangeSelection @@ -16,20 +17,23 @@ resolveOperationFocusAnchor, type OperationFocusAnchor } from '../../operations/focus'; - import type { - OperationAction, - OperationKind, - OperationRunDetail as OperationRunDetailModel, - OperationsURLState - } from '../../operations/models'; - import OperationLaneCards from './OperationLaneCards.svelte'; + import type { OperationAction, OperationKind, OperationsURLState } from '../../operations/models'; + import { + OPERATION_KIND_LABELS, + OPERATION_LANE_LABELS, + RELATED_STATUS_LABELS, + type OperationSettingsTarget, + type RelatedStatus + } from '../../operations/labels'; import OperationRelatedStatus from './OperationRelatedStatus.svelte'; import OperationRunDetail from './OperationRunDetail.svelte'; import OperationRunTable from './OperationRunTable.svelte'; + import OperationStatusList from './OperationStatusList.svelte'; import PageHeader from '../shell/PageHeader.svelte'; - type RelatedStatus = NonNullable; - type Controller = Pick; + type Controller = Pick< + OperationsController, 'snapshot' | 'refresh' | 'refreshStatus' | 'loadMore' | 'restart' | 'runAction' + >; let { controller, @@ -38,7 +42,8 @@ onStateChange = () => undefined, onNavigate = () => undefined, onAnnounce = () => undefined, - onConfigure = () => undefined + onConfigure = () => undefined, + onSetUp = () => undefined }: { controller: Controller; client?: APIClient; @@ -46,7 +51,8 @@ onStateChange?: (patch: Partial) => void; onNavigate?: (target: RelatedStatus) => void; onAnnounce?: (message: string) => void; - onConfigure?: (target: OperationStatusAuthority) => void; + onConfigure?: () => void; + onSetUp?: (target: OperationSettingsTarget) => void; } = $props(); let root = $state(); @@ -61,23 +67,11 @@ const laneOptions = [ { value: '', label: 'All lanes' }, - { value: 'messages', label: 'Messages' }, - { value: 'person_facts', label: 'Facts' }, - { value: 'contacts', label: 'Contacts' }, - { value: 'documents', label: 'Documents' }, - { value: 'visual_attachments', label: 'Attachments' } + ...Object.entries(OPERATION_LANE_LABELS).map(([value, label]) => ({ value, label })) ]; const kindOptions = [ { value: '', label: 'All kinds' }, - { value: 'source_sync', label: 'Source sync' }, - { value: 'message_embedding', label: 'Message embedding' }, - { value: 'person_sweep', label: 'Person fact sweep' }, - { value: 'person_embedding', label: 'Person embedding' }, - { value: 'person_enrichment', label: 'Person enrichment' }, - { value: 'carddav_sync', label: 'CardDAV sync' }, - { value: 'document_extraction', label: 'Document extraction' }, - { value: 'document_embedding', label: 'Document embedding' }, - { value: 'visual_embedding', label: 'Visual embedding' } + ...Object.entries(OPERATION_KIND_LABELS).map(([value, label]) => ({ value, label })) ]; const stateOptions = [ { value: '', label: 'All states' }, @@ -88,27 +82,11 @@ { value: 'failed', label: 'Failed' }, { value: 'cancelled', label: 'Cancelled' } ]; - const kindLabels: Record = { - source_sync: 'Source sync', - message_embedding: 'Message embedding', - person_sweep: 'Person fact sweep', - person_embedding: 'Person embedding', - person_enrichment: 'Person enrichment', - carddav_sync: 'CardDAV sync', - document_extraction: 'Document extraction', - document_embedding: 'Document embedding', - visual_embedding: 'Visual embedding' - }; const actionSuccess: Record = { carddav_sync: 'CardDAV sync request completed; current operation state was refreshed.', visual_build: 'Visual index build request completed; current operation state was refreshed.', visual_resume: 'Visual index resume request completed; current operation state was refreshed.' }; - const relatedStatusLabels: Record = { - getDocumentIndexStatus: 'Open Document index status', - getDocumentVectorStatus: 'Open Document vector status', - getVisualAttachmentStatus: 'Open Visual attachment status' - }; onMount(() => { const mediaQuery = window.matchMedia?.('(max-width: 760px)'); @@ -148,11 +126,15 @@ return undefined; } + function relatedStatusButton(target: OperationStatusAuthority): string { + return `Open ${RELATED_STATUS_LABELS[target]}`; + } + function navigateStatus(target: RelatedStatus, button: HTMLButtonElement): void { if (target === 'getDocumentIndexStatus' || target === 'getDocumentVectorStatus' || target === 'getVisualAttachmentStatus') { const buttons = Array.from(root?.querySelectorAll('button') ?? []); - const targetButtons = buttons.filter((candidate) => candidate.ariaLabel === relatedStatusLabels[target]); + const targetButtons = buttons.filter((candidate) => candidate.ariaLabel === relatedStatusButton(target)); statusFocus = { target, ordinal: Math.max(0, targetButtons.indexOf(button)), @@ -168,7 +150,7 @@ await tick(); await new Promise((resolve) => setTimeout(resolve, 0)); const buttons = Array.from(root?.querySelectorAll('button') ?? []); - const targetButtons = buttons.filter((button) => button.ariaLabel === relatedStatusLabels[focus.target]); + const targetButtons = buttons.filter((button) => button.ariaLabel === relatedStatusButton(focus.target)); (targetButtons[focus.ordinal] ?? buttons[focus.slot])?.focus(); statusFocus = undefined; } @@ -207,6 +189,19 @@ }); } + async function refreshStatus(): Promise { + const control = root?.querySelector('.kit-refresh-control'); + const hadFocus = Boolean(control?.contains(document.activeElement)); + await controller.refreshStatus(); + // Kit disables the button while busy, and a disabled button drops focus. + // Restore it only if the person has not moved focus elsewhere meanwhile. + if (!hadFocus) return; + await tick(); + const active = document.activeElement; + if (active && active !== document.body && !control?.contains(active)) return; + control?.querySelector('button')?.focus(); + } + async function runAction(action: OperationAction): Promise { const outcome = await controller.runAction(action); if (outcome === 'discarded') return; @@ -251,7 +246,7 @@ {#if current.unavailableKinds.length > 0}
{#each current.unavailableKinds as unavailable (unavailable.kind)} - {kindLabels[unavailable.kind]} history is unavailable. + {OPERATION_KIND_LABELS[unavailable.kind]} history is unavailable. {/each}
{/if} @@ -305,16 +300,22 @@ {:else} {#snippet actions()} - ' })); + const actions = createRawSnippet(() => ({ render: () => '' })); render(PageHeader, { title: 'Operations', description: 'Background work and its history.', actions }); expect(screen.getByRole('heading', { level: 1, name: 'Operations' })).toBeTruthy(); expect(screen.getByText('Background work and its history.')).toBeTruthy(); - expect(screen.getByRole('button', { name: 'Refresh operations' })).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Example action' })).toBeTruthy(); }); it('renders the view row below the title', () => { diff --git a/web/src/lib/components/sources/SourcesWorkspace.svelte b/web/src/lib/components/sources/SourcesWorkspace.svelte index cafc3586e..f8e328bdc 100644 --- a/web/src/lib/components/sources/SourcesWorkspace.svelte +++ b/web/src/lib/components/sources/SourcesWorkspace.svelte @@ -3,10 +3,15 @@ listSourceStatus as generatedListSourceStatus, triggerSync as generatedTriggerSync, } from '../../api/generated/api/api'; - import { Button, Chip, Spinner, Table, TableHeaderCell, type ChipTone } from '@kenn-io/kit-ui'; + import { Button, Chip, IconButton, Table, TableHeaderCell } from '@kenn-io/kit-ui'; + import ChevronDown from '@lucide/svelte/icons/chevron-down'; + import ChevronRight from '@lucide/svelte/icons/chevron-right'; import { onDestroy, onMount } from 'svelte'; + import { SvelteSet } from 'svelte/reactivity'; import type { APIClient } from '../../api/client'; import { scheduleSummary } from '../../settings/cron'; + import { sourceTypeLabel, syncStatusChip, syncUnavailableLabel } from '../../sources/labels'; + import { formatDateTime } from '../../util/format'; import PageHeader from '../shell/PageHeader.svelte'; import type { SourceStatus as GeneratedSourceStatus, @@ -34,6 +39,7 @@ onOpenOperations?: () => void; } = $props(); let sources = $state([]); + const expanded = new SvelteSet(); let loading = $state(true); let lockStatusStale = $state(false); let statusError = $state(''); @@ -260,25 +266,17 @@ function label(source: Source): string { return source.display_name || source.identifier; } - function statusLabel(run: SyncRun | null): string { - if (!run) return 'Never'; - if (run.status === 'completed') return 'Completed'; - if (run.status === 'failed') return 'Failed'; - return run.status.replaceAll('_', ' '); - } - function statusTone(run: SyncRun): ChipTone { - if (run.status === 'completed') return 'success'; - if (run.status === 'failed') return 'danger'; - return 'info'; - } function resultTimestamp(run: SyncRun | null | undefined): string | undefined { return run?.completed_at ?? run?.started_at; } - function formatTimestamp(value: string | null | undefined): string { - if (!value) return 'Not available'; - const date = new Date(value); - if (!Number.isFinite(date.getTime())) return value; - return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'short' }).format(date); + function hasDetails(source: Source): boolean { + return Boolean( + source.latest_sync?.error_message || source.latest_sync?.item_errors?.length || source.scheduler_last_error + ); + } + function toggleDetails(id: number): void { + if (expanded.has(id)) expanded.delete(id); + else expanded.add(id); } function staleLastResult(source: Source): boolean { const resultAt = source.latest_sync?.completed_at ?? source.latest_sync?.started_at; @@ -304,7 +302,7 @@
{#snippet actions()} -