diff --git a/docs/governance-incident-runbooks.md b/docs/governance-incident-runbooks.md index bc18e6bb96..9a35b2dad7 100644 --- a/docs/governance-incident-runbooks.md +++ b/docs/governance-incident-runbooks.md @@ -112,9 +112,9 @@ These are short operator checklists. Preserve evidence and identifiers, but do n 3. If raw query persistence, answer persistence, or provider response storage is enabled, disable the affected path. Verify all three controls are off before proceeding: ```bash railway variables --service clinical-kb-app \ - --set "RAG_PERSIST_RAW_QUERY=false" \ - --set "RAG_PERSIST_RAW_ANSWER=false" \ - --set "RAG_STORE_PROVIDER_RESPONSES=false" + --set "RAG_PERSIST_RAW_QUERY_TEXT=false" \ + --set "RAG_PERSIST_ANSWER_TEXT=false" \ + --set "OPENAI_STORE_RESPONSES=false" ``` 4. Inspect affected answer copies in `rag_response_cache.payload`; invalidate affected owners and purge incident-related rows where appropriate, then verify the bounded retention purge before assessing other cache and telemetry retention: ```sql diff --git a/scripts/deploy/write-release-manifest.mjs b/scripts/deploy/write-release-manifest.mjs index fbb38c3a11..9d1bfbf51f 100644 --- a/scripts/deploy/write-release-manifest.mjs +++ b/scripts/deploy/write-release-manifest.mjs @@ -29,16 +29,15 @@ function readGitHeadCommitSha(cwd = REPO_ROOT) { } export function resolveReleaseManifestIdentity(env = process.env, options = {}) { - const envCommitCandidate = - env.RAILWAY_GIT_COMMIT_SHA || - env.GITHUB_SHA || - env.SOURCE_COMMIT_SHA || - env.COMMIT_SHA || - env.VERCEL_GIT_COMMIT_SHA || - ""; + // Same candidate order as resolveDeploymentCommitSha in src/lib/observability/sentry-release.ts, + // and the first candidate that is a valid SHA wins, so a blank or malformed earlier variable + // cannot hide a valid later one. + const envCommitSha = + [env.RAILWAY_GIT_COMMIT_SHA, env.VERCEL_GIT_COMMIT_SHA, env.GITHUB_SHA, env.SOURCE_COMMIT_SHA, env.COMMIT_SHA] + .map(normalizeCommitSha) + .find(Boolean) ?? null; const commitSha = - normalizeCommitSha(envCommitCandidate) ?? - (options.resolveGitHead === false ? null : readGitHeadCommitSha(options.cwd ?? REPO_ROOT)); + envCommitSha ?? (options.resolveGitHead === false ? null : readGitHeadCommitSha(options.cwd ?? REPO_ROOT)); const explicitRelease = (env.SENTRY_RELEASE || env.NEXT_PUBLIC_SENTRY_RELEASE || "").trim(); const sentryRelease = explicitRelease || commitSha || "dev"; diff --git a/scripts/ops-digest.mjs b/scripts/ops-digest.mjs index 4b8398a3f0..0168bb12ed 100644 --- a/scripts/ops-digest.mjs +++ b/scripts/ops-digest.mjs @@ -108,7 +108,9 @@ export function updateHybridRpcHourlyEvidence(rawHistory, health, observedAt = n lastThree.every((entry, index) => { if (index === 0) return true; const gapMs = Date.parse(entry.hour) - Date.parse(lastThree[index - 1].hour); - return gapMs === hourlyWindowMs || gapMs === 6 * hourlyWindowMs; + // Scheduled runs are six hours apart, and a manual run between them shares the same history + // file, so any positive whole-hour gap up to six hours still counts as consecutive evidence. + return gapMs > 0 && gapMs <= 6 * hourlyWindowMs; }); const repeatedRpcNames = contiguous ? lastThree[0].rpcNames.filter((name) => lastThree.every((entry) => entry.rpcNames.includes(name))) diff --git a/scripts/trivy-image-scan.mjs b/scripts/trivy-image-scan.mjs index f13eb6939a..58740993b6 100644 --- a/scripts/trivy-image-scan.mjs +++ b/scripts/trivy-image-scan.mjs @@ -9,7 +9,7 @@ * daemon socket, which would be equivalent to root on the host. */ import { basename, dirname, join, resolve } from "node:path"; -import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; @@ -162,6 +162,43 @@ export function trivyDockerMounts({ workDir, cacheDir = null, outputDir = null } return mounts; } +/** + * Whether the cached vulnerability database must be downloaded before an offline scan. Scans pass + * --skip-db-update, so a persistent TRIVY_CACHE_DIR would otherwise keep using an expired database + * and report STATUS=ok on stale data. A missing, unreadable or undated metadata file, or a + * NextUpdate time that has passed, all require a fresh download. + * + * @param {string | null} metadataText contents of /db/metadata.json, or null when absent + * @param {Date} [now] + * @returns {boolean} + */ +export function trivyDbNeedsRefresh(metadataText, now = new Date()) { + if (typeof metadataText !== "string") return true; + let nextUpdate; + try { + nextUpdate = Date.parse(JSON.parse(metadataText)?.NextUpdate); + } catch { + return true; + } + return !Number.isFinite(nextUpdate) || nextUpdate <= now.getTime(); +} + +/** + * Whether a cached Trivy database directory needs downloading: its database file is missing (a + * partial cache), or its metadata says it is stale. The main database lives in `db/trivy.db` and + * the Java database in `java-db/trivy-java.db`, each beside its own metadata.json. + */ +export function cachedTrivyDbNeedsRefresh(cacheDir, subdir, dbFile, now = new Date()) { + if (!existsSync(join(cacheDir, subdir, dbFile))) return true; + let metadataText = null; + try { + metadataText = readFileSync(join(cacheDir, subdir, "metadata.json"), "utf8"); + } catch { + // Unreadable metadata is treated as absent. + } + return trivyDbNeedsRefresh(metadataText, now); +} + /** * @param {string} imageRef * @param {string[]} trivyArgs use "__IMAGE__" where the image ref belongs and @@ -179,7 +216,8 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null, needsVulne ? resolve(process.env.TRIVY_CACHE_DIR) : join(tmpdir(), `trivy-db-cache-${TRIVY_VERSION}`); mkdirSync(cacheDir, { recursive: true }); - if (!existsSync(join(cacheDir, "db", "metadata.json"))) { + // A failed refresh fails the scan rather than falling back to the expired database. + if (cachedTrivyDbNeedsRefresh(cacheDir, "db", "trivy.db")) { const dbDownload = run("docker", [ "run", "--rm", @@ -198,7 +236,10 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null, needsVulne stderr: dbDownload.stderr || dbDownload.stdout || "Trivy DB download failed", }; } - // Best-effort Java DB pre-population for Java archive inspection offline. + } + // The Java database expires on its own schedule, so it is checked separately. Its download + // stays best-effort: it only adds Java archive inspection, and a failure is retried next run. + if (cachedTrivyDbNeedsRefresh(cacheDir, "java-db", "trivy-java.db")) { run("docker", [ "run", "--rm", diff --git a/src/lib/roster/alerts/dispatch.ts b/src/lib/roster/alerts/dispatch.ts index c200df26f1..2b405ab680 100644 --- a/src/lib/roster/alerts/dispatch.ts +++ b/src/lib/roster/alerts/dispatch.ts @@ -236,7 +236,10 @@ async function runRosterAlertDispatch( ? new Set() : await recipientsOnNightNow(client, event.actorId, event.serviceId, eligible); const delivery = await sendRosterAlerts(client, eligible, type, quiet); - if (isOpenPostOffer && result.openShiftId && (delivery?.failed ?? 0) === 0) { + // Any successful send marks the offer delivered: a replay resends to the whole eligible list, so + // retrying a partial delivery would duplicate the offer for everyone it already reached. A gone + // device also counts as a failed send, so only a delivery that reached nobody is retried. + if (isOpenPostOffer && result.openShiftId && ((delivery?.sent ?? 0) > 0 || (delivery?.failed ?? 0) === 0)) { await markOpenPostAlertDelivered(client, event.serviceId, result.openShiftId); } } diff --git a/src/lib/teaching/repository.ts b/src/lib/teaching/repository.ts index 8932991fb3..55bd8558b9 100644 --- a/src/lib/teaching/repository.ts +++ b/src/lib/teaching/repository.ts @@ -475,21 +475,15 @@ export async function teachingServiceMutation( created_by?: string | null; teaching_occurrences?: { presenter_id?: string | null } | null; }>; - const ownRows = rows.filter((row) => row.created_by === actorId); - const otherRows = rows.filter((row) => !row.created_by || row.created_by !== actorId); - let isOrganiser = false; - if (rows.length === 0 || otherRows.length > 0) { - try { - await verifyActiveOrganiser(); - isOrganiser = true; - } catch (orgErr) { - if (ownRows.length === 0) { - throw orgErr; - } - } + // Revoking by occurrence revokes every active link or none. A link someone else created + // (or with no recorded creator) needs an active organiser; a non-organiser is refused + // outright rather than given a partial revoke that reports success while other links + // stay live. They can still revoke their own link by its token. + const hasOtherCreators = rows.some((row) => !row.created_by || row.created_by !== actorId); + if (rows.length === 0 || hasOtherCreators) { + await verifyActiveOrganiser(); } - const targetRows = isOrganiser ? rows : ownRows; - for (const row of targetRows) { + for (const row of rows) { if (row.link_hash) { await teachingCommand(client, actorId, serviceId, input.action, { linkHash: row.link_hash }); } diff --git a/tests/ops-digest.test.ts b/tests/ops-digest.test.ts index c9f54e4630..f56d20b4cd 100644 --- a/tests/ops-digest.test.ts +++ b/tests/ops-digest.test.ts @@ -77,6 +77,36 @@ describe("hybrid RPC hourly evidence", () => { } }); + it("counts windows from a manual run between scheduled runs as consecutive evidence", () => { + const result = updateHybridRpcHourlyEvidence( + { + version: 1, + windows: [ + { hour: "2026-08-23T00:00:00.000Z", rpcNames: ["hybrid_search"] }, + { hour: "2026-08-23T03:00:00.000Z", rpcNames: ["hybrid_search"] }, + ], + }, + { slo: { hybridRpcIdentityEvidenceComplete: true, hybridRpcErrorCounts: { hybrid_search: 1 } } }, + new Date("2026-08-23T06:20:00Z"), + ); + expect(result.repeatedRpcNames).toEqual(["hybrid_search"]); + }); + + it("does not join windows more than six hours apart", () => { + const result = updateHybridRpcHourlyEvidence( + { + version: 1, + windows: [ + { hour: "2026-08-23T00:00:00.000Z", rpcNames: ["hybrid_search"] }, + { hour: "2026-08-23T07:00:00.000Z", rpcNames: ["hybrid_search"] }, + ], + }, + { slo: { hybridRpcIdentityEvidenceComplete: true, hybridRpcErrorCounts: { hybrid_search: 1 } } }, + new Date("2026-08-23T08:20:00Z"), + ); + expect(result.repeatedRpcNames).toEqual([]); + }); + it("breaks the paging sequence when an hourly identity snapshot is incomplete", () => { const result = updateHybridRpcHourlyEvidence( { diff --git a/tests/roster-alerts-replay.test.ts b/tests/roster-alerts-replay.test.ts index 8237d521a8..966ca2bf71 100644 --- a/tests/roster-alerts-replay.test.ts +++ b/tests/roster-alerts-replay.test.ts @@ -100,6 +100,31 @@ describe("R22: durable open.post alert delivery tracking and replay recovery", ( expect(mocks.send).not.toHaveBeenCalled(); }); + it("treats a partial delivery as delivered so a replay does not resend the offer to everyone", async () => { + const OPEN_SHIFT_ID = "5e000000-0000-4000-8000-000000000299"; + const client = {} as Parameters[0]; + const action = makeOpenPostAction("5e000000-0000-4000-8000-000000000399"); + + // One recipient reached, one failed (for example a gone device): the offer has gone out. + mocks.send.mockResolvedValueOnce({ sent: 1, skipped: 0, failed: 1 }); + await dispatchRosterAlerts(client, { + serviceId: SERVICE, + actorId: MANAGER, + action, + result: { openShiftId: OPEN_SHIFT_ID, status: "open" }, + }); + expect(mocks.send).toHaveBeenCalledTimes(1); + + vi.clearAllMocks(); + await dispatchRosterAlerts(client, { + serviceId: SERVICE, + actorId: MANAGER, + action, + result: { openShiftId: OPEN_SHIFT_ID, status: "open", replayed: true }, + }); + expect(mocks.send).not.toHaveBeenCalled(); + }); + it("retries alert delivery on replay when sendRosterAlerts threw on the initial open.post", async () => { const OPEN_SHIFT_ID = "5e000000-0000-4000-8000-000000000202"; const client = {} as Parameters[0]; diff --git a/tests/sentry-release.test.ts b/tests/sentry-release.test.ts index bff908ffea..546130b124 100644 --- a/tests/sentry-release.test.ts +++ b/tests/sentry-release.test.ts @@ -104,6 +104,27 @@ describe("sentry-release and release-identity manifest", () => { }); }); + it("picks the manifest commit in the runtime resolver's order, skipping invalid candidates", () => { + const vercel = "b".repeat(40); + const github = "c".repeat(40); + expect( + resolveReleaseManifestIdentity( + { + RAILWAY_GIT_COMMIT_SHA: " ", + VERCEL_GIT_COMMIT_SHA: vercel, + GITHUB_SHA: github, + } as unknown as NodeJS.ProcessEnv, + { resolveGitHead: false }, + ).commitSha, + ).toBe(vercel); + expect( + resolveReleaseManifestIdentity( + { RAILWAY_GIT_COMMIT_SHA: "short-sha", GITHUB_SHA: github } as unknown as NodeJS.ProcessEnv, + { resolveGitHead: false }, + ).commitSha, + ).toBe(github); + }); + it("declares release identity build args and copies deploy/release-identity.json in both Dockerfiles", () => { const appDockerfile = readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"); const workerDockerfile = readFileSync(new URL("../Dockerfile.worker", import.meta.url), "utf8"); diff --git a/tests/teaching-display-revoke-auth.test.ts b/tests/teaching-display-revoke-auth.test.ts index 425d30ea3f..ed26b64b37 100644 --- a/tests/teaching-display-revoke-auth.test.ts +++ b/tests/teaching-display-revoke-auth.test.ts @@ -173,6 +173,51 @@ describe("teachingServiceMutation display.revoke authorization", () => { expect(rpc).not.toHaveBeenCalled(); }); + function occurrenceClient(rows: Array<{ id: string; link_hash: string; created_by: string | null }>) { + const base = createMockClient({ link: null, isOrganiser: false }); + const from = vi.fn((table: string) => { + if (table !== "teaching_display_links") return base.from(table); + const is = vi.fn().mockResolvedValue({ data: rows, error: null }); + const eq = vi.fn(() => ({ eq, is })); + return { select: vi.fn(() => ({ eq })) }; + }); + const client = { from, rpc: base.rpc } as unknown as Parameters[0]; + return { client, rpc: base.rpc }; + } + + it("refuses a non-organiser's revoke by occurrence when someone else created one of the links", async () => { + const { client, rpc } = occurrenceClient([ + { id: "link-own", link_hash: "b".repeat(64), created_by: actorId }, + { id: "link-other", link_hash: "c".repeat(64), created_by: "44444444-4444-4444-8444-444444444444" }, + ]); + + await expect( + teachingServiceMutation(client, actorId, serviceId, { + action: "display.revoke", + occurrenceId: "33333333-3333-4333-8333-333333333333", + }), + ).rejects.toMatchObject({ + status: 403, + details: { code: "teaching_role_denied" }, + } satisfies Partial); + + // Nothing is revoked, so the caller is never told a partial revoke succeeded. + expect(rpc).not.toHaveBeenCalled(); + }); + + it("lets a non-organiser revoke by occurrence when every active link is their own", async () => { + const { client, rpc } = occurrenceClient([{ id: "link-own", link_hash: "b".repeat(64), created_by: actorId }]); + + await expect( + teachingServiceMutation(client, actorId, serviceId, { + action: "display.revoke", + occurrenceId: "33333333-3333-4333-8333-333333333333", + }), + ).resolves.toEqual({}); + + expect(rpc).toHaveBeenCalledOnce(); + }); + it("allows the original creator even if not an organiser", async () => { const { client, rpc, from } = createMockClient({ link: { id: "link-1", created_by: actorId }, diff --git a/tests/trivy-image-scan.test.ts b/tests/trivy-image-scan.test.ts index ef50732c09..6558898efb 100644 --- a/tests/trivy-image-scan.test.ts +++ b/tests/trivy-image-scan.test.ts @@ -1,10 +1,14 @@ -import { readFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { evaluateTrivySummaryText, isCompatibleLocalTrivyVersion, TRIVY_IMAGE, TRIVY_VERSION, + cachedTrivyDbNeedsRefresh, + trivyDbNeedsRefresh, trivyDockerMounts, validateTrivyReportJson, } from "../scripts/trivy-image-scan.mjs"; @@ -149,3 +153,50 @@ describe("Trivy container mounts", () => { expect(trivyDockerMounts({ workDir: "/tmp/w" })).toEqual(["-v", "/tmp/w:/work:ro"]); }); }); + +describe("Trivy vulnerability database freshness", () => { + const now = new Date("2026-10-17T00:00:00Z"); + + it("refreshes a missing, unreadable or undated database", () => { + expect(trivyDbNeedsRefresh(null, now)).toBe(true); + expect(trivyDbNeedsRefresh("not json", now)).toBe(true); + expect(trivyDbNeedsRefresh(JSON.stringify({ Version: 2 }), now)).toBe(true); + }); + + it("refreshes a cached database whose NextUpdate has passed", () => { + expect(trivyDbNeedsRefresh(JSON.stringify({ NextUpdate: "2026-10-16T18:00:00Z" }), now)).toBe(true); + }); + + it("reuses a cached database that is still current", () => { + expect(trivyDbNeedsRefresh(JSON.stringify({ NextUpdate: "2026-10-17T06:00:00Z" }), now)).toBe(false); + }); +}); + +describe("Trivy cache directory freshness", () => { + const now = new Date("2026-10-17T00:00:00Z"); + const current = JSON.stringify({ NextUpdate: "2026-10-17T06:00:00Z" }); + + function cacheWith({ dbFile, metadata }: { dbFile: boolean; metadata: string | null }) { + const cacheDir = mkdtempSync(join(tmpdir(), "trivy-cache-test-")); + mkdirSync(join(cacheDir, "db"), { recursive: true }); + if (dbFile) writeFileSync(join(cacheDir, "db", "trivy.db"), ""); + if (metadata !== null) writeFileSync(join(cacheDir, "db", "metadata.json"), metadata); + return cacheDir; + } + + it("downloads when the database file is missing even though its metadata is current", () => { + expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: false, metadata: current }), "db", "trivy.db", now)).toBe( + true, + ); + }); + + it("downloads when the metadata is missing", () => { + expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: true, metadata: null }), "db", "trivy.db", now)).toBe(true); + }); + + it("reuses a complete, current cache", () => { + expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: true, metadata: current }), "db", "trivy.db", now)).toBe( + false, + ); + }); +}); diff --git a/tests/worker-lease-transition-contract.test.ts b/tests/worker-lease-transition-contract.test.ts index 632a8cdd4a..cd4694040e 100644 --- a/tests/worker-lease-transition-contract.test.ts +++ b/tests/worker-lease-transition-contract.test.ts @@ -130,10 +130,11 @@ describe("worker job transitions have no unfenced fallback", () => { // data-plane writes the longer one, both tied to the attempt's own signal. expect(mutationAdapter).toContain("runIngestionMutation("); expect(mutationAdapter).toContain("ingestionRpcClient"); - expect(mutationAdapter).toContain(".rpc(rpcName, payload)"); - expect(mutationAdapter).toContain("CONTROL_PLANE_RPCS.has(rpcName)"); - expect(mutationAdapter).toContain("controlPlaneAbortSignal(attempt.signal)"); - expect(mutationAdapter).toContain("dataPlaneAbortSignal(attempt.signal)"); + // The selected deadline must be attached to the RPC builder itself, not merely mentioned + // somewhere in the adapter. + expect(mutationAdapter).toMatch( + /\.rpc\(rpcName, payload\)\s*\.abortSignal\(\s*CONTROL_PLANE_RPCS\.has\(rpcName\)\s*\?\s*controlPlaneAbortSignal\(attempt\.signal\)\s*:\s*dataPlaneAbortSignal\(attempt\.signal\),?\s*\)/, + ); expect(fencedMutation).toContain("p_fence: attempt.fence"); expect(fencedMutation).toContain("if (result.error) throw new Error(result.error.message)"); expect(fencedMutation).toContain("attempt.accept(result.data)");