Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/governance-incident-runbooks.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ These are short operator checklists. Preserve evidence and identifiers, but do n
3. If raw query persistence, answer persistence, or provider response storage is enabled, disable the affected path. Verify all three controls are off before proceeding:
```bash
railway variables --service clinical-kb-app \
--set "RAG_PERSIST_RAW_QUERY=false" \
--set "RAG_PERSIST_RAW_ANSWER=false" \
--set "RAG_STORE_PROVIDER_RESPONSES=false"
--set "RAG_PERSIST_RAW_QUERY_TEXT=false" \
--set "RAG_PERSIST_ANSWER_TEXT=false" \
--set "OPENAI_STORE_RESPONSES=false"
```
4. Inspect affected answer copies in `rag_response_cache.payload`; invalidate affected owners and purge incident-related rows where appropriate, then verify the bounded retention purge before assessing other cache and telemetry retention:
```sql
Expand Down
17 changes: 8 additions & 9 deletions scripts/deploy/write-release-manifest.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,15 @@ function readGitHeadCommitSha(cwd = REPO_ROOT) {
}

export function resolveReleaseManifestIdentity(env = process.env, options = {}) {
const envCommitCandidate =
env.RAILWAY_GIT_COMMIT_SHA ||
env.GITHUB_SHA ||
env.SOURCE_COMMIT_SHA ||
env.COMMIT_SHA ||
env.VERCEL_GIT_COMMIT_SHA ||
"";
// Same candidate order as resolveDeploymentCommitSha in src/lib/observability/sentry-release.ts,
// and the first candidate that is a valid SHA wins, so a blank or malformed earlier variable
// cannot hide a valid later one.
const envCommitSha =
[env.RAILWAY_GIT_COMMIT_SHA, env.VERCEL_GIT_COMMIT_SHA, env.GITHUB_SHA, env.SOURCE_COMMIT_SHA, env.COMMIT_SHA]
.map(normalizeCommitSha)
.find(Boolean) ?? null;
const commitSha =
normalizeCommitSha(envCommitCandidate) ??
(options.resolveGitHead === false ? null : readGitHeadCommitSha(options.cwd ?? REPO_ROOT));
envCommitSha ?? (options.resolveGitHead === false ? null : readGitHeadCommitSha(options.cwd ?? REPO_ROOT));

const explicitRelease = (env.SENTRY_RELEASE || env.NEXT_PUBLIC_SENTRY_RELEASE || "").trim();
const sentryRelease = explicitRelease || commitSha || "dev";
Expand Down
4 changes: 3 additions & 1 deletion scripts/ops-digest.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ export function updateHybridRpcHourlyEvidence(rawHistory, health, observedAt = n
lastThree.every((entry, index) => {
if (index === 0) return true;
const gapMs = Date.parse(entry.hour) - Date.parse(lastThree[index - 1].hour);
return gapMs === hourlyWindowMs || gapMs === 6 * hourlyWindowMs;
// Scheduled runs are six hours apart, and a manual run between them shares the same history
// file, so any positive whole-hour gap up to six hours still counts as consecutive evidence.
return gapMs > 0 && gapMs <= 6 * hourlyWindowMs;
});
Comment thread
gitar-bot[bot] marked this conversation as resolved.
Resolved
const repeatedRpcNames = contiguous
? lastThree[0].rpcNames.filter((name) => lastThree.every((entry) => entry.rpcNames.includes(name)))
Expand Down
47 changes: 44 additions & 3 deletions scripts/trivy-image-scan.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* daemon socket, which would be equivalent to root on the host.
*/
import { basename, dirname, join, resolve } from "node:path";
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
Expand Down Expand Up @@ -162,6 +162,43 @@ export function trivyDockerMounts({ workDir, cacheDir = null, outputDir = null }
return mounts;
}

/**
* Whether the cached vulnerability database must be downloaded before an offline scan. Scans pass
* --skip-db-update, so a persistent TRIVY_CACHE_DIR would otherwise keep using an expired database
* and report STATUS=ok on stale data. A missing, unreadable or undated metadata file, or a
* NextUpdate time that has passed, all require a fresh download.
*
* @param {string | null} metadataText contents of <cache>/db/metadata.json, or null when absent
* @param {Date} [now]
* @returns {boolean}
*/
export function trivyDbNeedsRefresh(metadataText, now = new Date()) {
if (typeof metadataText !== "string") return true;
let nextUpdate;
try {
nextUpdate = Date.parse(JSON.parse(metadataText)?.NextUpdate);
} catch {
return true;
}
return !Number.isFinite(nextUpdate) || nextUpdate <= now.getTime();
}

/**
* Whether a cached Trivy database directory needs downloading: its database file is missing (a
* partial cache), or its metadata says it is stale. The main database lives in `db/trivy.db` and
* the Java database in `java-db/trivy-java.db`, each beside its own metadata.json.
*/
export function cachedTrivyDbNeedsRefresh(cacheDir, subdir, dbFile, now = new Date()) {
if (!existsSync(join(cacheDir, subdir, dbFile))) return true;
let metadataText = null;
try {
metadataText = readFileSync(join(cacheDir, subdir, "metadata.json"), "utf8");
} catch {
// Unreadable metadata is treated as absent.
}
return trivyDbNeedsRefresh(metadataText, now);
}

/**
* @param {string} imageRef
* @param {string[]} trivyArgs use "__IMAGE__" where the image ref belongs and
Expand All @@ -179,7 +216,8 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null, needsVulne
? resolve(process.env.TRIVY_CACHE_DIR)
: join(tmpdir(), `trivy-db-cache-${TRIVY_VERSION}`);
mkdirSync(cacheDir, { recursive: true });
if (!existsSync(join(cacheDir, "db", "metadata.json"))) {
// A failed refresh fails the scan rather than falling back to the expired database.
if (cachedTrivyDbNeedsRefresh(cacheDir, "db", "trivy.db")) {
const dbDownload = run("docker", [
"run",
"--rm",
Expand All @@ -198,7 +236,10 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null, needsVulne
stderr: dbDownload.stderr || dbDownload.stdout || "Trivy DB download failed",
};
}
// Best-effort Java DB pre-population for Java archive inspection offline.
}
// The Java database expires on its own schedule, so it is checked separately. Its download
// stays best-effort: it only adds Java archive inspection, and a failure is retried next run.
if (cachedTrivyDbNeedsRefresh(cacheDir, "java-db", "trivy-java.db")) {
run("docker", [
"run",
"--rm",
Expand Down
5 changes: 4 additions & 1 deletion src/lib/roster/alerts/dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,10 @@ async function runRosterAlertDispatch(
? new Set<string>()
: await recipientsOnNightNow(client, event.actorId, event.serviceId, eligible);
const delivery = await sendRosterAlerts(client, eligible, type, quiet);
if (isOpenPostOffer && result.openShiftId && (delivery?.failed ?? 0) === 0) {
// Any successful send marks the offer delivered: a replay resends to the whole eligible list, so
// retrying a partial delivery would duplicate the offer for everyone it already reached. A gone
// device also counts as a failed send, so only a delivery that reached nobody is retried.
if (isOpenPostOffer && result.openShiftId && ((delivery?.sent ?? 0) > 0 || (delivery?.failed ?? 0) === 0)) {
await markOpenPostAlertDelivered(client, event.serviceId, result.openShiftId);
}
}
Expand Down
22 changes: 8 additions & 14 deletions src/lib/teaching/repository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -475,21 +475,15 @@ export async function teachingServiceMutation(
created_by?: string | null;
teaching_occurrences?: { presenter_id?: string | null } | null;
}>;
const ownRows = rows.filter((row) => row.created_by === actorId);
const otherRows = rows.filter((row) => !row.created_by || row.created_by !== actorId);
let isOrganiser = false;
if (rows.length === 0 || otherRows.length > 0) {
try {
await verifyActiveOrganiser();
isOrganiser = true;
} catch (orgErr) {
if (ownRows.length === 0) {
throw orgErr;
}
}
// Revoking by occurrence revokes every active link or none. A link someone else created
// (or with no recorded creator) needs an active organiser; a non-organiser is refused
// outright rather than given a partial revoke that reports success while other links
// stay live. They can still revoke their own link by its token.
const hasOtherCreators = rows.some((row) => !row.created_by || row.created_by !== actorId);
if (rows.length === 0 || hasOtherCreators) {
await verifyActiveOrganiser();
}
const targetRows = isOrganiser ? rows : ownRows;
for (const row of targetRows) {
for (const row of rows) {
if (row.link_hash) {
await teachingCommand(client, actorId, serviceId, input.action, { linkHash: row.link_hash });
}
Expand Down
30 changes: 30 additions & 0 deletions tests/ops-digest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,36 @@ describe("hybrid RPC hourly evidence", () => {
}
});

it("counts windows from a manual run between scheduled runs as consecutive evidence", () => {
const result = updateHybridRpcHourlyEvidence(
{
version: 1,
windows: [
{ hour: "2026-08-23T00:00:00.000Z", rpcNames: ["hybrid_search"] },
{ hour: "2026-08-23T03:00:00.000Z", rpcNames: ["hybrid_search"] },
],
},
{ slo: { hybridRpcIdentityEvidenceComplete: true, hybridRpcErrorCounts: { hybrid_search: 1 } } },
new Date("2026-08-23T06:20:00Z"),
);
expect(result.repeatedRpcNames).toEqual(["hybrid_search"]);
});

it("does not join windows more than six hours apart", () => {
const result = updateHybridRpcHourlyEvidence(
{
version: 1,
windows: [
{ hour: "2026-08-23T00:00:00.000Z", rpcNames: ["hybrid_search"] },
{ hour: "2026-08-23T07:00:00.000Z", rpcNames: ["hybrid_search"] },
],
},
{ slo: { hybridRpcIdentityEvidenceComplete: true, hybridRpcErrorCounts: { hybrid_search: 1 } } },
new Date("2026-08-23T08:20:00Z"),
);
expect(result.repeatedRpcNames).toEqual([]);
});

it("breaks the paging sequence when an hourly identity snapshot is incomplete", () => {
const result = updateHybridRpcHourlyEvidence(
{
Expand Down
25 changes: 25 additions & 0 deletions tests/roster-alerts-replay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,31 @@ describe("R22: durable open.post alert delivery tracking and replay recovery", (
expect(mocks.send).not.toHaveBeenCalled();
});

it("treats a partial delivery as delivered so a replay does not resend the offer to everyone", async () => {
const OPEN_SHIFT_ID = "5e000000-0000-4000-8000-000000000299";
const client = {} as Parameters<typeof dispatchRosterAlerts>[0];
const action = makeOpenPostAction("5e000000-0000-4000-8000-000000000399");

// One recipient reached, one failed (for example a gone device): the offer has gone out.
mocks.send.mockResolvedValueOnce({ sent: 1, skipped: 0, failed: 1 });
await dispatchRosterAlerts(client, {
serviceId: SERVICE,
actorId: MANAGER,
action,
result: { openShiftId: OPEN_SHIFT_ID, status: "open" },
});
expect(mocks.send).toHaveBeenCalledTimes(1);

vi.clearAllMocks();
await dispatchRosterAlerts(client, {
serviceId: SERVICE,
actorId: MANAGER,
action,
result: { openShiftId: OPEN_SHIFT_ID, status: "open", replayed: true },
});
expect(mocks.send).not.toHaveBeenCalled();
});

it("retries alert delivery on replay when sendRosterAlerts threw on the initial open.post", async () => {
const OPEN_SHIFT_ID = "5e000000-0000-4000-8000-000000000202";
const client = {} as Parameters<typeof dispatchRosterAlerts>[0];
Expand Down
21 changes: 21 additions & 0 deletions tests/sentry-release.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,27 @@ describe("sentry-release and release-identity manifest", () => {
});
});

it("picks the manifest commit in the runtime resolver's order, skipping invalid candidates", () => {
const vercel = "b".repeat(40);
const github = "c".repeat(40);
expect(
resolveReleaseManifestIdentity(
{
RAILWAY_GIT_COMMIT_SHA: " ",
VERCEL_GIT_COMMIT_SHA: vercel,
GITHUB_SHA: github,
} as unknown as NodeJS.ProcessEnv,
{ resolveGitHead: false },
).commitSha,
).toBe(vercel);
expect(
resolveReleaseManifestIdentity(
{ RAILWAY_GIT_COMMIT_SHA: "short-sha", GITHUB_SHA: github } as unknown as NodeJS.ProcessEnv,
{ resolveGitHead: false },
).commitSha,
).toBe(github);
});

it("declares release identity build args and copies deploy/release-identity.json in both Dockerfiles", () => {
const appDockerfile = readFileSync(new URL("../Dockerfile", import.meta.url), "utf8");
const workerDockerfile = readFileSync(new URL("../Dockerfile.worker", import.meta.url), "utf8");
Expand Down
45 changes: 45 additions & 0 deletions tests/teaching-display-revoke-auth.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,51 @@ describe("teachingServiceMutation display.revoke authorization", () => {
expect(rpc).not.toHaveBeenCalled();
});

function occurrenceClient(rows: Array<{ id: string; link_hash: string; created_by: string | null }>) {
const base = createMockClient({ link: null, isOrganiser: false });
const from = vi.fn((table: string) => {
if (table !== "teaching_display_links") return base.from(table);
const is = vi.fn().mockResolvedValue({ data: rows, error: null });
const eq = vi.fn(() => ({ eq, is }));
return { select: vi.fn(() => ({ eq })) };
});
const client = { from, rpc: base.rpc } as unknown as Parameters<typeof teachingServiceMutation>[0];
return { client, rpc: base.rpc };
}

it("refuses a non-organiser's revoke by occurrence when someone else created one of the links", async () => {
const { client, rpc } = occurrenceClient([
{ id: "link-own", link_hash: "b".repeat(64), created_by: actorId },
{ id: "link-other", link_hash: "c".repeat(64), created_by: "44444444-4444-4444-8444-444444444444" },
]);

await expect(
teachingServiceMutation(client, actorId, serviceId, {
action: "display.revoke",
occurrenceId: "33333333-3333-4333-8333-333333333333",
}),
).rejects.toMatchObject({
status: 403,
details: { code: "teaching_role_denied" },
} satisfies Partial<PublicApiError>);

// Nothing is revoked, so the caller is never told a partial revoke succeeded.
expect(rpc).not.toHaveBeenCalled();
});

it("lets a non-organiser revoke by occurrence when every active link is their own", async () => {
const { client, rpc } = occurrenceClient([{ id: "link-own", link_hash: "b".repeat(64), created_by: actorId }]);

await expect(
teachingServiceMutation(client, actorId, serviceId, {
action: "display.revoke",
occurrenceId: "33333333-3333-4333-8333-333333333333",
}),
).resolves.toEqual({});

expect(rpc).toHaveBeenCalledOnce();
});

it("allows the original creator even if not an organiser", async () => {
const { client, rpc, from } = createMockClient({
link: { id: "link-1", created_by: actorId },
Expand Down
53 changes: 52 additions & 1 deletion tests/trivy-image-scan.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import { readFileSync } from "node:fs";
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
evaluateTrivySummaryText,
isCompatibleLocalTrivyVersion,
TRIVY_IMAGE,
TRIVY_VERSION,
cachedTrivyDbNeedsRefresh,
trivyDbNeedsRefresh,
trivyDockerMounts,
validateTrivyReportJson,
} from "../scripts/trivy-image-scan.mjs";
Expand Down Expand Up @@ -149,3 +153,50 @@ describe("Trivy container mounts", () => {
expect(trivyDockerMounts({ workDir: "/tmp/w" })).toEqual(["-v", "/tmp/w:/work:ro"]);
});
});

describe("Trivy vulnerability database freshness", () => {
const now = new Date("2026-10-17T00:00:00Z");

it("refreshes a missing, unreadable or undated database", () => {
expect(trivyDbNeedsRefresh(null, now)).toBe(true);
expect(trivyDbNeedsRefresh("not json", now)).toBe(true);
expect(trivyDbNeedsRefresh(JSON.stringify({ Version: 2 }), now)).toBe(true);
});

it("refreshes a cached database whose NextUpdate has passed", () => {
expect(trivyDbNeedsRefresh(JSON.stringify({ NextUpdate: "2026-10-16T18:00:00Z" }), now)).toBe(true);
});

it("reuses a cached database that is still current", () => {
expect(trivyDbNeedsRefresh(JSON.stringify({ NextUpdate: "2026-10-17T06:00:00Z" }), now)).toBe(false);
});
});

describe("Trivy cache directory freshness", () => {
const now = new Date("2026-10-17T00:00:00Z");
const current = JSON.stringify({ NextUpdate: "2026-10-17T06:00:00Z" });

function cacheWith({ dbFile, metadata }: { dbFile: boolean; metadata: string | null }) {
const cacheDir = mkdtempSync(join(tmpdir(), "trivy-cache-test-"));
mkdirSync(join(cacheDir, "db"), { recursive: true });
if (dbFile) writeFileSync(join(cacheDir, "db", "trivy.db"), "");
if (metadata !== null) writeFileSync(join(cacheDir, "db", "metadata.json"), metadata);
return cacheDir;
}

it("downloads when the database file is missing even though its metadata is current", () => {
expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: false, metadata: current }), "db", "trivy.db", now)).toBe(
true,
);
});

it("downloads when the metadata is missing", () => {
expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: true, metadata: null }), "db", "trivy.db", now)).toBe(true);
});

it("reuses a complete, current cache", () => {
expect(cachedTrivyDbNeedsRefresh(cacheWith({ dbFile: true, metadata: current }), "db", "trivy.db", now)).toBe(
false,
);
});
});
9 changes: 5 additions & 4 deletions tests/worker-lease-transition-contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,10 +130,11 @@ describe("worker job transitions have no unfenced fallback", () => {
// data-plane writes the longer one, both tied to the attempt's own signal.
expect(mutationAdapter).toContain("runIngestionMutation(");
expect(mutationAdapter).toContain("ingestionRpcClient");
expect(mutationAdapter).toContain(".rpc(rpcName, payload)");
expect(mutationAdapter).toContain("CONTROL_PLANE_RPCS.has(rpcName)");
expect(mutationAdapter).toContain("controlPlaneAbortSignal(attempt.signal)");
expect(mutationAdapter).toContain("dataPlaneAbortSignal(attempt.signal)");
// The selected deadline must be attached to the RPC builder itself, not merely mentioned
// somewhere in the adapter.
expect(mutationAdapter).toMatch(
/\.rpc\(rpcName, payload\)\s*\.abortSignal\(\s*CONTROL_PLANE_RPCS\.has\(rpcName\)\s*\?\s*controlPlaneAbortSignal\(attempt\.signal\)\s*:\s*dataPlaneAbortSignal\(attempt\.signal\),?\s*\)/,
);
expect(fencedMutation).toContain("p_fence: attempt.fence");
expect(fencedMutation).toContain("if (result.error) throw new Error(result.error.message)");
expect(fencedMutation).toContain("attempt.accept(result.data)");
Expand Down
Loading