diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 5797298c33..b05efef22a 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -178,30 +178,27 @@ jobs: if-no-files-found: warn - name: Vulnerability scan (HIGH,CRITICAL) - # Non-blocking by design — findings are reported, not merge-gating. - # The summary lines are kept so the next step can surface them. + # Non-blocking step wrapper so the evaluation step below can inspect the + # summary and fail closed with an explicit unassessed or finding error. continue-on-error: true run: | docker builder prune -af || true : > trivy-scan-summary.txt - node scripts/trivy-image-scan.mjs clinical-kb-app:ci | tee -a trivy-scan-summary.txt - node scripts/trivy-image-scan.mjs clinical-kb-worker:ci | tee -a trivy-scan-summary.txt + node scripts/trivy-image-scan.mjs clinical-kb-app:ci | tee -a trivy-scan-summary.txt || echo "Trivy scan failed (clinical-kb-app:ci): STATUS=unassessed reason=scanner_exit" | tee -a trivy-scan-summary.txt + node scripts/trivy-image-scan.mjs clinical-kb-worker:ci | tee -a trivy-scan-summary.txt || echo "Trivy scan failed (clinical-kb-worker:ci): STATUS=unassessed reason=scanner_exit" | tee -a trivy-scan-summary.txt - name: Fail on HIGH/CRITICAL image findings outside pull requests - # Audit M20: the worker image carries the Python parsers (PyMuPDF, - # Pillow, pytesseract, docling) that read attacker-supplied uploads, and - # until now a HIGH/CRITICAL finding reached only this run's log. Pull - # request runs stay advisory (a base-image CVE is not the PR author's - # change), but the weekly schedule, main pushes and manual dispatches - # now fail on any HIGH/CRITICAL so the run shows red in the Actions tab. - # A scanner that could not run (disk pressure, docker save) is still - # only a warning. + # Audit M20 + R13: the worker image carries the Python parsers (PyMuPDF, + # Pillow, pytesseract, docling) that read attacker-supplied uploads. + # Fail closed if trivy-scan-summary.txt is empty, contains STATUS=unassessed, + # or does not contain STATUS=ok lines for BOTH clinical-kb-app:ci and + # clinical-kb-worker:ci. Never report a broken/empty scan as clean. if: always() run: | set -euo pipefail if [ ! -s trivy-scan-summary.txt ]; then - echo "::warning::Trivy produced no scan summary; image vulnerabilities were not assessed this run." - exit 0 + echo "::error::Trivy scan unassessed: trivy-scan-summary.txt is empty; image vulnerabilities were not assessed." + exit 1 fi { echo "## Trivy image scan (HIGH,CRITICAL)" @@ -210,6 +207,14 @@ jobs: cat trivy-scan-summary.txt echo '```' } >> "$GITHUB_STEP_SUMMARY" + if grep -q 'STATUS=unassessed' trivy-scan-summary.txt; then + echo "::error::Trivy scan unassessed: one or more images reported STATUS=unassessed." + exit 1 + fi + if ! grep -q '^Trivy scan completed (clinical-kb-app:ci): STATUS=ok\b' trivy-scan-summary.txt || ! grep -q '^Trivy scan completed (clinical-kb-worker:ci): STATUS=ok\b' trivy-scan-summary.txt; then + echo "::error::Trivy scan unassessed: missing STATUS=ok evidence for both clinical-kb-app:ci and clinical-kb-worker:ci." + exit 1 + fi high="$(grep -oE 'HIGH=[0-9]+' trivy-scan-summary.txt | cut -d= -f2 | paste -sd+ - | bc || echo 0)" critical="$(grep -oE 'CRITICAL=[0-9]+' trivy-scan-summary.txt | cut -d= -f2 | paste -sd+ - | bc || echo 0)" high="${high:-0}" diff --git a/.github/workflows/ops-digest.yml b/.github/workflows/ops-digest.yml index 1b68baa86c..de8ac0d07b 100644 --- a/.github/workflows/ops-digest.yml +++ b/.github/workflows/ops-digest.yml @@ -1,10 +1,7 @@ # Ops deep-health digest + canary liveness. # -# Schedule is DAILY at 22:00 UTC (`0 22 * * *`), not hourly. Hybrid-RPC evidence -# steps still speak in "hourly" buckets for historical keying / future cadence -# bumps — that naming is intentional and is not the workflow cron. -# -# Keep workflow_dispatch for operator verification and the daily schedule for +# Schedule runs every 6 hours (`15 */6 * * *`) so trailing 60-minute SLO windows +# are sampled 4x/day and active warning/page alerts notify on the rolling issue. name: Ops Digest on: @@ -16,8 +13,8 @@ on: default: false type: boolean schedule: - # Once daily at 22:00 UTC. - - cron: "0 22 * * *" + # Every 6 hours at minute 15 UTC (4x/day sampling of 60-minute windows). + - cron: "15 */6 * * *" concurrency: group: ops-digest @@ -153,8 +150,6 @@ jobs: script: | const fs = require("fs"); const label = "ops-digest"; - const publishAttentionComment = - context.eventName === "workflow_dispatch" && process.env.OPS_DIGEST_PUBLISH_ALERT_COMMENT === "true"; const canaryStale = process.env.EVAL_CANARY_STALE === "true"; const canaryMessage = process.env.EVAL_CANARY_MESSAGE || ""; let body = fs.readFileSync("digest.md", "utf8"); @@ -167,6 +162,9 @@ jobs: const status = allowedStatuses.has(rawStatus) ? rawStatus : "unknown"; const rawSeverity = process.env.OPS_DIGEST_SEVERITY || "unknown"; const severity = allowedSeverities.has(rawSeverity) ? rawSeverity : "unknown"; + const publishAttentionComment = + (context.eventName === "workflow_dispatch" && process.env.OPS_DIGEST_PUBLISH_ALERT_COMMENT === "true") || + (context.eventName === "schedule" && (severity === "warning" || severity === "page")); let parsedSummary; try { parsedSummary = JSON.parse(process.env.OPS_DIGEST_ALERT_SUMMARY || "{}"); @@ -227,7 +225,7 @@ jobs: }); issueNumber = created.data.number; } - // Attention comments notify subscribers, so scheduled runs never publish them. + // Publish attention comments on confirmed manual dispatches or scheduled warning/page breaches. if (publishAttentionComment && (status !== "ok" || alerting)) { await github.rest.issues.createComment({ owner: context.repo.owner, @@ -236,7 +234,7 @@ jobs: body: `⚠ Attention (status=${status}, alerting=${alerting}, severity=${severity}${canaryStale ? ", eval-canary-stale=true" : ""}) — ${new Date().toISOString()}\n\nAlert summary: \`${alertSummary}\`\n\n${body}`, }); } else if (status !== "ok" || alerting) { - core.info("Attention comment skipped: manual operator confirmation was not supplied."); + core.info("Attention comment skipped: neither manual confirmation nor scheduled warning/page severity was met."); } if (status !== "ok") core.warning(`Ops digest status: ${status}`); if (canaryStale) core.warning(canaryMessage || "Eval-canary liveness reported stale."); diff --git a/Dockerfile b/Dockerfile index ae9eaa20ee..8d5685cd2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,10 @@ ARG MAX_UPLOAD_MB= # declares the matching build argument. This non-secret SHA keeps build-time # source maps and runtime Sentry events on the same release identity. ARG RAILWAY_GIT_COMMIT_SHA= +ARG GITHUB_SHA= +ARG SOURCE_COMMIT_SHA= +ARG COMMIT_SHA= +ARG SENTRY_RELEASE= # Next reads this during the build to stamp navigation and static asset URLs with # the operator-selected release identity instead of the commit SHA fallback. ARG NEXT_DEPLOYMENT_ID= @@ -95,8 +99,13 @@ ENV NEXT_PUBLIC_SUPABASE_STAGING_PROJECT_REF=${NEXT_PUBLIC_SUPABASE_STAGING_PROJ ENV NEXT_PUBLIC_SUPABASE_STAGING_PROJECT_NAME=${NEXT_PUBLIC_SUPABASE_STAGING_PROJECT_NAME} ENV NEXT_PUBLIC_MAX_UPLOAD_MB=${NEXT_PUBLIC_MAX_UPLOAD_MB} ENV RAILWAY_GIT_COMMIT_SHA=${RAILWAY_GIT_COMMIT_SHA} +ENV GITHUB_SHA=${GITHUB_SHA} +ENV SOURCE_COMMIT_SHA=${SOURCE_COMMIT_SHA} +ENV COMMIT_SHA=${COMMIT_SHA} +ENV SENTRY_RELEASE=${SENTRY_RELEASE} ENV NEXT_PUBLIC_SITE_URL=${NEXT_PUBLIC_SITE_URL} ENV RAILWAY_PUBLIC_DOMAIN=${RAILWAY_PUBLIC_DOMAIN} +RUN node scripts/deploy/write-release-manifest.mjs # The repo build script allocates an 8 GiB heap. Prefer builders with >= 10 GiB # locally (Docker Desktop hard-fails under the RAM guard by default). CI image # builds pass ALLOW_LOW_RAM_BUILD=1 because GitHub buildx runners report ~7–8 GiB @@ -123,9 +132,17 @@ RUN for attempt in 1 2 3; do \ FROM node-base AS runner WORKDIR /app +ARG RAILWAY_GIT_COMMIT_SHA= +ARG GITHUB_SHA= +ARG SOURCE_COMMIT_SHA= +ARG COMMIT_SHA= +ARG SENTRY_RELEASE= ENV NODE_ENV=production ENV NEXT_TELEMETRY_DISABLED=1 ENV PORT=3000 +ENV RAILWAY_GIT_COMMIT_SHA=${RAILWAY_GIT_COMMIT_SHA} +ENV COMMIT_SHA=${COMMIT_SHA} +ENV SENTRY_RELEASE=${SENTRY_RELEASE} COPY --from=prod-deps /app/node_modules ./node_modules COPY --from=build /app/.next ./.next # From the build stage, not the build context: `stamp-service-worker.mjs` rewrites @@ -141,6 +158,7 @@ COPY --from=build /app/src/data/therapy-catalogue-assets.ts ./src/data/therapy-c # Railway deploy pre-deploy gate (docs/worker-deploy-runbook.md §0): blocks/ # observes this service's deploy until migrations this build expects are live. COPY --from=build /app/deploy/expected-migrations.json ./deploy/expected-migrations.json +COPY --from=build /app/deploy/release-identity.json ./deploy/release-identity.json COPY --from=build /app/scripts/deploy/await-migrations.mjs ./scripts/deploy/await-migrations.mjs COPY --from=build /app/scripts/deploy/migration-versions.mjs ./scripts/deploy/migration-versions.mjs COPY package.json next.config.ts ./ diff --git a/Dockerfile.worker b/Dockerfile.worker index ed263175c8..aea61b0a88 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -42,11 +42,22 @@ RUN for attempt in 1 2 3; do \ sleep $((attempt * 10)); \ done COPY . . +ARG RAILWAY_GIT_COMMIT_SHA= +ARG GITHUB_SHA= +ARG SOURCE_COMMIT_SHA= +ARG COMMIT_SHA= +ARG SENTRY_RELEASE= +ENV RAILWAY_GIT_COMMIT_SHA=${RAILWAY_GIT_COMMIT_SHA} +ENV GITHUB_SHA=${GITHUB_SHA} +ENV SOURCE_COMMIT_SHA=${SOURCE_COMMIT_SHA} +ENV COMMIT_SHA=${COMMIT_SHA} +ENV SENTRY_RELEASE=${SENTRY_RELEASE} # supabase/migrations is only present in the build context, not the slim # runtime image, so the deploy pre-deploy gate's manifest of migration # versions this build expects is baked in now and copied into the runner # below (see scripts/deploy/write-migration-manifest.mjs). RUN node scripts/deploy/write-migration-manifest.mjs +RUN node scripts/deploy/write-release-manifest.mjs RUN node scripts/build-worker.mjs # Production-only node_modules for the runtime stage. The preinstall/ @@ -105,6 +116,11 @@ RUN /opt/docling-venv/bin/docling-tools models download --output-dir /opt/doclin --manifest /tmp/docling-verify/model-artifacts.json \ && chmod -R a+rX /opt/docling-models \ && rm -rf /tmp/docling-verify +ARG RAILWAY_GIT_COMMIT_SHA= +ARG GITHUB_SHA= +ARG SOURCE_COMMIT_SHA= +ARG COMMIT_SHA= +ARG SENTRY_RELEASE= ENV WORKER_DOCLING_PYTHON_BIN=/opt/docling-venv/bin/python ENV DOCLING_ARTIFACTS_PATH=/opt/docling-models ENV TORCHDYNAMO_DISABLE=1 @@ -113,11 +129,15 @@ ENV PATH="/opt/ocr-venv/bin:${PATH}" ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 ENV NODE_ENV=production +ENV RAILWAY_GIT_COMMIT_SHA=${RAILWAY_GIT_COMMIT_SHA} +ENV COMMIT_SHA=${COMMIT_SHA} +ENV SENTRY_RELEASE=${SENTRY_RELEASE} COPY --from=prod-deps /app/node_modules ./node_modules COPY --from=build /app/dist ./dist # Railway deploy pre-deploy gate (docs/worker-deploy-runbook.md §0): blocks/ # observes this service's deploy until migrations this build expects are live. COPY --from=build /app/deploy/expected-migrations.json ./deploy/expected-migrations.json +COPY --from=build /app/deploy/release-identity.json ./deploy/release-identity.json COPY --from=build /app/scripts/deploy/await-migrations.mjs ./scripts/deploy/await-migrations.mjs COPY --from=build /app/scripts/deploy/migration-versions.mjs ./scripts/deploy/migration-versions.mjs COPY worker/python worker/python diff --git a/data/repo-awareness-snapshot.json b/data/repo-awareness-snapshot.json index 4f835e9e43..27376a0901 100644 --- a/data/repo-awareness-snapshot.json +++ b/data/repo-awareness-snapshot.json @@ -1,7 +1,7 @@ { "version": "repo-awareness-snapshot-v3", "captured_revision": { - "committed_at": "2026-10-16" + "committed_at": "2026-10-17" }, "routes": { "modes": [ diff --git a/docs/codebase-index.md b/docs/codebase-index.md index babb1c161a..ab7f891614 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -961,7 +961,7 @@ Administrator source intake also records admission events in `document_admission `documents`, `document_pages`, `document_images`, `document_chunks`, `document_embedding_fields`, `document_index_units`, `document_table_facts`, `document_labels`, `document_summaries`, `document_sections`, `document_memory_cards`, `document_index_quality`, `document_title_words`, `document_publication_approvals`, `document_corpus_access_state`, `document_corpus_access_snapshots`, `ingestion_jobs`, `ingestion_job_stages`, `indexing_v3_agent_jobs`, `import_batches`, `image_caption_cache`, `rag_queries`, `rag_query_misses`, `rag_aliases`, `rag_response_cache`, `rag_retrieval_logs`, `rag_visual_eval_cases`, `rag_visual_eval_runs`, `rag_answer_feedback`, `clinical_registry_records`, `clinical_registry_record_sources`, `clinical_quality_feedback_triage`, `clinical_quality_feedback_triage_events`, `medication_records`, `differential_records`, `source_review_events`, `user_favourites`, `user_favourite_sets`, `user_preferences`, `work_admin_paperwork`, `work_backups`, `work_booking_courses`, `work_course_bookings`, `work_hospitals`, `work_hospital_teams`, `work_role_grants`, `alert_reminder_times`, `alert_brief_sent`, `api_rate_limits`, `api_rate_limit_subjects`, `audit_logs`, `storage_cleanup_jobs`, `on_call_entries`, `cme_years`, `cme_requirements`, `cme_routines`, `cme_entries`, `cme_allocations`, `cme_evidence`, `cme_year_snapshots`, `cme_year_amendments`, `cme_plan_goals`, `cme_plan_goal_carries`, `cme_entry_goals`, `cme_training_periods`, `cme_training_milestones`, `cme_missed_sessions`, `cme_entry_drafts`, `calendar_feed_tokens`, `on_call_shift_imports`, `on_call_shifts`, `on_call_services`, `on_call_service_sites`, `on_call_service_members`, `on_call_service_invitations`, `on_call_service_entries`, `on_call_service_reports`, `on_call_service_orientation`, `site_content_publications`, `site_content_reconciliation_plans`, `site_content_public_records`, `site_content_sync_state`, `site_content_sync_events`, `site_content_sync_event_plans`, `site_content_sync_worker_invocations`, `site_content_releases`, `site_content_release_records`, `site_content_release_receipts` -Team modes (Roster, Admin, Teaching): `admin_leave_balances`, `admin_settings`, `extra_time_records`, `on_call_service_member_events`, `registrar_assessments`, `registrar_training_milestones`, `roster_assignments`, `roster_calendar_links`, `roster_change_agreements`, `roster_changes`, `roster_draft_assignments`, `roster_drafts`, `roster_leave`, `roster_member_roles`, `roster_open_post_keys`, `roster_open_shifts`, `roster_publication_protections`, `roster_publication_seen`, `roster_publications`, `roster_rotation_preferences`, `roster_rotation_rounds`, `roster_shift_codes`, `roster_staffing_needs`, `roster_swaps`, `roster_team_settings`, `roster_unavailability`, `teaching_attendance`, `teaching_audit_events`, `teaching_calendar_optins`, `teaching_checkin_claims`, `teaching_collection_sections`, `teaching_collections`, `teaching_display_links`, `teaching_feedback_answers`, `teaching_feedback_replied`, `teaching_group_members`, `teaching_groups`, `teaching_member_roles`, `teaching_notice_reads`, `teaching_notices`, `teaching_occurrences`, `teaching_readiness`, `teaching_resource_saves`, `teaching_resources`, `teaching_series`, `teaching_supervision_entries`, `teaching_supervision_notes`, `teaching_supervision_pairings`, `teaching_team_settings`, `teaching_week_adds`, `web_push_subscriptions`. +Team modes (Roster, Admin, Teaching): `admin_leave_balances`, `admin_settings`, `extra_time_records`, `on_call_service_member_events`, `registrar_assessments`, `registrar_training_milestones`, `roster_assignments`, `roster_calendar_links`, `roster_change_agreements`, `roster_changes`, `roster_draft_assignments`, `roster_drafts`, `roster_leave`, `roster_member_roles`, `roster_open_post_alerts`, `roster_open_post_keys`, `roster_open_shifts`, `roster_publication_protections`, `roster_publication_seen`, `roster_publications`, `roster_rotation_preferences`, `roster_rotation_rounds`, `roster_shift_codes`, `roster_staffing_needs`, `roster_swaps`, `roster_team_settings`, `roster_unavailability`, `teaching_attendance`, `teaching_audit_events`, `teaching_calendar_optins`, `teaching_checkin_claims`, `teaching_collection_sections`, `teaching_collections`, `teaching_display_links`, `teaching_feedback_answers`, `teaching_feedback_replied`, `teaching_group_members`, `teaching_groups`, `teaching_member_roles`, `teaching_notice_reads`, `teaching_notices`, `teaching_occurrences`, `teaching_readiness`, `teaching_resource_saves`, `teaching_resources`, `teaching_series`, `teaching_supervision_entries`, `teaching_supervision_notes`, `teaching_supervision_pairings`, `teaching_team_settings`, `teaching_week_adds`, `web_push_subscriptions`. Public-source control-plane tables: `public_source_policy_entries`, `public_source_activation_events`, `public_source_versions`, `public_source_upload_attempts`, `public_source_activation_guards`, `public_source_cleanup_mutation_guards`. @@ -1055,7 +1055,7 @@ Public-source control-plane tables: `public_source_policy_entries`, `public_sour Golden retrieval fixture: `scripts/fixtures/rag-retrieval-golden.json` (other RAG, adversarial and ranking-snapshot fixtures sit beside it). -Subfolders: `scripts/lib/` (shared helpers, including the protected `clinical-aliases.ts`), `scripts/deploy/` (`await-migrations.mjs` Railway pre-deploy, `write-migration-manifest.mjs`), `scripts/sql/` (verify and parity SQL), `scripts/archive/` (historical, still referenced by two checks). `docs/scripts-index.md` is the full catalogue. +Subfolders: `scripts/lib/` (shared helpers, including the protected `clinical-aliases.ts`), `scripts/deploy/` (`await-migrations.mjs` Railway pre-deploy, `write-migration-manifest.mjs`, `write-release-manifest.mjs`), `scripts/sql/` (verify and parity SQL), `scripts/archive/` (historical, still referenced by two checks). `docs/scripts-index.md` is the full catalogue. --- diff --git a/docs/governance-incident-runbooks.md b/docs/governance-incident-runbooks.md index 298c84e322..bc18e6bb96 100644 --- a/docs/governance-incident-runbooks.md +++ b/docs/governance-incident-runbooks.md @@ -1,35 +1,147 @@ # Clinical governance incident runbooks -Last reviewed: 2026-07-17 +Last reviewed: 2026-10-16 These are short operator checklists. Preserve evidence and identifiers, but do not copy patient-identifying question or answer prose into tickets or chat. +## Escalation and backup-operator authority + +- **Primary clinical governance owner:** Clinical Lead (`Josh` / designated clinical governance reviewer). +- **Backup operator when primary owner is unavailable (> 15 minutes for P0/P1 clinical safety or privacy containment):** Any on-call platform operator with Railway and Supabase production access is authorised to execute immediate **containment actions** (setting `RAG_PROVIDER_MODE=offline`, setting `WORKER_PAUSED=true`, quarantining an implicated source, and purging `rag_response_cache`) without waiting for clinical sign-off. +- **Unblocking / restoration gate:** Backup operators may **contain and roll back** immediately, but **re-enabling** generative answer modes or lifting a `decommissioned` / `superseded` source quarantine still requires explicit review from the primary clinical owner or designated clinical deputy. + ## Adverse clinical output -1. Record the answer interaction ID, feedback category, cited source IDs, time, and affected workflow. -2. If harm is plausible, disable the affected answer mode or narrow its source scope; keep direct source browsing available. -3. Quarantine an implicated source through a `decommissioned` or `superseded` source-review event when appropriate. -4. Reproduce with a synthetic or minimised fixture, add a regression test, and identify whether retrieval, generation, rendering, or copy caused the defect. -5. Restore the feature only after the narrow regression and production-readiness checks pass and the clinical owner reviews material clinical-policy changes. +1. Record the answer interaction ID, feedback category, cited source IDs, time, and affected workflow: + ```sql + SELECT id, created_at, feedback_category, cited_document_ids, config_posture_version + FROM public.rag_query_feedback + WHERE id = ''; + ``` +2. If harm is plausible, disable the affected answer mode or narrow its source scope; keep direct source browsing available: + ```bash + # Switch answer generation to deterministic offline/source-only mode immediately + railway variables --service clinical-kb-app --set "RAG_PROVIDER_MODE=offline" + # Pause background worker ingestion/enrichment claims if source mutation is in flight + railway variables --service clinical-kb-worker --set "WORKER_PAUSED=true" + ``` +3. Quarantine an implicated source through a `decommissioned` or `superseded` source-review event when appropriate: + ```sql + UPDATE public.documents + SET source_review_status = 'decommissioned', + updated_at = timezone('utc', now()) + WHERE id = ''; + ``` +4. Reproduce with a synthetic or minimised fixture, add a regression test, and identify whether retrieval, generation, rendering, or copy caused the defect: + ```bash + npm run eval:rag:offline + npm run check:clinical-hazard-controls + ``` +5. Restore the feature only after the narrow regression and production-readiness checks pass and the clinical owner reviews material clinical-policy changes (if the primary clinical owner is unavailable, keep `RAG_PROVIDER_MODE=offline` active under backup-operator containment until clinical review completes). ## Source quarantine, recall, or replacement -1. Record an evidence-bearing review event with a reason and replacement document where applicable. -2. If immediate containment is required, disable the affected answer mode or narrow its source scope; do not treat review status alone as a retrieval exclusion. -3. Confirm answer caches for the owner are invalidated after the transaction commits. -4. Before declaring containment, verify the source remains directly browsable but cannot participate in retrieval or answer generation. -5. Re-run source-governance and focused answer tests. Live data repair or migration requires explicit approval. +1. Record an evidence-bearing review event with a reason and replacement document where applicable: + ```sql + BEGIN; + + INSERT INTO public.document_governance_events ( + document_id, + event_type, + review_status, + reason, + replacement_document_id + ) VALUES ( + '', + 'quarantine', + 'decommissioned', + '', + NULLIF('', '') + ); + + UPDATE public.documents + SET source_review_status = 'decommissioned', + updated_at = timezone('utc', now()) + WHERE id = ''; + + COMMIT; + ``` +2. If immediate containment is required, disable the affected answer mode or narrow its source scope; do not treat review status alone as a retrieval exclusion: + ```bash + railway variables --service clinical-kb-app --set "RAG_PROVIDER_MODE=offline" + railway variables --service clinical-kb-worker --set "WORKER_PAUSED=true" + ``` +3. Confirm answer caches for the owner are invalidated after the transaction commits: + ```sql + DELETE FROM public.rag_response_cache + WHERE owner_id = '' + OR payload::text LIKE '%%'; + ``` +4. Before declaring containment, verify the source remains directly browsable (`source_review_status = 'decommissioned'`), that retrieval exclusion is active (`RAG_PROVIDER_MODE=offline` or zero active retrieval chunks for the quarantined `document_id`), and that `rag_response_cache` contains zero entries referencing the quarantined document: + ```sql + SELECT d.id, + d.title, + d.source_review_status, + d.metadata->>'rag_indexed' AS rag_indexed, + (SELECT count(*) FROM public.document_chunks c WHERE c.document_id = d.id) AS chunk_count, + (SELECT count(*) FROM public.rag_response_cache r WHERE r.payload::text LIKE '%' || d.id::text || '%') AS cached_answer_count + FROM public.documents d + WHERE d.id = ''; + ``` + Confirm via `railway variables --service clinical-kb-app` (or a probe query against `/api/ask`) that the quarantined `document_id` is not returned in retrieval citations before lifting `RAG_PROVIDER_MODE=offline`. +5. Re-run source-governance and focused answer tests. Live data repair or migration requires explicit approval (if the primary owner is unavailable, the backup operator must leave the source quarantined and `rag_response_cache` purged until explicit approval is obtained): + ```bash + npm run audit:source-governance + npm run check:clinical-hazard-controls + ``` ## Privacy or provider incident -1. Preserve interaction IDs, hashes, timestamps, configuration-posture versions, and provider request IDs; do not preserve raw clinical prose unless an authorised incident process requires it. -2. Disable provider generation or switch to the source-only path if external processing must stop. -3. If raw query persistence, answer persistence, or provider response storage is enabled, disable the affected path. Verify all three controls are off before proceeding. -4. Inspect affected answer copies in `rag_response_cache.payload`; invalidate affected owners and purge incident-related rows where appropriate, then verify the bounded retention purge before assessing other cache and telemetry retention. -5. Escalate notification and breach decisions to the privacy owner; this runbook does not make that legal determination. +1. Preserve interaction IDs, hashes, timestamps, configuration-posture versions, and provider request IDs; do not preserve raw clinical prose unless an authorised incident process requires it: + ```sql + SELECT id, created_at, metadata + FROM public.rag_queries + WHERE created_at >= timezone('utc', now()) - interval '24 hours' + ORDER BY created_at DESC; + ``` +2. Disable provider generation or switch to the source-only path if external processing must stop: + ```bash + railway variables --service clinical-kb-app --set "RAG_PROVIDER_MODE=offline" + railway variables --service clinical-kb-worker --set "WORKER_PAUSED=true" + ``` +3. If raw query persistence, answer persistence, or provider response storage is enabled, disable the affected path. Verify all three controls are off before proceeding: + ```bash + railway variables --service clinical-kb-app \ + --set "RAG_PERSIST_RAW_QUERY=false" \ + --set "RAG_PERSIST_RAW_ANSWER=false" \ + --set "RAG_STORE_PROVIDER_RESPONSES=false" + ``` +4. Inspect affected answer copies in `rag_response_cache.payload`; invalidate affected owners and purge incident-related rows where appropriate, then verify the bounded retention purge before assessing other cache and telemetry retention: + ```sql + DELETE FROM public.rag_response_cache + WHERE owner_id = '' + OR created_at >= ''; + + SELECT public.purge_expired_rag_response_cache(); + ``` +5. Escalate notification and breach decisions to the privacy owner; this runbook does not make that legal determination. If the primary privacy owner is unavailable, the backup operator must complete steps 1–4 immediately, seal the audit metadata snapshot, and page the secondary governance contact. ## Answer-pipeline rollback -1. Revert or disable the smallest affected generation or ranking feature while retaining source browsing and deterministic source-only answers. +1. Revert or disable the smallest affected generation or ranking feature while retaining source browsing and deterministic source-only answers: + ```bash + railway variables --service clinical-kb-app --set "RAG_PROVIDER_MODE=offline" + railway variables --service clinical-kb-worker --set "WORKER_PAUSED=true" + ``` + Verify `/api/health` (`deploymentCommitSha`) and worker `/health` (`claimLoopPaused: true`, `deploymentCommitSha`) confirm the active release and paused worker loop: + ```bash + curl -fsS https:///api/health | jq '{status, version, deploymentCommitSha}' + curl -fsS http://:8080/health | jq '{ok, claimLoopPaused, deploymentCommitSha}' + ``` 2. Do not weaken danger-source exclusion, query and answer minimisation, or private-document access to restore availability. -3. Run the focused regression, offline RAG evaluation, production-readiness CI check, and local PR mirror before release. +3. Run the focused regression, offline RAG evaluation, production-readiness CI check, and local PR mirror before release (backup operators may roll back to the previous known-good release image immediately, then hold forward promotion until these checks pass): + ```bash + npm run eval:rag:offline + npm run check:production-readiness:ci + npm run verify:pr-local + ``` diff --git a/docs/organisation/systems/app-experience.json b/docs/organisation/systems/app-experience.json index 258428a30d..026ddea832 100644 --- a/docs/organisation/systems/app-experience.json +++ b/docs/organisation/systems/app-experience.json @@ -365,6 +365,7 @@ "tests/registry-retry.dom.test.tsx", "tests/rendered-text-formatting.test.ts", "tests/restore-focus-unless-moved.dom.test.tsx", + "tests/restored-answer-notice.dom.test.tsx", "tests/result-filter-url.test.ts", "tests/route-*", "tests/rsc-boundary.test.ts", diff --git a/docs/organisation/systems/data-platform.json b/docs/organisation/systems/data-platform.json index bb57d81cb8..6e2d0ce721 100644 --- a/docs/organisation/systems/data-platform.json +++ b/docs/organisation/systems/data-platform.json @@ -135,6 +135,7 @@ "tests/client-secret-surface.test.ts", "tests/clinical-ask-rate-limit.test.ts", "tests/clinical-quality-triage-schema.test.ts", + "tests/credential-revocation-and-rotation.test.ts", "tests/cross-tenant-*", "tests/document-admin-rate-limit.test.ts", "tests/document-corpus-access-mode.test.ts", diff --git a/docs/organisation/systems/delivery.json b/docs/organisation/systems/delivery.json index 994196d7c8..edca43595d 100644 --- a/docs/organisation/systems/delivery.json +++ b/docs/organisation/systems/delivery.json @@ -252,6 +252,7 @@ "tests/pr-batch-*", "tests/pr-mergeability-workflow-guard.test.ts", "tests/pr-policy-*", + "tests/pr-policy.test.ts", "tests/primary-checkout-lease.test.ts", "tests/production-readiness-offline.test.ts", "tests/property-*", @@ -279,6 +280,7 @@ "tests/tall-man-lettering.test.ts", "tests/test-*", "tests/text-source-provenance.test.ts", + "tests/trivy-image-scan.test.ts", "tests/trusted-document-admission.test.ts", "tests/tsx-server-only-runner.test.ts", "tests/typecheck-config-isolation.test.ts", @@ -288,6 +290,7 @@ "tests/webhooks-chat-notify.test.ts", "tests/webhooks-railway-route.test.ts", "tests/worktree-*", + "tests/xlsx-sheet-dictionary-projection.test.ts", "tests/xlsx-source-provenance.test.ts", "tests/yaml-contract.test.ts", "tsconfig.json", diff --git a/docs/organisation/systems/personal-practice.json b/docs/organisation/systems/personal-practice.json index 28855c4ac0..f2238d8732 100644 --- a/docs/organisation/systems/personal-practice.json +++ b/docs/organisation/systems/personal-practice.json @@ -137,6 +137,7 @@ "tests/work-screens-my-day-earlier-alerts.test.ts", "tests/work-search*", "tests/work-sync-client.dom.test.tsx", + "tests/work-sync-hardening.test.ts", "tests/work-sync.test.ts", "tests/work-text*", "tests/work-time-format.test.ts", diff --git a/docs/production-readiness.md b/docs/production-readiness.md index ed3e539ad8..37e82945ae 100644 --- a/docs/production-readiness.md +++ b/docs/production-readiness.md @@ -53,7 +53,7 @@ If any requirement remains `pending` or `partial`, the check exits with code 1 ( ## Required Operational Actions Before Production Release -The canonical register records four pending/partial items. The OpenAI and Railway agreement items were verified on 5 October 2026 for the first-release scope only (public clinical references and wholly invented examples, no real-patient cases or voice); see the [OpenAI](governance/openai-standard-agreement-evidence-2026-10-05.md) and [Railway](governance/railway-dpa-completion-evidence-2026-10-05.md) evidence records, which list what they do not cover. Use the [role-attestation pack](governance/privacy-role-attestation-pack-2026-09-01.md) for their evidence and decision requirements: +The canonical register records four pending/partial items (7 of 11 requirements are `verified`). The OpenAI and Railway agreement items were verified on 5 October 2026 (`PRIV-LEGAL-OPENAI-DPA` and `PRIV-LEGAL-RAILWAY-DPA` promoted to `verified` on `2026-10-05`, `reviewExpiresAt: 2026-12-01`) for the first-release scope only (public clinical references and wholly invented examples, no real-patient cases or voice); see the [OpenAI](governance/openai-standard-agreement-evidence-2026-10-05.md) and [Railway](governance/railway-dpa-completion-evidence-2026-10-05.md) evidence records, which list what they do not cover. Use the [role-attestation pack](governance/privacy-role-attestation-pack-2026-09-01.md) for their evidence and decision requirements: | Requirement | Recorded status | Remaining evidence or decision | | ------------------------------------ | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/scripts-index.md b/docs/scripts-index.md index 6d9245783a..0cfce64be3 100644 --- a/docs/scripts-index.md +++ b/docs/scripts-index.md @@ -1,6 +1,6 @@ # Scripts index -Curated map of `scripts/` (409 files) and the `package.json` script surface (329 entries), +Curated map of `scripts/` (415 files) and the `package.json` script surface (330 entries), grouped by purpose. This is orientation, not an exhaustive per-file listing — the authoritative command list is `package.json`, and `npm run docs:check-scripts` verifies every `npm run ` referenced in docs resolves to a real script. `npm run docs:update` refreshes the exact counts above. diff --git a/package.json b/package.json index f1a776701d..a25f6317ff 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "test:coverage": "node scripts/run-vitest.mjs run --coverage", "test:coverage:node": "node scripts/run-vitest.mjs run --project=node --coverage", "test:coverage:ui": "node scripts/run-vitest.mjs run --project=jsdom --coverage", - "test:ci-workflows": "node scripts/run-vitest.mjs run tests/ci-cache-safety.test.ts tests/organisation-weekly-workflow.test.ts tests/ci-main-failure-routing.test.ts tests/ci-main-tree-proof.test.ts tests/staging-tenancy-workflow.test.ts tests/site-content-freshness-workflow.test.ts tests/ci-audit-contracts.test.ts tests/secret-scan-merge-group.test.ts tests/ci-browser-matrix-coverage.test.ts tests/branch-review-index.test.ts tests/authenticated-live-workflow.test.ts tests/browser-test-plan.test.ts tests/chain-mirror-parity.test.ts tests/pr-batch-github.test.ts tests/pr-mergeability-workflow-guard.test.ts tests/pr-policy-merge-queue.test.ts tests/eval-canary-workflow.test.ts tests/live-drift-workflow.test.ts tests/live-domain-monitor-workflow.test.ts tests/ops-digest.test.ts tests/container-ci-contract.test.ts tests/test-runner-safety.test.ts tests/installed-lock-parity.test.ts tests/railway-config.test.ts tests/ingestion-autopilot.test.ts tests/ingestion-autopilot-workflow.test.ts tests/reindex-reaper-workflow.test.ts tests/check-lighthouse-budget.test.ts tests/claude-review-workflow.test.ts tests/live-web-vitals-inputs.test.ts tests/offline-release-profile.test.ts tests/bundle-budget-refresh-workflow.test.ts tests/ranking-snapshot-refresh-workflow.test.ts tests/dependency-report-workflow.test.ts tests/eval-canary-case-summary.test.ts", + "test:ci-workflows": "node scripts/run-vitest.mjs run tests/ci-cache-safety.test.ts tests/organisation-weekly-workflow.test.ts tests/ci-main-failure-routing.test.ts tests/ci-main-tree-proof.test.ts tests/staging-tenancy-workflow.test.ts tests/site-content-freshness-workflow.test.ts tests/ci-audit-contracts.test.ts tests/secret-scan-merge-group.test.ts tests/ci-browser-matrix-coverage.test.ts tests/branch-review-index.test.ts tests/authenticated-live-workflow.test.ts tests/browser-test-plan.test.ts tests/chain-mirror-parity.test.ts tests/pr-batch-github.test.ts tests/pr-mergeability-workflow-guard.test.ts tests/pr-policy-merge-queue.test.ts tests/eval-canary-workflow.test.ts tests/live-drift-workflow.test.ts tests/live-domain-monitor-workflow.test.ts tests/ops-digest.test.ts tests/container-ci-contract.test.ts tests/test-runner-safety.test.ts tests/installed-lock-parity.test.ts tests/railway-config.test.ts tests/ingestion-autopilot.test.ts tests/ingestion-autopilot-workflow.test.ts tests/reindex-reaper-workflow.test.ts tests/check-lighthouse-budget.test.ts tests/claude-review-workflow.test.ts tests/live-web-vitals-inputs.test.ts tests/offline-release-profile.test.ts tests/bundle-budget-refresh-workflow.test.ts tests/ranking-snapshot-refresh-workflow.test.ts tests/dependency-report-workflow.test.ts tests/eval-canary-case-summary.test.ts tests/trivy-image-scan.test.ts", "test:e2e": "node scripts/run-playwright.mjs", "test:e2e:all": "node scripts/run-playwright.mjs", "test:e2e:headed": "node scripts/run-playwright.mjs --headed", diff --git a/scripts/check-maintainability-budgets.mjs b/scripts/check-maintainability-budgets.mjs index ef867f002a..e616e4a0f7 100644 --- a/scripts/check-maintainability-budgets.mjs +++ b/scripts/check-maintainability-budgets.mjs @@ -4,8 +4,8 @@ import { readFileSync } from "node:fs"; const budgets = new Map([ // Chrome ownership/reporting lives in use-dashboard-chrome-coordinator; keep // the reclaimed monolith budget so it cannot silently drift back to 4160. - // 3889: per-document actions moved to use-dashboard-document-actions.ts. - ["src/components/ClinicalDashboard.tsx", 3889], + // 3600: ratcheted down toward actual line count (~3521) to eliminate slack. + ["src/components/ClinicalDashboard.tsx", 3600], // Evidence coverage, per-request hydration, and second-stage ranking live in // focused rag modules; keep the reclaimed budget so it cannot silently drift back. // 4362: soft-tail answer-cache skip call site (logic in rag-query-guard.ts). @@ -13,19 +13,19 @@ const budgets = new Map([ ["src/components/DocumentViewer.tsx", 1734], ["supabase/functions/indexing-v3-agent/index.ts", 2191], // Added 2026-10-06 with no prior cap. The search header and the extractive - // answer builder are capped at their current size; split them, never grow them. - ["src/components/clinical-dashboard/master-search-header.tsx", 3241], + // answer builder are capped near their current size; split them, never grow them. + ["src/components/clinical-dashboard/master-search-header.tsx", 2900], // 2026-10-16: split into four layered modules behind rag-extractive-answer.ts, which now only // re-exports. Each part is capped at its size on the split. ["src/lib/rag/rag-extractive-answer.ts", 72], ["src/lib/rag/rag-extractive-signals.ts", 1002], - ["src/lib/rag/rag-answer-shape-checks.ts", 414], - ["src/lib/rag/rag-extractive-builder.ts", 2335], + ["src/lib/rag/rag-answer-shape-checks.ts", 425], + ["src/lib/rag/rag-extractive-builder.ts", 2345], ["src/lib/rag/rag-answer-quality-gate.ts", 1690], // globals.css keeps a little headroom because most UI changes still add CSS // here and dozens of contract tests read this file directly. Ratchet it down // as area stylesheets move out behind @import (see mode-band.css). - ["src/app/globals.css", 6700], + ["src/app/globals.css", 6550], ]); function sourceLineCount(file) { diff --git a/scripts/check-runtime.ts b/scripts/check-runtime.ts index 22f40ef3b1..1d407c0195 100644 --- a/scripts/check-runtime.ts +++ b/scripts/check-runtime.ts @@ -1,7 +1,11 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + export type RuntimeCheckResult = { ok: boolean; expectedMajor: number; actualVersion: string; + driftDetected?: boolean; message: string; }; @@ -106,8 +110,80 @@ export function checkNpmRuntime( return runtimeResult("npm", version, expectedMajor); } +export function readInstalledNodeTypesVersion( + packageJsonPath = resolve(process.cwd(), "node_modules/@types/node/package.json"), +): string { + try { + const raw = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { version?: unknown }; + return typeof raw.version === "string" && raw.version.trim() ? raw.version.trim() : "unknown"; + } catch { + return "unknown"; + } +} + +/** + * Validates installed `@types/node` against the supported Node 24 runtime. + * Fails closed if `@types/node` is older than Node 24 (`< 24`), and emits a + * surfaced warning (`driftDetected: true`) when `@types/node` is ahead of + * Node 24 (`> 24`, e.g. `@types/node` 25.x in `package-lock.json`) unless + * `strictMajor` (`CHECK_NODE_TYPES_STRICT_MAJOR=1`) is enabled. + */ +export function checkNodeTypesRuntime( + version = readInstalledNodeTypesVersion(), + expectedMajor = 24, + options?: { strictMajor?: boolean }, +): RuntimeCheckResult { + if (version === "unknown") { + return { + ok: true, + expectedMajor, + actualVersion: "unknown", + message: `@types/node package version was not detected; skipping @types/node ${expectedMajor}.x check.`, + }; + } + + const actualMajor = Number(version.split(".")[0]); + if (!Number.isFinite(actualMajor)) { + return { + ok: false, + expectedMajor, + actualVersion: version, + message: `Could not parse @types/node version "${version}". Expected @types/node ${expectedMajor}.x.`, + }; + } + + if (actualMajor < expectedMajor) { + return { + ok: false, + expectedMajor, + actualVersion: version, + driftDetected: true, + message: `@types/node ${version} is older than required Node ${expectedMajor}.x runtime types.`, + }; + } + + if (actualMajor > expectedMajor) { + const strictMajor = options?.strictMajor ?? process.env.CHECK_NODE_TYPES_STRICT_MAJOR === "1"; + return { + ok: !strictMajor, + expectedMajor, + actualVersion: version, + driftDetected: true, + message: `@types/node ${version} (major ${actualMajor}) is ahead of the Node ${expectedMajor}.x runtime target; avoid relying on Node >${expectedMajor} APIs.`, + }; + } + + return { + ok: true, + expectedMajor, + actualVersion: version, + driftDetected: false, + message: `@types/node ${version} matches required Node ${expectedMajor}.x runtime target.`, + }; +} + function main() { - const results = [checkNodeRuntime(process.versions.node), checkNpmRuntime()]; + const results = [checkNodeRuntime(process.versions.node), checkNpmRuntime(), checkNodeTypesRuntime()]; for (const result of results) { console.log(`[Runtime Check] ${result.ok ? "PASS" : "FAIL"}: ${result.message}`); } diff --git a/scripts/deploy/write-release-manifest.mjs b/scripts/deploy/write-release-manifest.mjs new file mode 100644 index 0000000000..fbb38c3a11 --- /dev/null +++ b/scripts/deploy/write-release-manifest.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { mkdirSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(__dirname, "..", ".."); +const DEFAULT_OUTPUT_PATH = join(REPO_ROOT, "deploy", "release-identity.json"); +const SHA_RE = /^[0-9a-f]{40}$/i; + +function normalizeCommitSha(raw) { + const trimmed = typeof raw === "string" ? raw.trim() : ""; + return SHA_RE.test(trimmed) ? trimmed.toLowerCase() : null; +} + +function readGitHeadCommitSha(cwd = REPO_ROOT) { + try { + const stdout = execFileSync("git", ["rev-parse", "HEAD"], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + return normalizeCommitSha(stdout); + } catch { + return null; + } +} + +export function resolveReleaseManifestIdentity(env = process.env, options = {}) { + const envCommitCandidate = + env.RAILWAY_GIT_COMMIT_SHA || + env.GITHUB_SHA || + env.SOURCE_COMMIT_SHA || + env.COMMIT_SHA || + env.VERCEL_GIT_COMMIT_SHA || + ""; + const commitSha = + normalizeCommitSha(envCommitCandidate) ?? + (options.resolveGitHead === false ? null : readGitHeadCommitSha(options.cwd ?? REPO_ROOT)); + + const explicitRelease = (env.SENTRY_RELEASE || env.NEXT_PUBLIC_SENTRY_RELEASE || "").trim(); + const sentryRelease = explicitRelease || commitSha || "dev"; + + return { + commitSha, + sentryRelease, + }; +} + +export function writeReleaseManifest(outputPath = DEFAULT_OUTPUT_PATH, env = process.env, options = {}) { + const payload = resolveReleaseManifestIdentity(env, options); + mkdirSync(dirname(outputPath), { recursive: true }); + // Write to a temporary file beside the target, then rename, so a crash or a concurrent reader + // never sees a partial manifest. + const temporaryPath = `${outputPath}.${process.pid}.tmp`; + writeFileSync(temporaryPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + renameSync(temporaryPath, outputPath); + return payload; +} + +function isDirectEntrypoint(metaUrl) { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(metaUrl) === entry; + } catch { + return false; + } +} + +if (isDirectEntrypoint(import.meta.url)) { + const payload = writeReleaseManifest(); + console.log( + `[write-release-manifest] Wrote commitSha=${payload.commitSha ?? "null"} sentryRelease=${payload.sentryRelease} to ${DEFAULT_OUTPUT_PATH}`, + ); +} diff --git a/scripts/lib/operational-alerts.mjs b/scripts/lib/operational-alerts.mjs index 725347ed06..6012b01fe1 100644 --- a/scripts/lib/operational-alerts.mjs +++ b/scripts/lib/operational-alerts.mjs @@ -6,6 +6,9 @@ const RUNBOOKS = { hybridPage: "docs/observability-slos.md#ops_hybrid_rpc_error_rate_page", degradedWarning: "docs/observability-slos.md#ops_degraded_answer_rate_warning", degradedPage: "docs/observability-slos.md#ops_degraded_answer_rate_page", + sourceGapWarning: "docs/observability-slos.md#ops_source_gap_rate_warning", + unsupportedWarning: "docs/observability-slos.md#ops_unsupported_answer_rate_warning", + latencyBudgetWarning: "docs/observability-slos.md#ops_answer_latency_budget_warning", spend: "docs/observability-slos.md#ops_projected_daily_spend_warning", canary: "docs/observability-slos.md#ops_eval_canary_stale_warning", }; @@ -62,6 +65,27 @@ const policies = { escalationOwner: "Clinical safety owner", runbook: RUNBOOKS.degradedPage, }, + sourceGapWarning: { + code: "OPS_SOURCE_GAP_RATE_WARNING", + severity: "warning", + owner: "Clinical evaluation owner", + escalationOwner: "Clinical safety owner", + runbook: RUNBOOKS.sourceGapWarning, + }, + unsupportedWarning: { + code: "OPS_UNSUPPORTED_ANSWER_RATE_WARNING", + severity: "warning", + owner: "AI platform owner", + escalationOwner: "Clinical safety owner", + runbook: RUNBOOKS.unsupportedWarning, + }, + latencyBudgetWarning: { + code: "OPS_ANSWER_LATENCY_BUDGET_WARNING", + severity: "warning", + owner: "Platform operations", + escalationOwner: "AI platform owner", + runbook: RUNBOOKS.latencyBudgetWarning, + }, spend: { code: "OPS_PROJECTED_DAILY_SPEND_WARNING", severity: "warning", @@ -178,6 +202,37 @@ export function evaluateOperationalAlerts(health, context = {}) { }), ); } + + if (slo.totalQueries >= 20) { + const windowMinutes = finiteNumber(slo.windowMinutes) ? slo.windowMinutes : 60; + if (validRate(slo.sourceGapRate) && slo.sourceGapRate > 0.15) { + alerts.push( + alert(policies.sourceGapWarning, { + observedValue: slo.sourceGapRate, + threshold: { operator: ">", value: 0.15, unit: "rate" }, + windowMinutes, + }), + ); + } + if (validRate(slo.unsupportedRate) && slo.unsupportedRate > 0.15) { + alerts.push( + alert(policies.unsupportedWarning, { + observedValue: slo.unsupportedRate, + threshold: { operator: ">", value: 0.15, unit: "rate" }, + windowMinutes, + }), + ); + } + if (validRate(slo.latencyBudgetBreachRate) && slo.latencyBudgetBreachRate > 0.15) { + alerts.push( + alert(policies.latencyBudgetWarning, { + observedValue: slo.latencyBudgetBreachRate, + threshold: { operator: ">", value: 0.15, unit: "rate" }, + windowMinutes, + }), + ); + } + } } if (health?.spend?.alerting) { diff --git a/scripts/lib/tenancy-scan.mjs b/scripts/lib/tenancy-scan.mjs index 5b0381d8f3..26e4f81bb8 100644 --- a/scripts/lib/tenancy-scan.mjs +++ b/scripts/lib/tenancy-scan.mjs @@ -1378,10 +1378,10 @@ export const SCOPE_EXEMPTIONS = [ file: "src/lib/observability/answer-slo.ts", table: "rag_queries", fn: "answerSloSnapshot", - queries: 1, + queries: 2, proof: PROOF_KINDS.REVIEWED_INDIRECT, reason: - "Deliberate cross-tenant operator aggregate: reads only `metadata` for rows carrying a hybrid_rpc_errors map, to name which retrieval RPC degraded. Same HEALTH_DEEP_PROBE_SECRET gate; no query text, answer text, or owner identity leaves the probe.", + "Deliberate cross-tenant operator aggregate: two reads of only `metadata` (the trailing-window sample, plus a dedicated sample of rows carrying a hybrid_rpc_errors map when the first sample missed some), to name which retrieval RPC degraded. Same HEALTH_DEEP_PROBE_SECRET gate; no query text, answer text, or owner identity leaves the probe.", }, { file: "src/lib/observability/spend-metrics.ts", diff --git a/scripts/operator-explain-documents-indexes.sql b/scripts/operator-explain-documents-indexes.sql index 5b62210698..7b60b5ef9f 100644 --- a/scripts/operator-explain-documents-indexes.sql +++ b/scripts/operator-explain-documents-indexes.sql @@ -87,3 +87,68 @@ LIMIT 5000; -- FROM pg_index i -- JOIN pg_stat_user_indexes s ON s.indexrelid = i.indexrelid -- WHERE s.relname = 'documents'; + +-- ---------------------------------------------------------------------------- +-- Step 4: DB Advisor foreign-key index diagnostics (read-only) +-- Candidates: +-- * public.storage_cleanup_jobs(document_id) +-- * public.rag_visual_eval_cases(document_id) +-- * public.rag_visual_eval_runs(document_id) +-- ---------------------------------------------------------------------------- + +-- 4a: Inspect existing indexes on foreign-key advisor candidate tables +SELECT + schemaname, + tablename, + indexname, + indexdef +FROM pg_indexes +WHERE schemaname = 'public' + AND tablename IN ('storage_cleanup_jobs', 'rag_visual_eval_cases', 'rag_visual_eval_runs') +ORDER BY tablename, indexname; + +-- 4b: Check index scan statistics and unused/low-scan indexes via pg_stat_user_indexes +SELECT + s.schemaname, + s.relname AS table_name, + s.indexrelname AS index_name, + s.idx_scan, + s.idx_tup_read, + s.idx_tup_fetch, + pg_size_pretty(pg_relation_size(s.indexrelid)) AS index_size +FROM pg_stat_user_indexes s +WHERE s.schemaname = 'public' + AND s.relname IN ( + 'documents', + 'storage_cleanup_jobs', + 'rag_visual_eval_cases', + 'rag_visual_eval_runs' + ) +ORDER BY s.relname, s.idx_scan ASC, s.indexrelname; + +-- 4c: Verify whether foreign-key columns referencing public.documents(id) are covered by a leading index column +SELECT + c.conrelid::regclass AS referencing_table, + c.conname AS constraint_name, + a.attname AS fk_column, + EXISTS ( + SELECT 1 + FROM pg_index i + WHERE i.indrelid = c.conrelid + AND i.indisvalid + AND i.indpred IS NULL + AND i.indkey[0] = a.attnum + ) AS has_leading_column_index +FROM pg_constraint c +JOIN pg_attribute a + ON a.attrelid = c.conrelid + AND a.attnum = c.conkey[1] +WHERE c.contype = 'f' + AND c.confrelid = 'public.documents'::regclass + AND c.conrelid IN ( + 'public.storage_cleanup_jobs'::regclass, + 'public.rag_visual_eval_cases'::regclass, + 'public.rag_visual_eval_runs'::regclass + ) +ORDER BY referencing_table::text, fk_column; + diff --git a/scripts/ops-digest.mjs b/scripts/ops-digest.mjs index 72145de699..4b8398a3f0 100644 --- a/scripts/ops-digest.mjs +++ b/scripts/ops-digest.mjs @@ -105,10 +105,11 @@ export function updateHybridRpcHourlyEvidence(rawHistory, health, observedAt = n const contiguous = lastThree.length === 3 && lastThree.every((entry) => Array.isArray(entry.rpcNames)) && - lastThree.every( - (entry, index) => - index === 0 || Date.parse(entry.hour) - Date.parse(lastThree[index - 1].hour) === hourlyWindowMs, - ); + lastThree.every((entry, index) => { + if (index === 0) return true; + const gapMs = Date.parse(entry.hour) - Date.parse(lastThree[index - 1].hour); + return gapMs === hourlyWindowMs || gapMs === 6 * hourlyWindowMs; + }); const repeatedRpcNames = contiguous ? lastThree[0].rpcNames.filter((name) => lastThree.every((entry) => entry.rpcNames.includes(name))) : []; diff --git a/scripts/pr-policy.mjs b/scripts/pr-policy.mjs index 57828d41ee..59192e77e2 100644 --- a/scripts/pr-policy.mjs +++ b/scripts/pr-policy.mjs @@ -273,12 +273,27 @@ const ragRankingPatterns = [ /^src\/lib\/(?:retrieval-rpc-rollout|clinical-evidence-haystack|cross-document-synthesis|corpus-grounding|keyword-query)\.ts$/, /^src\/lib\/(?:document-enrichment|visual-intelligence|image-filtering)\.ts$/, /^worker\/(?:embedding-fields|table-facts|assertion-tagging)\.ts$/, + /^worker\/(?:candidate-chunks|managed-local-memory-producer|managed-admin-text-terminal|managed-prepared-supplier|published-admin-source-layout|scoped-candidate-projection|whole-original-supplier)\.ts$/, /^scripts\/(?:eval-retrieval|build-ranking-snapshot|tune-search-weights)\.ts$/, /^scripts\/lib\/(?:clinical-aliases|ranking-tuning|ranking-snapshot-builder)\.ts$/, /^scripts\/fixtures\/(?:rag-retrieval-golden|rag-ranking-candidate-snapshot\.v1)\.json$/, // The contract-pinning tests are protected too: weakening them is the evasion route. /^tests\/(?:rag-fast-path-ordering|ranking-tuning|retrieval-selection|rag-second-stage-ranking|eval-retrieval|rag-imputation-contract)\.test\.ts$/, ]; +export const protectedPathSample = Object.freeze([ + "src/lib/released-search-order.ts", + "src/lib/chunking.ts", + "worker/embedding-fields.ts", + "worker/table-facts.ts", + "worker/assertion-tagging.ts", + "worker/candidate-chunks.ts", + "worker/managed-local-memory-producer.ts", + "worker/managed-admin-text-terminal.ts", + "worker/managed-prepared-supplier.ts", + "worker/published-admin-source-layout.ts", + "worker/scoped-candidate-projection.ts", + "worker/whole-original-supplier.ts", +]); export const safetyPathLists = Object.freeze({ ragRanking: Object.freeze(ragRankingPatterns.map((p) => new RegExp(p.source, p.flags))), clinicalRisk: Object.freeze(clinicalRiskPatterns.map((p) => new RegExp(p.source, p.flags))), @@ -1514,6 +1529,13 @@ function selfTest() { "worker/embedding-fields.ts", "worker/table-facts.ts", "worker/assertion-tagging.ts", + "worker/candidate-chunks.ts", + "worker/managed-local-memory-producer.ts", + "worker/managed-admin-text-terminal.ts", + "worker/managed-prepared-supplier.ts", + "worker/published-admin-source-layout.ts", + "worker/scoped-candidate-projection.ts", + "worker/whole-original-supplier.ts", ]) { assert.equal(classifyPullRequestFiles([file]).ragRanking, true, file); } diff --git a/scripts/trivy-image-scan.mjs b/scripts/trivy-image-scan.mjs index 773e357aac..f13eb6939a 100644 --- a/scripts/trivy-image-scan.mjs +++ b/scripts/trivy-image-scan.mjs @@ -9,13 +9,17 @@ * daemon socket, which would be equivalent to root on the host. */ import { basename, dirname, join, resolve } from "node:path"; -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; // Pin to a multi-platform image index digest. Update with: // node scripts/resolve-oci-image-digest.mjs aquasec/trivy:0.59.1 -const TRIVY_IMAGE = "aquasec/trivy:0.59.1@sha256:029e990b328d149bf0a9ffe355919041e1f86192db2df47e217f8a36dd42ceac"; +export const TRIVY_VERSION = "0.59.1"; +export const TRIVY_IMAGE = + "aquasec/trivy:0.59.1@sha256:029e990b328d149bf0a9ffe355919041e1f86192db2df47e217f8a36dd42ceac"; +export const REQUIRED_TRIVY_SCAN_IMAGES = Object.freeze(["clinical-kb-app:ci", "clinical-kb-worker:ci"]); function run(cmd, args, opts = {}) { const result = spawnSync(cmd, args, { encoding: "utf8", stdio: "pipe", ...opts }); @@ -23,22 +27,192 @@ function run(cmd, args, opts = {}) { return result; } -function localTrivy() { - const result = run(process.platform === "win32" ? "where" : "which", ["trivy"]); - return result.stdout?.split(/\r?\n/)[0]?.trim() || null; +export function isCompatibleLocalTrivyVersion(versionOutput, expectedVersion = TRIVY_VERSION) { + if (typeof versionOutput !== "string" || !versionOutput.trim()) return false; + const escaped = expectedVersion.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`\\bVersion:\\s*v?${escaped}\\b`, "i").test(versionOutput); +} + +function localTrivy(env = process.env) { + try { + const result = run(process.platform === "win32" ? "where" : "which", ["trivy"]); + const binary = result.stdout?.split(/\r?\n/)[0]?.trim() || null; + if (!binary) return null; + if (env.TRIVY_PREFER_LOCAL === "1" || env.TRIVY_PREFER_LOCAL === "true") { + return binary; + } + const versionCheck = run(binary, ["--version"]); + if (versionCheck.status === 0 && isCompatibleLocalTrivyVersion(versionCheck.stdout, TRIVY_VERSION)) { + return binary; + } + return null; + } catch { + return null; + } +} + +export function validateTrivyReportJson(rawStdout) { + if (typeof rawStdout !== "string" || !rawStdout.trim()) { + return { ok: false, reason: "invalid_trivy_json" }; + } + let report; + try { + report = JSON.parse(rawStdout); + } catch { + return { ok: false, reason: "invalid_trivy_json" }; + } + if (!report || typeof report !== "object" || Array.isArray(report)) { + return { ok: false, reason: "invalid_trivy_json" }; + } + if (typeof report.SchemaVersion !== "number" || !Number.isFinite(report.SchemaVersion)) { + return { ok: false, reason: "invalid_trivy_json" }; + } + const hasResultsArray = Array.isArray(report.Results); + const hasVerifiedMetadata = + report.Metadata !== null && + typeof report.Metadata === "object" && + !Array.isArray(report.Metadata) && + (typeof report.Metadata.ImageID === "string" || + (report.Metadata.OS !== null && typeof report.Metadata.OS === "object") || + Array.isArray(report.Metadata.DiffIDs)); + if (!hasResultsArray && !hasVerifiedMetadata) { + return { ok: false, reason: "invalid_trivy_json" }; + } + + const results = hasResultsArray ? report.Results : []; + const counts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0, UNKNOWN: 0 }; + let totalVulnerabilities = 0; + + for (const entry of results) { + if (!entry || typeof entry !== "object") continue; + const vulns = Array.isArray(entry.Vulnerabilities) ? entry.Vulnerabilities : []; + for (const vuln of vulns) { + totalVulnerabilities += 1; + const sev = typeof vuln?.Severity === "string" && vuln.Severity in counts ? vuln.Severity : "UNKNOWN"; + counts[sev] += 1; + } + } + + return { + ok: true, + report, + counts, + totalVulnerabilities, + }; +} + +export function evaluateTrivySummaryText(summaryText, requiredImages = REQUIRED_TRIVY_SCAN_IMAGES) { + const text = typeof summaryText === "string" ? summaryText.trim() : ""; + if (!text) { + return { + assessed: false, + ok: false, + high: 0, + critical: 0, + reason: "empty_summary", + }; + } + if (/\bSTATUS=unassessed\b/.test(text)) { + return { + assessed: false, + ok: false, + high: 0, + critical: 0, + reason: "contains_unassessed_status", + }; + } + for (const imageRef of requiredImages) { + const escaped = imageRef.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const okLineRe = new RegExp(`^Trivy scan completed \\(${escaped}\\): STATUS=ok\\b`, "m"); + if (!okLineRe.test(text)) { + return { + assessed: false, + ok: false, + high: 0, + critical: 0, + reason: `missing_status_ok_for_${imageRef}`, + }; + } + } + + const high = [...text.matchAll(/\bHIGH=(\d+)\b/g)].reduce((sum, match) => sum + Number(match[1]), 0); + const critical = [...text.matchAll(/\bCRITICAL=(\d+)\b/g)].reduce((sum, match) => sum + Number(match[1]), 0); + return { + assessed: true, + ok: high === 0 && critical === 0, + high, + critical, + reason: high === 0 && critical === 0 ? null : "high_or_critical_vulnerabilities_found", + }; +} + +/** + * Docker volume arguments for a containerised Trivy run. The saved image tarball stays read-only. + * Only the throwaway vulnerability-database cache is writable: Trivy opens its database file + * read-write, and a read-only mount made every scan fail with "read-only file system", which is + * reported as unassessed. The container still runs with no network. + * + * @param {{ workDir: string; cacheDir?: string | null; outputDir?: string | null }} options + * @returns {string[]} + */ +export function trivyDockerMounts({ workDir, cacheDir = null, outputDir = null }) { + const mounts = ["-v", `${workDir}:/work:ro`]; + if (cacheDir) mounts.push("-v", `${cacheDir}:/cache`); + if (outputDir) mounts.push("-v", `${outputDir}:/out`); + return mounts; } /** * @param {string} imageRef * @param {string[]} trivyArgs use "__IMAGE__" where the image ref belongs and * "__OUT__" where an output path belongs (when outputHostPath is set) - * @param {{ outputHostPath?: string | null }} [opts] + * @param {{ outputHostPath?: string | null; needsVulnerabilityDb?: boolean }} [opts] */ -function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null } = {}) { +function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null, needsVulnerabilityDb = false } = {}) { const workDir = mkdtempSync(join(tmpdir(), "trivy-scan-")); const tarName = "image.tar"; const tarPath = join(workDir, tarName); try { + let cacheDir = null; + if (needsVulnerabilityDb) { + cacheDir = process.env.TRIVY_CACHE_DIR + ? resolve(process.env.TRIVY_CACHE_DIR) + : join(tmpdir(), `trivy-db-cache-${TRIVY_VERSION}`); + mkdirSync(cacheDir, { recursive: true }); + if (!existsSync(join(cacheDir, "db", "metadata.json"))) { + const dbDownload = run("docker", [ + "run", + "--rm", + "-v", + `${cacheDir}:/cache`, + TRIVY_IMAGE, + "image", + "--download-db-only", + "--cache-dir", + "/cache", + ]); + if (dbDownload.status !== 0) { + return { + status: dbDownload.status ?? 1, + stdout: dbDownload.stdout, + stderr: dbDownload.stderr || dbDownload.stdout || "Trivy DB download failed", + }; + } + // Best-effort Java DB pre-population for Java archive inspection offline. + run("docker", [ + "run", + "--rm", + "-v", + `${cacheDir}:/cache`, + TRIVY_IMAGE, + "image", + "--download-java-db-only", + "--cache-dir", + "/cache", + ]); + } + } + // Keep Docker's save temp files inside workDir so finally{} can reclaim them // and CI runners do not fill the shared /tmp between sequential scans. const saveEnv = { ...process.env, TMPDIR: workDir, DOCKER_TMPDIR: workDir }; @@ -55,10 +229,11 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null } = {}) { }; } - const mounts = ["-v", `${workDir}:/work:ro`]; - if (outputHostPath) { - mounts.push("-v", `${dirname(resolve(outputHostPath))}:/out`); - } + const mounts = trivyDockerMounts({ + workDir, + cacheDir, + outputDir: outputHostPath ? dirname(resolve(outputHostPath)) : null, + }); const dockerArgs = ["run", "--rm", "--network=none", ...mounts, TRIVY_IMAGE]; for (const arg of trivyArgs) { @@ -70,6 +245,9 @@ function runDockerTrivy(imageRef, trivyArgs, { outputHostPath = null } = {}) { dockerArgs.push(arg); } } + if (cacheDir) { + dockerArgs.push("--cache-dir", "/cache", "--skip-db-update", "--skip-java-db-update"); + } return run("docker", dockerArgs); } finally { @@ -116,31 +294,39 @@ function main() { const result = trivy ? run(trivy, ["image", "--format", "json", "--exit-code", "0", "--severity", severity, imageRef]) - : runDockerTrivy(imageRef, ["image", "--format", "json", "--exit-code", "0", "--severity", severity, "__IMAGE__"]); + : runDockerTrivy(imageRef, ["image", "--format", "json", "--exit-code", "0", "--severity", severity, "__IMAGE__"], { + needsVulnerabilityDb: true, + }); if (result.status !== 0) { console.error(result.stderr || result.stdout || `Trivy failed to scan ${imageRef}`); - process.exit(1); + console.log(`Trivy scan failed (${imageRef}): STATUS=unassessed reason=scanner_exit_${result.status ?? 1}`); + process.exit(2); + } + + const validated = validateTrivyReportJson(result.stdout); + if (!validated.ok) { + console.error(`Trivy scan failed (${imageRef}): STATUS=unassessed reason=${validated.reason}`); + console.log(`Trivy scan failed (${imageRef}): STATUS=unassessed reason=${validated.reason}`); + process.exit(2); } - let summary = null; + const { counts } = validated; + console.log( + `Trivy scan completed (${imageRef}): STATUS=ok HIGH=${counts.HIGH} CRITICAL=${counts.CRITICAL} MEDIUM=${counts.MEDIUM} LOW=${counts.LOW} UNKNOWN=${counts.UNKNOWN}`, + ); +} + +function isDirectEntrypoint(metaUrl) { + const entry = process.argv[1]; + if (!entry) return false; try { - const report = JSON.parse(result.stdout || "{}"); - const results = report.Results || []; - const vulnerabilities = results.flatMap((r) => r.Vulnerabilities || []); - const counts = { CRITICAL: 0, HIGH: 0, MEDIUM: 0, LOW: 0, UNKNOWN: 0 }; - for (const v of vulnerabilities) { - counts[v.Severity ?? "UNKNOWN"] = (counts[v.Severity ?? "UNKNOWN"] ?? 0) + 1; - } - summary = `vulnerabilities=${vulnerabilities.length} (${Object.entries(counts) - .filter(([, n]) => n > 0) - .map(([sev, n]) => `${sev}=${n}`) - .join(", ")})`; + return fileURLToPath(metaUrl) === resolve(entry); } catch { - summary = "could not parse Trivy JSON"; + return false; } - - console.log(`Trivy scan completed for ${imageRef}: ${summary}`); } -main(); +if (isDirectEntrypoint(import.meta.url)) { + main(); +} diff --git a/src/app/(search-app)/cme/plan/page.tsx b/src/app/(search-app)/cme/plan/page.tsx index 78ffd7a81a..8152216c5e 100644 --- a/src/app/(search-app)/cme/plan/page.tsx +++ b/src/app/(search-app)/cme/plan/page.tsx @@ -39,6 +39,7 @@ export default async function CmePlanRoute({ searchParams }: { searchParams: Pro now={data.now} nextYearConfirmed={data.nextYearConfirmed} nextYearGoals={data.nextYearGoals} + ownerId={data.ownerId ?? undefined} />