From 62c013a3219e0dd1334c7902e019614a0b60420f Mon Sep 17 00:00:00 2001 From: Philippe Loulidi <78689302+LipLoody@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:31:51 -0600 Subject: [PATCH 1/2] [ecr] add Jev shadow evaluation --- .expo-agents/code-review/config.jsonc | 10 + .github/workflows/expo-code-review.yml | 2 + README.md | 36 ++- bun.lock | 3 + llp/0000-expo-code-review-cli.explainer.md | 4 +- llp/0002-review-engine-pipeline.explainer.md | 4 +- ...config-schema-loading-routing.explainer.md | 10 +- ...option-templates-and-ci-workflows.guide.md | 9 + llp/0014-jev-shadow-evaluation.explainer.md | 159 +++++++++++ package.json | 1 + src/__tests__/jev.test.ts | 152 +++++++++++ src/config/load.ts | 14 +- src/config/schema.ts | 36 ++- src/core/auth.ts | 1 + src/core/jev.ts | 257 ++++++++++++++++++ src/core/log.ts | 3 + src/core/review.ts | 32 ++- templates/atlantis.yml | 2 + templates/command.yml | 2 + templates/config.jsonc | 15 + templates/workflow.yml | 2 + 21 files changed, 745 insertions(+), 9 deletions(-) create mode 100644 llp/0014-jev-shadow-evaluation.explainer.md create mode 100644 src/__tests__/jev.test.ts create mode 100644 src/core/jev.ts diff --git a/.expo-agents/code-review/config.jsonc b/.expo-agents/code-review/config.jsonc index 42f9eaa..7c4befb 100644 --- a/.expo-agents/code-review/config.jsonc +++ b/.expo-agents/code-review/config.jsonc @@ -31,6 +31,16 @@ "timeoutMs": 30000 }, + // Shadow-only Jev second opinion. Keep disabled until TYPESAFE_API_KEY is added + // as a repository secret; enabling it changes only reviews.jsonl telemetry. + "jev": { + "enabled": false, + "model": "jev-1.13.0", + "maxFindings": 20, + "timeoutMs": 10000, + "maxPatchChars": 30000 + }, + // Large diffs are split into focused chunks by changed-line count, plus a // cross-cutting pass for multi-file issues. Diffs under maxChangedLines are one // full-context pass. Defaults shown; raise/lower per your model + PR sizes. diff --git a/.github/workflows/expo-code-review.yml b/.github/workflows/expo-code-review.yml index b597458..9c85abb 100644 --- a/.github/workflows/expo-code-review.yml +++ b/.github/workflows/expo-code-review.yml @@ -90,5 +90,7 @@ jobs: CLAUDE_CODE_REVIEW_SHARED_API_TOKEN: ${{ secrets.CLAUDE_CODE_REVIEW_SHARED_API_TOKEN }} # Search-only credential used for fixed, site-scoped documentation discovery. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} + # Shadow-only Jev evaluator; unused while config.jev.enabled is false. + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} diff --git a/README.md b/README.md index 946e019..7f70703 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,38 @@ The boundary, in brief: Full detail — providers, query grammar, `fetch_platform_doc` modes, provenance and citation grounding: [LLP 0013](./llp/0013-platform-research.explainer.md). +## Jev shadow evaluation (experimental) + +ECR can ask [TypeSafe AI's Jev](https://docs.typesafe.ai/) for a typed second opinion +on each final finding. This integration is deliberately observation-only: Jev's +support and severity probabilities go to `.runs/reviews.jsonl`, but they never alter +the posted findings or decision. That makes real review traffic the calibration set +before any future filtering or routing policy is considered. + +Enable it in the root config and provide `TYPESAFE_API_KEY` (the scaffolded workflows +already map the optional repository secret): + +```jsonc +{ + "jev": { + "enabled": true, + "model": "jev-1.13.0", + "maxFindings": 20, + "timeoutMs": 10000, + "maxPatchChars": 30000 + } +} +``` + +Each request contains one finding and only the bounded patch for its file. The model +is pinned because probability thresholds must be calibrated against a stable version. +Jev is an evaluator, not a generator: it cannot explain a defect or propose a fix, +and its confidence is distribution concentration rather than proof that an answer is +correct. It is also not a prompt-injection or security boundary. Missing credentials, +timeouts, and provider errors leave the review unchanged and are recorded as reduced +shadow coverage. See [LLP 0014](./llp/0014-jev-shadow-evaluation.explainer.md) for the +research, threat model, and promotion criteria. + ## Monorepos (routing manifest) A monorepo can route different subtrees to different reviewer rosters from a single @@ -355,9 +387,9 @@ your-monorepo/ Enforced in code and by an independent CI guard step, not by convention: -- **auth and research are locked to the root.** `tokenEnv` is honored only in the +- **auth, research, and Jev are locked to the root.** `tokenEnv` is honored only in the root `config.jsonc` / `routing.jsonc` `defaults.auth`; a scope config declaring - `auth`/`breakGlass`/`research` fails to parse, and the CI guard sweeps every + `auth`/`breakGlass`/`research`/`jev` fails to parse, and the CI guard sweeps every config file repo-wide and refuses to run unless `tokenEnv` appears exactly once, root-owned, equal to `ECR_EXPECTED_TOKEN_ENV`. Routing globs choose *which roster* reviews a file, never *which secret* is sent. diff --git a/bun.lock b/bun.lock index d7bfa81..866d91d 100644 --- a/bun.lock +++ b/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "1.30.0", "@opencode-ai/sdk": "1.18.4", + "@typesafe-ai/sdk": "0.6.0", "cheerio": "1.1.2", "minisearch": "7.2.0", "opencode-ai": "1.18.4", @@ -109,6 +110,8 @@ "@types/node": ["@types/node@20.14.8", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA=="], + "@typesafe-ai/sdk": ["@typesafe-ai/sdk@0.6.0", "", {}, "sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw=="], + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], diff --git a/llp/0000-expo-code-review-cli.explainer.md b/llp/0000-expo-code-review-cli.explainer.md index 1199881..d23728f 100644 --- a/llp/0000-expo-code-review-cli.explainer.md +++ b/llp/0000-expo-code-review-cli.explainer.md @@ -6,7 +6,7 @@ **Author:** Philippe Loulidi / Claude **Date:** 2026-07-30 **Role:** Root -**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0003](0003-model-runtimes-and-credentials.explainer.md), [LLP 0004](0004-diff-noise-and-prompts.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0007](0007-cli-commands-and-ci.explainer.md), [LLP 0008](0008-sources-and-reporters.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md), [LLP 0013](0013-platform-research.explainer.md) +**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0003](0003-model-runtimes-and-credentials.explainer.md), [LLP 0004](0004-diff-noise-and-prompts.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0007](0007-cli-commands-and-ci.explainer.md), [LLP 0008](0008-sources-and-reporters.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md), [LLP 0013](0013-platform-research.explainer.md), [LLP 0014](0014-jev-shadow-evaluation.explainer.md) This is the root map for `@expo/code-review-cli` (`ecr`). It records what the system does, how the subsystems divide, the invariants that hold across all of them, and where each concern's rationale lives. It is intentionally thin: every subsystem's detail belongs in the doc that owns it. Read this first, then follow the link for the part you care about. @@ -29,6 +29,7 @@ The code divides by concern. Each directory maps to the doc that owns its ration - `src/sources/` (`local-git`, `github-pr`) + `src/reporters/` (`terminal`, `github`) — where the diff comes from and where findings go [observed: AGENTS.md:14-15; src/sources/, src/reporters/]. Owned by [LLP 0008](0008-sources-and-reporters.explainer.md). - `templates/` + the scaffolded CI workflow — the files `ecr init` writes into an adopting repo; they define adopter-facing behavior and must stay in sync with the code [observed: AGENTS.md:19-20]. Owned by [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md). - `src/research-mcp/` + `research/` — the bundled read-only documentation MCP, scoped remote discovery and allowlisted fetch adapters, optional index builder, source catalog, and host-side evidence prepass. Owned by [LLP 0013](0013-platform-research.explainer.md). +- `src/core/jev.ts` — optional, observation-only typed evaluation of final findings against bounded file patches. Owned by [LLP 0014](0014-jev-shadow-evaluation.explainer.md). - The trust model spans all of the above and is stated as principles in [LLP 0001](0001-trust-model.principles.md). ## Cross-Cutting Invariants @@ -70,5 +71,6 @@ Start here, then follow the owning doc. | What do the CLI commands do, and how does CI orchestrate them? | [LLP 0007](0007-cli-commands-and-ci.explainer.md) | `src/cli.ts`, `src/commands/*` | | Where does the diff come from and where do findings go? | [LLP 0008](0008-sources-and-reporters.explainer.md) | `src/sources/*`, `src/reporters/*` | | What does `ecr init` scaffold into an adopting repo? | [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md) | `templates/`, scaffolded CI workflow | +| How does Jev evaluate findings without changing review decisions? | [LLP 0014](0014-jev-shadow-evaluation.explainer.md) | `src/core/jev.ts`, `src/core/log.ts` | Non-doc sources of truth: `AGENTS.md` holds the working conventions (import style, error handling, the `opencode-ai`/`@opencode-ai/sdk` pinning rule, security invariants); `ROADMAP.md` lists open and deferred items — incremental review, inline PR comments, result-level caching, and a Codex token rotator among them are still open and must not be assumed implemented [observed: ROADMAP.md]; and `templates/` is the adopter-facing contract, so treat it as behavior, not sample text. diff --git a/llp/0002-review-engine-pipeline.explainer.md b/llp/0002-review-engine-pipeline.explainer.md index 31ad1d3..fede0dc 100644 --- a/llp/0002-review-engine-pipeline.explainer.md +++ b/llp/0002-review-engine-pipeline.explainer.md @@ -11,7 +11,7 @@ ## Pipeline Stages -`runReview()` runs a fixed sequence: filter noise → resolve the read root (the trust boundary) → start the engines → preflight that every model id resolves → chunk the diff → run agent×chunk passes plus one cross-cutting pass through a growable bounded queue → coordinate → verify → suppress → reconcile the summary → log. The whole run's resource lifecycle — engine handles, auth material, cwd, and the read-root worktree — is released in a single `finally` (`review.ts:947-951`), so no partial-startup path leaks a server or a worktree. +`runReview()` runs a fixed sequence: filter noise → resolve the read root (the trust boundary) → start the engines → preflight that every model id resolves → chunk the diff → run agent×chunk passes plus one cross-cutting pass through a growable bounded queue → coordinate → verify → suppress → reconcile the summary → optional Jev shadow observation → log. Jev runs only after the final finding set exists and cannot feed values back into it; [LLP 0014](0014-jev-shadow-evaluation.explainer.md) owns that boundary. The whole run's resource lifecycle — engine handles, auth material, cwd, and the read-root worktree — is released in a single `finally` (`review.ts:947-951`), so no partial-startup path leaks a server or a worktree. Two ordering decisions inside that lifecycle are load-bearing and easy to break by "cleaning up": @@ -77,7 +77,7 @@ Rate-limit evidence is tracked per engine separately and surfaced with the cause `writeRunLog()` (`src/core/log.ts`) appends one JSON line per run — cost, tokens (run-wide and per bucket), findings, coverage notes, decision — so runs are auditable and cost/latency measurable later. The record deliberately excludes PR title and body text, keeping only `baseRef`/`headRef`, to avoid persisting secrets that might appear in author-controlled text [observed] (`log.ts:13-18`). Findings may quote changed source lines, but only content the review already publishes verbatim in the PR comment. -Each successful reviewer pass may also return a bounded conclusion-only trace: up to three concrete checks and two unresolved questions, with each note capped at 240 characters. Trace parsing is deliberately fail-soft: malformed optional diagnostics are dropped while valid findings from the same response survive, so observability can never create a coverage gap. `runReview()` merges and de-duplicates valid notes per reviewer, sorts reviewer ids, caps both the stored roster and the complete decoded trace (6 KB), labels the result `unverified-model-diagnostics`, and attaches it to `CoordinatorOutput.reviewTrace` only after the decision pipeline finishes. `attachReviewTrace` strips any coordinator-authored value before attaching the engine-built trace; even a malformed injected coordinator value parses as absent rather than failing consolidation. The trace therefore never reaches coordination, verification, suppression, or decision logic. Because the GitHub comment already embeds the full coordinator output as durable state, downstream agents can recover the trace without adding visible review prose or storing raw transcripts and chain-of-thought. The same trace rides the run log for short-lived operational inspection [observed: `schema.ts` `ReviewerTraceNotesSchema`/`ReviewTraceSchema`/`ReviewerOutputSchema`; `review.ts` `mergeTraceNotes`/`buildReviewTrace`/`attachReviewTrace`; `render.ts` embeds the full review]. +Each successful reviewer pass may also return a bounded conclusion-only trace: up to three concrete checks and two unresolved questions, with each note capped at 240 characters. Trace parsing is deliberately fail-soft: malformed optional diagnostics are dropped while valid findings from the same response survive, so observability can never create a coverage gap. `runReview()` merges and de-duplicates valid notes per reviewer, sorts reviewer ids, caps both the stored roster and the complete decoded trace (6 KB), labels the result `unverified-model-diagnostics`, and attaches it to `CoordinatorOutput.reviewTrace` only after the decision pipeline finishes. `attachReviewTrace` strips any coordinator-authored value before attaching the engine-built trace; even a malformed injected coordinator value parses as absent rather than failing consolidation. The trace therefore never reaches coordination, verification, suppression, or decision logic. Because the GitHub comment already embeds the full coordinator output as durable state, downstream agents can recover the trace without adding visible review prose or storing raw transcripts and chain-of-thought. The same trace rides the run log for short-lived operational inspection [observed: `schema.ts` `ReviewerTraceNotesSchema`/`ReviewTraceSchema`/`ReviewerOutputSchema`; `review.ts` `mergeTraceNotes`/`buildReviewTrace`/`attachReviewTrace`; `render.ts` embeds the full review]. Optional Jev observations are a second log-only sink: they contain fingerprints and typed probabilities, never extra source text, and cannot alter the trace or result. Observability must never break a review, so both output sinks swallow their own errors: `safeLog` wraps `writeRunLog` and discards any write error [observed] (`review.ts:1269-1275`), and `appendStepSummary` is a no-op outside Actions and swallows append failures [observed] (`step-summary.ts:9-19`). A consequence worth stating: a missing or corrupt run-log entry is therefore **not** an error signal — the log's presence or absence proves nothing about whether the review succeeded. diff --git a/llp/0006-config-schema-loading-routing.explainer.md b/llp/0006-config-schema-loading-routing.explainer.md index e052557..d9b88f2 100644 --- a/llp/0006-config-schema-loading-routing.explainer.md +++ b/llp/0006-config-schema-loading-routing.explainer.md @@ -19,7 +19,8 @@ caller, [LLP 0007](0007-cli-commands-and-ci.explainer.md) and [LLP 0001](0001-tr ## Root vs Scope Config A scope config is the root schema minus the centrally locked keys. `ScopeReviewConfigSchema` -takes `ReviewConfigSchema`, omits `auth`, `breakGlass`, and `commentTag`, and re-adds each as +takes `ReviewConfigSchema`, omits centrally owned keys including `auth`, `breakGlass`, +`commentTag`, `research`, and `jev`, and re-adds each as `z.never().optional()` so declaring any of them is a hard parse error, not a later runtime check [observed: `schema.ts:232-247`]. The point of failing at the Zod level is that an IDE or `doctor` catches it before CI does [observed: `schema.ts:224-225` comment]. The remaining @@ -84,6 +85,13 @@ commit `43b31a4` follow-up]. The actual secret values never enter this subsystem owns the wiring and the `FORBIDDEN_TOKEN_ENVS` list, and schema/load only ever touch the env-var NAME [observed: `AGENTS.md` security invariants; `src/core/auth.ts:28,45,206`]. +`jev` is root-only for the same trust reason as research: enabling it sends bounded +repository-derived text to another provider. A scope may neither enable it nor alter +its model or limits. Nested scopes inherit the root's fully resolved Jev settings, so +one PR has one centrally owned disclosure policy [observed: `schema.ts` +`ScopeReviewConfigSchema`; `load.ts` `loadScopeConfig`]. The data boundary and +shadow-only behavior live in [LLP 0014](0014-jev-shadow-evaluation.explainer.md). + ## Model Resolution The `REVIEWER_MODEL` env override is applied on top of frontmatter/default model resolution using diff --git a/llp/0009-adoption-templates-and-ci-workflows.guide.md b/llp/0009-adoption-templates-and-ci-workflows.guide.md index 0234c2b..b011fd8 100644 --- a/llp/0009-adoption-templates-and-ci-workflows.guide.md +++ b/llp/0009-adoption-templates-and-ci-workflows.guide.md @@ -64,6 +64,15 @@ credential actually selected by trusted config. The default remains Anthropic an `dismiss.yml` still receives no model credential [observed: `templates/config.jsonc`; `templates/workflow.yml`, `command.yml`, and `atlantis.yml`]. +**Jev is an independent optional secret.** Each review-running workflow maps +`TYPESAFE_API_KEY`, while the root config keeps `jev.enabled` false. GitHub supplies an +empty value when the repository secret is absent, so the default scaffold does not +gain a working external call. Once enabled from trusted root config, ECR sends bounded +finding/file-patch state directly to TypeSafe and writes only probabilities to the run +log; the key never enters `auth.tokenEnv` or the reviewer engines. `dismiss.yml` still +receives no model or Jev credential. [LLP 0014](0014-jev-shadow-evaluation.explainer.md) +owns the disclosure and promotion rules. + **Keep the `**/*` catch-all first in `routing.jsonc`.** Scopes match last-match-wins, so a `**/*` catch-all must come first and more specific scopes come after to override it [observed: `templates/routing.jsonc:27-31`; `AGENTS.md:17-18`]. Nothing validates the order — reversing it silently changes which config applies to already-routed files [inferred: no code checks that the `**/*` catch-all comes first, so reversing it silently changes routing]. **`scope-config.jsonc` deliberately omits `auth` and `commentTag`.** It is a *different* template from the root `config.jsonc`, not a copy. A scope config carries no `auth` block — credentials are root-only, and the loader plus the CI guard reject a scope config that declares one [observed: `templates/scope-config.jsonc:1-3`; `AGENTS.md:51`]. It also omits `commentTag`, because a scope's PR-comment marker is always derived as `:` so `ecr ci` and `ecr review --scope --post` target the same comment; declaring one is rejected by the scope schema [observed: `templates/scope-config.jsonc:23-25`]. Reusing the root `config.jsonc` template for a scope — a tempting "simplification" — would break the trust boundary (LLP 0006). diff --git a/llp/0014-jev-shadow-evaluation.explainer.md b/llp/0014-jev-shadow-evaluation.explainer.md new file mode 100644 index 0000000..c6b4f00 --- /dev/null +++ b/llp/0014-jev-shadow-evaluation.explainer.md @@ -0,0 +1,159 @@ +# LLP 0014: Jev Shadow Evaluation + +**Type:** Explainer +**Status:** Active +**Systems:** Engine, Config, Security, Observability, Templates +**Author:** Expo +**Date:** 2026-09-21 +**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md) + +Jev is a discriminative model from TypeSafe AI. It accepts a state plus typed +questions and returns probabilities for choices, boolean “Noul” questions, or an +ordered score. It does not generate explanations, remediation text, or code. ECR +therefore uses Jev as an optional shadow evaluator of the final finding set, not as +a reviewer or replacement for the coordinator and verifier. + +## Evidence Survey + +The primary sources are TypeSafe's [Jev announcement](https://typesafe.ai/blog/introducing-system-one-models-and-jev), +[documentation](https://docs.typesafe.ai/), and open-source +[JavaScript SDK](https://github.com/typesafe-ai/typesafe-sdk-js). As of this design, +the current dated model is `jev-1.13.0`; the service also offers moving aliases. +The SDK is at `0.6.0`, supports Node 20+, reads `TYPESAFE_API_KEY`, defaults to a +10-second per-attempt timeout and two retries, and exposes each answer's probability +distribution plus token usage. + +The vendor describes a 64K-token request limit, with at most 32K tokens for state, +text-only input, up to 255 Choice labels, and up to 10 Score levels. Published pricing +is $0.042 per million input tokens with no output-token charge. Published account +limits are 250,000 tokens per second and 1,200 requests per minute, subject to account +changes. These are service facts, not ECR capacity targets; ECR keeps lower local +bounds. + +The vendor benchmark measures agreement with answers from frontier generative models, +not correctness against independently adjudicated human labels. Its reported speed +and cost advantages therefore justify an experiment, not authority over findings. +TypeSafe's own guidance also says that confidence measures concentration of the +returned distribution rather than correctness, thresholds require calibration on the +application's data, irrelevant context degrades results, question wording matters, +and Jev is weak at arithmetic, counting, date comparison, and complex indirection. +Customer requests are not used for training under the standard policy, while zero +data retention is an enterprise control rather than the default. + +Community experiments support a narrow role. The open-source +[`jev-review`](https://github.com/devagrawal09/jev-review) workflow uses typed stages +for file profiling, evidence selection, mechanism, severity, and routing, while +explicitly treating outputs as review leads rather than proof. Other early projects +such as [`pi-warden`](https://github.com/DevMortimer/pi-warden) and +[`pi-jev`](https://github.com/y0usaf/pi-jev) emphasize paired evaluation, calibration, +and enforcement disabled by default. An independent +[hands-on review](https://jevaiguide.com/jev-review/) reports low latency on small +requests but also early documentation/API rough edges and no public research paper. +Those reports are useful implementation anecdotes, not independent validation of code +review accuracy. + +## Fit With ECR + +Jev fits three bounded jobs: + +1. **Finding calibration:** ask whether the changed-code patch supports a final + finding and compare its severity distribution with ECR's result. +2. **Routing:** classify a small diff or candidate finding before assigning expensive + generative passes, once a repo-specific recall benchmark exists. +3. **Evidence triage:** rank retrieved documentation passages or verifier candidates + before a generative model explains them. + +It does not fit jobs that require synthesis: discovering an open-ended bug, tracing a +large execution path with tools, explaining causality, writing remediation, or +consolidating prose. Existing generative reviewers remain responsible for those jobs. +Deterministic code remains responsible for path confinement, schema validation, +credential boundaries, exact quote checks, protected severity floors, and all other +security invariants. + +Jev must never be a prompt-injection detector or security boundary. The state is +attacker-controlled code and finding prose, and a discriminative model can be steered +by adversarial content. No Jev answer may override ECR's critical/secrets floors or +turn a failed run into approval. + +## Shadow-First Integration + +The root-only `jev` config is disabled by default. When enabled, ECR waits until +verification, suppression, requalification, citation handling, and feedback handling +have produced the final finding set. It then sends one request per finding, capped by +`maxFindings`, with two Choice questions: + +- whether the supplied patch supports, contradicts, or is insufficient to judge the + exact finding; +- the appropriate `suggestion`, `warning`, or `critical` severity if the finding is + real. + +The request state contains the finding's category, severity, file, line, title, +rationale, and evidence plus only that file's unified patch. `maxPatchChars` bounds +the patch; when possible, truncation centers on the cited evidence. ECR sends no PR +title/body, unrelated file, surrounding repository tree, credential, agent transcript, +or author reply. Four requests run concurrently. `maxFindings`, the SDK timeout, and +one retry bound time and provider load. + +The API origin and dated model are explicit in code/config. Ambient +`TYPESAFE_BASE_URL` and `TYPESAFE_DEFAULT_MODEL` cannot redirect the integration or +silently move its calibration target. The SDK logger is off because debug bodies would +contain source. `TYPESAFE_API_KEY` is a dedicated workflow secret and is also in +`FORBIDDEN_TOKEN_ENVS`, so an `auth.tokenEnv` cannot forward it to a model provider. + +The observation stores no additional source text. `.runs/reviews.jsonl` receives the +finding fingerprint, actual model, selected labels, full probability distributions, +token totals, duration, and coverage counts. Provider errors are counted without +persisting their text because an error may echo request details. A missing key produces +one explicit availability error. None of these values enters `CoordinatorOutput`, the +reporter, the decision calculation, or a later review prompt. + +The integration is fail-open by design because it is telemetry: construction, +credential, request, timeout, and response failures leave the review byte-for-byte +unchanged. This differs from review coverage. A failed Jev observation is not a missing +review pass and must not add an `incomplete` review note. + +## Calibration and Promotion Gates + +“Supported” is not ground truth, and agreement with ECR can mean both systems made the +same mistake. Before Jev gains any decision effect, evaluation must join shadow records +with human outcomes such as accepted fixes, explicit dismissals, and independently +adjudicated sampled findings. Measure by category and severity, not only in aggregate: + +- recall on confirmed critical, secrets, and security findings; +- precision and false-negative rate for `contradicted` and `insufficient` labels; +- calibration curves for each probability, pinned model version, and question text; +- latency, failures, retries, input tokens, and cost per reviewed PR; +- drift after any Jev model, SDK, rubric, state shape, or truncation change. + +Promotion proceeds in separate changes: + +1. **Shadow finding evaluator:** current phase; log only. +2. **Visible advisory:** optionally show a clearly labeled second opinion without + changing decisions, after measured calibration. +3. **Cost routing:** let Jev skip only low-risk expensive work when a held-out benchmark + proves the required recall; always-run security and deterministic checks remain. +4. **Finding suppression:** last and highest bar. It requires human-labeled data, + category-specific thresholds, an audit trail, a kill switch, and hard bypasses for + critical/secrets/security findings. No current evidence meets this bar. + +Moving aliases are unsuitable for a calibrated gate. A promotion must pin a dated +model and treat every version or question change as a new classifier requiring fresh +validation. Confidence alone is never a threshold justification. + +## Rejected Alternatives + +**Replace the verifier.** Jev cannot inspect the repository with tools or explain the +source-level contradiction. The current verifier can re-read source and fails open; +replacing it would reduce evidence and debuggability. + +**Pre-filter reviewer findings before coordination.** This creates an early silent +false-negative path exactly where ECR requires coverage honesty. Shadow evaluation of +final findings is measurable and reversible. + +**Send the entire diff or repository context.** More context raises disclosure, cost, +and context-rot risk without improving the exact local judgment. One bounded file +patch is the smallest useful state. + +**Use Jev for prompt-injection detection.** An attacker controls the state being +classified. Only deterministic isolation and trust boundaries can protect credentials +and execution. diff --git a/package.json b/package.json index f3f727f..c47af6f 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "1.30.0", "@opencode-ai/sdk": "1.18.4", + "@typesafe-ai/sdk": "0.6.0", "cheerio": "1.1.2", "minisearch": "7.2.0", "opencode-ai": "1.18.4", diff --git a/src/__tests__/jev.test.ts b/src/__tests__/jev.test.ts new file mode 100644 index 0000000..a94201e --- /dev/null +++ b/src/__tests__/jev.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from "bun:test"; +import { TypeSafeClient } from "@typesafe-ai/sdk"; + +import { ReviewConfigSchema, ScopeReviewConfigSchema } from "../config/schema.js"; +import { FORBIDDEN_TOKEN_ENVS } from "../core/auth.js"; +import { buildJevState, observeFindingsWithJev, withoutJevCredential } from "../core/jev.js"; +import type { Finding } from "../core/schema.js"; + +const finding: Finding = { + severity: "warning", + category: "correctness", + file: "src/example.ts", + line: 2, + title: "The fallback returns the wrong value", + rationale: "The changed branch returns false when the caller requires true.", + evidence: "return false", +}; + +const config = { + enabled: true, + model: "jev-1.13.0", + maxFindings: 20, + timeoutMs: 10_000, + maxPatchChars: 30_000, +}; + +describe("Jev config", () => { + test("is disabled and pinned by default", () => { + const parsed = ReviewConfigSchema.parse({}); + expect(parsed.jev).toEqual({ ...config, enabled: false }); + }); + + test("is root-only", () => { + expect(ScopeReviewConfigSchema.safeParse({ jev: { enabled: true } }).success).toBe(false); + }); + + test("cannot be repurposed as a reviewer provider credential", () => { + expect(FORBIDDEN_TOKEN_ENVS.has("TYPESAFE_API_KEY")).toBe(true); + }); +}); + +describe("observeFindingsWithJev", () => { + test("does nothing when disabled", async () => { + expect( + await observeFindingsWithJev({ + config: { ...config, enabled: false }, + findings: [finding], + files: [], + env: {}, + }), + ).toBeUndefined(); + }); + + test("records a missing key without throwing", async () => { + const observation = await observeFindingsWithJev({ + config, + findings: [finding], + files: [], + env: {}, + }); + expect(observation?.error).toBe("TYPESAFE_API_KEY is not set"); + expect(observation?.findings).toEqual([]); + }); + + test("records typed probabilities without source text", async () => { + const requests: unknown[] = []; + const client = new TypeSafeClient({ + apiKey: "test-key", + defaultModel: config.model, + retry: { maxRetries: 0 }, + fetch: async (_input, init) => { + requests.push(JSON.parse(String(init?.body))); + return new Response( + JSON.stringify({ + model: "jev-1.13.0", + answers: { + support: { + type: "choice", + choice: "supported", + confidence: 0.91, + probabilities: { supported: 0.91, insufficient: 0.07, contradicted: 0.02 }, + }, + severity: { + type: "choice", + choice: "warning", + confidence: 0.8, + probabilities: { suggestion: 0.1, warning: 0.8, critical: 0.1 }, + }, + }, + usage: { input_tokens: 123, output_tokens: 0 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }, + }); + + const observation = await observeFindingsWithJev({ + config, + findings: [finding], + files: [{ path: finding.file, patch: "@@ -1 +1 @@\n-return true\n+return false" }], + client, + env: {}, + }); + + expect(requests).toHaveLength(1); + expect(observation?.evaluated).toBe(1); + expect(observation?.inputTokens).toBe(123); + expect(observation?.findings[0]?.support).toBe("supported"); + expect(observation?.findings[0]?.severity).toBe("warning"); + expect(JSON.stringify(observation)).not.toContain("return false"); + }); + + test("caps findings and fails open on provider errors", async () => { + const client = new TypeSafeClient({ + apiKey: "test-key", + retry: { maxRetries: 0 }, + fetch: async () => { + throw new Error("request contained sensitive source"); + }, + }); + const observation = await observeFindingsWithJev({ + config: { ...config, maxFindings: 1 }, + findings: [finding, { ...finding, title: "Another finding" }], + files: [], + client, + env: {}, + }); + + expect(observation?.failed).toBe(1); + expect(observation?.skipped).toBe(1); + expect(observation?.error).toBeUndefined(); + expect(JSON.stringify(observation)).not.toContain("sensitive source"); + }); +}); + +test("buildJevState bounds large patches around the cited evidence", () => { + const state = buildJevState( + finding, + `before-${"x".repeat(2_000)}return false${"y".repeat(2_000)}`, + 1_000, + ); + expect(state.changedCodePatch.length).toBeLessThanOrEqual(1_000); + expect(state.changedCodePatch).toContain("return false"); + expect(state.changedCodePatch).toContain("patch omitted by ecr"); +}); + +test("withoutJevCredential restores the key after hiding it from a child spawn", async () => { + const env = { TYPESAFE_API_KEY: "secret", KEEP: "visible" }; + const seen = await withoutJevCredential(async () => ({ ...env }), env); + expect(seen).toEqual({ KEEP: "visible" }); + expect(env.TYPESAFE_API_KEY).toBe("secret"); +}); diff --git a/src/config/load.ts b/src/config/load.ts index 0942864..f705a21 100644 --- a/src/config/load.ts +++ b/src/config/load.ts @@ -71,6 +71,15 @@ const RESEARCH_CONFIG_DEFAULTS: LoadedConfig["research"] = { timeoutMs: 30_000, }; +/** Jev defaults for a scope load (where `jev` is schema-rejected). */ +const JEV_CONFIG_DEFAULTS: LoadedConfig["jev"] = { + enabled: false, + model: "jev-1.13.0", + maxFindings: 20, + timeoutMs: 10_000, + maxPatchChars: 30_000, +}; + /** Default OpenCode tool toggles for a reviewer: read the repo, never mutate it. */ const DEFAULT_AGENT_TOOLS = toolMap(["read", "grep", "glob", "list"]); @@ -112,7 +121,7 @@ export interface LoadConfigOptions { /** Parsed config with the centrally-locked keys optional (scope configs omit them). */ type ParsedConfig = Omit< RawReviewConfig, - "auth" | "breakGlass" | "commentTag" | "stack" | "feedback" | "research" | "inline" + "auth" | "breakGlass" | "commentTag" | "stack" | "feedback" | "research" | "jev" | "inline" > & { auth?: RawReviewConfig["auth"]; breakGlass?: RawReviewConfig["breakGlass"]; @@ -120,6 +129,7 @@ type ParsedConfig = Omit< stack?: RawReviewConfig["stack"]; feedback?: RawReviewConfig["feedback"]; research?: RawReviewConfig["research"]; + jev?: RawReviewConfig["jev"]; inline?: RawReviewConfig["inline"]; }; @@ -240,6 +250,7 @@ async function loadConfigDir( // Root-only: scope schemas reject research configuration, so an untrusted // subtree cannot select the index or alter the network-facing runtime. research: parsed.research ?? RESEARCH_CONFIG_DEFAULTS, + jev: parsed.jev ?? JEV_CONFIG_DEFAULTS, // parsed.breakGlass/auth are always present for the root schema (defaults) and // absent for the scope schema; loadScopeConfig overrides both afterwards. breakGlassMarker: parsed.breakGlass?.marker ?? "/skip-review", @@ -427,6 +438,7 @@ export async function loadScopeConfig( feedback: rootConfig.feedback, inline: rootConfig.inline, research: rootConfig.research, + jev: rootConfig.jev, scopeName: scope.name, }; } diff --git a/src/config/schema.ts b/src/config/schema.ts index 367b923..543bf40 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -87,6 +87,25 @@ export const ReviewConfigSchema = z.object({ resultsPerQuery: 2, timeoutMs: 30_000, }), + // Optional Jev second opinion. This is deliberately observation-only: its + // answers are written to the run log and never alter findings or decisions. + // ROOT-ONLY because it sends bounded source context to a separate provider. + // @ref LLP 0014#shadow-first-integration [implements] — collect calibration data before granting Jev decision authority + jev: z + .object({ + enabled: z.boolean().default(false), + model: z.string().min(1).default("jev-1.13.0"), + maxFindings: z.number().int().min(1).max(100).default(20), + timeoutMs: z.number().int().min(1000).max(60_000).default(10_000), + maxPatchChars: z.number().int().min(1000).max(30_000).default(30_000), + }) + .default({ + enabled: false, + model: "jev-1.13.0", + maxFindings: 20, + timeoutMs: 10_000, + maxPatchChars: 30_000, + }), breakGlass: z .object({ marker: z.string().default("/skip-review") }) .default({ marker: "/skip-review" }), @@ -334,7 +353,7 @@ export type RoutingDefaults = RoutingManifest["defaults"]; * Scope config = root config MINUS the centrally locked keys. Allowlist of * scope-overridable keys (Turborepo-style, graft 6): model, policy, chunk, * noise (+ the prompt files living beside it: shared.md, coordinator.md, - * agents/). NEVER auth, breakGlass, or research — declaring one fails parsing at the + * agents/). NEVER auth, breakGlass, research, or jev — declaring one fails parsing at the * Zod level so IDE/doctor catch it before CI. commentTag is also locked: a * scope's comment marker is always DERIVED (`:`; the default * scope keeps the root tag) so `ecr ci`'s post/clear/reconcile paths and a @@ -349,6 +368,7 @@ export const ScopeReviewConfigSchema = ReviewConfigSchema.omit({ stack: true, feedback: true, research: true, + jev: true, inline: true, }).extend({ auth: z @@ -379,6 +399,12 @@ export const ScopeReviewConfigSchema = ReviewConfigSchema.omit({ "research is locked to the root config because it starts a trusted host process; remove it from this scope config", }) .optional(), + jev: z + .never({ + error: + "jev is locked to the root config because it sends bounded source context to an external provider; remove it from this scope config", + }) + .optional(), inline: z .never({ error: @@ -462,6 +488,14 @@ export interface LoadedConfig { resultsPerQuery: number; timeoutMs: number; }; + /** Root-only, observation-only Jev evaluation settings. */ + jev: { + enabled: boolean; + model: string; + maxFindings: number; + timeoutMs: number; + maxPatchChars: number; + }; breakGlassMarker: string; commentTag: string; /** Every configured provider credential (one entry for the legacy single shape). */ diff --git a/src/core/auth.ts b/src/core/auth.ts index b067fc9..860872c 100644 --- a/src/core/auth.ts +++ b/src/core/auth.ts @@ -50,6 +50,7 @@ export const FORBIDDEN_TOKEN_ENVS = new Set([ "GITHUB_TOKEN", "GH_TOKEN", "BRAVE_SEARCH_API_KEY", + "TYPESAFE_API_KEY", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_ID_TOKEN_REQUEST_TOKEN", "AWS_ACCESS_KEY_ID", diff --git a/src/core/jev.ts b/src/core/jev.ts new file mode 100644 index 0000000..a807a7f --- /dev/null +++ b/src/core/jev.ts @@ -0,0 +1,257 @@ +// @ref LLP 0014#shadow-first-integration [implements] — Jev records a bounded second opinion without entering the decision path +import { TypeSafeClient, choice } from "@typesafe-ai/sdk"; +import { z } from "zod"; + +import type { LoadedConfig } from "../config/schema.js"; +import type { DiffEntry, Finding } from "./schema.js"; +import { fingerprintFinding } from "./schema.js"; + +const TYPESAFE_API_URL = "https://api.typesafe.ai"; +const CONCURRENCY = 4; +const MAX_FILE_CHARS = 1_000; +const MAX_TITLE_CHARS = 1_000; +const MAX_RATIONALE_CHARS = 6_000; +const MAX_EVIDENCE_CHARS = 6_000; + +const SUPPORT_CRITERIA = { + supported: "The supplied patch contains concrete evidence for the reported problem.", + insufficient: "The supplied patch does not contain enough evidence to decide.", + contradicted: "The supplied patch shows that the reported problem is not present.", +} as const; + +const SEVERITY_CRITERIA = { + suggestion: "A non-blocking improvement with no demonstrated shipped failure.", + warning: "A real defect with a plausible user, reliability, or maintainability impact.", + critical: "A severe security, secret-exposure, data-loss, or broadly breaking defect.", +} as const; + +const probability = z.number().min(0).max(1); +const JevResponseSchema = z.object({ + model: z.string().min(1), + answers: z.object({ + support: z.object({ + type: z.literal("choice"), + choice: z.enum(["supported", "insufficient", "contradicted"]), + confidence: probability, + probabilities: z.object({ + supported: probability, + insufficient: probability, + contradicted: probability, + }), + }), + severity: z.object({ + type: z.literal("choice"), + choice: z.enum(["suggestion", "warning", "critical"]), + confidence: probability, + probabilities: z.object({ + suggestion: probability, + warning: probability, + critical: probability, + }), + }), + }), + usage: z.object({ + input_tokens: z.number().int().nonnegative(), + output_tokens: z.number().int().nonnegative(), + }), +}); + +export interface JevFindingObservation { + fingerprint: string; + model: string; + support: keyof typeof SUPPORT_CRITERIA; + supportConfidence: number; + supportProbabilities: Record; + severity: keyof typeof SEVERITY_CRITERIA; + severityConfidence: number; + severityProbabilities: Record; +} + +export interface JevObservation { + configuredModel: string; + durationMs: number; + evaluated: number; + skipped: number; + failed: number; + inputTokens: number; + outputTokens: number; + findings: JevFindingObservation[]; + error?: string; +} + +export interface ObserveWithJevOptions { + config: LoadedConfig["jev"]; + findings: readonly Finding[]; + files: readonly DiffEntry[]; + env?: NodeJS.ProcessEnv; + /** Explicit key captured before reviewer engines start; never forwarded to them. */ + apiKey?: string; + client?: TypeSafeClient; +} + +/** Run a reviewer-engine spawn without exposing the unrelated Jev credential. */ +export async function withoutJevCredential( + run: () => Promise, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const apiKey = env.TYPESAFE_API_KEY; + try { + delete env.TYPESAFE_API_KEY; + return await run(); + } finally { + if (apiKey === undefined) { + delete env.TYPESAFE_API_KEY; + } else { + env.TYPESAFE_API_KEY = apiKey; + } + } +} + +/** + * Ask Jev typed questions about final findings. The returned record is telemetry + * only: callers must never use it to mutate a finding or review decision. + */ +export async function observeFindingsWithJev({ + config, + findings, + files, + env = process.env, + apiKey: explicitApiKey, + client, +}: ObserveWithJevOptions): Promise { + if (!config.enabled) return undefined; + + const started = Date.now(); + const apiKey = (explicitApiKey ?? env.TYPESAFE_API_KEY)?.trim(); + if (!client && !apiKey) { + return emptyObservation(config, started, "TYPESAFE_API_KEY is not set"); + } + + let resolvedClient: TypeSafeClient; + try { + resolvedClient = + client ?? + new TypeSafeClient({ + apiKey, + baseURL: TYPESAFE_API_URL, + defaultModel: config.model, + timeout: config.timeoutMs, + retry: { maxRetries: 1 }, + logLevel: "off", + }); + } catch { + return emptyObservation(config, started, "Jev client could not be initialized"); + } + + const selected = findings.slice(0, config.maxFindings); + const patches = new Map(files.map((file) => [file.path, file.patch])); + const results: JevFindingObservation[] = []; + let failed = 0; + let inputTokens = 0; + let outputTokens = 0; + let next = 0; + + const worker = async (): Promise => { + while (next < selected.length) { + const finding = selected[next++]; + if (!finding) continue; + try { + const response = JevResponseSchema.parse( + await resolvedClient.systemOne({ + model: config.model, + state: buildJevState(finding, patches.get(finding.file) ?? "", config.maxPatchChars), + questions: { + support: choice( + "Does the supplied changed-code patch support this exact code-review finding? Judge only the supplied evidence; do not assume missing repository context.", + SUPPORT_CRITERIA, + ), + severity: choice( + "If the finding is real, what is its appropriate review severity?", + SEVERITY_CRITERIA, + ), + }, + }), + ); + inputTokens += response.usage.input_tokens; + outputTokens += response.usage.output_tokens; + results.push({ + fingerprint: fingerprintFinding(finding), + model: response.model, + support: response.answers.support.choice, + supportConfidence: response.answers.support.confidence, + supportProbabilities: response.answers.support.probabilities, + severity: response.answers.severity.choice, + severityConfidence: response.answers.severity.confidence, + severityProbabilities: response.answers.severity.probabilities, + }); + } catch { + // Observation is fail-open and provider errors may contain request details. + failed += 1; + } + } + }; + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, selected.length) }, () => worker())); + + return { + configuredModel: config.model, + durationMs: Date.now() - started, + evaluated: results.length, + skipped: Math.max(0, findings.length - selected.length), + failed, + inputTokens, + outputTokens, + findings: results.sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)), + }; +} + +/** Build the only state sent to TypeSafe: finding metadata plus its bounded file patch. */ +export function buildJevState(finding: Finding, patch: string, maxPatchChars: number) { + return { + finding: { + severity: finding.severity, + category: finding.category, + file: boundText(finding.file, MAX_FILE_CHARS), + line: finding.line ?? null, + title: boundText(finding.title, MAX_TITLE_CHARS), + rationale: boundText(finding.rationale, MAX_RATIONALE_CHARS), + evidence: finding.evidence ? boundText(finding.evidence, MAX_EVIDENCE_CHARS) : null, + }, + changedCodePatch: boundPatch(patch, finding.evidence, maxPatchChars), + }; +} + +function boundText(value: string, maxChars: number): string { + return value.length <= maxChars ? value : `${value.slice(0, maxChars - 1)}…`; +} + +function boundPatch(patch: string, evidence: string | undefined, maxChars: number): string { + if (patch.length <= maxChars) return patch; + const omitted = "\n... patch omitted by ecr ...\n"; + const budget = Math.max(1, maxChars - omitted.length); + const evidenceAt = evidence ? patch.indexOf(evidence) : -1; + if (evidenceAt >= 0) { + const start = Math.max(0, evidenceAt - Math.floor(budget / 2)); + return patch.slice(start, start + budget) + omitted; + } + const half = Math.floor(budget / 2); + return patch.slice(0, half) + omitted + patch.slice(-half); +} + +function emptyObservation( + config: LoadedConfig["jev"], + started: number, + error: string, +): JevObservation { + return { + configuredModel: config.model, + durationMs: Date.now() - started, + evaluated: 0, + skipped: 0, + failed: 0, + inputTokens: 0, + outputTokens: 0, + findings: [], + error, + }; +} diff --git a/src/core/log.ts b/src/core/log.ts index 4869d8b..d8f5b6a 100644 --- a/src/core/log.ts +++ b/src/core/log.ts @@ -6,6 +6,7 @@ import type { CoordinatorOutput, Finding, ReviewMetadata, ReviewTrace } from "./ import type { FilteredFile } from "./noise.js"; import type { TokenUsage } from "./opencode.js"; import type { ResearchProvenance } from "./research.js"; +import type { JevObservation } from "./jev.js"; export interface RunLogRecord { timestamp: string; @@ -21,6 +22,8 @@ export interface RunLogRecord { filteredFiles: FilteredFile[]; /** Bounded queries/results plus grounded research decisions and final usefulness metrics. */ research?: ResearchProvenance; + /** Observation-only Jev results; these never influence findings or decisions. */ + jev?: JevObservation; agentCosts: Record; totalCost: number; // Aggregate token usage across all agent + coordinator requests, for cache diff --git a/src/core/review.ts b/src/core/review.ts index 9ccf96d..27ad644 100644 --- a/src/core/review.ts +++ b/src/core/review.ts @@ -84,6 +84,8 @@ import { summarizeResearchUsefulness, } from "./research.js"; import type { ResearchEvidence, ResearchMcpRuntime, ResearchProvenance } from "./research.js"; +import { observeFindingsWithJev, withoutJevCredential } from "./jev.js"; +import type { JevObservation } from "./jev.js"; export interface ReviewRunOptions { config: LoadedConfig; @@ -297,6 +299,7 @@ export async function runReview( let researchEvidence: ResearchEvidence[] = []; let researchRecord: ResearchProvenance | undefined; + let jevRecord: JevObservation | undefined; // Materialize the PR-head tree (not the current checkout) when the source can, so // the agents' surrounding-source reads and the verifier's re-reads see the versions @@ -387,9 +390,16 @@ export async function runReview( // close it. Two separate try blocks keep the precise per-engine error messages. let opencodeHandle: OpencodeHandle | null = null; let claudeHandle: ClaudeCodeHandle | null = null; + // OpenCode's SDK copies process.env into its server. Withhold the unrelated Jev + // credential at spawn time, then restore the parent immediately; Claude uses an + // allowlisted child env already. The captured value is passed directly to Jev. + // @ref LLP 0014#shadow-first-integration [constrained-by] — the reviewer engines never receive the Jev credential + const jevApiKey = process.env.TYPESAFE_API_KEY; try { if (usesOpencode) { - opencodeHandle = await startOpencode(buildOpencodeConfig(config, researchRuntime)); + opencodeHandle = await withoutJevCredential(() => + startOpencode(buildOpencodeConfig(config, researchRuntime)), + ); } } catch (error) { await auth.cleanup(); @@ -1328,6 +1338,24 @@ export async function runReview( await appendStepSummary(renderResearchUsefulnessMarkdown(researchRecord)); } + // @ref LLP 0014#shadow-first-integration [implements] — run after the final + // finding set exists and never feed the answers back into `output`. + jevRecord = await observeFindingsWithJev({ + config: config.jev, + findings: output.findings, + files: kept, + apiKey: jevApiKey, + }); + if (jevRecord) { + if (jevRecord.error) { + progress(`Jev shadow evaluation unavailable (${jevRecord.error}); review unchanged.`); + } else { + progress( + `Jev shadow evaluation: ${jevRecord.evaluated} observed, ${jevRecord.failed} failed, ${jevRecord.skipped} over cap; review unchanged.`, + ); + } + } + // Surface provider throttling as a fact about the run: passes already waited or // backed off, but the operator should still SEE that it happened (a run that // was rate-limited is slower and may carry partial passes — that's the cause). @@ -1370,6 +1398,7 @@ export async function runReview( await safeLog(logPath, { ...baseRecord, ...(researchRecord ? { research: researchRecord } : {}), + ...(jevRecord ? { jev: jevRecord } : {}), agentCosts, totalCost: sum(agentCosts), tokens: tokenTotals, @@ -1408,6 +1437,7 @@ export async function runReview( await safeLog(logPath, { ...baseRecord, ...(researchRecord ? { research: researchRecord } : {}), + ...(jevRecord ? { jev: jevRecord } : {}), agentCosts, totalCost: sum(agentCosts), tokens: tokenTotals, diff --git a/templates/atlantis.yml b/templates/atlantis.yml index 2202aae..9c239f8 100644 --- a/templates/atlantis.yml +++ b/templates/atlantis.yml @@ -114,6 +114,8 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} + # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} run: | diff --git a/templates/command.yml b/templates/command.yml index e3a58e4..f6f7ddd 100644 --- a/templates/command.yml +++ b/templates/command.yml @@ -152,6 +152,8 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} + # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} AGENTS: ${{ steps.cmd.outputs.agents }} diff --git a/templates/config.jsonc b/templates/config.jsonc index 364ec57..aff51a8 100644 --- a/templates/config.jsonc +++ b/templates/config.jsonc @@ -48,6 +48,21 @@ // "timeoutMs": 30000 // }, + // Optional Jev shadow evaluation (ROOT-ONLY; off by default). Jev receives each + // final finding plus a bounded patch for that finding's file and returns typed + // support/severity probabilities. ECR writes those observations to reviews.jsonl + // but never changes, hides, or re-severities a finding from them. Pin the model + // while calibrating: aliases such as jev-latest can move underneath thresholds. + // Set TYPESAFE_API_KEY in the environment (the workflow templates show the + // optional secret mapping) before enabling this. + // "jev": { + // "enabled": true, + // "model": "jev-1.13.0", + // "maxFindings": 20, + // "timeoutMs": 10000, + // "maxPatchChars": 30000 + // }, + // Large diffs are split into focused chunks by changed-line count, plus a // cross-cutting pass for multi-file issues. Diffs under maxChangedLines are one // full-context pass. Defaults shown; raise/lower per your model + PR sizes. diff --git a/templates/workflow.yml b/templates/workflow.yml index f49ccc4..fdbabab 100644 --- a/templates/workflow.yml +++ b/templates/workflow.yml @@ -118,6 +118,8 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} + # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} From da80f6d3ca9a9fea74f3b353662d40a214b9ba67 Mon Sep 17 00:00:00 2001 From: Philippe Loulidi <78689302+LipLoody@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:22:48 -0600 Subject: [PATCH 2/2] [ecr] make Jev an active verification cascade --- .expo-agents/code-review/config.jsonc | 9 +- README.md | 35 ++- llp/0000-expo-code-review-cli.explainer.md | 6 +- llp/0002-review-engine-pipeline.explainer.md | 4 +- ...config-schema-loading-routing.explainer.md | 10 +- ...option-templates-and-ci-workflows.guide.md | 14 +- ...14-jev-selective-verification.explainer.md | 156 ++++++++++++ llp/0014-jev-shadow-evaluation.explainer.md | 159 ------------ src/__tests__/jev.test.ts | 128 +++++----- src/__tests__/verify.test.ts | 63 +++++ src/config/load.ts | 11 +- src/config/schema.ts | 31 +-- src/core/jev.ts | 226 +++++++++--------- src/core/log.ts | 6 +- src/core/review.ts | 42 ++-- src/core/verify.ts | 85 ++++++- templates/atlantis.yml | 2 +- templates/command.yml | 2 +- templates/config.jsonc | 17 +- templates/workflow.yml | 2 +- 20 files changed, 565 insertions(+), 443 deletions(-) create mode 100644 llp/0014-jev-selective-verification.explainer.md delete mode 100644 llp/0014-jev-shadow-evaluation.explainer.md diff --git a/.expo-agents/code-review/config.jsonc b/.expo-agents/code-review/config.jsonc index 7c4befb..96d9784 100644 --- a/.expo-agents/code-review/config.jsonc +++ b/.expo-agents/code-review/config.jsonc @@ -31,14 +31,13 @@ "timeoutMs": 30000 }, - // Shadow-only Jev second opinion. Keep disabled until TYPESAFE_API_KEY is added - // as a repository secret; enabling it changes only reviews.jsonl telemetry. + // Active Jev verification cascade. Confident local judgments act immediately; + // ambiguous and protected findings defer to the reasoning verifier. "jev": { - "enabled": false, "model": "jev-1.13.0", - "maxFindings": 20, + "minConfidence": 0.9, "timeoutMs": 10000, - "maxPatchChars": 30000 + "maxContextChars": 30000 }, // Large diffs are split into focused chunks by changed-line count, plus a diff --git a/README.md b/README.md index 7f70703..97ebf47 100644 --- a/README.md +++ b/README.md @@ -283,37 +283,34 @@ The boundary, in brief: Full detail — providers, query grammar, `fetch_platform_doc` modes, provenance and citation grounding: [LLP 0013](./llp/0013-platform-research.explainer.md). -## Jev shadow evaluation (experimental) +## Jev selective verification -ECR can ask [TypeSafe AI's Jev](https://docs.typesafe.ai/) for a typed second opinion -on each final finding. This integration is deliberately observation-only: Jev's -support and severity probabilities go to `.runs/reviews.jsonl`, but they never alter -the posted findings or decision. That makes real review traffic the calibration set -before any future filtering or routing policy is considered. +ECR can use [TypeSafe AI's Jev](https://docs.typesafe.ai/) as the fast first stage of +finding verification. Generative agents still discover and explain problems. Jev +answers one narrow typed question against bounded local source, then either keeps a +clearly supported finding, drops a clearly contradicted ordinary finding, or defers +to the existing reasoning verifier. -Enable it in the root config and provide `TYPESAFE_API_KEY` (the scaffolded workflows -already map the optional repository secret): +There is no shadow mode or separate `enabled` flag. Add the root-only block and provide +`TYPESAFE_API_KEY`; removing the block restores the original verification path: ```jsonc { "jev": { - "enabled": true, "model": "jev-1.13.0", - "maxFindings": 20, + "minConfidence": 0.9, "timeoutMs": 10000, - "maxPatchChars": 30000 + "maxContextChars": 30000 } } ``` -Each request contains one finding and only the bounded patch for its file. The model -is pinned because probability thresholds must be calibrated against a stable version. -Jev is an evaluator, not a generator: it cannot explain a defect or propose a fix, -and its confidence is distribution concentration rather than proof that an answer is -correct. It is also not a prompt-injection or security boundary. Missing credentials, -timeouts, and provider errors leave the review unchanged and are recorded as reduced -shadow coverage. See [LLP 0014](./llp/0014-jev-shadow-evaluation.explainer.md) for the -research, threat model, and promotion criteria. +Each request contains one finding and bounded source from its file. Critical, security, +secrets, and cited findings are never dropped by Jev alone. Low confidence, missing +context, credentials, timeouts, and provider errors fall back to the reasoning or +original verification path. Confidence is distribution concentration, not proof, so +the model and threshold are explicit. See [LLP 0014](./llp/0014-jev-selective-verification.explainer.md) +for the research, decision table, and threat model. ## Monorepos (routing manifest) diff --git a/llp/0000-expo-code-review-cli.explainer.md b/llp/0000-expo-code-review-cli.explainer.md index d23728f..2e17faf 100644 --- a/llp/0000-expo-code-review-cli.explainer.md +++ b/llp/0000-expo-code-review-cli.explainer.md @@ -6,7 +6,7 @@ **Author:** Philippe Loulidi / Claude **Date:** 2026-07-30 **Role:** Root -**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0003](0003-model-runtimes-and-credentials.explainer.md), [LLP 0004](0004-diff-noise-and-prompts.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0007](0007-cli-commands-and-ci.explainer.md), [LLP 0008](0008-sources-and-reporters.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md), [LLP 0013](0013-platform-research.explainer.md), [LLP 0014](0014-jev-shadow-evaluation.explainer.md) +**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0003](0003-model-runtimes-and-credentials.explainer.md), [LLP 0004](0004-diff-noise-and-prompts.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0007](0007-cli-commands-and-ci.explainer.md), [LLP 0008](0008-sources-and-reporters.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md), [LLP 0013](0013-platform-research.explainer.md), [LLP 0014](0014-jev-selective-verification.explainer.md) This is the root map for `@expo/code-review-cli` (`ecr`). It records what the system does, how the subsystems divide, the invariants that hold across all of them, and where each concern's rationale lives. It is intentionally thin: every subsystem's detail belongs in the doc that owns it. Read this first, then follow the link for the part you care about. @@ -29,7 +29,7 @@ The code divides by concern. Each directory maps to the doc that owns its ration - `src/sources/` (`local-git`, `github-pr`) + `src/reporters/` (`terminal`, `github`) — where the diff comes from and where findings go [observed: AGENTS.md:14-15; src/sources/, src/reporters/]. Owned by [LLP 0008](0008-sources-and-reporters.explainer.md). - `templates/` + the scaffolded CI workflow — the files `ecr init` writes into an adopting repo; they define adopter-facing behavior and must stay in sync with the code [observed: AGENTS.md:19-20]. Owned by [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md). - `src/research-mcp/` + `research/` — the bundled read-only documentation MCP, scoped remote discovery and allowlisted fetch adapters, optional index builder, source catalog, and host-side evidence prepass. Owned by [LLP 0013](0013-platform-research.explainer.md). -- `src/core/jev.ts` — optional, observation-only typed evaluation of final findings against bounded file patches. Owned by [LLP 0014](0014-jev-shadow-evaluation.explainer.md). +- `src/core/jev.ts` — optional typed first stage of finding verification against bounded local source. Owned by [LLP 0014](0014-jev-selective-verification.explainer.md). - The trust model spans all of the above and is stated as principles in [LLP 0001](0001-trust-model.principles.md). ## Cross-Cutting Invariants @@ -71,6 +71,6 @@ Start here, then follow the owning doc. | What do the CLI commands do, and how does CI orchestrate them? | [LLP 0007](0007-cli-commands-and-ci.explainer.md) | `src/cli.ts`, `src/commands/*` | | Where does the diff come from and where do findings go? | [LLP 0008](0008-sources-and-reporters.explainer.md) | `src/sources/*`, `src/reporters/*` | | What does `ecr init` scaffold into an adopting repo? | [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md) | `templates/`, scaffolded CI workflow | -| How does Jev evaluate findings without changing review decisions? | [LLP 0014](0014-jev-shadow-evaluation.explainer.md) | `src/core/jev.ts`, `src/core/log.ts` | +| How does Jev make narrow decisions and defer uncertain findings? | [LLP 0014](0014-jev-selective-verification.explainer.md) | `src/core/jev.ts`, `src/core/verify.ts` | Non-doc sources of truth: `AGENTS.md` holds the working conventions (import style, error handling, the `opencode-ai`/`@opencode-ai/sdk` pinning rule, security invariants); `ROADMAP.md` lists open and deferred items — incremental review, inline PR comments, result-level caching, and a Codex token rotator among them are still open and must not be assumed implemented [observed: ROADMAP.md]; and `templates/` is the adopter-facing contract, so treat it as behavior, not sample text. diff --git a/llp/0002-review-engine-pipeline.explainer.md b/llp/0002-review-engine-pipeline.explainer.md index fede0dc..dc66f57 100644 --- a/llp/0002-review-engine-pipeline.explainer.md +++ b/llp/0002-review-engine-pipeline.explainer.md @@ -11,7 +11,7 @@ ## Pipeline Stages -`runReview()` runs a fixed sequence: filter noise → resolve the read root (the trust boundary) → start the engines → preflight that every model id resolves → chunk the diff → run agent×chunk passes plus one cross-cutting pass through a growable bounded queue → coordinate → verify → suppress → reconcile the summary → optional Jev shadow observation → log. Jev runs only after the final finding set exists and cannot feed values back into it; [LLP 0014](0014-jev-shadow-evaluation.explainer.md) owns that boundary. The whole run's resource lifecycle — engine handles, auth material, cwd, and the read-root worktree — is released in a single `finally` (`review.ts:947-951`), so no partial-startup path leaks a server or a worktree. +`runReview()` runs a fixed sequence: filter noise → resolve the read root (the trust boundary) → start the engines → preflight that every model id resolves → chunk the diff → run agent×chunk passes plus one cross-cutting pass through a growable bounded queue → coordinate → verify → suppress → reconcile the summary → log. Optional Jev classification is the first stage inside verification: clear local judgments act and uncertainty defers to the reasoning verifier. [LLP 0014](0014-jev-selective-verification.explainer.md) owns that boundary. The whole run's resource lifecycle — engine handles, auth material, cwd, and the read-root worktree — is released in a single `finally` (`review.ts:947-951`), so no partial-startup path leaks a server or a worktree. Two ordering decisions inside that lifecycle are load-bearing and easy to break by "cleaning up": @@ -77,7 +77,7 @@ Rate-limit evidence is tracked per engine separately and surfaced with the cause `writeRunLog()` (`src/core/log.ts`) appends one JSON line per run — cost, tokens (run-wide and per bucket), findings, coverage notes, decision — so runs are auditable and cost/latency measurable later. The record deliberately excludes PR title and body text, keeping only `baseRef`/`headRef`, to avoid persisting secrets that might appear in author-controlled text [observed] (`log.ts:13-18`). Findings may quote changed source lines, but only content the review already publishes verbatim in the PR comment. -Each successful reviewer pass may also return a bounded conclusion-only trace: up to three concrete checks and two unresolved questions, with each note capped at 240 characters. Trace parsing is deliberately fail-soft: malformed optional diagnostics are dropped while valid findings from the same response survive, so observability can never create a coverage gap. `runReview()` merges and de-duplicates valid notes per reviewer, sorts reviewer ids, caps both the stored roster and the complete decoded trace (6 KB), labels the result `unverified-model-diagnostics`, and attaches it to `CoordinatorOutput.reviewTrace` only after the decision pipeline finishes. `attachReviewTrace` strips any coordinator-authored value before attaching the engine-built trace; even a malformed injected coordinator value parses as absent rather than failing consolidation. The trace therefore never reaches coordination, verification, suppression, or decision logic. Because the GitHub comment already embeds the full coordinator output as durable state, downstream agents can recover the trace without adding visible review prose or storing raw transcripts and chain-of-thought. The same trace rides the run log for short-lived operational inspection [observed: `schema.ts` `ReviewerTraceNotesSchema`/`ReviewTraceSchema`/`ReviewerOutputSchema`; `review.ts` `mergeTraceNotes`/`buildReviewTrace`/`attachReviewTrace`; `render.ts` embeds the full review]. Optional Jev observations are a second log-only sink: they contain fingerprints and typed probabilities, never extra source text, and cannot alter the trace or result. +Each successful reviewer pass may also return a bounded conclusion-only trace: up to three concrete checks and two unresolved questions, with each note capped at 240 characters. Trace parsing is deliberately fail-soft: malformed optional diagnostics are dropped while valid findings from the same response survive, so observability can never create a coverage gap. `runReview()` merges and de-duplicates valid notes per reviewer, sorts reviewer ids, caps both the stored roster and the complete decoded trace (6 KB), labels the result `unverified-model-diagnostics`, and attaches it to `CoordinatorOutput.reviewTrace` only after the decision pipeline finishes. `attachReviewTrace` strips any coordinator-authored value before attaching the engine-built trace; even a malformed injected coordinator value parses as absent rather than failing consolidation. The trace therefore never reaches coordination, verification, suppression, or decision logic. Because the GitHub comment already embeds the full coordinator output as durable state, downstream agents can recover the trace without adding visible review prose or storing raw transcripts and chain-of-thought. The same trace rides the run log for short-lived operational inspection [observed: `schema.ts` `ReviewerTraceNotesSchema`/`ReviewTraceSchema`/`ReviewerOutputSchema`; `review.ts` `mergeTraceNotes`/`buildReviewTrace`/`attachReviewTrace`; `render.ts` embeds the full review]. Optional Jev aggregate usage is a second log sink; individual suppressions use the ordinary `verifierDropped` audit trail, and neither contains extra source text. Observability must never break a review, so both output sinks swallow their own errors: `safeLog` wraps `writeRunLog` and discards any write error [observed] (`review.ts:1269-1275`), and `appendStepSummary` is a no-op outside Actions and swallows append failures [observed] (`step-summary.ts:9-19`). A consequence worth stating: a missing or corrupt run-log entry is therefore **not** an error signal — the log's presence or absence proves nothing about whether the review succeeded. diff --git a/llp/0006-config-schema-loading-routing.explainer.md b/llp/0006-config-schema-loading-routing.explainer.md index d9b88f2..92bc501 100644 --- a/llp/0006-config-schema-loading-routing.explainer.md +++ b/llp/0006-config-schema-loading-routing.explainer.md @@ -85,12 +85,12 @@ commit `43b31a4` follow-up]. The actual secret values never enter this subsystem owns the wiring and the `FORBIDDEN_TOKEN_ENVS` list, and schema/load only ever touch the env-var NAME [observed: `AGENTS.md` security invariants; `src/core/auth.ts:28,45,206`]. -`jev` is root-only for the same trust reason as research: enabling it sends bounded -repository-derived text to another provider. A scope may neither enable it nor alter -its model or limits. Nested scopes inherit the root's fully resolved Jev settings, so +`jev` is root-only for the same trust reason as research: adding it sends bounded +repository-derived text to another provider. A scope may neither activate it nor alter +its model, threshold, or limits. Nested scopes inherit the root's resolved settings, so one PR has one centrally owned disclosure policy [observed: `schema.ts` -`ScopeReviewConfigSchema`; `load.ts` `loadScopeConfig`]. The data boundary and -shadow-only behavior live in [LLP 0014](0014-jev-shadow-evaluation.explainer.md). +`ScopeReviewConfigSchema`; `load.ts` `loadScopeConfig`]. The data boundary and active +selective cascade live in [LLP 0014](0014-jev-selective-verification.explainer.md). ## Model Resolution diff --git a/llp/0009-adoption-templates-and-ci-workflows.guide.md b/llp/0009-adoption-templates-and-ci-workflows.guide.md index b011fd8..76ab5c1 100644 --- a/llp/0009-adoption-templates-and-ci-workflows.guide.md +++ b/llp/0009-adoption-templates-and-ci-workflows.guide.md @@ -65,13 +65,13 @@ credential actually selected by trusted config. The default remains Anthropic an `templates/workflow.yml`, `command.yml`, and `atlantis.yml`]. **Jev is an independent optional secret.** Each review-running workflow maps -`TYPESAFE_API_KEY`, while the root config keeps `jev.enabled` false. GitHub supplies an -empty value when the repository secret is absent, so the default scaffold does not -gain a working external call. Once enabled from trusted root config, ECR sends bounded -finding/file-patch state directly to TypeSafe and writes only probabilities to the run -log; the key never enters `auth.tokenEnv` or the reviewer engines. `dismiss.yml` still -receives no model or Jev credential. [LLP 0014](0014-jev-shadow-evaluation.explainer.md) -owns the disclosure and promotion rules. +`TYPESAFE_API_KEY`; adding `jev` to trusted root config activates the selective cascade. +GitHub supplies an empty value when the repository secret is absent, so the original +verification path remains available. ECR sends bounded finding/file source directly +to TypeSafe and logs aggregate usage; the key never enters `auth.tokenEnv` or reviewer +engines. `dismiss.yml` still receives no model or Jev credential. +[LLP 0014](0014-jev-selective-verification.explainer.md) owns the disclosure and +fallback rules. **Keep the `**/*` catch-all first in `routing.jsonc`.** Scopes match last-match-wins, so a `**/*` catch-all must come first and more specific scopes come after to override it [observed: `templates/routing.jsonc:27-31`; `AGENTS.md:17-18`]. Nothing validates the order — reversing it silently changes which config applies to already-routed files [inferred: no code checks that the `**/*` catch-all comes first, so reversing it silently changes routing]. diff --git a/llp/0014-jev-selective-verification.explainer.md b/llp/0014-jev-selective-verification.explainer.md new file mode 100644 index 0000000..8fcb9d5 --- /dev/null +++ b/llp/0014-jev-selective-verification.explainer.md @@ -0,0 +1,156 @@ +# LLP 0014: Jev Selective Verification + +**Type:** Explainer +**Status:** Active +**Systems:** Engine, Config, Security, Observability, Templates +**Author:** Expo +**Date:** 2026-09-21 +**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md) + +Jev is a discriminative model from TypeSafe AI. It answers typed questions over +provided state and returns a probability distribution; it does not browse a repo, +discover open-ended defects, explain causality, or write remediation. ECR therefore +uses Jev inside verification, after generative reviewers have produced findings. + +## Decision + +ECR uses an active `generate → classify → defer` cascade. Jev is neither a shadow +observer nor another general reviewer agent: + +1. generative reviewers discover and explain candidate defects; +2. deterministic code confines paths and grades quoted evidence; +3. Jev makes one atomic local support judgment per finding; +4. high-confidence support keeps the finding, while high-confidence contradiction + drops only ordinary correctness/quality findings; +5. uncertainty and protected findings defer to the existing reasoning verifier. + +Adding the root `jev` config activates the cascade. There is no separate `enabled` +flag and no shadow phase. If the config, credential, or a usable response is absent, +the finding follows the pre-Jev verification rules. + +## Evidence Survey + +The primary implementation sources are TypeSafe's [Jev announcement](https://typesafe.ai/blog/introducing-system-one-models-and-jev), +[System One guide](https://docs.typesafe.ai/concepts/how-to-build-with-system-one), +[confidence guidance](https://docs.typesafe.ai/confidence), +[confidence-routing pattern](https://docs.typesafe.ai/patterns/confidence-routing), +[citation-check cookbook](https://docs.typesafe.ai/cookbooks/citation_check), +[software-development cascade](https://docs.typesafe.ai/cookbooks/sde_cascade), +[Jev 1.13 model notes](https://docs.typesafe.ai/model-jaggedness/jev-1.13), and the +open-source [JavaScript SDK](https://github.com/typesafe-ai/typesafe-sdk-js). + +The consistent vendor guidance is to keep deterministic control flow in code, ask +narrow independent questions over relevant state, act only above a risk-appropriate +confidence threshold, and route uncertain results to a stronger reasoner. Confidence +measures concentration of the answer distribution, not factual correctness. Jev is +also documented as weak on arithmetic, counting, dates, complex indirection, and +irrelevant context. These limits rule out using it as the only verifier. + +The architecture also follows older work on selective prediction and abstention: +[SelectiveNet](https://proceedings.mlr.press/v97/geifman19a), +[Learning to Defer](https://proceedings.mlr.press/v119/mozannar20b.html), +[Calibrated Learning to Defer](https://proceedings.mlr.press/v162/verma22c.html), and +[Language Model Cascades](https://arxiv.org/abs/2207.10342). Their shared idea is +that a useful classifier should act on its competence region and reject or defer the +rest. [Generative Verifiers](https://openreview.net/forum?id=Ccwp4tFEtE) further +supports retaining a generative verifier for multi-step reasoning rather than treating +a discriminative score as universal proof. + +Code-review-specific evidence argues for conservative authority. Research on +[LLM code-review overcorrection](https://arxiv.org/abs/2603.00539), +[security code review](https://arxiv.org/abs/2401.16310), and +[judge calibration and bias](https://aclanthology.org/2025.acl-long.808/) shows that +binary judgments can be confidently wrong and that security claims need stronger +grounding. Vendor benchmarks measure agreement with other model outputs rather than +human-adjudicated ECR findings, so they establish speed and structured behavior, not +an acceptable false-negative rate for this repository. + +The useful philosophical analogy is dual-process reasoning: Jev supplies a fast, +bounded System 1 judgment; the tool-using verifier supplies slower System 2 analysis. +The relevant engineering principle is a reject option, not majority voting. Jev's +different model family adds diversity, but agreement between two models is not ground +truth and disagreement is a reason to inspect, not to average blindly. + +## Active Selective Cascade + +The policy is intentionally asymmetric: + +| Jev result | Ordinary correctness/quality | Critical, security, secrets, or cited | +| --- | --- | --- | +| supported at or above threshold | keep | cited findings defer; other protected findings keep | +| contradicted at or above threshold | drop with an audit reason | defer to reasoning verifier | +| `needs_reasoning` or low confidence | defer to reasoning verifier | defer to reasoning verifier | +| missing response or provider failure | use original verification path | use original verification path | + +“Protected” controls the dangerous direction: Jev may help retain such a finding, but +cannot suppress it alone. Cited findings always defer because Jev receives repository +source, not the audited external passage whose support must be checked. + +The default `minConfidence` is `0.9`. It is a conservative operating point, not a +claim of 90% correctness. Repositories may change it explicitly, but model version, +question text, state shape, threshold, and truncation policy form one classifier and +must be reviewed together. A dated model is preferred over a moving alias. + +## State and Security Boundaries + +Each request includes finding category, severity, file, line, title, rationale, and +quoted evidence plus bounded source from that file. ECR sends no PR title/body, +unrelated file, agent transcript, author reply, or credential. `maxContextChars` +bounds disclosure and context rot; truncation centers on the quoted evidence where +possible. Four requests run concurrently, with a bounded timeout and one retry. + +The API origin is explicit. Ambient TypeSafe URL/model variables cannot redirect the +request. SDK logging is off because bodies contain source. `TYPESAFE_API_KEY` is +captured before reviewer startup, withheld from OpenCode, and forbidden as a reviewer +provider credential. Paths remain confined to the materialized review tree before +any source is read. + +Jev is not a prompt-injection detector or security boundary. The code and finding +text are attacker-controlled inputs, and model output is schema-validated before the +policy sees it. Deterministic code continues to own credential isolation, path +confinement, evidence matching, category floors, and final decision derivation. + +## Failure and Observability + +The cascade fails back, not closed. Missing credentials disable only Jev for that run. +An individual timeout, malformed response, or provider error sends that finding +through the original verification path. Error text is not persisted because providers +may echo source. The run log stores aggregate model, duration, token, cost, evaluated, +and failure counts; any finding dropped by Jev appears in the ordinary +`verifierDropped` audit trail with its confidence. + +Published pricing for the pinned integration is $0.042 per million input tokens with +no output-token charge. Cost is recorded in the `jev-verifier` bucket. Provider limits +are not treated as local capacity guarantees. + +## What Jev Replaces + +Jev replaces only straightforward finding verification calls. It does not replace: + +- reviewer agents that discover bugs and supply explanations; +- the coordinator that merges and prioritizes findings; +- deterministic evidence, trust, and security checks; +- the reasoning verifier for ambiguity, multi-file behavior, runtime semantics, + citations, or protected negative judgments. + +This division gives Jev real authority and cost benefit without pretending that a +fast typed classifier is a complete code reviewer. + +## Rejected Alternatives + +**A standalone `jev.md` reviewer agent.** Jev cannot use ECR's browsing tools or emit +the finding schema's explanation and remediation. Presenting it as a peer reviewer +would hide a different and narrower capability behind the same abstraction. + +**Shadow-only operation.** It adds latency and cost without improving the review. The +safe boundary comes from abstention, protected categories, and fallback behavior, not +from making every result inert. + +**Replace all verification.** Local classification cannot resolve missing repository +context, multi-step execution, arithmetic, temporal behavior, or external citations. + +**Use confidence as proof.** A concentrated distribution can still be wrong. The +threshold is a routing policy and protected contradictions still require reasoning. + +**Send the entire diff or repository.** Unrelated context increases disclosure and +context rot. One bounded file is the smallest state that supports the local question. diff --git a/llp/0014-jev-shadow-evaluation.explainer.md b/llp/0014-jev-shadow-evaluation.explainer.md deleted file mode 100644 index c6b4f00..0000000 --- a/llp/0014-jev-shadow-evaluation.explainer.md +++ /dev/null @@ -1,159 +0,0 @@ -# LLP 0014: Jev Shadow Evaluation - -**Type:** Explainer -**Status:** Active -**Systems:** Engine, Config, Security, Observability, Templates -**Author:** Expo -**Date:** 2026-09-21 -**Related:** [LLP 0001](0001-trust-model.principles.md), [LLP 0002](0002-review-engine-pipeline.explainer.md), [LLP 0005](0005-verification-fingerprints-rendering.explainer.md), [LLP 0006](0006-config-schema-loading-routing.explainer.md), [LLP 0009](0009-adoption-templates-and-ci-workflows.guide.md) - -Jev is a discriminative model from TypeSafe AI. It accepts a state plus typed -questions and returns probabilities for choices, boolean “Noul” questions, or an -ordered score. It does not generate explanations, remediation text, or code. ECR -therefore uses Jev as an optional shadow evaluator of the final finding set, not as -a reviewer or replacement for the coordinator and verifier. - -## Evidence Survey - -The primary sources are TypeSafe's [Jev announcement](https://typesafe.ai/blog/introducing-system-one-models-and-jev), -[documentation](https://docs.typesafe.ai/), and open-source -[JavaScript SDK](https://github.com/typesafe-ai/typesafe-sdk-js). As of this design, -the current dated model is `jev-1.13.0`; the service also offers moving aliases. -The SDK is at `0.6.0`, supports Node 20+, reads `TYPESAFE_API_KEY`, defaults to a -10-second per-attempt timeout and two retries, and exposes each answer's probability -distribution plus token usage. - -The vendor describes a 64K-token request limit, with at most 32K tokens for state, -text-only input, up to 255 Choice labels, and up to 10 Score levels. Published pricing -is $0.042 per million input tokens with no output-token charge. Published account -limits are 250,000 tokens per second and 1,200 requests per minute, subject to account -changes. These are service facts, not ECR capacity targets; ECR keeps lower local -bounds. - -The vendor benchmark measures agreement with answers from frontier generative models, -not correctness against independently adjudicated human labels. Its reported speed -and cost advantages therefore justify an experiment, not authority over findings. -TypeSafe's own guidance also says that confidence measures concentration of the -returned distribution rather than correctness, thresholds require calibration on the -application's data, irrelevant context degrades results, question wording matters, -and Jev is weak at arithmetic, counting, date comparison, and complex indirection. -Customer requests are not used for training under the standard policy, while zero -data retention is an enterprise control rather than the default. - -Community experiments support a narrow role. The open-source -[`jev-review`](https://github.com/devagrawal09/jev-review) workflow uses typed stages -for file profiling, evidence selection, mechanism, severity, and routing, while -explicitly treating outputs as review leads rather than proof. Other early projects -such as [`pi-warden`](https://github.com/DevMortimer/pi-warden) and -[`pi-jev`](https://github.com/y0usaf/pi-jev) emphasize paired evaluation, calibration, -and enforcement disabled by default. An independent -[hands-on review](https://jevaiguide.com/jev-review/) reports low latency on small -requests but also early documentation/API rough edges and no public research paper. -Those reports are useful implementation anecdotes, not independent validation of code -review accuracy. - -## Fit With ECR - -Jev fits three bounded jobs: - -1. **Finding calibration:** ask whether the changed-code patch supports a final - finding and compare its severity distribution with ECR's result. -2. **Routing:** classify a small diff or candidate finding before assigning expensive - generative passes, once a repo-specific recall benchmark exists. -3. **Evidence triage:** rank retrieved documentation passages or verifier candidates - before a generative model explains them. - -It does not fit jobs that require synthesis: discovering an open-ended bug, tracing a -large execution path with tools, explaining causality, writing remediation, or -consolidating prose. Existing generative reviewers remain responsible for those jobs. -Deterministic code remains responsible for path confinement, schema validation, -credential boundaries, exact quote checks, protected severity floors, and all other -security invariants. - -Jev must never be a prompt-injection detector or security boundary. The state is -attacker-controlled code and finding prose, and a discriminative model can be steered -by adversarial content. No Jev answer may override ECR's critical/secrets floors or -turn a failed run into approval. - -## Shadow-First Integration - -The root-only `jev` config is disabled by default. When enabled, ECR waits until -verification, suppression, requalification, citation handling, and feedback handling -have produced the final finding set. It then sends one request per finding, capped by -`maxFindings`, with two Choice questions: - -- whether the supplied patch supports, contradicts, or is insufficient to judge the - exact finding; -- the appropriate `suggestion`, `warning`, or `critical` severity if the finding is - real. - -The request state contains the finding's category, severity, file, line, title, -rationale, and evidence plus only that file's unified patch. `maxPatchChars` bounds -the patch; when possible, truncation centers on the cited evidence. ECR sends no PR -title/body, unrelated file, surrounding repository tree, credential, agent transcript, -or author reply. Four requests run concurrently. `maxFindings`, the SDK timeout, and -one retry bound time and provider load. - -The API origin and dated model are explicit in code/config. Ambient -`TYPESAFE_BASE_URL` and `TYPESAFE_DEFAULT_MODEL` cannot redirect the integration or -silently move its calibration target. The SDK logger is off because debug bodies would -contain source. `TYPESAFE_API_KEY` is a dedicated workflow secret and is also in -`FORBIDDEN_TOKEN_ENVS`, so an `auth.tokenEnv` cannot forward it to a model provider. - -The observation stores no additional source text. `.runs/reviews.jsonl` receives the -finding fingerprint, actual model, selected labels, full probability distributions, -token totals, duration, and coverage counts. Provider errors are counted without -persisting their text because an error may echo request details. A missing key produces -one explicit availability error. None of these values enters `CoordinatorOutput`, the -reporter, the decision calculation, or a later review prompt. - -The integration is fail-open by design because it is telemetry: construction, -credential, request, timeout, and response failures leave the review byte-for-byte -unchanged. This differs from review coverage. A failed Jev observation is not a missing -review pass and must not add an `incomplete` review note. - -## Calibration and Promotion Gates - -“Supported” is not ground truth, and agreement with ECR can mean both systems made the -same mistake. Before Jev gains any decision effect, evaluation must join shadow records -with human outcomes such as accepted fixes, explicit dismissals, and independently -adjudicated sampled findings. Measure by category and severity, not only in aggregate: - -- recall on confirmed critical, secrets, and security findings; -- precision and false-negative rate for `contradicted` and `insufficient` labels; -- calibration curves for each probability, pinned model version, and question text; -- latency, failures, retries, input tokens, and cost per reviewed PR; -- drift after any Jev model, SDK, rubric, state shape, or truncation change. - -Promotion proceeds in separate changes: - -1. **Shadow finding evaluator:** current phase; log only. -2. **Visible advisory:** optionally show a clearly labeled second opinion without - changing decisions, after measured calibration. -3. **Cost routing:** let Jev skip only low-risk expensive work when a held-out benchmark - proves the required recall; always-run security and deterministic checks remain. -4. **Finding suppression:** last and highest bar. It requires human-labeled data, - category-specific thresholds, an audit trail, a kill switch, and hard bypasses for - critical/secrets/security findings. No current evidence meets this bar. - -Moving aliases are unsuitable for a calibrated gate. A promotion must pin a dated -model and treat every version or question change as a new classifier requiring fresh -validation. Confidence alone is never a threshold justification. - -## Rejected Alternatives - -**Replace the verifier.** Jev cannot inspect the repository with tools or explain the -source-level contradiction. The current verifier can re-read source and fails open; -replacing it would reduce evidence and debuggability. - -**Pre-filter reviewer findings before coordination.** This creates an early silent -false-negative path exactly where ECR requires coverage honesty. Shadow evaluation of -final findings is measurable and reversible. - -**Send the entire diff or repository context.** More context raises disclosure, cost, -and context-rot risk without improving the exact local judgment. One bounded file -patch is the smallest useful state. - -**Use Jev for prompt-injection detection.** An attacker controls the state being -classified. Only deterministic isolation and trust boundaries can protect credentials -and execution. diff --git a/src/__tests__/jev.test.ts b/src/__tests__/jev.test.ts index a94201e..f53c4ae 100644 --- a/src/__tests__/jev.test.ts +++ b/src/__tests__/jev.test.ts @@ -3,7 +3,12 @@ import { TypeSafeClient } from "@typesafe-ai/sdk"; import { ReviewConfigSchema, ScopeReviewConfigSchema } from "../config/schema.js"; import { FORBIDDEN_TOKEN_ENVS } from "../core/auth.js"; -import { buildJevState, observeFindingsWithJev, withoutJevCredential } from "../core/jev.js"; +import { + buildJevState, + evaluateFindingsWithJev, + jevDisposition, + withoutJevCredential, +} from "../core/jev.js"; import type { Finding } from "../core/schema.js"; const finding: Finding = { @@ -17,21 +22,20 @@ const finding: Finding = { }; const config = { - enabled: true, model: "jev-1.13.0", - maxFindings: 20, + minConfidence: 0.9, timeoutMs: 10_000, - maxPatchChars: 30_000, + maxContextChars: 30_000, }; describe("Jev config", () => { - test("is disabled and pinned by default", () => { - const parsed = ReviewConfigSchema.parse({}); - expect(parsed.jev).toEqual({ ...config, enabled: false }); + test("is activated by presence and absent by default", () => { + expect(ReviewConfigSchema.parse({}).jev).toBeUndefined(); + expect(ReviewConfigSchema.parse({ jev: {} }).jev).toEqual(config); }); test("is root-only", () => { - expect(ScopeReviewConfigSchema.safeParse({ jev: { enabled: true } }).success).toBe(false); + expect(ScopeReviewConfigSchema.safeParse({ jev: {} }).success).toBe(false); }); test("cannot be repurposed as a reviewer provider credential", () => { @@ -39,30 +43,18 @@ describe("Jev config", () => { }); }); -describe("observeFindingsWithJev", () => { - test("does nothing when disabled", async () => { - expect( - await observeFindingsWithJev({ - config: { ...config, enabled: false }, - findings: [finding], - files: [], - env: {}, - }), - ).toBeUndefined(); - }); - - test("records a missing key without throwing", async () => { - const observation = await observeFindingsWithJev({ +describe("evaluateFindingsWithJev", () => { + test("falls back cleanly when the key is missing", async () => { + const result = await evaluateFindingsWithJev({ config, - findings: [finding], - files: [], + candidates: [{ finding, sourceContext: "return false" }], env: {}, }); - expect(observation?.error).toBe("TYPESAFE_API_KEY is not set"); - expect(observation?.findings).toEqual([]); + expect(result.summary.unavailable).toBe("TYPESAFE_API_KEY is not set"); + expect(result.evaluations.size).toBe(0); }); - test("records typed probabilities without source text", async () => { + test("returns a typed atomic judgment without persisting source", async () => { const requests: unknown[] = []; const client = new TypeSafeClient({ apiKey: "test-key", @@ -78,13 +70,7 @@ describe("observeFindingsWithJev", () => { type: "choice", choice: "supported", confidence: 0.91, - probabilities: { supported: 0.91, insufficient: 0.07, contradicted: 0.02 }, - }, - severity: { - type: "choice", - choice: "warning", - confidence: 0.8, - probabilities: { suggestion: 0.1, warning: 0.8, critical: 0.1 }, + probabilities: { supported: 0.91, needs_reasoning: 0.07, contradicted: 0.02 }, }, }, usage: { input_tokens: 123, output_tokens: 0 }, @@ -94,23 +80,21 @@ describe("observeFindingsWithJev", () => { }, }); - const observation = await observeFindingsWithJev({ + const result = await evaluateFindingsWithJev({ config, - findings: [finding], - files: [{ path: finding.file, patch: "@@ -1 +1 @@\n-return true\n+return false" }], + candidates: [{ finding, sourceContext: "export function fallback() { return false }" }], client, env: {}, }); expect(requests).toHaveLength(1); - expect(observation?.evaluated).toBe(1); - expect(observation?.inputTokens).toBe(123); - expect(observation?.findings[0]?.support).toBe("supported"); - expect(observation?.findings[0]?.severity).toBe("warning"); - expect(JSON.stringify(observation)).not.toContain("return false"); + expect(result.summary.evaluated).toBe(1); + expect(result.summary.inputTokens).toBe(123); + expect([...result.evaluations.values()][0]?.support).toBe("supported"); + expect(JSON.stringify(result.summary)).not.toContain("return false"); }); - test("caps findings and fails open on provider errors", async () => { + test("fails open per finding on provider errors", async () => { const client = new TypeSafeClient({ apiKey: "test-key", retry: { maxRetries: 0 }, @@ -118,30 +102,64 @@ describe("observeFindingsWithJev", () => { throw new Error("request contained sensitive source"); }, }); - const observation = await observeFindingsWithJev({ - config: { ...config, maxFindings: 1 }, - findings: [finding, { ...finding, title: "Another finding" }], - files: [], + const result = await evaluateFindingsWithJev({ + config, + candidates: [{ finding, sourceContext: "secret source" }], client, env: {}, }); - expect(observation?.failed).toBe(1); - expect(observation?.skipped).toBe(1); - expect(observation?.error).toBeUndefined(); - expect(JSON.stringify(observation)).not.toContain("sensitive source"); + expect(result.summary.failed).toBe(1); + expect(result.summary.unavailable).toBeUndefined(); + expect(JSON.stringify(result.summary)).not.toContain("sensitive source"); + }); +}); + +describe("jevDisposition", () => { + const evaluation = ( + support: "supported" | "needs_reasoning" | "contradicted", + confidence = 0.95, + ) => ({ + fingerprint: "fp", + model: config.model, + support, + confidence, + probabilities: { supported: 0.02, needs_reasoning: 0.03, contradicted: 0.95 }, + }); + + test("acts on clear ordinary findings and defers uncertainty", () => { + expect(jevDisposition(finding, evaluation("supported"), 0.9)).toBe("keep"); + expect(jevDisposition(finding, evaluation("contradicted"), 0.9)).toBe("drop"); + expect(jevDisposition(finding, evaluation("needs_reasoning"), 0.9)).toBe("defer"); + expect(jevDisposition(finding, evaluation("contradicted", 0.89), 0.9)).toBe("defer"); + }); + + test("never drops protected or cited findings by itself", () => { + expect( + jevDisposition({ ...finding, category: "security" }, evaluation("contradicted"), 0.9), + ).toBe("defer"); + expect( + jevDisposition({ ...finding, severity: "critical" }, evaluation("contradicted"), 0.9), + ).toBe("defer"); + expect( + jevDisposition( + { ...finding, sources: [{ title: "Docs", url: "https://example.com/docs" }] }, + evaluation("supported"), + 0.9, + ), + ).toBe("defer"); }); }); -test("buildJevState bounds large patches around the cited evidence", () => { +test("buildJevState bounds large source around the cited evidence", () => { const state = buildJevState( finding, `before-${"x".repeat(2_000)}return false${"y".repeat(2_000)}`, 1_000, ); - expect(state.changedCodePatch.length).toBeLessThanOrEqual(1_000); - expect(state.changedCodePatch).toContain("return false"); - expect(state.changedCodePatch).toContain("patch omitted by ecr"); + expect(state.sourceContext.length).toBeLessThanOrEqual(1_000); + expect(state.sourceContext).toContain("return false"); + expect(state.sourceContext).toContain("source omitted by ecr"); }); test("withoutJevCredential restores the key after hiding it from a child spawn", async () => { diff --git a/src/__tests__/verify.test.ts b/src/__tests__/verify.test.ts index 6c2a1fc..dcbff8e 100644 --- a/src/__tests__/verify.test.ts +++ b/src/__tests__/verify.test.ts @@ -1,4 +1,5 @@ import { test, expect, beforeAll, afterAll } from "bun:test"; +import { TypeSafeClient } from "@typesafe-ai/sdk"; import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -82,6 +83,34 @@ const finding = (over: Partial): Finding => ({ // (keep the finding). Grounded non-criticals never reach the LLM at all. const handle = {} as OpencodeHandle; +const contradictedJev = new TypeSafeClient({ + apiKey: "test-key", + retry: { maxRetries: 0 }, + fetch: async () => + new Response( + JSON.stringify({ + model: "jev-1.13.0", + answers: { + support: { + type: "choice", + choice: "contradicted", + confidence: 0.97, + probabilities: { supported: 0.01, needs_reasoning: 0.02, contradicted: 0.97 }, + }, + }, + usage: { input_tokens: 20, output_tokens: 0 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), +}); + +const jevConfig = { + model: "jev-1.13.0", + minConfidence: 0.9, + timeoutMs: 10_000, + maxContextChars: 30_000, +}; + test("grounded non-critical is kept without an LLM call", async () => { const res = await verifyFindings( handle, @@ -92,6 +121,40 @@ test("grounded non-critical is kept without an LLM call", async () => { expect(res.dropped).toEqual([]); }); +test("Jev actively drops a confidently contradicted ordinary finding", async () => { + const res = await verifyFindings( + handle, + [finding({ title: "contradicted", evidence: "return items[next++]!;" })], + "/", + undefined, + [], + { config: jevConfig, client: contradictedJev }, + ); + expect(res.kept).toEqual([]); + expect(res.dropped[0]?.reason).toContain("Jev directly contradicted"); + expect(res.jev?.evaluated).toBe(1); +}); + +test("Jev defers a contradicted security finding to the reasoning verifier", async () => { + const res = await verifyFindings( + handle, + [ + finding({ + title: "protected", + category: "security", + evidence: "return items[next++]!;", + }), + ], + "/", + undefined, + [], + { config: jevConfig, client: contradictedJev }, + ); + // The dummy reasoning handle fails, and the existing fail-open rule keeps it. + expect(res.kept.map((item) => item.title)).toEqual(["protected"]); + expect(res.dropped).toEqual([]); +}); + test("absent evidence is ESCALATED, not hard-dropped (fail-open keeps it here)", async () => { // The key regression guard: an imperfect/absent quote must not be a silent drop. // With the dummy handle the escalated LLM call throws → fail-open → kept. diff --git a/src/config/load.ts b/src/config/load.ts index f705a21..410c562 100644 --- a/src/config/load.ts +++ b/src/config/load.ts @@ -71,15 +71,6 @@ const RESEARCH_CONFIG_DEFAULTS: LoadedConfig["research"] = { timeoutMs: 30_000, }; -/** Jev defaults for a scope load (where `jev` is schema-rejected). */ -const JEV_CONFIG_DEFAULTS: LoadedConfig["jev"] = { - enabled: false, - model: "jev-1.13.0", - maxFindings: 20, - timeoutMs: 10_000, - maxPatchChars: 30_000, -}; - /** Default OpenCode tool toggles for a reviewer: read the repo, never mutate it. */ const DEFAULT_AGENT_TOOLS = toolMap(["read", "grep", "glob", "list"]); @@ -250,7 +241,7 @@ async function loadConfigDir( // Root-only: scope schemas reject research configuration, so an untrusted // subtree cannot select the index or alter the network-facing runtime. research: parsed.research ?? RESEARCH_CONFIG_DEFAULTS, - jev: parsed.jev ?? JEV_CONFIG_DEFAULTS, + ...(parsed.jev ? { jev: parsed.jev } : {}), // parsed.breakGlass/auth are always present for the root schema (defaults) and // absent for the scope schema; loadScopeConfig overrides both afterwards. breakGlassMarker: parsed.breakGlass?.marker ?? "/skip-review", diff --git a/src/config/schema.ts b/src/config/schema.ts index 543bf40..500a442 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -87,25 +87,19 @@ export const ReviewConfigSchema = z.object({ resultsPerQuery: 2, timeoutMs: 30_000, }), - // Optional Jev second opinion. This is deliberately observation-only: its - // answers are written to the run log and never alter findings or decisions. - // ROOT-ONLY because it sends bounded source context to a separate provider. - // @ref LLP 0014#shadow-first-integration [implements] — collect calibration data before granting Jev decision authority + // Optional Jev verification cascade. Presence enables it: confident local + // judgments act immediately, while ambiguous or protected findings defer to + // the reasoning verifier. ROOT-ONLY because it sends bounded source context + // to a separate provider. + // @ref LLP 0014#active-selective-cascade [implements] — narrow decisions act; uncertainty defers jev: z .object({ - enabled: z.boolean().default(false), model: z.string().min(1).default("jev-1.13.0"), - maxFindings: z.number().int().min(1).max(100).default(20), + minConfidence: z.number().min(0.5).max(1).default(0.9), timeoutMs: z.number().int().min(1000).max(60_000).default(10_000), - maxPatchChars: z.number().int().min(1000).max(30_000).default(30_000), + maxContextChars: z.number().int().min(1000).max(30_000).default(30_000), }) - .default({ - enabled: false, - model: "jev-1.13.0", - maxFindings: 20, - timeoutMs: 10_000, - maxPatchChars: 30_000, - }), + .optional(), breakGlass: z .object({ marker: z.string().default("/skip-review") }) .default({ marker: "/skip-review" }), @@ -488,13 +482,12 @@ export interface LoadedConfig { resultsPerQuery: number; timeoutMs: number; }; - /** Root-only, observation-only Jev evaluation settings. */ - jev: { - enabled: boolean; + /** Root-only Jev selective-verification settings; absent means disabled. */ + jev?: { model: string; - maxFindings: number; + minConfidence: number; timeoutMs: number; - maxPatchChars: number; + maxContextChars: number; }; breakGlassMarker: string; commentTag: string; diff --git a/src/core/jev.ts b/src/core/jev.ts index a807a7f..d208a6e 100644 --- a/src/core/jev.ts +++ b/src/core/jev.ts @@ -1,9 +1,9 @@ -// @ref LLP 0014#shadow-first-integration [implements] — Jev records a bounded second opinion without entering the decision path +// @ref LLP 0014#active-selective-cascade [implements] — Jev handles narrow judgments and defers uncertainty import { TypeSafeClient, choice } from "@typesafe-ai/sdk"; import { z } from "zod"; import type { LoadedConfig } from "../config/schema.js"; -import type { DiffEntry, Finding } from "./schema.js"; +import type { Finding } from "./schema.js"; import { fingerprintFinding } from "./schema.js"; const TYPESAFE_API_URL = "https://api.typesafe.ai"; @@ -12,17 +12,14 @@ const MAX_FILE_CHARS = 1_000; const MAX_TITLE_CHARS = 1_000; const MAX_RATIONALE_CHARS = 6_000; const MAX_EVIDENCE_CHARS = 6_000; +const INPUT_COST_PER_MILLION = 0.042; const SUPPORT_CRITERIA = { - supported: "The supplied patch contains concrete evidence for the reported problem.", - insufficient: "The supplied patch does not contain enough evidence to decide.", - contradicted: "The supplied patch shows that the reported problem is not present.", -} as const; - -const SEVERITY_CRITERIA = { - suggestion: "A non-blocking improvement with no demonstrated shipped failure.", - warning: "A real defect with a plausible user, reliability, or maintainability impact.", - critical: "A severe security, secret-exposure, data-loss, or broadly breaking defect.", + supported: "The supplied source context directly demonstrates the exact reported problem.", + needs_reasoning: + "The context is insufficient, or the claim needs multi-file, runtime, arithmetic, temporal, or indirect reasoning.", + contradicted: + "The supplied source context directly demonstrates that the reported problem is not present.", } as const; const probability = z.number().min(0).max(1); @@ -31,24 +28,14 @@ const JevResponseSchema = z.object({ answers: z.object({ support: z.object({ type: z.literal("choice"), - choice: z.enum(["supported", "insufficient", "contradicted"]), + choice: z.enum(["supported", "needs_reasoning", "contradicted"]), confidence: probability, probabilities: z.object({ supported: probability, - insufficient: probability, + needs_reasoning: probability, contradicted: probability, }), }), - severity: z.object({ - type: z.literal("choice"), - choice: z.enum(["suggestion", "warning", "critical"]), - confidence: probability, - probabilities: z.object({ - suggestion: probability, - warning: probability, - critical: probability, - }), - }), }), usage: z.object({ input_tokens: z.number().int().nonnegative(), @@ -56,35 +43,40 @@ const JevResponseSchema = z.object({ }), }); -export interface JevFindingObservation { +export interface JevCandidate { + finding: Finding; + sourceContext: string; +} + +export interface JevEvaluation { fingerprint: string; model: string; support: keyof typeof SUPPORT_CRITERIA; - supportConfidence: number; - supportProbabilities: Record; - severity: keyof typeof SEVERITY_CRITERIA; - severityConfidence: number; - severityProbabilities: Record; + confidence: number; + probabilities: Record; } -export interface JevObservation { +export interface JevVerificationSummary { configuredModel: string; + actualModel?: string; durationMs: number; evaluated: number; - skipped: number; failed: number; inputTokens: number; outputTokens: number; - findings: JevFindingObservation[]; - error?: string; + cost: number; + unavailable?: string; +} + +export interface JevVerificationResult { + evaluations: Map; + summary: JevVerificationSummary; } -export interface ObserveWithJevOptions { - config: LoadedConfig["jev"]; - findings: readonly Finding[]; - files: readonly DiffEntry[]; +export interface EvaluateWithJevOptions { + config: NonNullable; + candidates: readonly JevCandidate[]; env?: NodeJS.ProcessEnv; - /** Explicit key captured before reviewer engines start; never forwarded to them. */ apiKey?: string; client?: TypeSafeClient; } @@ -99,33 +91,22 @@ export async function withoutJevCredential( delete env.TYPESAFE_API_KEY; return await run(); } finally { - if (apiKey === undefined) { - delete env.TYPESAFE_API_KEY; - } else { - env.TYPESAFE_API_KEY = apiKey; - } + if (apiKey === undefined) delete env.TYPESAFE_API_KEY; + else env.TYPESAFE_API_KEY = apiKey; } } -/** - * Ask Jev typed questions about final findings. The returned record is telemetry - * only: callers must never use it to mutate a finding or review decision. - */ -export async function observeFindingsWithJev({ +/** Classify candidates with one atomic question; failures become missing evaluations. */ +export async function evaluateFindingsWithJev({ config, - findings, - files, + candidates, env = process.env, apiKey: explicitApiKey, client, -}: ObserveWithJevOptions): Promise { - if (!config.enabled) return undefined; - +}: EvaluateWithJevOptions): Promise { const started = Date.now(); const apiKey = (explicitApiKey ?? env.TYPESAFE_API_KEY)?.trim(); - if (!client && !apiKey) { - return emptyObservation(config, started, "TYPESAFE_API_KEY is not set"); - } + if (!client && !apiKey) return emptyResult(config, started, "TYPESAFE_API_KEY is not set"); let resolvedClient: TypeSafeClient; try { @@ -140,73 +121,97 @@ export async function observeFindingsWithJev({ logLevel: "off", }); } catch { - return emptyObservation(config, started, "Jev client could not be initialized"); + return emptyResult(config, started, "Jev client could not be initialized"); } - const selected = findings.slice(0, config.maxFindings); - const patches = new Map(files.map((file) => [file.path, file.patch])); - const results: JevFindingObservation[] = []; + const evaluations = new Map(); let failed = 0; let inputTokens = 0; let outputTokens = 0; + let actualModel: string | undefined; let next = 0; const worker = async (): Promise => { - while (next < selected.length) { - const finding = selected[next++]; - if (!finding) continue; + while (next < candidates.length) { + const candidate = candidates[next++]; + if (!candidate) continue; try { const response = JevResponseSchema.parse( await resolvedClient.systemOne({ model: config.model, - state: buildJevState(finding, patches.get(finding.file) ?? "", config.maxPatchChars), + state: buildJevState( + candidate.finding, + candidate.sourceContext, + config.maxContextChars, + ), questions: { support: choice( - "Does the supplied changed-code patch support this exact code-review finding? Judge only the supplied evidence; do not assume missing repository context.", + "Does this source context directly support or contradict this exact code-review finding? Select needs_reasoning whenever the answer depends on omitted context or non-local reasoning.", SUPPORT_CRITERIA, ), - severity: choice( - "If the finding is real, what is its appropriate review severity?", - SEVERITY_CRITERIA, - ), }, }), ); inputTokens += response.usage.input_tokens; outputTokens += response.usage.output_tokens; - results.push({ - fingerprint: fingerprintFinding(finding), + actualModel = response.model; + const fingerprint = fingerprintFinding(candidate.finding); + evaluations.set(fingerprint, { + fingerprint, model: response.model, support: response.answers.support.choice, - supportConfidence: response.answers.support.confidence, - supportProbabilities: response.answers.support.probabilities, - severity: response.answers.severity.choice, - severityConfidence: response.answers.severity.confidence, - severityProbabilities: response.answers.severity.probabilities, + confidence: response.answers.support.confidence, + probabilities: response.answers.support.probabilities, }); } catch { - // Observation is fail-open and provider errors may contain request details. failed += 1; } } }; - await Promise.all(Array.from({ length: Math.min(CONCURRENCY, selected.length) }, () => worker())); + await Promise.all( + Array.from({ length: Math.min(CONCURRENCY, candidates.length) }, () => worker()), + ); return { - configuredModel: config.model, - durationMs: Date.now() - started, - evaluated: results.length, - skipped: Math.max(0, findings.length - selected.length), - failed, - inputTokens, - outputTokens, - findings: results.sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)), + evaluations, + summary: { + configuredModel: config.model, + ...(actualModel ? { actualModel } : {}), + durationMs: Date.now() - started, + evaluated: evaluations.size, + failed, + inputTokens, + outputTokens, + cost: (inputTokens * INPUT_COST_PER_MILLION) / 1_000_000, + }, }; } -/** Build the only state sent to TypeSafe: finding metadata plus its bounded file patch. */ -export function buildJevState(finding: Finding, patch: string, maxPatchChars: number) { +/** High-confidence local answers act; sensitive or uncertain answers defer. */ +export function jevDisposition( + finding: Finding, + evaluation: JevEvaluation | undefined, + minConfidence: number, +): "keep" | "drop" | "defer" { + if (!evaluation || evaluation.confidence < minConfidence) return "defer"; + // Jev sees repository source, not the audited external passage. The reasoning + // verifier remains responsible for citation support in either direction. + if (finding.sources?.length) return "defer"; + if (evaluation.support === "supported") return "keep"; + if (evaluation.support === "needs_reasoning") return "defer"; + if ( + finding.severity === "critical" || + finding.category === "security" || + finding.category === "secrets" + ) { + return "defer"; + } + return "drop"; +} + +/** Build the only state sent to TypeSafe: finding metadata plus bounded local source. */ +export function buildJevState(finding: Finding, sourceContext: string, maxContextChars: number) { return { finding: { severity: finding.severity, @@ -217,7 +222,7 @@ export function buildJevState(finding: Finding, patch: string, maxPatchChars: nu rationale: boundText(finding.rationale, MAX_RATIONALE_CHARS), evidence: finding.evidence ? boundText(finding.evidence, MAX_EVIDENCE_CHARS) : null, }, - changedCodePatch: boundPatch(patch, finding.evidence, maxPatchChars), + sourceContext: boundContext(sourceContext, finding.evidence, maxContextChars), }; } @@ -225,33 +230,36 @@ function boundText(value: string, maxChars: number): string { return value.length <= maxChars ? value : `${value.slice(0, maxChars - 1)}…`; } -function boundPatch(patch: string, evidence: string | undefined, maxChars: number): string { - if (patch.length <= maxChars) return patch; - const omitted = "\n... patch omitted by ecr ...\n"; +function boundContext(context: string, evidence: string | undefined, maxChars: number): string { + if (context.length <= maxChars) return context; + const omitted = "\n... source omitted by ecr ...\n"; const budget = Math.max(1, maxChars - omitted.length); - const evidenceAt = evidence ? patch.indexOf(evidence) : -1; + const evidenceAt = evidence ? context.indexOf(evidence) : -1; if (evidenceAt >= 0) { const start = Math.max(0, evidenceAt - Math.floor(budget / 2)); - return patch.slice(start, start + budget) + omitted; + return context.slice(start, start + budget) + omitted; } - const half = Math.floor(budget / 2); - return patch.slice(0, half) + omitted + patch.slice(-half); + const first = Math.floor(budget / 2); + const last = budget - first; + return context.slice(0, first) + omitted + context.slice(-last); } -function emptyObservation( - config: LoadedConfig["jev"], +function emptyResult( + config: NonNullable, started: number, - error: string, -): JevObservation { + unavailable: string, +): JevVerificationResult { return { - configuredModel: config.model, - durationMs: Date.now() - started, - evaluated: 0, - skipped: 0, - failed: 0, - inputTokens: 0, - outputTokens: 0, - findings: [], - error, + evaluations: new Map(), + summary: { + configuredModel: config.model, + durationMs: Date.now() - started, + evaluated: 0, + failed: 0, + inputTokens: 0, + outputTokens: 0, + cost: 0, + unavailable, + }, }; } diff --git a/src/core/log.ts b/src/core/log.ts index d8f5b6a..4943066 100644 --- a/src/core/log.ts +++ b/src/core/log.ts @@ -6,7 +6,7 @@ import type { CoordinatorOutput, Finding, ReviewMetadata, ReviewTrace } from "./ import type { FilteredFile } from "./noise.js"; import type { TokenUsage } from "./opencode.js"; import type { ResearchProvenance } from "./research.js"; -import type { JevObservation } from "./jev.js"; +import type { JevVerificationSummary } from "./jev.js"; export interface RunLogRecord { timestamp: string; @@ -22,8 +22,8 @@ export interface RunLogRecord { filteredFiles: FilteredFile[]; /** Bounded queries/results plus grounded research decisions and final usefulness metrics. */ research?: ResearchProvenance; - /** Observation-only Jev results; these never influence findings or decisions. */ - jev?: JevObservation; + /** Active Jev cascade usage and availability; individual drops use verifierDropped. */ + jevVerification?: JevVerificationSummary; agentCosts: Record; totalCost: number; // Aggregate token usage across all agent + coordinator requests, for cache diff --git a/src/core/review.ts b/src/core/review.ts index 27ad644..ee4b62f 100644 --- a/src/core/review.ts +++ b/src/core/review.ts @@ -84,8 +84,8 @@ import { summarizeResearchUsefulness, } from "./research.js"; import type { ResearchEvidence, ResearchMcpRuntime, ResearchProvenance } from "./research.js"; -import { observeFindingsWithJev, withoutJevCredential } from "./jev.js"; -import type { JevObservation } from "./jev.js"; +import { withoutJevCredential } from "./jev.js"; +import type { JevVerificationSummary } from "./jev.js"; export interface ReviewRunOptions { config: LoadedConfig; @@ -299,7 +299,7 @@ export async function runReview( let researchEvidence: ResearchEvidence[] = []; let researchRecord: ResearchProvenance | undefined; - let jevRecord: JevObservation | undefined; + let jevRecord: JevVerificationSummary | undefined; // Materialize the PR-head tree (not the current checkout) when the source can, so // the agents' surrounding-source reads and the verifier's re-reads see the versions @@ -393,7 +393,7 @@ export async function runReview( // OpenCode's SDK copies process.env into its server. Withhold the unrelated Jev // credential at spawn time, then restore the parent immediately; Claude uses an // allowlisted child env already. The captured value is passed directly to Jev. - // @ref LLP 0014#shadow-first-integration [constrained-by] — the reviewer engines never receive the Jev credential + // @ref LLP 0014#active-selective-cascade [constrained-by] — reviewer engines never receive the Jev credential const jevApiKey = process.env.TYPESAFE_API_KEY; try { if (usesOpencode) { @@ -1099,6 +1099,9 @@ export async function runReview( process.cwd(), progress, researchEvidence, + config.jev + ? { config: config.jev, ...(jevApiKey ? { apiKey: jevApiKey } : {}) } + : undefined, ); agentCosts["verifier"] = verification.cost; trackTokens("verifier", verification.tokens); @@ -1108,6 +1111,15 @@ export async function runReview( config.agents[0]?.model ?? config.coordinator.model, verification.model, ); + if (verification.jev) { + jevRecord = verification.jev; + agentCosts["jev-verifier"] = verification.jev.cost; + trackTokens("jev-verifier", { + input: verification.jev.inputTokens, + output: verification.jev.outputTokens, + }); + trackModel("jev-verifier", verification.jev.configuredModel, verification.jev.actualModel); + } verifierDropped = verification.dropped; citationStrips = verification.citationStripped; if (verification.dropped.length > 0 || verification.citationStripped.length > 0) { @@ -1338,24 +1350,6 @@ export async function runReview( await appendStepSummary(renderResearchUsefulnessMarkdown(researchRecord)); } - // @ref LLP 0014#shadow-first-integration [implements] — run after the final - // finding set exists and never feed the answers back into `output`. - jevRecord = await observeFindingsWithJev({ - config: config.jev, - findings: output.findings, - files: kept, - apiKey: jevApiKey, - }); - if (jevRecord) { - if (jevRecord.error) { - progress(`Jev shadow evaluation unavailable (${jevRecord.error}); review unchanged.`); - } else { - progress( - `Jev shadow evaluation: ${jevRecord.evaluated} observed, ${jevRecord.failed} failed, ${jevRecord.skipped} over cap; review unchanged.`, - ); - } - } - // Surface provider throttling as a fact about the run: passes already waited or // backed off, but the operator should still SEE that it happened (a run that // was rate-limited is slower and may carry partial passes — that's the cause). @@ -1398,7 +1392,7 @@ export async function runReview( await safeLog(logPath, { ...baseRecord, ...(researchRecord ? { research: researchRecord } : {}), - ...(jevRecord ? { jev: jevRecord } : {}), + ...(jevRecord ? { jevVerification: jevRecord } : {}), agentCosts, totalCost: sum(agentCosts), tokens: tokenTotals, @@ -1437,7 +1431,7 @@ export async function runReview( await safeLog(logPath, { ...baseRecord, ...(researchRecord ? { research: researchRecord } : {}), - ...(jevRecord ? { jev: jevRecord } : {}), + ...(jevRecord ? { jevVerification: jevRecord } : {}), agentCosts, totalCost: sum(agentCosts), tokens: tokenTotals, diff --git a/src/core/verify.ts b/src/core/verify.ts index aea102a..67f862b 100644 --- a/src/core/verify.ts +++ b/src/core/verify.ts @@ -2,11 +2,15 @@ // @ref LLP 0005#verifier-confinement-and-fail-open import { readFile } from "node:fs/promises"; import path from "node:path"; +import type { TypeSafeClient } from "@typesafe-ai/sdk"; import { pathInside } from "./exec.js"; +import type { LoadedConfig } from "../config/schema.js"; +import { evaluateFindingsWithJev, jevDisposition } from "./jev.js"; +import type { JevVerificationSummary } from "./jev.js"; import type { ResearchEvidence } from "./research.js"; import type { Finding } from "./schema.js"; -import { parseVerdict } from "./schema.js"; +import { fingerprintFinding, parseVerdict } from "./schema.js"; import { addTokenUsage, promptAndParse, VERIFIER_AGENT } from "./opencode.js"; import type { OpencodeHandle, TokenUsage } from "./opencode.js"; import { buildVerifierSystem, buildVerifierTask } from "./prompts.js"; @@ -34,6 +38,8 @@ export interface VerificationResult { tokens: TokenUsage; /** provider/model that actually answered the verify calls (see PromptResult.model). */ model?: string; + /** Active selective-verification usage; absent when Jev is not configured. */ + jev?: JevVerificationSummary; } /** The audited passages behind a finding's grounded citations, bounded per source. */ @@ -94,10 +100,13 @@ export function matchEvidence(evidence: string, content: string): "present" | "a // @ref LLP 0005#verifier-confinement-and-fail-open [implements] — pathInside gate: out-of-tree reads (and their present/absent verdict) refused /** Read the cited file and grade the evidence against it (see matchEvidence). */ -async function evidencePresence( +async function inspectEvidence( finding: Finding, cwd: string, -): Promise<"present" | "absent" | "unknown"> { +): Promise<{ + presence: "present" | "absent" | "unknown"; + sourceContext: string; +}> { // finding.file is an unconstrained, LLM-authored string produced over untrusted PR // content, so a prompt-injected finding could point it at a host secret. path.resolve // IGNORES cwd when finding.file is already absolute (e.g. ~/.claude/.credentials.json), @@ -107,15 +116,15 @@ async function evidencePresence( // materialized PR-head tree); anything outside is uncheckable, never read. const resolved = path.resolve(cwd, finding.file); if (!pathInside(resolved, cwd)) { - return "unknown"; + return { presence: "unknown", sourceContext: "" }; } let content: string; try { content = await readFile(resolved, "utf8"); } catch { - return "unknown"; + return { presence: "unknown", sourceContext: "" }; } - return matchEvidence(finding.evidence ?? "", content); + return { presence: matchEvidence(finding.evidence ?? "", content), sourceContext: content }; } // @ref LLP 0005#verifier-confinement-and-fail-open [constrained-by] — fails open: a verify error/timeout keeps the finding, never drops it @@ -145,6 +154,12 @@ export async function verifyFindings( onProgress?: (message: string) => void, /** This run's audited research evidence, for findings that cite documentation. */ researchEvidence: ResearchEvidence[] = [], + jev?: { + config: NonNullable; + apiKey?: string; + /** Test seam; production constructs the pinned client inside jev.ts. */ + client?: TypeSafeClient; + }, ): Promise { const dropped: Array<{ finding: Finding; reason: string }> = []; const citationStripped: Array<{ finding: Finding; reason: string }> = []; @@ -156,9 +171,29 @@ export async function verifyFindings( // Phase 1 — deterministic quote-grounding for every finding. const checked = await Promise.all( - findings.map(async (finding) => ({ finding, presence: await evidencePresence(finding, cwd) })), + findings.map(async (finding) => ({ finding, ...(await inspectEvidence(finding, cwd)) })), ); + // Phase 2 — active selective classification. A confident local answer can + // avoid the slower verifier; uncertainty is explicitly handed to it. + const jevResult = jev + ? await evaluateFindingsWithJev({ + config: jev.config, + candidates: checked.map(({ finding, sourceContext }) => ({ finding, sourceContext })), + ...(jev.apiKey ? { apiKey: jev.apiKey } : {}), + ...(jev.client ? { client: jev.client } : {}), + }) + : undefined; + if (jevResult?.summary.unavailable) { + onProgress?.( + ` Jev unavailable (${jevResult.summary.unavailable}); using normal verification.`, + ); + } else if (jevResult) { + onProgress?.( + ` Jev classified ${jevResult.summary.evaluated} finding(s); ${jevResult.summary.failed} fell back.`, + ); + } + // Decide which findings need an LLM check vs. can be kept directly. A finding // that cites documentation always gets an LLM check: the repo alone cannot // confirm an external-behavior claim, and the verifier must judge whether the @@ -171,14 +206,36 @@ export async function verifyFindings( }> = []; for (const { finding, presence } of checked) { const citedSources = citedSourcesFor(finding, researchEvidence); - if (presence === "absent" || finding.severity === "critical" || citedSources) { + const evaluation = jevResult?.evaluations.get(fingerprintFinding(finding)); + const disposition = evaluation + ? jevDisposition(finding, evaluation, jev?.config.minConfidence ?? 1) + : undefined; + if (disposition === "keep") { + verdicts.set(finding, "keep"); + continue; + } + if (disposition === "drop") { + verdicts.set(finding, "drop"); + dropped.push({ + finding, + reason: `Jev directly contradicted the finding (${Math.round(evaluation!.confidence * 100)}% confidence)`, + }); + onProgress?.(` Jev: dropped ${finding.severity} "${finding.title}" as contradicted.`); + continue; + } + if ( + disposition === "defer" || + presence === "absent" || + finding.severity === "critical" || + citedSources + ) { toVerify.push({ finding, presence, ...(citedSources ? { citedSources } : {}) }); } else { verdicts.set(finding, "keep"); // grounded (or uncheckable) non-critical } } - // Phase 2 — LLM verify (parallel). Refuted → drop; verified or errored → keep. + // Phase 3 — reasoning verify (parallel). Refuted → drop; verified or errored → keep. await Promise.all( toVerify.map(async ({ finding, presence, citedSources }, index) => { try { @@ -242,5 +299,13 @@ export async function verifyFindings( const kept = findings .filter((finding) => verdicts.get(finding) === "keep") .map((finding) => replacements.get(finding) ?? finding); - return { kept, dropped, citationStripped, cost, tokens, model }; + return { + kept, + dropped, + citationStripped, + cost, + tokens, + model, + ...(jevResult ? { jev: jevResult.summary } : {}), + }; } diff --git a/templates/atlantis.yml b/templates/atlantis.yml index 9c239f8..cdac473 100644 --- a/templates/atlantis.yml +++ b/templates/atlantis.yml @@ -114,7 +114,7 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} - # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + # Optional Jev verification credential (used when the root config has `jev`). TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} diff --git a/templates/command.yml b/templates/command.yml index f6f7ddd..74ac1da 100644 --- a/templates/command.yml +++ b/templates/command.yml @@ -152,7 +152,7 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} - # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + # Optional Jev verification credential (used when the root config has `jev`). TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }} diff --git a/templates/config.jsonc b/templates/config.jsonc index aff51a8..341a5c9 100644 --- a/templates/config.jsonc +++ b/templates/config.jsonc @@ -48,19 +48,16 @@ // "timeoutMs": 30000 // }, - // Optional Jev shadow evaluation (ROOT-ONLY; off by default). Jev receives each - // final finding plus a bounded patch for that finding's file and returns typed - // support/severity probabilities. ECR writes those observations to reviews.jsonl - // but never changes, hides, or re-severities a finding from them. Pin the model - // while calibrating: aliases such as jev-latest can move underneath thresholds. - // Set TYPESAFE_API_KEY in the environment (the workflow templates show the - // optional secret mapping) before enabling this. + // Optional Jev verification cascade (ROOT-ONLY). Adding this block activates it: + // confident local support keeps a finding, confident contradiction drops an + // ordinary finding, and uncertainty or protected findings defer to the reasoning + // verifier. Pin the model because moving aliases can invalidate the threshold. + // Set TYPESAFE_API_KEY in the environment before adding the block. // "jev": { - // "enabled": true, // "model": "jev-1.13.0", - // "maxFindings": 20, + // "minConfidence": 0.9, // "timeoutMs": 10000, - // "maxPatchChars": 30000 + // "maxContextChars": 30000 // }, // Large diffs are split into focused chunks by changed-line count, plus a diff --git a/templates/workflow.yml b/templates/workflow.yml index fdbabab..ae0dbfe 100644 --- a/templates/workflow.yml +++ b/templates/workflow.yml @@ -118,7 +118,7 @@ jobs: # META_API_KEY: ${{ secrets.META_API_KEY }} # Optional search-only credential for trusted platform documentation research. BRAVE_SEARCH_API_KEY: ${{ secrets.BRAVE_SEARCH_API_KEY }} - # Optional Jev shadow-evaluation credential (only used when config.jev.enabled). + # Optional Jev verification credential (used when the root config has `jev`). TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} # Optional: override the model for every agent. REVIEWER_MODEL: ${{ vars.REVIEWER_MODEL }}