From 34a5b289ac373ec85a022b54b28448cbc7c56891 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Sun, 6 Sep 2026 11:48:59 -0500 Subject: [PATCH 1/2] Require a real model catalog lookup before a Postgres kernel is built The lookup family has a two-layer posture that deps.py states in prose: the make_*_kernel primitives default to disarmed stubs so an integration test never seeds what it does not exercise, and the fail-loud requirement lives one layer up, in build_kernel's Postgres branch, which raises when a factory a real deployment must bind is missing. language_model_lookup had its first half and never got its second. The obligation is recent: until seed_agent started checking a seeded Agent's model, the seeds appended directly and no seeded agent was gated at all, so a missing lookup was genuinely harmless. Arming that gate is what created the Layer B obligation, and nothing discharged it. The stub does not weaken this gate, it removes it. AlwaysApproved answers every identity with an Approved entry, so define_agent and seed_agent both keep passing and nothing anywhere records that no catalog was consulted. A disarmed gate and an approving one are indistinguishable from the outside, and this gate is the only place the facility's model-provenance constraint is enforced. So the absence stops startup instead of reading as agreement. Operationally this costs nothing: main.py binds the factory in source, not configuration, so no operator can leave it unset. The guard fires only if that binding is deleted, which is what it is for. Three docstrings claimed the settled posture was permissive. The kernel field's said it "mirrors the spend_lookup opt-in posture", citing as authority for permissiveness the one field in the family with no default at all and a named place in this same guard. Reading half that precedent is the plausible route to the gap, so all three now state both halves. Verified by mutation: disabling the guard fails exactly the new test and nothing else. 52,318 unit + architecture passing. Not covered, and deliberately left for the chain test: no CI check asserts that main.py binds this factory, or the financial pair either. Deleting the binding is caught at boot, not in the suite. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/src/cora/infrastructure/deps.py | 28 +++++++++++++++---- apps/api/src/cora/infrastructure/kernel.py | 19 +++++++------ .../ports/language_model_lookup.py | 19 ++++++++----- apps/api/tests/unit/test_deps.py | 28 ++++++++++++++++++- 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/apps/api/src/cora/infrastructure/deps.py b/apps/api/src/cora/infrastructure/deps.py index 298394ca2a3..5535d4d3c95 100644 --- a/apps/api/src/cora/infrastructure/deps.py +++ b/apps/api/src/cora/infrastructure/deps.py @@ -337,7 +337,11 @@ def make_postgres_kernel( a catalog keep the define_agent gate disarmed. Production's `build_kernel` injects the real `PostgresLanguageModelLookup` via the `language_model_lookup_factory` argument; gate-specific tests - override here explicitly. + override here explicitly. Like the financial pair, the permissive + default here is only the first half of the posture: the fail-loud + requirement lives one layer up, in `build_kernel`'s Postgres branch, + which raises when the factory is missing so a real deployment can + never silently answer every identity Approved. `model_usage_lookup` defaults to `AlwaysEmptyModelUsageLookup` (no recorded call touched any model) so tests that don't exercise the @@ -1227,6 +1231,21 @@ async def build_kernel( "financial lookup would silently meter zero and disarm the " "spend envelope" ) + if language_model_lookup_factory is None: + # Same layer, same reason, different stake. The always-approved stub + # answers every identity with an Approved entry, so falling back to it + # in a Postgres deployment does not weaken the model-approval gate, it + # removes it: define_agent and seed_agent both keep passing, and an + # agent can be registered on a model the facility never approved with + # nothing anywhere recording that no catalog was consulted. The gate is + # how the provenance constraint is enforced at all, so its absence must + # stop startup rather than read as agreement. + raise ValueError( + "build_kernel requires language_model_lookup_factory in a Postgres " + "deployment; the always-approved stub would answer every identity " + "Approved, silently disarming the model-approval gate that " + "define_agent and seed_agent both rely on" + ) pool = await create_pool( settings.database_url, min_size=settings.db_pool_min_size, @@ -1283,11 +1302,8 @@ async def build_kernel( ) # Non-None guaranteed by the fail-loud guard above. spend_lookup: SpendLookup = spend_lookup_factory(pool) - language_model_lookup: LanguageModelLookup = ( - language_model_lookup_factory(pool) - if language_model_lookup_factory is not None - else AlwaysApprovedLanguageModelLookup() - ) + # Non-None guaranteed by the fail-loud guard above. + language_model_lookup: LanguageModelLookup = language_model_lookup_factory(pool) model_usage_lookup: ModelUsageLookup = ( model_usage_lookup_factory(pool) if model_usage_lookup_factory is not None diff --git a/apps/api/src/cora/infrastructure/kernel.py b/apps/api/src/cora/infrastructure/kernel.py index 1245322ee8c..430bb333178 100644 --- a/apps/api/src/cora/infrastructure/kernel.py +++ b/apps/api/src/cora/infrastructure/kernel.py @@ -187,9 +187,12 @@ class Kernel: `PostgresLanguageModelLookup` as the production adapter (reads `proj_agent_language_model_summary`). Defaults to `AlwaysApprovedLanguageModelLookup` (every identity - Approved) so tests and catalog-less deployments keep the - pre-catalog behavior; standing up a real catalog is what arms the - gate. Mirrors the `spend_lookup` opt-in posture. + Approved) so tests keep the pre-catalog behavior. Mirrors the + `spend_lookup` posture, BOTH halves of it: permissive at this + layer, and required one layer up, where `build_kernel` refuses a + Postgres deployment that did not bind the real adapter. Reading + only the first half is what left this field out of that guard + while `define_agent` and `seed_agent` were both gated on it. `model_usage_lookup`: cross-BC port consumed by Agent BC's `list_at_risk_results` read slice to enumerate the Decisions whose @@ -476,11 +479,11 @@ class Kernel: default_factory=AlwaysApprovedLanguageModelLookup ) """Resolve a model identity (provider + model) to its catalog entry. - Defaults to the always-approved stub so tests and deployments without - a catalog keep the pre-catalog `define_agent` behavior; the - composition root binds the Agent BC's `PostgresLanguageModelLookup` - over `proj_agent_language_model_summary` when a pool exists, arming - the Approved-entry gate.""" + Defaults to the always-approved stub so tests keep the pre-catalog + `define_agent` behavior; the composition root binds the Agent BC's + `PostgresLanguageModelLookup` over `proj_agent_language_model_summary`, + arming the Approved-entry gate, and refuses to build a Postgres kernel + without it.""" model_usage_lookup: ModelUsageLookup = field(default_factory=AlwaysEmptyModelUsageLookup) """Enumerate the Decisions whose recorded LLM calls touched one diff --git a/apps/api/src/cora/infrastructure/ports/language_model_lookup.py b/apps/api/src/cora/infrastructure/ports/language_model_lookup.py index 688df8ad095..15d2daf3026 100644 --- a/apps/api/src/cora/infrastructure/ports/language_model_lookup.py +++ b/apps/api/src/cora/infrastructure/ports/language_model_lookup.py @@ -21,13 +21,18 @@ ## Failure direction -The kernel default is the always-approved stub, so tests and -deployments that have not stood up a catalog are unaffected (the same -opt-in posture as every lookup in the family: declaring a catalog is -what arms the gate). The Postgres adapter answers with the newest -APPROVED entry only; None means "nothing currently approved for this -identity" (never cataloged, or every entry for it is Defined or -terminal), which the gate treats as refusal once a catalog exists. +The kernel default is the always-approved stub, so tests that have not +stood up a catalog are unaffected. That permissiveness is scoped to the +kernel layer only: `build_kernel` refuses a Postgres deployment that did +not bind the real adapter, because the stub does not weaken this gate, +it removes it. Silence from a disarmed gate is indistinguishable from +approval, and the gate is the only place the facility's model-provenance +constraint is enforced. + +The Postgres adapter answers with the newest APPROVED entry only; None +means "nothing currently approved for this identity" (never cataloged, +or every entry for it is Defined or terminal), which the gate treats as +refusal. """ from dataclasses import dataclass diff --git a/apps/api/tests/unit/test_deps.py b/apps/api/tests/unit/test_deps.py index c43956970a0..d01ed6bd528 100644 --- a/apps/api/tests/unit/test_deps.py +++ b/apps/api/tests/unit/test_deps.py @@ -22,7 +22,12 @@ from cora.infrastructure.adapters.in_memory_idempotency_store import InMemoryIdempotencyStore from cora.infrastructure.config import Settings from cora.infrastructure.deps import build_kernel -from cora.infrastructure.ports import AllowAllAuthorize, FakeLLM +from cora.infrastructure.ports import ( + AllowAllAuthorize, + AlwaysZeroSpendLookup, + FakeLLM, + NoActiveAllocationLookup, +) from cora.trust import build_authorize from cora.trust.authorize import TrustAuthorize @@ -39,6 +44,27 @@ async def test_build_kernel_refuses_a_postgres_deployment_without_financial_look await build_kernel(authorize_factory=build_authorize, settings=settings) +@pytest.mark.unit +async def test_build_kernel_refuses_a_postgres_deployment_without_the_model_catalog_lookup() -> ( + None +): + """A missing catalog binding does not weaken the model-approval gate, + it removes it: `AlwaysApprovedLanguageModelLookup` answers every + identity Approved, so `define_agent` and `seed_agent` both keep + passing and nothing records that no catalog was consulted. Financial + factories are supplied here so the guard under test is the one that + fires, not the financial one above it.""" + settings = Settings(app_env="production") # type: ignore[call-arg] + + with pytest.raises(ValueError, match="language_model_lookup_factory"): + await build_kernel( + authorize_factory=build_authorize, + settings=settings, + spend_lookup_factory=lambda pool: AlwaysZeroSpendLookup(), + allocation_lookup_factory=lambda pool: NoActiveAllocationLookup(), + ) + + @pytest.mark.unit async def test_build_kernel_uses_in_memory_stores_in_test_env( monkeypatch: pytest.MonkeyPatch, From 6ca94464f3f307bee4028c18702167e4a51e10d2 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:15:05 -0500 Subject: [PATCH 2/2] Test the model-approval chain at its joins, not link by link Seeding a LanguageModel-brained Agent reaches its verdict through five links: seed_language_models appends the catalog streams, the projection turns those events into proj_agent_language_model_summary rows, drain_projections advances the bookmark so the rows are visible, PostgresLanguageModelLookup reads them, and seed_agent's gate consumes the answer. Every link had a test. Nothing crossed a join. The adapter's own integration test is the clearest case: it fills the read model by hand-written INSERT, so it proves the SQL reads a correct table and cannot see whether anything fills it. The drain covering THIS projection was asserted nowhere, because both architecture guards read main.py and assert about main.py, which can show a drain call sits between the two seeds but not that it reaches the table the gate will read. So the whole sequence could break with a green suite. Dropping the projection from register_agent_projections is the one-line version, and until now it failed nothing while making a fresh boot refuse both shipped agents that declare real Anthropic models. Three tests, reproducing the composition root's real sequence with the REAL adapter bound, which build_postgres_deps otherwise leaves at the always-approved stub: the chain admits CautionDrafter; the same kernel REFUSES when the drain is removed, which is what makes the drain load-bearing rather than incidental; and the drain fills the specific identity the gate asks for, so a failure names the broken link. Verified by mutation, both directions. Dropping the projection from the registry fails two of the three while the architecture guard and the adapter integration test both stay GREEN, which is the gap stated as an experiment. Swapping the real adapter back to the stub fails exactly the refusal test. That second mutation also shows the end-to-end test passes under the stub on its own, so its docstring now says it is only meaningful paired with the refusal test rather than letting a later reader take it for an independent check. Adds a language_model_lookup passthrough to build_postgres_deps; the binding had no way in before. 1,377 integration + 52,350 unit and architecture passing. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/tests/integration/_helpers.py | 3 + ...t_language_model_catalog_chain_postgres.py | 176 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 apps/api/tests/integration/test_language_model_catalog_chain_postgres.py diff --git a/apps/api/tests/integration/_helpers.py b/apps/api/tests/integration/_helpers.py index 6772204e102..2360d9a4e0a 100644 --- a/apps/api/tests/integration/_helpers.py +++ b/apps/api/tests/integration/_helpers.py @@ -63,6 +63,7 @@ FixedIdGenerator, IdempotencyStore, IdGenerator, + LanguageModelLookup, ProfileStore, RoleLookup, RunActorInvolvementLookup, @@ -97,6 +98,7 @@ def build_postgres_deps( run_actor_involvement_lookup: RunActorInvolvementLookup | None = None, consequence_lookup: ConsequenceLookup | None = None, profile_store: ProfileStore | None = None, + language_model_lookup: LanguageModelLookup | None = None, llm: LLM | None = None, ) -> Kernel: """Build a Kernel for integration-test handler invocation against real Postgres. @@ -140,6 +142,7 @@ def build_postgres_deps( run_actor_involvement_lookup=run_actor_involvement_lookup, consequence_lookup=consequence_lookup, profile_store=profile_store, + language_model_lookup=language_model_lookup, llm=llm, ) diff --git a/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py b/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py new file mode 100644 index 00000000000..8f76f799870 --- /dev/null +++ b/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py @@ -0,0 +1,176 @@ +"""The model-approval chain, joined up, against real Postgres. + +Seeding a LanguageModel-brained Agent reaches its verdict through five +links: `seed_language_models` appends the catalog streams, the projection +turns those events into `proj_agent_language_model_summary` rows, +`drain_projections` advances the bookmark so the rows are visible, +`PostgresLanguageModelLookup` reads them, and `seed_agent`'s gate consumes +the answer. Every link already has a test. None of them crosses a join. + +The clearest case is the adapter's own integration test, which populates +the read model by hand-written INSERT. That proves the SQL reads a correct +table; it cannot see whether anything fills it. And `drain_projections` +covering THIS projection is asserted nowhere at all: the two architecture +guards read `main.py` and assert about the order of calls in `main.py`, so +they can prove a drain sits between the two seeds and not that the drain +reaches the table the gate will read. + +So this file asserts the joins rather than the links, reproducing the +composition root's real sequence (main.py's `seed_language_models` -> +`register_agent_projections` -> `drain_projections` -> `seed_*_agent`) +with the REAL `PostgresLanguageModelLookup` bound, which is the binding +`build_postgres_deps` otherwise leaves at the always-approved stub. + +CautionDrafter is the subject because it is one of the two shipped agents +that declare a real Anthropic model, so it is one of the two whose boot a +broken chain actually stops. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import uuid4 + +import asyncpg +import pytest +import pytest_asyncio +from testcontainers.postgres import PostgresContainer + +from cora.agent import register_agent_projections +from cora.agent.adapters import PostgresLanguageModelLookup +from cora.agent.aggregates.agent.read import load_agent +from cora.agent.aggregates.language_model.state import LanguageModelNotApprovedError +from cora.agent.prompts.caution_drafter import DEFAULT_CAUTION_DRAFTER_MODEL +from cora.agent.seed_caution_drafter import ( + CAUTION_DRAFTER_AGENT_ID, + seed_caution_drafter_agent, +) +from cora.agent.seed_language_models import seed_language_models +from cora.infrastructure.kernel import Kernel +from cora.infrastructure.postgres.pool import create_pool +from cora.infrastructure.projection.drain import drain_projections +from cora.infrastructure.projection.registry import ProjectionRegistry +from tests._postgres import normalize_async_url +from tests.integration._helpers import build_postgres_deps + +pytestmark = pytest.mark.integration + +_NOW = datetime(2026, 9, 6, tzinfo=UTC) + + +@pytest_asyncio.fixture +async def chain_pool( + postgres_container: PostgresContainer, + template_database: str, +): + """A per-test database, because the sequence under test writes the + catalog streams and the read model the next link reads back.""" + test_db = f"lmchain_{uuid4().hex[:12]}" + admin_url = normalize_async_url(postgres_container.get_connection_url(), database="postgres") + admin = await asyncpg.connect(admin_url) + try: + await admin.execute(f'CREATE DATABASE "{test_db}" TEMPLATE "{template_database}"') + finally: + await admin.close() + + test_url = normalize_async_url(postgres_container.get_connection_url(), database=test_db) + pool = await create_pool(test_url, min_size=1, max_size=4) + try: + yield pool + finally: + await pool.close() + admin = await asyncpg.connect(admin_url) + try: + await admin.execute(f'DROP DATABASE "{test_db}"') + finally: + await admin.close() + + +def _kernel_with_the_real_lookup(pool: asyncpg.Pool) -> Kernel: + """The gate's production binding, which every other test leaves as + the always-approved stub. Without this the assertions below pass on + any database, chain or no chain.""" + return build_postgres_deps( + pool, + now=_NOW, + language_model_lookup=PostgresLanguageModelLookup(pool), + ) + + +async def _drain_the_catalog(pool: asyncpg.Pool, kernel: Kernel) -> None: + """main.py's own three lines, verbatim in shape.""" + registry = ProjectionRegistry() + register_agent_projections(registry, kernel) + await drain_projections(pool, registry, deadline_seconds=5.0) + + +async def test_seeding_the_catalog_then_draining_admits_an_llm_brained_agent( + chain_pool: asyncpg.Pool, +) -> None: + """The whole sequence, end to end: after the catalog is seeded and + drained, the gate reading the real projection admits the shipped + agent whose brain is a real Anthropic model. + + This one passes under the always-approved stub too, so on its own it + would not prove the gate was consulted at all. What makes it mean + something is the test below: the same helper's kernel REFUSES when the + read model is empty, so the admission here is the catalog answering + rather than a stub agreeing. Read the two as a pair. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + + await seed_language_models(kernel) + await _drain_the_catalog(chain_pool, kernel) + await seed_caution_drafter_agent(kernel) + + agent = await load_agent(kernel.event_store, CAUTION_DRAFTER_AGENT_ID) + assert agent is not None, "the gate refused an agent whose model the catalog approves" + + +async def test_the_drain_is_what_makes_the_approval_visible( + chain_pool: asyncpg.Pool, +) -> None: + """The join the static guards cannot assert. + + Same sequence with the drain removed. The catalog streams exist, so + every event the approval is derived from is already written; only the + read model the gate consults is unpopulated. The gate must refuse, + which is what makes the drain load-bearing rather than incidental. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + + await seed_language_models(kernel) + + with pytest.raises(LanguageModelNotApprovedError): + await seed_caution_drafter_agent(kernel) + + +async def test_the_drain_fills_the_read_model_the_gate_reads( + chain_pool: asyncpg.Pool, +) -> None: + """The middle join on its own, so a failure says WHICH link broke. + + The test above proves the gate refuses without a drain; this one + proves the drain is what fills the specific identity the gate will + ask for, rather than merely writing some rows. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + lookup = PostgresLanguageModelLookup(chain_pool) + + await seed_language_models(kernel) + assert ( + await lookup.find_by_model( + provider=DEFAULT_CAUTION_DRAFTER_MODEL.provider, + model=DEFAULT_CAUTION_DRAFTER_MODEL.model, + ) + is None + ), "the read model answered before any drain ran" + + await _drain_the_catalog(chain_pool, kernel) + + entry = await lookup.find_by_model( + provider=DEFAULT_CAUTION_DRAFTER_MODEL.provider, + model=DEFAULT_CAUTION_DRAFTER_MODEL.model, + ) + assert entry is not None, "the drain did not reach proj_agent_language_model_summary" + assert entry.status == "Approved"