diff --git a/apps/api/src/cora/infrastructure/deps.py b/apps/api/src/cora/infrastructure/deps.py index 298394ca2a3..5535d4d3c95 100644 --- a/apps/api/src/cora/infrastructure/deps.py +++ b/apps/api/src/cora/infrastructure/deps.py @@ -337,7 +337,11 @@ def make_postgres_kernel( a catalog keep the define_agent gate disarmed. Production's `build_kernel` injects the real `PostgresLanguageModelLookup` via the `language_model_lookup_factory` argument; gate-specific tests - override here explicitly. + override here explicitly. Like the financial pair, the permissive + default here is only the first half of the posture: the fail-loud + requirement lives one layer up, in `build_kernel`'s Postgres branch, + which raises when the factory is missing so a real deployment can + never silently answer every identity Approved. `model_usage_lookup` defaults to `AlwaysEmptyModelUsageLookup` (no recorded call touched any model) so tests that don't exercise the @@ -1227,6 +1231,21 @@ async def build_kernel( "financial lookup would silently meter zero and disarm the " "spend envelope" ) + if language_model_lookup_factory is None: + # Same layer, same reason, different stake. The always-approved stub + # answers every identity with an Approved entry, so falling back to it + # in a Postgres deployment does not weaken the model-approval gate, it + # removes it: define_agent and seed_agent both keep passing, and an + # agent can be registered on a model the facility never approved with + # nothing anywhere recording that no catalog was consulted. The gate is + # how the provenance constraint is enforced at all, so its absence must + # stop startup rather than read as agreement. + raise ValueError( + "build_kernel requires language_model_lookup_factory in a Postgres " + "deployment; the always-approved stub would answer every identity " + "Approved, silently disarming the model-approval gate that " + "define_agent and seed_agent both rely on" + ) pool = await create_pool( settings.database_url, min_size=settings.db_pool_min_size, @@ -1283,11 +1302,8 @@ async def build_kernel( ) # Non-None guaranteed by the fail-loud guard above. spend_lookup: SpendLookup = spend_lookup_factory(pool) - language_model_lookup: LanguageModelLookup = ( - language_model_lookup_factory(pool) - if language_model_lookup_factory is not None - else AlwaysApprovedLanguageModelLookup() - ) + # Non-None guaranteed by the fail-loud guard above. + language_model_lookup: LanguageModelLookup = language_model_lookup_factory(pool) model_usage_lookup: ModelUsageLookup = ( model_usage_lookup_factory(pool) if model_usage_lookup_factory is not None diff --git a/apps/api/src/cora/infrastructure/kernel.py b/apps/api/src/cora/infrastructure/kernel.py index 1245322ee8c..430bb333178 100644 --- a/apps/api/src/cora/infrastructure/kernel.py +++ b/apps/api/src/cora/infrastructure/kernel.py @@ -187,9 +187,12 @@ class Kernel: `PostgresLanguageModelLookup` as the production adapter (reads `proj_agent_language_model_summary`). Defaults to `AlwaysApprovedLanguageModelLookup` (every identity - Approved) so tests and catalog-less deployments keep the - pre-catalog behavior; standing up a real catalog is what arms the - gate. Mirrors the `spend_lookup` opt-in posture. + Approved) so tests keep the pre-catalog behavior. Mirrors the + `spend_lookup` posture, BOTH halves of it: permissive at this + layer, and required one layer up, where `build_kernel` refuses a + Postgres deployment that did not bind the real adapter. Reading + only the first half is what left this field out of that guard + while `define_agent` and `seed_agent` were both gated on it. `model_usage_lookup`: cross-BC port consumed by Agent BC's `list_at_risk_results` read slice to enumerate the Decisions whose @@ -476,11 +479,11 @@ class Kernel: default_factory=AlwaysApprovedLanguageModelLookup ) """Resolve a model identity (provider + model) to its catalog entry. - Defaults to the always-approved stub so tests and deployments without - a catalog keep the pre-catalog `define_agent` behavior; the - composition root binds the Agent BC's `PostgresLanguageModelLookup` - over `proj_agent_language_model_summary` when a pool exists, arming - the Approved-entry gate.""" + Defaults to the always-approved stub so tests keep the pre-catalog + `define_agent` behavior; the composition root binds the Agent BC's + `PostgresLanguageModelLookup` over `proj_agent_language_model_summary`, + arming the Approved-entry gate, and refuses to build a Postgres kernel + without it.""" model_usage_lookup: ModelUsageLookup = field(default_factory=AlwaysEmptyModelUsageLookup) """Enumerate the Decisions whose recorded LLM calls touched one diff --git a/apps/api/src/cora/infrastructure/ports/language_model_lookup.py b/apps/api/src/cora/infrastructure/ports/language_model_lookup.py index 688df8ad095..15d2daf3026 100644 --- a/apps/api/src/cora/infrastructure/ports/language_model_lookup.py +++ b/apps/api/src/cora/infrastructure/ports/language_model_lookup.py @@ -21,13 +21,18 @@ ## Failure direction -The kernel default is the always-approved stub, so tests and -deployments that have not stood up a catalog are unaffected (the same -opt-in posture as every lookup in the family: declaring a catalog is -what arms the gate). The Postgres adapter answers with the newest -APPROVED entry only; None means "nothing currently approved for this -identity" (never cataloged, or every entry for it is Defined or -terminal), which the gate treats as refusal once a catalog exists. +The kernel default is the always-approved stub, so tests that have not +stood up a catalog are unaffected. That permissiveness is scoped to the +kernel layer only: `build_kernel` refuses a Postgres deployment that did +not bind the real adapter, because the stub does not weaken this gate, +it removes it. Silence from a disarmed gate is indistinguishable from +approval, and the gate is the only place the facility's model-provenance +constraint is enforced. + +The Postgres adapter answers with the newest APPROVED entry only; None +means "nothing currently approved for this identity" (never cataloged, +or every entry for it is Defined or terminal), which the gate treats as +refusal. """ from dataclasses import dataclass diff --git a/apps/api/tests/integration/_helpers.py b/apps/api/tests/integration/_helpers.py index 6772204e102..2360d9a4e0a 100644 --- a/apps/api/tests/integration/_helpers.py +++ b/apps/api/tests/integration/_helpers.py @@ -63,6 +63,7 @@ FixedIdGenerator, IdempotencyStore, IdGenerator, + LanguageModelLookup, ProfileStore, RoleLookup, RunActorInvolvementLookup, @@ -97,6 +98,7 @@ def build_postgres_deps( run_actor_involvement_lookup: RunActorInvolvementLookup | None = None, consequence_lookup: ConsequenceLookup | None = None, profile_store: ProfileStore | None = None, + language_model_lookup: LanguageModelLookup | None = None, llm: LLM | None = None, ) -> Kernel: """Build a Kernel for integration-test handler invocation against real Postgres. @@ -140,6 +142,7 @@ def build_postgres_deps( run_actor_involvement_lookup=run_actor_involvement_lookup, consequence_lookup=consequence_lookup, profile_store=profile_store, + language_model_lookup=language_model_lookup, llm=llm, ) diff --git a/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py b/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py new file mode 100644 index 00000000000..8f76f799870 --- /dev/null +++ b/apps/api/tests/integration/test_language_model_catalog_chain_postgres.py @@ -0,0 +1,176 @@ +"""The model-approval chain, joined up, against real Postgres. + +Seeding a LanguageModel-brained Agent reaches its verdict through five +links: `seed_language_models` appends the catalog streams, the projection +turns those events into `proj_agent_language_model_summary` rows, +`drain_projections` advances the bookmark so the rows are visible, +`PostgresLanguageModelLookup` reads them, and `seed_agent`'s gate consumes +the answer. Every link already has a test. None of them crosses a join. + +The clearest case is the adapter's own integration test, which populates +the read model by hand-written INSERT. That proves the SQL reads a correct +table; it cannot see whether anything fills it. And `drain_projections` +covering THIS projection is asserted nowhere at all: the two architecture +guards read `main.py` and assert about the order of calls in `main.py`, so +they can prove a drain sits between the two seeds and not that the drain +reaches the table the gate will read. + +So this file asserts the joins rather than the links, reproducing the +composition root's real sequence (main.py's `seed_language_models` -> +`register_agent_projections` -> `drain_projections` -> `seed_*_agent`) +with the REAL `PostgresLanguageModelLookup` bound, which is the binding +`build_postgres_deps` otherwise leaves at the always-approved stub. + +CautionDrafter is the subject because it is one of the two shipped agents +that declare a real Anthropic model, so it is one of the two whose boot a +broken chain actually stops. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import uuid4 + +import asyncpg +import pytest +import pytest_asyncio +from testcontainers.postgres import PostgresContainer + +from cora.agent import register_agent_projections +from cora.agent.adapters import PostgresLanguageModelLookup +from cora.agent.aggregates.agent.read import load_agent +from cora.agent.aggregates.language_model.state import LanguageModelNotApprovedError +from cora.agent.prompts.caution_drafter import DEFAULT_CAUTION_DRAFTER_MODEL +from cora.agent.seed_caution_drafter import ( + CAUTION_DRAFTER_AGENT_ID, + seed_caution_drafter_agent, +) +from cora.agent.seed_language_models import seed_language_models +from cora.infrastructure.kernel import Kernel +from cora.infrastructure.postgres.pool import create_pool +from cora.infrastructure.projection.drain import drain_projections +from cora.infrastructure.projection.registry import ProjectionRegistry +from tests._postgres import normalize_async_url +from tests.integration._helpers import build_postgres_deps + +pytestmark = pytest.mark.integration + +_NOW = datetime(2026, 9, 6, tzinfo=UTC) + + +@pytest_asyncio.fixture +async def chain_pool( + postgres_container: PostgresContainer, + template_database: str, +): + """A per-test database, because the sequence under test writes the + catalog streams and the read model the next link reads back.""" + test_db = f"lmchain_{uuid4().hex[:12]}" + admin_url = normalize_async_url(postgres_container.get_connection_url(), database="postgres") + admin = await asyncpg.connect(admin_url) + try: + await admin.execute(f'CREATE DATABASE "{test_db}" TEMPLATE "{template_database}"') + finally: + await admin.close() + + test_url = normalize_async_url(postgres_container.get_connection_url(), database=test_db) + pool = await create_pool(test_url, min_size=1, max_size=4) + try: + yield pool + finally: + await pool.close() + admin = await asyncpg.connect(admin_url) + try: + await admin.execute(f'DROP DATABASE "{test_db}"') + finally: + await admin.close() + + +def _kernel_with_the_real_lookup(pool: asyncpg.Pool) -> Kernel: + """The gate's production binding, which every other test leaves as + the always-approved stub. Without this the assertions below pass on + any database, chain or no chain.""" + return build_postgres_deps( + pool, + now=_NOW, + language_model_lookup=PostgresLanguageModelLookup(pool), + ) + + +async def _drain_the_catalog(pool: asyncpg.Pool, kernel: Kernel) -> None: + """main.py's own three lines, verbatim in shape.""" + registry = ProjectionRegistry() + register_agent_projections(registry, kernel) + await drain_projections(pool, registry, deadline_seconds=5.0) + + +async def test_seeding_the_catalog_then_draining_admits_an_llm_brained_agent( + chain_pool: asyncpg.Pool, +) -> None: + """The whole sequence, end to end: after the catalog is seeded and + drained, the gate reading the real projection admits the shipped + agent whose brain is a real Anthropic model. + + This one passes under the always-approved stub too, so on its own it + would not prove the gate was consulted at all. What makes it mean + something is the test below: the same helper's kernel REFUSES when the + read model is empty, so the admission here is the catalog answering + rather than a stub agreeing. Read the two as a pair. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + + await seed_language_models(kernel) + await _drain_the_catalog(chain_pool, kernel) + await seed_caution_drafter_agent(kernel) + + agent = await load_agent(kernel.event_store, CAUTION_DRAFTER_AGENT_ID) + assert agent is not None, "the gate refused an agent whose model the catalog approves" + + +async def test_the_drain_is_what_makes_the_approval_visible( + chain_pool: asyncpg.Pool, +) -> None: + """The join the static guards cannot assert. + + Same sequence with the drain removed. The catalog streams exist, so + every event the approval is derived from is already written; only the + read model the gate consults is unpopulated. The gate must refuse, + which is what makes the drain load-bearing rather than incidental. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + + await seed_language_models(kernel) + + with pytest.raises(LanguageModelNotApprovedError): + await seed_caution_drafter_agent(kernel) + + +async def test_the_drain_fills_the_read_model_the_gate_reads( + chain_pool: asyncpg.Pool, +) -> None: + """The middle join on its own, so a failure says WHICH link broke. + + The test above proves the gate refuses without a drain; this one + proves the drain is what fills the specific identity the gate will + ask for, rather than merely writing some rows. + """ + kernel = _kernel_with_the_real_lookup(chain_pool) + lookup = PostgresLanguageModelLookup(chain_pool) + + await seed_language_models(kernel) + assert ( + await lookup.find_by_model( + provider=DEFAULT_CAUTION_DRAFTER_MODEL.provider, + model=DEFAULT_CAUTION_DRAFTER_MODEL.model, + ) + is None + ), "the read model answered before any drain ran" + + await _drain_the_catalog(chain_pool, kernel) + + entry = await lookup.find_by_model( + provider=DEFAULT_CAUTION_DRAFTER_MODEL.provider, + model=DEFAULT_CAUTION_DRAFTER_MODEL.model, + ) + assert entry is not None, "the drain did not reach proj_agent_language_model_summary" + assert entry.status == "Approved" diff --git a/apps/api/tests/unit/test_deps.py b/apps/api/tests/unit/test_deps.py index c43956970a0..d01ed6bd528 100644 --- a/apps/api/tests/unit/test_deps.py +++ b/apps/api/tests/unit/test_deps.py @@ -22,7 +22,12 @@ from cora.infrastructure.adapters.in_memory_idempotency_store import InMemoryIdempotencyStore from cora.infrastructure.config import Settings from cora.infrastructure.deps import build_kernel -from cora.infrastructure.ports import AllowAllAuthorize, FakeLLM +from cora.infrastructure.ports import ( + AllowAllAuthorize, + AlwaysZeroSpendLookup, + FakeLLM, + NoActiveAllocationLookup, +) from cora.trust import build_authorize from cora.trust.authorize import TrustAuthorize @@ -39,6 +44,27 @@ async def test_build_kernel_refuses_a_postgres_deployment_without_financial_look await build_kernel(authorize_factory=build_authorize, settings=settings) +@pytest.mark.unit +async def test_build_kernel_refuses_a_postgres_deployment_without_the_model_catalog_lookup() -> ( + None +): + """A missing catalog binding does not weaken the model-approval gate, + it removes it: `AlwaysApprovedLanguageModelLookup` answers every + identity Approved, so `define_agent` and `seed_agent` both keep + passing and nothing records that no catalog was consulted. Financial + factories are supplied here so the guard under test is the one that + fires, not the financial one above it.""" + settings = Settings(app_env="production") # type: ignore[call-arg] + + with pytest.raises(ValueError, match="language_model_lookup_factory"): + await build_kernel( + authorize_factory=build_authorize, + settings=settings, + spend_lookup_factory=lambda pool: AlwaysZeroSpendLookup(), + allocation_lookup_factory=lambda pool: NoActiveAllocationLookup(), + ) + + @pytest.mark.unit async def test_build_kernel_uses_in_memory_stores_in_test_env( monkeypatch: pytest.MonkeyPatch,