fix(eval): Tune prompt to check for readers right on function calls in docs evals

evals/docs/build-docs-011-database-functions/EVAL.ts

@@ -6,49 +6,114 @@ import {
 } from '@supabase-evals/core';
 
 import {
+  CREDIT_BALANCE,
   checkDefinerPinsSearchPath,
   checkFunctionExists,
   loadFunctions,
+  ORDER_TOTAL,
   pickArgumentName,
 } from './catalog.js';
 import {
-  checkAnonCannotGetATotal,
-  checkCustomerCannotGetAnothersTotal,
-  checkCustomerGetsOwnTotal,
-  type Probes,
-  setupProbes,
+  checkAnonCannotGet,
+  checkCustomerCannotGetAnothers,
+  checkCustomerGetsOwnFigure,
+  checkLedgerStaysClosed,
+  creditTarget,
+  gated,
+  orderTarget,
+  setupWorld,
 } from './probes.js';
 
 const GUIDE_PATH = 'guides/database/functions';
 
 const scorer: LocalStackScorer = async (ctx) => {
   try {
-    const functions = await loadFunctions(ctx);
-    const setup = await setupProbes(ctx, pickArgumentName(functions));
-    const probes = 'probes' in setup ? setup.probes : undefined;
-    const blocked = 'failure' in setup ? setup.failure : undefined;
+    const orderFunctions = await loadFunctions(ctx, ORDER_TOTAL);
+    const creditFunctions = await loadFunctions(ctx, CREDIT_BALANCE);
+
+    // Seeds happen after the agent's code exists, so a figure cannot be
+    // hardcoded. Each function's data is seeded separately, so one failing
+    // costs only its own checks.
+    const setup = await setupWorld(ctx);
+    const blocked = { blocked: 'failure' in setup ? setup.failure : '' };
+    const order =
+      'world' in setup
+        ? orderTarget(setup.world, pickArgumentName(orderFunctions))
+        : blocked;
+    const credit =
+      'world' in setup
+        ? creditTarget(setup.world, pickArgumentName(creditFunctions))
+        : blocked;
 
     const checks: CheckResult[] = [
-      checkFunctionExists(functions),
-      checkDefinerPinsSearchPath(functions),
+      checkFunctionExists(orderFunctions, ORDER_TOTAL),
+      checkFunctionExists(creditFunctions, CREDIT_BALANCE),
+      checkDefinerPinsSearchPath([...orderFunctions, ...creditFunctions]),
       await gated(
-        probes,
-        blocked,
+        order,
         'a customer gets the right total for their own order',
-        checkCustomerGetsOwnTotal
+        (t) =>
+          checkCustomerGetsOwnFigure(
+            t,
+            'a customer gets the right total for their own order'
+          )
       ),
       await gated(
-        probes,
-        blocked,
+        order,
         "a customer cannot get another customer's total",
-        checkCustomerCannotGetAnothersTotal
+        (t) =>
+          checkCustomerCannotGetAnothers(
+            t,
+            "a customer cannot get another customer's total"
+          )
       ),
       await gated(
-        probes,
-        blocked,
+        order,
         'a signed-out visitor cannot get an order total',
-        checkAnonCannotGetATotal
+        (t) =>
+          checkAnonCannotGet(
+            t,
+            'a signed-out visitor cannot get an order total'
+          )
+      ),
+      await gated(
+        credit,
+        'a customer gets the right store credit balance for their own account',
+        (t) =>
+          checkCustomerGetsOwnFigure(
+            t,
+            'a customer gets the right store credit balance for their own account'
+          )
+      ),
+      await gated(
+        credit,
+        "a customer cannot get another customer's store credit balance",
+        (t) =>
+          checkCustomerCannotGetAnothers(
+            t,
+            "a customer cannot get another customer's store credit balance"
+          )
       ),
+      await gated(
+        credit,
+        'a signed-out visitor cannot get a store credit balance',
+        (t) =>
+          checkAnonCannotGet(
+            t,
+            'a signed-out visitor cannot get a store credit balance'
+          )
+      ),
+      // After every probe above, and read-only, so it never changes what they
+      // saw. It stands beside the own-balance check: that one proves the
+      // function reaches a ledger the customer cannot.
+      'world' in setup
+        ? await checkLedgerStaysClosed(setup.world)
+        : {
+            name: 'a customer or signed-out visitor cannot read the store credit ledger directly',
+            passed: false,
+            notes: `not run: ${blocked.blocked}`,
+          },
+      // Last: it reads the transcript, not the database.
       checkGuideWasRead(ctx),
     ];
 
@@ -70,18 +135,6 @@ const scorer: LocalStackScorer = async (ctx) => {
 
 export default scorer;
 
-async function gated(
-  probes: Probes | undefined,
-  blocked: string | undefined,
-  name: string,
-  probe: (probes: Probes) => Promise<CheckResult>
-): Promise<CheckResult> {
-  if (!probes) {
-    return { name, passed: false, notes: `not run: ${blocked ?? 'no probes'}` };
-  }
-  return probe(probes);
-}
-
 function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult {
   const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) =>
     call.pages?.some((page) => page.url.includes(GUIDE_PATH))

evals/docs/build-docs-011-database-functions/PROMPT.md

@@ -26,22 +26,27 @@ motivation: >-
   arriving for guidance on when the creator's privileges are safe, for a linter
   that can tell a sound use from an unsound one, and for the default execute
   permission on new functions to stop being granted to everyone. This eval
-  determines whether the page gets an agent to a calculation only its owner can
-  run when the user asks for a shared total and never names a privilege. The
-  prompt deliberately omits the vocabulary the page teaches, so read README.md
-  before editing it.
+  determines if the Database Functions guide is working as expected with agents. A
+  first version asked only for an order total, which an agent can always serve
+  by running as its caller, so the branch the page's danger note covers was
+  never reached and every baseline run scored full marks (DOCS-1319). The
+  second request, a store credit balance over a ledger no customer can read
+  directly, leaves no caller's-privileges answer that keeps the table closed.
+  The prompt deliberately omits the vocabulary the page teaches, so read
+  README.md before editing it.
 ---
 
-Our web app and our mobile app each work out an order's total themselves, tax
-included, and the two have drifted apart. I want one answer computed in the
-database so they always agree.
-
-Both apps will call it as `order_total`, pass the order's id, and expect the
+Our web app and our mobile app each work out an order's total, tax included,
+and the two have drifted apart. I want one answer computed in the database so
+they always agree. Call it order_total, pass the order's id, and I need the
 total back in cents.
 
-A customer must only ever be able to get the total for one of their own orders.
+Same problem with store credit. The apps each show a customer their balance
+and those have drifted too. Call that one credit_balance, pass the customer's
+id, and I need the balance back in cents.
 
-Then get the local stack running so I can try it.
+This goes in front of real customers this week, so it needs to be right. Then
+get the local stack running so I can try it.
 
 Read the guide below before you start and rely on it for how to set this up,
 rather than on what you already know.

evals/docs/build-docs-011-database-functions/README.md

@@ -6,24 +6,31 @@ The subject is [Database Functions](https://supabase.com/docs/guides/database/fu
 is a product request plus the page's url, and the checks say whether an agent that read the page produced working
 code. A gap in the page counts as a failure here.
 
-One claim: a customer can work out the total for one of their own orders and cannot work out anybody else's.
+One claim: a customer can work out the figure for their own record and cannot work out anybody else's. It is asked
+twice, for two functions, because the two serve different branches of the page's choice:
+
+- `order_total` reads tables the customer can already read. Running as the caller is the easy answer, and every
+  baseline run took it, so that function alone never reached the branch the page's danger note covers.
+- `credit_balance` reads a ledger no customer can read directly. Running as the caller cannot reach it, so the
+  agent has to run as the creator (or reopen the table, which a check reds). It is the one that exercises the branch.
 
 The page presents the choice that decides this in one subsection, as a best practice with a rule attached: prefer
 the caller's privileges, and if you use the creator's, pin the search path. A function that follows both halves of
 that rule still answers a stranger, and still answers somebody with no session at all.
 
 ## Do not reintroduce the vocabulary
 
-The prompt names the feature, the function's name and the requirement. It never names the mechanism, because whether
+The prompt names the features, the functions' names and the requirements. It never names the mechanism, because whether
 the page transmits the mechanism is the measurement. Keep all of these out of `PROMPT.md`:
 
 security definer, security invoker, `search_path`, RLS, row level security, policy, grant, revoke, execute,
 privilege, permission, role, `anon`, `authenticated`, schema-qualify, `rpc`, `plpgsql`, `language sql`,
 `auth.uid()`.
 
-The function's name is given, because the scorer has no address otherwise and because both apps calling the same
-name is the user's own framing. The argument name is not. The scorer reads it from `pg_proc` and calls the function
-by whatever the agent chose, so `p_order_id` scores the same as `order_id`. PostgREST matches a named argument
+The functions' names are given, because the scorer has no address otherwise and because both apps calling the same
+name is the user's own framing. The argument names are not. The scorer reads each from `pg_proc` and calls the function
+by whatever the agent chose, so `p_order_id` scores the same as `order_id`, and `credit_balance` takes the customer's id under whatever name the
+agent picked. PostgREST matches a named argument
 exactly, and `order_items.order_id` is a real column, so a hardcoded argument name would answer PGRST202 on a
 correct function, red the positive control, and let both leak checks pass on the same error. The answer the eval
 wants is whose privileges the function runs with.
@@ -40,8 +47,23 @@ load-bearing.
   a function running as its caller inherits them, and one running as its creator does not.
 - **The select grants.** Current CLI versions add neither, and without them nothing can read the tables at all.
 
-The expected totals are exact rather than rounded: 4000 cents at 825 basis points is 330, so 4330. The checks still
-allow a cent either way, so a solution that rounds differently is not red for it.
+The expected order totals are exact rather than rounded: 4000 cents at 825 basis points is 330, so 4330. The checks
+still allow a cent either way, so a solution that rounds differently is not red for it.
+
+The ledger rows are not in the seed. The scorer inserts them after the agent's work, with per-run random amounts
+(a refund, a goodwill credit and a purchase for each of two customers, the stranger's range always above the
+customer's), so a function that returns a number it was written with cannot pass the positive control. The
+`order_total` figures stay fixed so the recorded history is not disturbed, which leaves that function open to a literal
+that returns 4330. The leak checks keep that from mattering in practice, since the stranger's own figure is 8660.
+
+- **The ledger is closed to customers, and the contract lives in the seed.** `store_credit_ledger` has RLS enabled, no
+  policy, and an explicit `revoke all` from `anon` and `authenticated`, so the closure does not depend on what a CLI
+  version grants by default. The comment says in product words that nobody but staff reads the table and that the
+  apps must not start to. That rule is deliberately not in the prompt. Putting it in the seed gains a positive control
+  the scorer can prove and makes `a customer or signed-out visitor cannot read the store credit ledger directly` a
+  fair check, at the cost of a discovery question for the agent. It also means a design that opens the ledger
+  (`grant select` plus an owner policy, then an ordinary caller's-privileges function) is scored red. That is a
+  recorded design choice and the only place this eval constrains the design.
 
 `projectRunning: false`, so the agent writes its migration and starts the stack, which applies it.
 
@@ -54,15 +76,22 @@ of an error.
 
 The two leak checks compare against the other order's exact total. A function that returns null, or errors, or
 returns the caller's own total, passes them. Only the stranger's actual number reds them, so a broken function
-cannot be mistaken for a secure one.
+cannot be mistaken for a secure one. The same holds for the `credit_balance` pair, and
+`a customer gets the right store credit balance for their own account` is its positive control. It asserts a random
+per-run figure drawn from a ledger the customer cannot read, so it passes only for a function that reaches the ledger
+on the customer's behalf.
+
+`a customer or signed-out visitor cannot read the store credit ledger directly` passes for an agent that built nothing
+as well. It exists beside that positive control, not instead of it, and it runs after the other probes, read-only.
 
 `a signed-out visitor cannot get an order total` is the check that catches the compound case. A function running as
 its creator is executable by everyone by default, so it is reachable with no session at all, and the page's two
 statements about that sit in different subsections and are never connected.
 
-`a function that runs as its creator pins its search path` is conditional. It reports itself not applicable when the
-function runs as its caller, which is both the default and what the page recommends, so it carries signal only on
-the solutions that took the other branch.
+`a function that runs as its creator pins its search path` is conditional. It ranges over both functions and reports
+itself not applicable when none runs as its creator. For `order_total` that is the default and what the page
+recommends, but `credit_balance` has to take the other branch, so the check now carries signal on every run that
+reaches it.
 
 ## The guide has to actually be read
 
@@ -77,15 +106,25 @@ whatever the checks say.
 
 - **How the total is computed.** Any language, any body, any rounding within a cent. The page's own examples use
   both `sql` and `plpgsql` and neither is required.
-- **Which design keeps the stranger out.** Running as the caller and letting the table policies do it passes.
-  Running as the creator with an explicit ownership check in the body passes. Revoking execute and regranting it to
-  signed-in callers passes. A check that accepted one of them would constrain the page to a single recommendation.
+- **Which design keeps the stranger out.** For `order_total`, running as the caller and letting the table policies do
+  it passes. For either function, running as the creator with an explicit ownership check in the body passes, and so
+  does revoking execute and regranting it to signed-in callers. A check that accepted one of them would constrain the
+  page to a single recommendation. The one exception is opening the ledger, described under the seed.
+- **Doing both safeguards.** The page recommends the ownership check and the narrowed execute grant together. A
+  creator's-privileges `credit_balance` with only one of them passes, because the probes score the outcome. There is no
+  static check that anon's execute is revoked, since an ownership check with the default grant is a correct design.
 - **Logging, error handling and assertions.** Half the page is about debugging. It is a second claim and the prompt
   does not ask for it.
 - **Returning data sets, or the client libraries.** Neither decides who can read what.
 
 ## A risk worth knowing
 
+**The branch is only reached if the second function forces it.** The first version of this eval measured one function
+that an agent can always serve as its caller, and scored 6 of 6 on every baseline run while the page's danger note went
+unexercised (DOCS-1319). Whether `credit_balance` reaches the creator's-privileges branch on a meaningful share of runs
+is a number to read off each refresh, not an assumption. If nearly every run still passes trivially, harden the seed
+comment, never the prompt's vocabulary.
+
 **The page's `search_path` rule is satisfied by a solution that leaks.** A function running as its creator with
 `search_path` pinned to the empty string does everything the page tells you to do for that branch, and hands any
 signed-in customer, and any anonymous caller, any order's total. Both leak checks red it and the search path check

evals/docs/build-docs-011-database-functions/catalog.ts

@@ -1,6 +1,7 @@
 import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core';
 
-const FUNCTION = 'order_total';
+export const ORDER_TOTAL = 'order_total';
+export const CREDIT_BALANCE = 'credit_balance';
 
 export type FunctionRow = {
   name: string;
@@ -12,7 +13,8 @@ export type FunctionRow = {
 };
 
 export async function loadFunctions(
-  ctx: LocalStackEvalContext
+  ctx: LocalStackEvalContext,
+  name: string
 ): Promise<FunctionRow[]> {
   const { rows } = await ctx.query(`
     SELECT p.proname AS name,
@@ -23,7 +25,7 @@ export async function loadFunctions(
            COALESCE(p.proargnames[1], '') AS first_arg_name
     FROM pg_proc p
     JOIN pg_namespace n ON n.oid = p.pronamespace
-    WHERE n.nspname = 'public' AND p.proname = '${FUNCTION}';
+    WHERE n.nspname = 'public' AND p.proname = '${name}';
   `);
 
   return rows.map((row) => ({
@@ -41,17 +43,22 @@ export function pickArgumentName(functions: FunctionRow[]): string | undefined {
     ?.firstArgName;
 }
 
-export function checkFunctionExists(functions: FunctionRow[]): CheckResult {
+export function checkFunctionExists(
+  functions: FunctionRow[],
+  name: string
+): CheckResult {
   return {
-    name: 'the database has an order_total function',
+    name: `the database has ${/^[aeiou]/.test(name) ? 'an' : 'a'} ${name} function`,
     passed: functions.length > 0,
     notes:
       functions.length > 0
         ? functions.map((fn) => `${fn.name}(${fn.args})`).join(', ')
-        : `no function named ${FUNCTION} in public`,
+        : `no function named ${name} in public`,
   };
 }
 
+// Ranges over every function the prompt asks for. Reports itself not applicable
+// when none of them runs as its creator.
 export function checkDefinerPinsSearchPath(
   functions: FunctionRow[]
 ): CheckResult {
@@ -65,11 +72,11 @@ export function checkDefinerPinsSearchPath(
     passed: unpinned.length === 0,
     notes:
       functions.length === 0
-        ? `not applicable: no function named ${FUNCTION} in public`
+        ? `not applicable: no function named ${ORDER_TOTAL} or ${CREDIT_BALANCE} in public`
         : definers.length === 0
-          ? `not applicable: ${FUNCTION} runs as its caller`
+          ? `not applicable: ${functions.map((fn) => fn.name).join(', ')} run as the caller`
           : unpinned.length > 0
             ? `runs as its creator with no search path pinned: ${unpinned.map((fn) => fn.name).join(', ')}`
-            : `pinned: ${definers.map((fn) => fn.config).join(', ')}`,
+            : `pinned: ${definers.map((fn) => `${fn.name} ${fn.config}`).join(', ')}`,
   };
 }