diff --git a/apps/web/src/data/docs-eval-results.json b/apps/web/src/data/docs-eval-results.json index 2425db0fe..fbf599d77 100644 --- a/apps/web/src/data/docs-eval-results.json +++ b/apps/web/src/data/docs-eval-results.json @@ -12207,7 +12207,6 @@ }, { "experiment": "codex-gpt-6-luna-no-skills", - "experimentSuite": "docs", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", @@ -12233,12 +12232,17 @@ { "name": "the database has an order_total function", "passed": true, - "notes": "order_total(order_id bigint)" + "notes": "order_total(p_order_id bigint)" + }, + { + "name": "the database has a credit_balance function", + "passed": true, + "notes": "credit_balance(p_customer_id uuid)" }, { "name": "a function that runs as its creator pins its search path", "passed": true, - "notes": "not applicable: order_total runs as its caller" + "notes": "pinned: credit_balance search_path=\"\"" }, { "name": "a customer gets the right total for their own order", @@ -12255,6 +12259,26 @@ "passed": true, "notes": "permission denied for function order_total" }, + { + "name": "a customer gets the right store credit balance for their own account", + "passed": true, + "notes": "4631 cents" + }, + { + "name": "a customer cannot get another customer's store credit balance", + "passed": true, + "notes": "got null" + }, + { + "name": "a signed-out visitor cannot get a store credit balance", + "passed": true, + "notes": "permission denied for function credit_balance" + }, + { + "name": "a customer or signed-out visitor cannot read the store credit ledger directly", + "passed": true, + "notes": "permission denied for table store_credit_ledger" + }, { "name": "the agent read the Database Functions guide the prompt referenced", "passed": true, @@ -12279,103 +12303,62 @@ }, { "source": "search_docs", - "query": "query { searchDocs(query: \"database functions create function security invoker auth.uid access control\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"database functions SQL security definer search_path grant execute\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/functions", "title": "Database functions" }, - { - "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0002_auth_users_exposed", - "title": "Database Advisor: Lint 0002_auth_users_exposed" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks", - "title": "Auth Hooks" - } - ], - "resultChars": 105508 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Functions security invoker RLS security definer auth.uid execute privileges\", limit: 8) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", - "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" - }, - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and data" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" + "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", + "title": "Foreign Data Wrappers" }, { - "url": "https://supabase.com/docs/guides/api/custom-claims-and-role-based-access-control-rbac", - "title": "Custom Claims & Role-based Access Control (RBAC)" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-hooks", + "title": "Configure Auth Hooks" }, { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" } ], - "resultChars": 161050 + "resultChars": 75992 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 645898, - "cacheReadInputTokens": 594484, - "cacheWriteInputTokens": 51206, - "outputTokens": 9460 + "inputTokens": 1470211, + "cacheReadInputTokens": 1403782, + "cacheWriteInputTokens": 62105, + "outputTokens": 22335 } ], - "stepCount": 23, - "toolCallCount": 21, - "agentRunDurationMs": 177405, + "stepCount": 43, + "toolCallCount": 28, + "agentRunDurationMs": 409767, "sandboxUsage": { - "activeCpuDurationMs": 309790, - "duration": 279481, + "activeCpuDurationMs": 301455, + "duration": 509998, "memory": 8192, "networkTransfer": { - "ingress": 1653999863, - "egress": 10794005 + "ingress": 1634359596, + "egress": 18114756 } }, - "prompt": "Our web app and our mobile app each work out an order's total themselves, tax\nincluded, and the two have drifted apart. I want one answer computed in the\ndatabase so they always agree.\n\nBoth apps will call it as `order_total`, pass the order's id, and expect the\ntotal back in cents.\n\nA customer must only ever be able to get the total for one of their own orders.\n\nThen get the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", + "prompt": "Our web app and our mobile app each work out an order's total, tax included,\nand the two have drifted apart. I want one answer computed in the database so\nthey always agree. Call it order_total, pass the order's id, and I need the\ntotal back in cents.\n\nSame problem with store credit. The apps each show a customer their balance\nand those have drifted too. Call that one credit_balance, pass the customer's\nid, and I need the balance back in cents.\n\nThis goes in front of real customers this week, so it needs to be right. Then\nget the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", "promptSourcePath": "evals/docs/build-docs-011-database-functions/PROMPT.md", "run": 1, - "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-1/result.json" + "sourcePath": "docs/codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-1/result.json" }, { "experiment": "codex-gpt-6-luna-no-skills", - "experimentSuite": "docs", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", @@ -12401,12 +12384,17 @@ { "name": "the database has an order_total function", "passed": true, - "notes": "order_total(order_id bigint)" + "notes": "order_total(p_order_id bigint)" + }, + { + "name": "the database has a credit_balance function", + "passed": true, + "notes": "credit_balance(p_customer_id uuid)" }, { "name": "a function that runs as its creator pins its search path", "passed": true, - "notes": "not applicable: order_total runs as its caller" + "notes": "pinned: credit_balance search_path=\"\"" }, { "name": "a customer gets the right total for their own order", @@ -12423,10 +12411,30 @@ "passed": true, "notes": "permission denied for function order_total" }, + { + "name": "a customer gets the right store credit balance for their own account", + "passed": true, + "notes": "5435 cents" + }, + { + "name": "a customer cannot get another customer's store credit balance", + "passed": true, + "notes": "got null: not authorized" + }, + { + "name": "a signed-out visitor cannot get a store credit balance", + "passed": true, + "notes": "permission denied for function credit_balance" + }, + { + "name": "a customer or signed-out visitor cannot read the store credit ledger directly", + "passed": true, + "notes": "permission denied for table store_credit_ledger" + }, { "name": "the agent read the Database Functions guide the prompt referenced", "passed": true, - "notes": "web_search, search_docs, web_search" + "notes": "web_search, search_docs" } ], "skills": { @@ -12445,108 +12453,64 @@ } ] }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/database/functions Supabase Database Functions create function SQL security definer", - "pages": [] - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Database Functions create function SQL invoker security definer grants Supabase\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"database functions create function SQL SECURITY INVOKER SECURITY DEFINER rpc grant execute\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content methodName } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks", - "title": "Auth Hooks" - }, { "url": "https://supabase.com/docs/guides/database/functions", "title": "Database functions" }, - { - "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", - "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" - }, { "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", "title": "Foreign Data Wrappers" }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 84148 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/database/functions", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/functions" - } - ] - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Database Functions invoke functions security invoker default Supabase postgres RLS grants execute\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", - "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, { "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0028_anon_security_definer_function_executable", "title": "Database Advisor: Lint 0028_anon_security_definer_function_executable" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", + "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" } ], - "resultChars": 68303 + "resultChars": 50324 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 661290, - "cacheReadInputTokens": 595559, - "cacheWriteInputTokens": 49626, - "outputTokens": 9507 + "inputTokens": 807022, + "cacheReadInputTokens": 751841, + "cacheWriteInputTokens": 50896, + "outputTokens": 12627 } ], - "stepCount": 24, + "stepCount": 29, "toolCallCount": 17, - "agentRunDurationMs": 185155, + "agentRunDurationMs": 263613, "sandboxUsage": { - "activeCpuDurationMs": 312756, - "duration": 288441, + "activeCpuDurationMs": 253551, + "duration": 359364, "memory": 8192, "networkTransfer": { - "ingress": 1654338452, - "egress": 11246066 + "ingress": 1631031753, + "egress": 12187411 } }, - "prompt": "Our web app and our mobile app each work out an order's total themselves, tax\nincluded, and the two have drifted apart. I want one answer computed in the\ndatabase so they always agree.\n\nBoth apps will call it as `order_total`, pass the order's id, and expect the\ntotal back in cents.\n\nA customer must only ever be able to get the total for one of their own orders.\n\nThen get the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", + "prompt": "Our web app and our mobile app each work out an order's total, tax included,\nand the two have drifted apart. I want one answer computed in the database so\nthey always agree. Call it order_total, pass the order's id, and I need the\ntotal back in cents.\n\nSame problem with store credit. The apps each show a customer their balance\nand those have drifted too. Call that one credit_balance, pass the customer's\nid, and I need the balance back in cents.\n\nThis goes in front of real customers this week, so it needs to be right. Then\nget the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", "promptSourcePath": "evals/docs/build-docs-011-database-functions/PROMPT.md", "run": 2, - "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-2/result.json" + "sourcePath": "docs/codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-2/result.json" }, { "experiment": "codex-gpt-6-luna-no-skills", - "experimentSuite": "docs", "experimentDisplay": { "agent": "codex", "modelProvider": "openai", @@ -12572,12 +12536,17 @@ { "name": "the database has an order_total function", "passed": true, - "notes": "order_total(order_id bigint)" + "notes": "order_total(p_order_id bigint)" + }, + { + "name": "the database has a credit_balance function", + "passed": true, + "notes": "credit_balance(p_customer_id uuid)" }, { "name": "a function that runs as its creator pins its search path", "passed": true, - "notes": "not applicable: order_total runs as its caller" + "notes": "pinned: credit_balance search_path=\"\"" }, { "name": "a customer gets the right total for their own order", @@ -12594,10 +12563,30 @@ "passed": true, "notes": "permission denied for function order_total" }, + { + "name": "a customer gets the right store credit balance for their own account", + "passed": true, + "notes": "5683 cents" + }, + { + "name": "a customer cannot get another customer's store credit balance", + "passed": true, + "notes": "got null" + }, + { + "name": "a signed-out visitor cannot get a store credit balance", + "passed": true, + "notes": "permission denied for function credit_balance" + }, + { + "name": "a customer or signed-out visitor cannot read the store credit ledger directly", + "passed": true, + "notes": "permission denied for table store_credit_ledger" + }, { "name": "the agent read the Database Functions guide the prompt referenced", "passed": true, - "notes": "web_search, search_docs" + "notes": "web_search, search_docs, search_docs" } ], "skills": { @@ -12616,194 +12605,89 @@ } ] }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/database/functions Supabase Database Functions SQL functions security invoker", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"database functions PostgreSQL create function security invoker Supabase RPC\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "{ searchDocs(query: \"Postgres database functions SQL create function RPC security invoker grants search_path\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/functions", "title": "Database functions" }, - { - "url": "https://supabase.com/docs/guides/database/functions#passing-parameters", - "title": "Passing parameters" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#suggestions", - "title": "Suggestions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#database-functions-vs-edge-functions", - "title": "Database Functions vs Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#security-definer-vs-invoker", - "title": "Security definer vs invoker" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#function-privileges", - "title": "Function privileges" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#debugging-functions", - "title": "Debugging functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#general-logging", - "title": "General logging" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#error-handling", - "title": "Error handling" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#advanced-logging", - "title": "Advanced logging" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#deep-dive", - "title": "Deep dive" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#create-database-functions", - "title": "Create Database Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#call-database-functions-using-javascript", - "title": "Call Database Functions using JavaScript" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#using-database-functions-to-call-an-external-api", - "title": "Using Database Functions to call an external API" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#quick-demo", - "title": "Quick demo" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#getting-started", - "title": "Getting started" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#simple-functions", - "title": "Simple functions" - }, - { - "url": "https://supabase.com/docs/guides/database/functions#returning-data-sets", - "title": "Returning data sets" - }, { "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview", "title": "Foreign Data Wrappers" }, { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#security", - "title": "Security" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#concepts", - "title": "Concepts" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#remote-servers", - "title": "Remote servers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#foreign-tables", - "title": "Foreign tables" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#etl-with-wrappers", - "title": "ETL with Wrappers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#on-demand-etl-with-wrappers", - "title": "On-demand ETL with Wrappers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#batch-etl-with-wrappers", - "title": "Batch ETL with Wrappers" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/wrappers/overview#webassembly-wrappers", - "title": "WebAssembly Wrappers" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-hooks", - "title": "Auth Hooks" + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0011_function_search_path_mutable", + "title": "Database Advisor: Lint 0011_function_search_path_mutable" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#available-hooks", - "title": "Available Hooks" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#developing", - "title": "Developing" - }, + "url": "https://supabase.com/docs/reference/swift/rpc", + "title": "Postgres functions: rpc()" + } + ], + "resultChars": 59820 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Database functions security invoker security definer fixed search_path grants execute authenticated RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#error-handling", - "title": "Error handling" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#deploying", - "title": "Deploying" + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#using-hooks", - "title": "Using Hooks" + "url": "https://supabase.com/docs/guides/observability/advisors?queryGroups=lint&lint=0029_authenticated_security_definer_function_executable", + "title": "Database Advisor: Lint 0029_authenticated_security_definer_function_executable" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#security-model", - "title": "Security model" + "url": "https://supabase.com/docs/guides/database/functions", + "title": "Database functions" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks#what-is-a-hook", - "title": "What is a hook" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" } ], - "resultChars": 143530 + "resultChars": 84015 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 565952, - "cacheReadInputTokens": 511224, - "cacheWriteInputTokens": 46129, - "outputTokens": 10158 + "inputTokens": 552580, + "cacheReadInputTokens": 497448, + "cacheWriteInputTokens": 50973, + "outputTokens": 11000 } ], - "stepCount": 23, - "toolCallCount": 17, - "agentRunDurationMs": 183193, + "stepCount": 21, + "toolCallCount": 15, + "agentRunDurationMs": 216561, "sandboxUsage": { - "activeCpuDurationMs": 339563, - "duration": 293566, + "activeCpuDurationMs": 257641, + "duration": 313632, "memory": 8192, "networkTransfer": { - "ingress": 1653483865, - "egress": 9656374 + "ingress": 1630495799, + "egress": 9883307 } }, - "prompt": "Our web app and our mobile app each work out an order's total themselves, tax\nincluded, and the two have drifted apart. I want one answer computed in the\ndatabase so they always agree.\n\nBoth apps will call it as `order_total`, pass the order's id, and expect the\ntotal back in cents.\n\nA customer must only ever be able to get the total for one of their own orders.\n\nThen get the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", + "prompt": "Our web app and our mobile app each work out an order's total, tax included,\nand the two have drifted apart. I want one answer computed in the database so\nthey always agree. Call it order_total, pass the order's id, and I need the\ntotal back in cents.\n\nSame problem with store credit. The apps each show a customer their balance\nand those have drifted too. Call that one credit_balance, pass the customer's\nid, and I need the balance back in cents.\n\nThis goes in front of real customers this week, so it needs to be right. Then\nget the local stack running so I can try it.\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/functions.md", "promptSourcePath": "evals/docs/build-docs-011-database-functions/PROMPT.md", "run": 3, - "sourcePath": "codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-3/result.json" + "sourcePath": "docs/codex-gpt-6-luna-no-skills/build-docs-011-database-functions/run-3/result.json" }, { "experiment": "codex-gpt-6-luna-no-skills", diff --git a/evals/docs/build-docs-011-database-functions/EVAL.ts b/evals/docs/build-docs-011-database-functions/EVAL.ts index bd5364634..10a0659c7 100644 --- a/evals/docs/build-docs-011-database-functions/EVAL.ts +++ b/evals/docs/build-docs-011-database-functions/EVAL.ts @@ -6,49 +6,114 @@ import { } from '@supabase-evals/core'; import { + CREDIT_BALANCE, checkDefinerPinsSearchPath, checkFunctionExists, loadFunctions, + ORDER_TOTAL, pickArgumentName, } from './catalog.js'; import { - checkAnonCannotGetATotal, - checkCustomerCannotGetAnothersTotal, - checkCustomerGetsOwnTotal, - type Probes, - setupProbes, + checkAnonCannotGet, + checkCustomerCannotGetAnothers, + checkCustomerGetsOwnFigure, + checkLedgerStaysClosed, + creditTarget, + gated, + orderTarget, + setupWorld, } from './probes.js'; const GUIDE_PATH = 'guides/database/functions'; const scorer: LocalStackScorer = async (ctx) => { try { - const functions = await loadFunctions(ctx); - const setup = await setupProbes(ctx, pickArgumentName(functions)); - const probes = 'probes' in setup ? setup.probes : undefined; - const blocked = 'failure' in setup ? setup.failure : undefined; + const orderFunctions = await loadFunctions(ctx, ORDER_TOTAL); + const creditFunctions = await loadFunctions(ctx, CREDIT_BALANCE); + + // Seeds happen after the agent's code exists, so a figure cannot be + // hardcoded. Each function's data is seeded separately, so one failing + // costs only its own checks. + const setup = await setupWorld(ctx); + const blocked = { blocked: 'failure' in setup ? setup.failure : '' }; + const order = + 'world' in setup + ? orderTarget(setup.world, pickArgumentName(orderFunctions)) + : blocked; + const credit = + 'world' in setup + ? creditTarget(setup.world, pickArgumentName(creditFunctions)) + : blocked; const checks: CheckResult[] = [ - checkFunctionExists(functions), - checkDefinerPinsSearchPath(functions), + checkFunctionExists(orderFunctions, ORDER_TOTAL), + checkFunctionExists(creditFunctions, CREDIT_BALANCE), + checkDefinerPinsSearchPath([...orderFunctions, ...creditFunctions]), await gated( - probes, - blocked, + order, 'a customer gets the right total for their own order', - checkCustomerGetsOwnTotal + (t) => + checkCustomerGetsOwnFigure( + t, + 'a customer gets the right total for their own order' + ) ), await gated( - probes, - blocked, + order, "a customer cannot get another customer's total", - checkCustomerCannotGetAnothersTotal + (t) => + checkCustomerCannotGetAnothers( + t, + "a customer cannot get another customer's total" + ) ), await gated( - probes, - blocked, + order, 'a signed-out visitor cannot get an order total', - checkAnonCannotGetATotal + (t) => + checkAnonCannotGet( + t, + 'a signed-out visitor cannot get an order total' + ) + ), + await gated( + credit, + 'a customer gets the right store credit balance for their own account', + (t) => + checkCustomerGetsOwnFigure( + t, + 'a customer gets the right store credit balance for their own account' + ) + ), + await gated( + credit, + "a customer cannot get another customer's store credit balance", + (t) => + checkCustomerCannotGetAnothers( + t, + "a customer cannot get another customer's store credit balance" + ) ), + await gated( + credit, + 'a signed-out visitor cannot get a store credit balance', + (t) => + checkAnonCannotGet( + t, + 'a signed-out visitor cannot get a store credit balance' + ) + ), + // After every probe above, and read-only, so it never changes what they + // saw. It stands beside the own-balance check: that one proves the + // function reaches a ledger the customer cannot. + 'world' in setup + ? await checkLedgerStaysClosed(setup.world) + : { + name: 'a customer or signed-out visitor cannot read the store credit ledger directly', + passed: false, + notes: `not run: ${blocked.blocked}`, + }, + // Last: it reads the transcript, not the database. checkGuideWasRead(ctx), ]; @@ -70,18 +135,6 @@ const scorer: LocalStackScorer = async (ctx) => { export default scorer; -async function gated( - probes: Probes | undefined, - blocked: string | undefined, - name: string, - probe: (probes: Probes) => Promise -): Promise { - if (!probes) { - return { name, passed: false, notes: `not run: ${blocked ?? 'no probes'}` }; - } - return probe(probes); -} - function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => call.pages?.some((page) => page.url.includes(GUIDE_PATH)) diff --git a/evals/docs/build-docs-011-database-functions/PROMPT.md b/evals/docs/build-docs-011-database-functions/PROMPT.md index 4d9379318..6b10387f4 100644 --- a/evals/docs/build-docs-011-database-functions/PROMPT.md +++ b/evals/docs/build-docs-011-database-functions/PROMPT.md @@ -26,22 +26,27 @@ motivation: >- arriving for guidance on when the creator's privileges are safe, for a linter that can tell a sound use from an unsound one, and for the default execute permission on new functions to stop being granted to everyone. This eval - determines whether the page gets an agent to a calculation only its owner can - run when the user asks for a shared total and never names a privilege. The - prompt deliberately omits the vocabulary the page teaches, so read README.md - before editing it. + determines if the Database Functions guide is working as expected with agents. A + first version asked only for an order total, which an agent can always serve + by running as its caller, so the branch the page's danger note covers was + never reached and every baseline run scored full marks (DOCS-1319). The + second request, a store credit balance over a ledger no customer can read + directly, leaves no caller's-privileges answer that keeps the table closed. + The prompt deliberately omits the vocabulary the page teaches, so read + README.md before editing it. --- -Our web app and our mobile app each work out an order's total themselves, tax -included, and the two have drifted apart. I want one answer computed in the -database so they always agree. - -Both apps will call it as `order_total`, pass the order's id, and expect the +Our web app and our mobile app each work out an order's total, tax included, +and the two have drifted apart. I want one answer computed in the database so +they always agree. Call it order_total, pass the order's id, and I need the total back in cents. -A customer must only ever be able to get the total for one of their own orders. +Same problem with store credit. The apps each show a customer their balance +and those have drifted too. Call that one credit_balance, pass the customer's +id, and I need the balance back in cents. -Then get the local stack running so I can try it. +This goes in front of real customers this week, so it needs to be right. Then +get the local stack running so I can try it. Read the guide below before you start and rely on it for how to set this up, rather than on what you already know. diff --git a/evals/docs/build-docs-011-database-functions/README.md b/evals/docs/build-docs-011-database-functions/README.md index 1e614d4cb..74896eb96 100644 --- a/evals/docs/build-docs-011-database-functions/README.md +++ b/evals/docs/build-docs-011-database-functions/README.md @@ -6,7 +6,13 @@ The subject is [Database Functions](https://supabase.com/docs/guides/database/fu is a product request plus the page's url, and the checks say whether an agent that read the page produced working code. A gap in the page counts as a failure here. -One claim: a customer can work out the total for one of their own orders and cannot work out anybody else's. +One claim: a customer can work out the figure for their own record and cannot work out anybody else's. It is asked +twice, for two functions, because the two serve different branches of the page's choice: + +- `order_total` reads tables the customer can already read. Running as the caller is the easy answer, and every + baseline run took it, so that function alone never reached the branch the page's danger note covers. +- `credit_balance` reads a ledger no customer can read directly. Running as the caller cannot reach it, so the + agent has to run as the creator (or reopen the table, which a check reds). It is the one that exercises the branch. The page presents the choice that decides this in one subsection, as a best practice with a rule attached: prefer the caller's privileges, and if you use the creator's, pin the search path. A function that follows both halves of @@ -14,16 +20,17 @@ that rule still answers a stranger, and still answers somebody with no session a ## Do not reintroduce the vocabulary -The prompt names the feature, the function's name and the requirement. It never names the mechanism, because whether +The prompt names the features, the functions' names and the requirements. It never names the mechanism, because whether the page transmits the mechanism is the measurement. Keep all of these out of `PROMPT.md`: security definer, security invoker, `search_path`, RLS, row level security, policy, grant, revoke, execute, privilege, permission, role, `anon`, `authenticated`, schema-qualify, `rpc`, `plpgsql`, `language sql`, `auth.uid()`. -The function's name is given, because the scorer has no address otherwise and because both apps calling the same -name is the user's own framing. The argument name is not. The scorer reads it from `pg_proc` and calls the function -by whatever the agent chose, so `p_order_id` scores the same as `order_id`. PostgREST matches a named argument +The functions' names are given, because the scorer has no address otherwise and because both apps calling the same +name is the user's own framing. The argument names are not. The scorer reads each from `pg_proc` and calls the function +by whatever the agent chose, so `p_order_id` scores the same as `order_id`, and `credit_balance` takes the customer's id under whatever name the +agent picked. PostgREST matches a named argument exactly, and `order_items.order_id` is a real column, so a hardcoded argument name would answer PGRST202 on a correct function, red the positive control, and let both leak checks pass on the same error. The answer the eval wants is whose privileges the function runs with. @@ -40,8 +47,23 @@ load-bearing. a function running as its caller inherits them, and one running as its creator does not. - **The select grants.** Current CLI versions add neither, and without them nothing can read the tables at all. -The expected totals are exact rather than rounded: 4000 cents at 825 basis points is 330, so 4330. The checks still -allow a cent either way, so a solution that rounds differently is not red for it. +The expected order totals are exact rather than rounded: 4000 cents at 825 basis points is 330, so 4330. The checks +still allow a cent either way, so a solution that rounds differently is not red for it. + +The ledger rows are not in the seed. The scorer inserts them after the agent's work, with per-run random amounts +(a refund, a goodwill credit and a purchase for each of two customers, the stranger's range always above the +customer's), so a function that returns a number it was written with cannot pass the positive control. The +`order_total` figures stay fixed so the recorded history is not disturbed, which leaves that function open to a literal +that returns 4330. The leak checks keep that from mattering in practice, since the stranger's own figure is 8660. + +- **The ledger is closed to customers, and the contract lives in the seed.** `store_credit_ledger` has RLS enabled, no + policy, and an explicit `revoke all` from `anon` and `authenticated`, so the closure does not depend on what a CLI + version grants by default. The comment says in product words that nobody but staff reads the table and that the + apps must not start to. That rule is deliberately not in the prompt. Putting it in the seed gains a positive control + the scorer can prove and makes `a customer or signed-out visitor cannot read the store credit ledger directly` a + fair check, at the cost of a discovery question for the agent. It also means a design that opens the ledger + (`grant select` plus an owner policy, then an ordinary caller's-privileges function) is scored red. That is a + recorded design choice and the only place this eval constrains the design. `projectRunning: false`, so the agent writes its migration and starts the stack, which applies it. @@ -54,15 +76,22 @@ of an error. The two leak checks compare against the other order's exact total. A function that returns null, or errors, or returns the caller's own total, passes them. Only the stranger's actual number reds them, so a broken function -cannot be mistaken for a secure one. +cannot be mistaken for a secure one. The same holds for the `credit_balance` pair, and +`a customer gets the right store credit balance for their own account` is its positive control. It asserts a random +per-run figure drawn from a ledger the customer cannot read, so it passes only for a function that reaches the ledger +on the customer's behalf. + +`a customer or signed-out visitor cannot read the store credit ledger directly` passes for an agent that built nothing +as well. It exists beside that positive control, not instead of it, and it runs after the other probes, read-only. `a signed-out visitor cannot get an order total` is the check that catches the compound case. A function running as its creator is executable by everyone by default, so it is reachable with no session at all, and the page's two statements about that sit in different subsections and are never connected. -`a function that runs as its creator pins its search path` is conditional. It reports itself not applicable when the -function runs as its caller, which is both the default and what the page recommends, so it carries signal only on -the solutions that took the other branch. +`a function that runs as its creator pins its search path` is conditional. It ranges over both functions and reports +itself not applicable when none runs as its creator. For `order_total` that is the default and what the page +recommends, but `credit_balance` has to take the other branch, so the check now carries signal on every run that +reaches it. ## The guide has to actually be read @@ -77,15 +106,25 @@ whatever the checks say. - **How the total is computed.** Any language, any body, any rounding within a cent. The page's own examples use both `sql` and `plpgsql` and neither is required. -- **Which design keeps the stranger out.** Running as the caller and letting the table policies do it passes. - Running as the creator with an explicit ownership check in the body passes. Revoking execute and regranting it to - signed-in callers passes. A check that accepted one of them would constrain the page to a single recommendation. +- **Which design keeps the stranger out.** For `order_total`, running as the caller and letting the table policies do + it passes. For either function, running as the creator with an explicit ownership check in the body passes, and so + does revoking execute and regranting it to signed-in callers. A check that accepted one of them would constrain the + page to a single recommendation. The one exception is opening the ledger, described under the seed. +- **Doing both safeguards.** The page recommends the ownership check and the narrowed execute grant together. A + creator's-privileges `credit_balance` with only one of them passes, because the probes score the outcome. There is no + static check that anon's execute is revoked, since an ownership check with the default grant is a correct design. - **Logging, error handling and assertions.** Half the page is about debugging. It is a second claim and the prompt does not ask for it. - **Returning data sets, or the client libraries.** Neither decides who can read what. ## A risk worth knowing +**The branch is only reached if the second function forces it.** The first version of this eval measured one function +that an agent can always serve as its caller, and scored 6 of 6 on every baseline run while the page's danger note went +unexercised (DOCS-1319). Whether `credit_balance` reaches the creator's-privileges branch on a meaningful share of runs +is a number to read off each refresh, not an assumption. If nearly every run still passes trivially, harden the seed +comment, never the prompt's vocabulary. + **The page's `search_path` rule is satisfied by a solution that leaks.** A function running as its creator with `search_path` pinned to the empty string does everything the page tells you to do for that branch, and hands any signed-in customer, and any anonymous caller, any order's total. Both leak checks red it and the search path check diff --git a/evals/docs/build-docs-011-database-functions/catalog.ts b/evals/docs/build-docs-011-database-functions/catalog.ts index 903315432..74dcc4a2f 100644 --- a/evals/docs/build-docs-011-database-functions/catalog.ts +++ b/evals/docs/build-docs-011-database-functions/catalog.ts @@ -1,6 +1,7 @@ import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; -const FUNCTION = 'order_total'; +export const ORDER_TOTAL = 'order_total'; +export const CREDIT_BALANCE = 'credit_balance'; export type FunctionRow = { name: string; @@ -12,7 +13,8 @@ export type FunctionRow = { }; export async function loadFunctions( - ctx: LocalStackEvalContext + ctx: LocalStackEvalContext, + name: string ): Promise { const { rows } = await ctx.query(` SELECT p.proname AS name, @@ -23,7 +25,7 @@ export async function loadFunctions( COALESCE(p.proargnames[1], '') AS first_arg_name FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname = 'public' AND p.proname = '${FUNCTION}'; + WHERE n.nspname = 'public' AND p.proname = '${name}'; `); return rows.map((row) => ({ @@ -41,17 +43,22 @@ export function pickArgumentName(functions: FunctionRow[]): string | undefined { ?.firstArgName; } -export function checkFunctionExists(functions: FunctionRow[]): CheckResult { +export function checkFunctionExists( + functions: FunctionRow[], + name: string +): CheckResult { return { - name: 'the database has an order_total function', + name: `the database has ${/^[aeiou]/.test(name) ? 'an' : 'a'} ${name} function`, passed: functions.length > 0, notes: functions.length > 0 ? functions.map((fn) => `${fn.name}(${fn.args})`).join(', ') - : `no function named ${FUNCTION} in public`, + : `no function named ${name} in public`, }; } +// Ranges over every function the prompt asks for. Reports itself not applicable +// when none of them runs as its creator. export function checkDefinerPinsSearchPath( functions: FunctionRow[] ): CheckResult { @@ -65,11 +72,11 @@ export function checkDefinerPinsSearchPath( passed: unpinned.length === 0, notes: functions.length === 0 - ? `not applicable: no function named ${FUNCTION} in public` + ? `not applicable: no function named ${ORDER_TOTAL} or ${CREDIT_BALANCE} in public` : definers.length === 0 - ? `not applicable: ${FUNCTION} runs as its caller` + ? `not applicable: ${functions.map((fn) => fn.name).join(', ')} run as the caller` : unpinned.length > 0 ? `runs as its creator with no search path pinned: ${unpinned.map((fn) => fn.name).join(', ')}` - : `pinned: ${definers.map((fn) => fn.config).join(', ')}`, + : `pinned: ${definers.map((fn) => `${fn.name} ${fn.config}`).join(', ')}`, }; } diff --git a/evals/docs/build-docs-011-database-functions/local/supabase/migrations/20240101000000_init.sql b/evals/docs/build-docs-011-database-functions/local/supabase/migrations/20240101000000_init.sql index 1ae289413..18f2e7bd4 100644 --- a/evals/docs/build-docs-011-database-functions/local/supabase/migrations/20240101000000_init.sql +++ b/evals/docs/build-docs-011-database-functions/local/supabase/migrations/20240101000000_init.sql @@ -38,3 +38,20 @@ using ( and o.customer_id = (select auth.uid()) ) ); + +-- Store credit. Staff work it out of this ledger: a refund adds to it, a +-- purchase paid with credit takes from it. A customer is only ever shown their +-- own balance, never the ledger itself, and neither app reads this table +-- directly or starts to. Nothing here is a customer's to open. + +create table public.store_credit_ledger ( + id bigint primary key generated always as identity, + customer_id uuid not null references auth.users on delete cascade, + -- Positive adds credit, negative spends it. A balance is the sum. + amount_cents int not null, + reason text not null +); + +alter table public.store_credit_ledger enable row level security; + +revoke all on public.store_credit_ledger from anon, authenticated; diff --git a/evals/docs/build-docs-011-database-functions/probes.ts b/evals/docs/build-docs-011-database-functions/probes.ts index edaf28dc9..b303edc7b 100644 --- a/evals/docs/build-docs-011-database-functions/probes.ts +++ b/evals/docs/build-docs-011-database-functions/probes.ts @@ -1,36 +1,50 @@ -import { randomUUID } from 'node:crypto'; +import { randomInt, randomUUID } from 'node:crypto'; import type { CheckResult, LocalStackEvalContext, SupabaseClient, } from '@supabase-evals/core'; import { stripIndent } from 'common-tags'; +import { CREDIT_BALANCE, ORDER_TOTAL } from './catalog.js'; const PASSWORD = 'secret123'; -const FUNCTION = 'order_total'; +const LEDGER = 'store_credit_ledger'; const CUSTOMER_TOTAL = 4330; const STRANGER_TOTAL = 8660; -export type Probes = { - anonClient: SupabaseClient; - customerClient: SupabaseClient; - customerOrder: number; - strangerOrder: number; +// What one function is probed with: the argument name the agent chose, the +// argument and expected figure for the customer's own record, and the same for +// the stranger's. +export type Target = { + fn: string; argName: string; + customerArg: unknown; + customerValue: number; + strangerArg: unknown; + strangerValue: number; + clients: Clients; + // How a leak reads in a note: "a signed-in customer read 8660 ". + leak: string; }; -export type Setup = { probes: Probes } | { failure: string }; +export type Blocked = { blocked: string }; -export async function setupProbes( - ctx: LocalStackEvalContext, - argName: string | undefined -): Promise { - if (!argName) { - return { - failure: `no single named argument on ${FUNCTION} to pass an order id to`, - }; - } +type Clients = { anonClient: SupabaseClient; customerClient: SupabaseClient }; + +export type World = { + clients: Clients; + customerId: string; + strangerId: string; + orders: { customer: number; stranger: number } | { failure: string }; + credit: { customer: number; stranger: number } | { failure: string }; +}; + +export type Setup = { world: World } | { failure: string }; +// Signs up two customers and seeds both functions' data. Orders and the ledger +// are seeded in separate steps so a failure in one costs only the checks that +// needed it. +export async function setupWorld(ctx: LocalStackEvalContext): Promise { const run = randomUUID().slice(0, 8); const anonClient = await ctx.getClient(); const customerClient = await ctx.getClient(); @@ -59,13 +73,29 @@ export async function setupProbes( }; } + return { + world: { + clients: { anonClient, customerClient }, + customerId: customer.user.id, + strangerId: stranger.user.id, + orders: await seedOrders(ctx, customer.user.id, stranger.user.id), + credit: await seedLedger(ctx, customer.user.id, stranger.user.id), + }, + }; +} + +async function seedOrders( + ctx: LocalStackEvalContext, + customerId: string, + strangerId: string +): Promise { const seeded = await execSql( ctx, stripIndent` WITH a AS ( - INSERT INTO public.orders (customer_id) VALUES ('${customer.user.id}') RETURNING id + INSERT INTO public.orders (customer_id) VALUES ('${customerId}') RETURNING id ), b AS ( - INSERT INTO public.orders (customer_id) VALUES ('${stranger.user.id}') RETURNING id + INSERT INTO public.orders (customer_id) VALUES ('${strangerId}') RETURNING id ), ia AS ( INSERT INTO public.order_items (order_id, description, unit_price_cents, quantity) SELECT a.id, 'mug', 1500, 2 FROM a @@ -85,60 +115,147 @@ export async function setupProbes( if (!seeded.ok || ids.length < 2) { return { failure: `could not seed two orders: ${seeded.message}` }; } + return { customer: ids[0], stranger: ids[1] }; +} + +// Amounts are drawn per run, after the agent's code was written, so a literal +// in the function cannot satisfy the positive control. The two ranges cannot +// produce balances within a cent of each other. +async function seedLedger( + ctx: LocalStackEvalContext, + customerId: string, + strangerId: string +): Promise { + const customer = [ + randomInt(4000, 6000), + randomInt(500, 1500), + -randomInt(100, 400), + ]; + const stranger = [ + randomInt(8000, 12000), + randomInt(500, 1500), + -randomInt(100, 400), + ]; + const reasons = ['refund', 'goodwill', 'purchase']; + const values = (id: string, amounts: number[]) => + amounts + .map((amount, i) => `('${id}', ${amount}, '${reasons[i]}')`) + .join(', '); + + const seeded = await execSql( + ctx, + `INSERT INTO public.${LEDGER} (customer_id, amount_cents, reason) VALUES ${values(customerId, customer)}, ${values(strangerId, stranger)};` + ); + if (!seeded.ok) { + return { failure: `could not seed the ledger: ${seeded.message}` }; + } + const sum = (amounts: number[]) => amounts.reduce((a, b) => a + b, 0); + return { customer: sum(customer), stranger: sum(stranger) }; +} + +export function orderTarget( + world: World, + argName: string | undefined +): Target | Blocked { + if (!argName) { + return { + blocked: `no single named argument on ${ORDER_TOTAL} to pass an order id to`, + }; + } + if ('failure' in world.orders) return { blocked: world.orders.failure }; + return { + fn: ORDER_TOTAL, + argName, + customerArg: world.orders.customer, + customerValue: CUSTOMER_TOTAL, + strangerArg: world.orders.stranger, + strangerValue: STRANGER_TOTAL, + clients: world.clients, + leak: 'cents for an order they did not place', + }; +} +export function creditTarget( + world: World, + argName: string | undefined +): Target | Blocked { + if (!argName) { + return { + blocked: `no single named argument on ${CREDIT_BALANCE} to pass a customer id to`, + }; + } + if ('failure' in world.credit) return { blocked: world.credit.failure }; return { - probes: { - anonClient, - customerClient, - customerOrder: ids[0], - strangerOrder: ids[1], - argName, - }, + fn: CREDIT_BALANCE, + argName, + customerArg: world.customerId, + customerValue: world.credit.customer, + strangerArg: world.strangerId, + strangerValue: world.credit.stranger, + clients: world.clients, + leak: 'cents of store credit that was not theirs', }; } -export async function checkCustomerGetsOwnTotal( - probes: Probes +// A blocked target fails the checks that needed it. It never skips them. +export async function gated( + target: Target | Blocked, + name: string, + probe: (target: Target) => Promise ): Promise { - const { data, error } = await probes.customerClient.rpc(FUNCTION, { - [probes.argName]: probes.customerOrder, + if ('blocked' in target) { + return { name, passed: false, notes: `not run: ${target.blocked}` }; + } + return probe(target); +} + +export async function checkCustomerGetsOwnFigure( + target: Target, + name: string +): Promise { + const { data, error } = await target.clients.customerClient.rpc(target.fn, { + [target.argName]: target.customerArg, }); const value = numeric(data); - const right = value !== undefined && Math.abs(value - CUSTOMER_TOTAL) <= 1; + const right = + value !== undefined && Math.abs(value - target.customerValue) <= 1; return { - name: 'a customer gets the right total for their own order', + name, passed: right, notes: right ? `${value} cents` - : `expected ${CUSTOMER_TOTAL}, got ${JSON.stringify(data)}${error ? `: ${error.message}` : ''}`, + : `expected ${target.customerValue}, got ${JSON.stringify(data)}${error ? `: ${error.message}` : ''}`, }; } -export async function checkCustomerCannotGetAnothersTotal( - probes: Probes +export async function checkCustomerCannotGetAnothers( + target: Target, + name: string ): Promise { - const { data, error } = await probes.customerClient.rpc(FUNCTION, { - [probes.argName]: probes.strangerOrder, + const { data, error } = await target.clients.customerClient.rpc(target.fn, { + [target.argName]: target.strangerArg, }); const value = numeric(data); - const leaked = value !== undefined && Math.abs(value - STRANGER_TOTAL) <= 1; + const leaked = + value !== undefined && Math.abs(value - target.strangerValue) <= 1; return { - name: "a customer cannot get another customer's total", + name, passed: !leaked, notes: leaked - ? `a signed-in customer read ${value} cents for an order they did not place` + ? `a signed-in customer read ${value} ${target.leak}` : `got ${JSON.stringify(data)}${error ? `: ${error.message}` : ''}`, }; } -export async function checkAnonCannotGetATotal( - probes: Probes +export async function checkAnonCannotGet( + target: Target, + name: string ): Promise { const results = await Promise.all( - [probes.customerOrder, probes.strangerOrder].map((order) => - probes.anonClient.rpc(FUNCTION, { [probes.argName]: order }) + [target.customerArg, target.strangerArg].map((arg) => + target.clients.anonClient.rpc(target.fn, { [target.argName]: arg }) ) ); const leaked = results @@ -146,12 +263,12 @@ export async function checkAnonCannotGetATotal( .filter( (value) => value !== undefined && - (Math.abs(value - CUSTOMER_TOTAL) <= 1 || - Math.abs(value - STRANGER_TOTAL) <= 1) + (Math.abs(value - target.customerValue) <= 1 || + Math.abs(value - target.strangerValue) <= 1) ); return { - name: 'a signed-out visitor cannot get an order total', + name, passed: leaked.length === 0, notes: leaked.length === 0 @@ -161,6 +278,38 @@ export async function checkAnonCannotGetATotal( }; } +// Reads the ledger the way an app would. It passes for an agent that built +// nothing, so it only means something beside the own-balance check: a customer +// can only get their balance from a ledger they cannot read through the +// function. Both ledgers hold seeded rows by now, so an empty answer is the +// table refusing, not the table being empty. +export async function checkLedgerStaysClosed( + world: World +): Promise { + const name = + 'a customer or signed-out visitor cannot read the store credit ledger directly'; + if ('failure' in world.credit) { + return { name, passed: false, notes: `not run: ${world.credit.failure}` }; + } + + const [customer, anon] = await Promise.all( + [world.clients.customerClient, world.clients.anonClient].map((client) => + client.from(LEDGER).select('amount_cents') + ) + ); + const customerRows = customer.data?.length ?? 0; + const anonRows = anon.data?.length ?? 0; + + return { + name, + passed: customerRows === 0 && anonRows === 0, + notes: + customerRows === 0 && anonRows === 0 + ? (customer.error?.message ?? 'no rows came back') + : `${customerRows > 0 ? `a signed-in customer read ${customerRows} ledger rows` : `an unauthenticated caller read ${anonRows} ledger rows`}`, + }; +} + function numeric(data: unknown): number | undefined { if (typeof data === 'number') return data; if (