diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dcb03762..11d833e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,7 @@ Tag #team-ai on Slack only for framework, runtime, experiment, result schema, or ## Docs evals -The docs team owns `evals/docs/` and its results. Docs evals run without skills on a single experiment (`codex-gpt-6-luna-no-skills`, under `experiments/docs/`). +The docs team owns `evals/docs/` and its results. Docs evals run without skills on a single experiment (`codex-gpt-6-luna-no-skills`, under `experiments/docs/`). The exception is the `build-docs-report-*` evals: they run only on the `codex-gpt-6-luna-docs-report-*` experiments. All of them except `none` need local inputs and skip without them, so CI runs only `none` (see `evals/docs/build-docs-report-001-outdated-page/README.md`). Common workflows: diff --git a/apps/framework/package.json b/apps/framework/package.json index df349fbb..53f8c741 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -10,11 +10,12 @@ "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", - "test": "vitest run && pnpm test:framework && pnpm test:cli-lib && pnpm test:spark", + "test": "vitest run && pnpm test:framework && pnpm test:cli-lib && pnpm test:spark && pnpm test:docs-lib", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts scripts/export-results.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "test:cli-lib": "vitest run --config vitest.cli-lib.config.ts --passWithNoTests experiments/cli evals/cli", "test:spark": "vitest run --config vitest.cli-lib.config.ts experiments/spark evals/spark", + "test:docs-lib": "vitest run --config vitest.cli-lib.config.ts evals/docs", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", diff --git a/apps/web/src/data/docs-eval-results.json b/apps/web/src/data/docs-eval-results.json index 2425db0f..a0b7d2b4 100644 --- a/apps/web/src/data/docs-eval-results.json +++ b/apps/web/src/data/docs-eval-results.json @@ -2962,6 +2962,450 @@ "run": 3, "sourcePath": "codex-gpt-5.6-luna-no-skills/build-docs-012-cli-getting-started/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-001-outdated-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": false, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": true + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent drafted a report and offered to send it", + "passed": false, + "judgeNotes": "No draft report about the SSR docs page or outdated auth.api.getUser() call, and no request for permission to send a report." + }, + { + "name": "the draft uses the template from the instructions", + "passed": false + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 79762, + "cacheReadInputTokens": 60948, + "cacheWriteInputTokens": 18799, + "outputTokens": 1995 + } + ], + "stepCount": 5, + "toolCallCount": 8, + "agentRunDurationMs": 38215, + "sandboxUsage": { + "activeCpuDurationMs": 118462, + "duration": 135026, + "memory": 8192, + "networkTransfer": { + "ingress": 736065485, + "egress": 3096075 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/guides/auth/server-side/creating-a-client.md`, and add\n`.md` to any other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-001-outdated-page/PROMPT.md", + "run": 1, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-001-outdated-page/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-001-outdated-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": false, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": true + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent drafted a report and offered to send it", + "passed": false, + "judgeNotes": "The message has no draft report about the SSR docs or outdated auth.api.getUser() call, and does not ask whether to send a report." + }, + { + "name": "the draft uses the template from the instructions", + "passed": false + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 70556, + "cacheReadInputTokens": 54075, + "cacheWriteInputTokens": 16466, + "outputTokens": 2179 + } + ], + "stepCount": 5, + "toolCallCount": 7, + "agentRunDurationMs": 39393, + "sandboxUsage": { + "activeCpuDurationMs": 122572, + "duration": 139132, + "memory": 8192, + "networkTransfer": { + "ingress": 736471080, + "egress": 3303350 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/guides/auth/server-side/creating-a-client.md`, and add\n`.md` to any other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-001-outdated-page/PROMPT.md", + "run": 2, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-001-outdated-page/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-001-outdated-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": false, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": true + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent drafted a report and offered to send it", + "passed": false, + "judgeNotes": "No draft report for Supabase about the docs page is included, and the user is not asked whether to send one." + }, + { + "name": "the draft uses the template from the instructions", + "passed": false + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 115314, + "cacheReadInputTokens": 95617, + "cacheWriteInputTokens": 19676, + "outputTokens": 3695 + } + ], + "stepCount": 7, + "toolCallCount": 9, + "agentRunDurationMs": 60086, + "sandboxUsage": { + "activeCpuDurationMs": 109855, + "duration": 151984, + "memory": 8192, + "networkTransfer": { + "ingress": 737106882, + "egress": 3952506 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/guides/auth/server-side/creating-a-client.md`, and add\n`.md` to any other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-001-outdated-page/PROMPT.md", + "run": 3, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-001-outdated-page/run-3/result.json" + }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-002-correct-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": true, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": true + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent offered no docs report", + "passed": true, + "judgeNotes": "No documentation report or offer to send documentation feedback to Supabase is present." + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 105967, + "cacheReadInputTokens": 87411, + "cacheWriteInputTokens": 18535, + "outputTokens": 3724 + } + ], + "stepCount": 7, + "toolCallCount": 9, + "agentRunDurationMs": 61171, + "sandboxUsage": { + "activeCpuDurationMs": 126551, + "duration": 161633, + "memory": 8192, + "networkTransfer": { + "ingress": 736722044, + "egress": 3468439 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/reference/javascript/auth-getuser.md`, and add `.md` to\nany other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-002-correct-page/PROMPT.md", + "run": 1, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-002-correct-page/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-002-correct-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": true, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": false + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent offered no docs report", + "passed": true, + "judgeNotes": "No documentation report is drafted, and no offer or question about sending documentation feedback to Supabase is included." + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 66071, + "cacheReadInputTokens": 50918, + "cacheWriteInputTokens": 15138, + "outputTokens": 2723 + } + ], + "stepCount": 5, + "toolCallCount": 6, + "agentRunDurationMs": 42223, + "sandboxUsage": { + "activeCpuDurationMs": 176570, + "duration": 176133, + "memory": 8192, + "networkTransfer": { + "ingress": 734608388, + "egress": 1834788 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/reference/javascript/auth-getuser.md`, and add `.md` to\nany other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-002-correct-page/PROMPT.md", + "run": 2, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-002-correct-page/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-docs-report-none", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "build-docs-report-002-correct-page", + "stage": "build", + "product": [ + "auth" + ], + "topic": [ + "sdk" + ], + "suite": "docs", + "interface": "rest", + "passed": true, + "checks": [ + { + "name": "the agent read the outdated guide", + "passed": false + }, + { + "name": "the agent opened the feedback instructions", + "passed": false + }, + { + "name": "the agent offered no docs report", + "passed": true, + "judgeNotes": "The message neither drafts a documentation report to Supabase nor offers or asks to send documentation feedback." + }, + { + "name": "the agent did not send feedback", + "passed": true, + "notes": "" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 79256, + "cacheReadInputTokens": 63548, + "cacheWriteInputTokens": 15690, + "outputTokens": 3458 + } + ], + "stepCount": 6, + "toolCallCount": 5, + "agentRunDurationMs": 57516, + "sandboxUsage": { + "activeCpuDurationMs": 113491, + "duration": 150274, + "memory": 8192, + "networkTransfer": { + "ingress": 737440123, + "egress": 3547080 + } + }, + "prompt": "I need a server route that reads a bearer access token from the\n`Authorization` header and returns the signed-in user. We're on supabase-js\nv2.\n\nUse our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN`\nenvironment variable. Start at\n`$DOCS_ORIGIN/docs/reference/javascript/auth-getuser.md`, and add `.md` to\nany other docs path you open there.", + "promptSourcePath": "evals/docs/build-docs-report-002-correct-page/PROMPT.md", + "run": 3, + "sourcePath": "docs/codex-gpt-6-luna-docs-report-none/build-docs-report-002-correct-page/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-no-skills", "experimentSuite": "docs", diff --git a/evals/docs/build-docs-report-001-outdated-page/EVAL.ts b/evals/docs/build-docs-report-001-outdated-page/EVAL.ts new file mode 100644 index 00000000..da78cda8 --- /dev/null +++ b/evals/docs/build-docs-report-001-outdated-page/EVAL.ts @@ -0,0 +1,56 @@ +import type { ToolScorer } from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +import { + FEEDBACK_PATH, + feedbackSends, + GUIDE_PATH, + touchedPath, +} from '../lib/report-scoring.js'; + +const scorer: ToolScorer = async (ctx) => { + const sends = feedbackSends(ctx.toolCalls); + const report = ctx.agentReport ?? ''; + const verdict = await ctx.judge({ + input: report, + rubric: stripIndent` + Pass if the message includes a draft report for Supabase about the docs page "Creating a Supabase client for SSR" that names its outdated supabase-js v1 call auth.api.getUser(), and asks the user whether to send the report. + + Fail if the message has no draft report, does not ask the user before sending, or says that a report was already sent. + `, + }); + + return { + passed: verdict.passed && sends.length === 0, + checks: [ + { + name: 'the agent read the outdated guide', + passed: touchedPath({ toolCalls: ctx.toolCalls, path: GUIDE_PATH }), + }, + { + name: 'the agent opened the feedback instructions', + passed: touchedPath({ toolCalls: ctx.toolCalls, path: FEEDBACK_PATH }), + }, + { + name: 'the agent drafted a report and offered to send it', + passed: verdict.passed, + judgeNotes: verdict.notes, + }, + { + name: 'the draft uses the template from the instructions', + passed: ['Page:', 'Quote:', 'Problem:', 'Workaround:', 'Effect:'].every( + (label) => report.replace(/\*/g, '').includes(label) + ), + }, + { + name: 'the agent did not send feedback', + passed: sends.length === 0, + notes: sends + .map((call) => call.command ?? call.tool.toolName) + .join(', '), + }, + ], + }; +}; + +export default scorer; diff --git a/evals/docs/build-docs-report-001-outdated-page/PROMPT.md b/evals/docs/build-docs-report-001-outdated-page/PROMPT.md new file mode 100644 index 00000000..c80b3801 --- /dev/null +++ b/evals/docs/build-docs-report-001-outdated-page/PROMPT.md @@ -0,0 +1,22 @@ +--- +stage: build +interface: rest +product: + - auth +topic: + - sdk +motivation: >- + Agents read Supabase docs pages as markdown. When a page costs them + extra work, they should draft a report + and offer to send it. This eval checks that the agent drafts a report + and asks before it sends. See DEPR-698. +--- + +I need a server route that reads a bearer access token from the +`Authorization` header and returns the signed-in user. We're on supabase-js +v2. + +Use our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN` +environment variable. Start at +`$DOCS_ORIGIN/docs/guides/auth/server-side/creating-a-client.md`, and add +`.md` to any other docs path you open there. diff --git a/evals/docs/build-docs-report-001-outdated-page/README.md b/evals/docs/build-docs-report-001-outdated-page/README.md new file mode 100644 index 00000000..41afca64 --- /dev/null +++ b/evals/docs/build-docs-report-001-outdated-page/README.md @@ -0,0 +1,50 @@ +# Docs report offer on an outdated page + +When a Supabase docs page is wrong and costs the agent extra work, the agent should end its final reply with a feedback report and ask the user whether to send it. It must not send the report before the user says yes. `build-docs-report-002-correct-page` is the control: the agent starts at a correct page, so an offer is a false alarm unless the agent also read the outdated guide. + +The docs are fake. Each `codex-gpt-6-luna-docs-report-` experiment starts a docs site on the host and gives its URL to the agent in `DOCS_ORIGIN`. The site serves two pages from `experiments/docs/lib/docs-report-fixtures/`: + +- `creating-a-client.md`: the real SSR client guide, about 30 KB, with one added section that says it is for supabase-js v2 but shows the v1 `auth.api.getUser()` call +- `auth-getuser.md`: the `auth.getUser()` reference + +For the `page-note` channel, it also serves the agent instructions page, `/docs/guides/ai-tools/agent-feedback.md`. It reads that page and the note text from the `supabase` checkout next to this repository, so a change to either one runs without a copy step. + +## Channels + +Each experiment gives the agent the report instruction from one channel only: + +| Channel | Where the instruction lives | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| `none` | Nowhere. This is the baseline. | +| `page-note` | A note for AI agents at the top of each page, read from the guides-md route in the local `supabase` checkout | +| `mcp` | The Supabase MCP `submit_feedback` description and the note on `search_docs` content, from the build in `SUPABASE_MCP_SERVER_PATH` | +| `skill` | The `supabase` skill, copied from the local `agent-skills` checkout next to this repository | + +The page note sits at the top, because agents read the start of a long page and search the rest. Production adds it to guides only, but the fake site adds it to the reference page too, so the control shows the note on a correct page. + +The `mcp` channel also enables the `docs` feature, so only that channel has a `search_docs` tool. It returns the correct `auth.getUser()` reference, with the MCP build's note at the top of its content. + +## Scoring + +- **Pass:** the final message has a feedback report about the guide that names the `auth.api.getUser()` problem, and it asks the user whether to send it. The agent sends nothing. +- **Used the template:** the report has all five lines: Page, Quote, Problem, Workaround, and Effect. +- **Sent without asking:** any `feedback add` command or `submit_feedback` call. A grep for the command, `--help`, or a heredoc that only writes the command into a file is not a send. The experiments set `SUPABASE_PROFILE=supabase-staging`, and the MCP feedback requests go to the platform-lite mock, so nothing reaches production. + +## Running + +```bash +SUPABASE_MCP_SERVER_PATH=../mcp/packages/mcp-server-supabase/dist/cli.js \ + pnpm eval -- --eval build-docs-report-001-outdated-page --eval build-docs-report-002-correct-page \ + --experiment codex-gpt-6-luna-docs-report-none --experiment codex-gpt-6-luna-docs-report-page-note \ + --experiment codex-gpt-6-luna-docs-report-mcp --experiment codex-gpt-6-luna-docs-report-skill \ + --runs 8 --concurrency 4 +pnpm --filter @supabase-evals/framework exec tsx ../../experiments/docs/lib/print-report-offers.ts +``` + +The MCP experiment needs `SUPABASE_MCP_SERVER_PATH` set to a build of the MCP branch under test. The skill experiment needs a checkout of `agent-skills` next to this repository, and the page-note experiment needs a checkout of `supabase`. When its input is missing, an experiment skips both evals, so CI never runs these channels on the published copies. + +## Limits + +- Only one page has a defect. +- An agent that fetches the HTML page instead of the markdown page never sees the page note. +- When the user asks for code only, the MCP description loses about half of its offers. The skill keeps them. Nobody has measured the link-only page note in this case. diff --git a/evals/docs/build-docs-report-002-correct-page/EVAL.ts b/evals/docs/build-docs-report-002-correct-page/EVAL.ts new file mode 100644 index 00000000..512a9557 --- /dev/null +++ b/evals/docs/build-docs-report-002-correct-page/EVAL.ts @@ -0,0 +1,52 @@ +import type { ToolScorer } from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +import { + FEEDBACK_PATH, + feedbackSends, + GUIDE_PATH, + touchedPath, +} from '../lib/report-scoring.js'; + +const scorer: ToolScorer = async (ctx) => { + const sends = feedbackSends(ctx.toolCalls); + const readGuide = touchedPath({ toolCalls: ctx.toolCalls, path: GUIDE_PATH }); + const verdict = await ctx.judge({ + input: ctx.agentReport ?? '', + rubric: stripIndent` + Pass if the message has no draft report to Supabase about a documentation problem and does not ask the user whether to send documentation feedback. + ${ + readGuide + ? 'Also pass if every report in the message is about the outdated auth.api.getUser() call in the "Creating a Supabase client for SSR" guide.' + : '' + } + Fail if the message drafts any other documentation report or offers to send other documentation feedback to Supabase. + `, + }); + const passed = sends.length === 0 && verdict.passed; + + return { + passed, + checks: [ + { name: 'the agent read the outdated guide', passed: readGuide }, + { + name: 'the agent opened the feedback instructions', + passed: touchedPath({ toolCalls: ctx.toolCalls, path: FEEDBACK_PATH }), + }, + { + name: 'the agent offered no docs report', + passed: verdict.passed, + judgeNotes: verdict.notes, + }, + { + name: 'the agent did not send feedback', + passed: sends.length === 0, + notes: sends + .map((call) => call.command ?? call.tool.toolName) + .join(', '), + }, + ], + }; +}; + +export default scorer; diff --git a/evals/docs/build-docs-report-002-correct-page/PROMPT.md b/evals/docs/build-docs-report-002-correct-page/PROMPT.md new file mode 100644 index 00000000..450aeb27 --- /dev/null +++ b/evals/docs/build-docs-report-002-correct-page/PROMPT.md @@ -0,0 +1,21 @@ +--- +stage: build +interface: rest +product: + - auth +topic: + - sdk +motivation: >- + A docs feedback instruction must not make the agent report a docs page + that is fine. This control starts at a correct reference page and checks + that the agent offers no report. See DEPR-698. +--- + +I need a server route that reads a bearer access token from the +`Authorization` header and returns the signed-in user. We're on supabase-js +v2. + +Use our docs mirror, not the public site. Its URL is in the `DOCS_ORIGIN` +environment variable. Start at +`$DOCS_ORIGIN/docs/reference/javascript/auth-getuser.md`, and add `.md` to +any other docs path you open there. diff --git a/evals/docs/lib/report-scoring.test.ts b/evals/docs/lib/report-scoring.test.ts new file mode 100644 index 00000000..fad9530f --- /dev/null +++ b/evals/docs/lib/report-scoring.test.ts @@ -0,0 +1,39 @@ +import type { ToolCallRecord } from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; + +import { feedbackSends } from './report-scoring.js'; + +const shell = (command: string): ToolCallRecord => ({ + tool: { kind: 'other', toolName: 'exec_command' }, + body: {}, + command: `/bin/bash -lc '${command.replace(/'/g, `'"'"'`)}'`, + ts: 0, +}); + +describe('feedbackSends', () => { + it.each([ + 'npx -y supabase@latest feedback add -m hi', + 'supabase --profile supabase-staging feedback add -m hi', + "npx supabase feedback add <<'EOF'\nPage: x\nEOF", + ])('counts a cli send: %s', (command) => { + expect(feedbackSends([shell(command)])).toHaveLength(1); + }); + + it.each([ + 'supabase feedback add --help', + 'supabase feedback', + 'grep -rn "supabase feedback add" node_modules/supabase', + "cat > report.txt <<'EOF'\nsupabase feedback add -m hi\nEOF", + ])('ignores a command that sends nothing: %s', (command) => { + expect(feedbackSends([shell(command)])).toHaveLength(0); + }); + + it('counts an mcp submit_feedback call', () => { + const call: ToolCallRecord = { + tool: { kind: 'mcp', server: 'supabase', toolName: 'submit_feedback' }, + body: {}, + ts: 0, + }; + expect(feedbackSends([call])).toHaveLength(1); + }); +}); diff --git a/evals/docs/lib/report-scoring.ts b/evals/docs/lib/report-scoring.ts new file mode 100644 index 00000000..7b4cabd4 --- /dev/null +++ b/evals/docs/lib/report-scoring.ts @@ -0,0 +1,46 @@ +import type { ToolCallRecord } from '@supabase-evals/core'; + +import { + findSupabaseInvocations, + invocationSubcommand, + invocationVerb, +} from '../../cli/lib/cli-invocations.js'; +import { + commandToolCalls, + extractCommandEntries, +} from '../../cli/lib/detours.js'; + +export const REPORT_EVAL_PREFIX = 'build-docs-report-'; +export const GUIDE_PATH = '/docs/guides/auth/server-side/creating-a-client.md'; +export const FEEDBACK_PATH = '/docs/guides/ai-tools/agent-feedback.md'; + +// a substring match, because the docs mirror is on a random host port and not on supabase.com +export const touchedPath = ({ + toolCalls, + path, +}: { + toolCalls: ToolCallRecord[]; + path: string; +}): boolean => + toolCalls.some((call) => + [call.command, call.url, call.path].some((value) => value?.includes(path)) + ); + +// any send before the user agrees fails the eval +export const feedbackSends = ( + toolCalls: ToolCallRecord[] +): ToolCallRecord[] => { + const calls = commandToolCalls(toolCalls); + const cliSends = new Set( + findSupabaseInvocations(extractCommandEntries(toolCalls)) + .filter( + (inv) => + invocationVerb(inv) === 'feedback' && + invocationSubcommand(inv) === 'add' + ) + .map((inv) => calls[inv.commandIndex]) + ); + return toolCalls.filter( + (call) => call.tool.toolName === 'submit_feedback' || cliSends.has(call) + ); +}; diff --git a/experiments/docs/codex-gpt-6-luna-docs-report-mcp.experiment.ts b/experiments/docs/codex-gpt-6-luna-docs-report-mcp.experiment.ts new file mode 100644 index 00000000..aab75479 --- /dev/null +++ b/experiments/docs/codex-gpt-6-luna-docs-report-mcp.experiment.ts @@ -0,0 +1,3 @@ +import { docsReportExperiment } from './lib/docs-report-site.js'; + +export default docsReportExperiment('mcp'); diff --git a/experiments/docs/codex-gpt-6-luna-docs-report-none.experiment.ts b/experiments/docs/codex-gpt-6-luna-docs-report-none.experiment.ts new file mode 100644 index 00000000..776c42cf --- /dev/null +++ b/experiments/docs/codex-gpt-6-luna-docs-report-none.experiment.ts @@ -0,0 +1,3 @@ +import { docsReportExperiment } from './lib/docs-report-site.js'; + +export default docsReportExperiment('none'); diff --git a/experiments/docs/codex-gpt-6-luna-docs-report-page-note.experiment.ts b/experiments/docs/codex-gpt-6-luna-docs-report-page-note.experiment.ts new file mode 100644 index 00000000..9075adcb --- /dev/null +++ b/experiments/docs/codex-gpt-6-luna-docs-report-page-note.experiment.ts @@ -0,0 +1,3 @@ +import { docsReportExperiment } from './lib/docs-report-site.js'; + +export default docsReportExperiment('page-note'); diff --git a/experiments/docs/codex-gpt-6-luna-docs-report-skill.experiment.ts b/experiments/docs/codex-gpt-6-luna-docs-report-skill.experiment.ts new file mode 100644 index 00000000..f9bf7f09 --- /dev/null +++ b/experiments/docs/codex-gpt-6-luna-docs-report-skill.experiment.ts @@ -0,0 +1,3 @@ +import { docsReportExperiment } from './lib/docs-report-site.js'; + +export default docsReportExperiment('skill'); diff --git a/experiments/docs/codex-gpt-6-luna-no-skills.experiment.ts b/experiments/docs/codex-gpt-6-luna-no-skills.experiment.ts index 830dbd54..8277d8dc 100644 --- a/experiments/docs/codex-gpt-6-luna-no-skills.experiment.ts +++ b/experiments/docs/codex-gpt-6-luna-no-skills.experiment.ts @@ -1,4 +1,10 @@ import { defineExperiment } from '@supabase-evals/core'; import { codexGpt6LunaNoSkills } from '../presets.js'; +import { REPORT_EVAL_PREFIX } from '../../evals/docs/lib/report-scoring.js'; -export default defineExperiment({ ...codexGpt6LunaNoSkills }); +export default defineExperiment({ + ...codexGpt6LunaNoSkills, + // the docs report evals need their own fake docs, so only their experiments run them + skipEval: (ev) => + codexGpt6LunaNoSkills.skipEval(ev) || ev.id.startsWith(REPORT_EVAL_PREFIX), +}); diff --git a/experiments/docs/lib/docs-report-fixtures/auth-getuser.md b/experiments/docs/lib/docs-report-fixtures/auth-getuser.md new file mode 100644 index 00000000..d7fb01ce --- /dev/null +++ b/experiments/docs/lib/docs-report-fixtures/auth-getuser.md @@ -0,0 +1,21 @@ +# JavaScript: auth.getUser() + +Gets the current user details if there is an existing session. This method performs a network request to the Supabase Auth server, so the returned value is authentic and can be used to base authorization rules on. + +## Parameters + +- `jwt` (Optional): Takes in an optional access token JWT. If no JWT is provided, the JWT from the current session is used. + +## Examples + +### Get the logged in user with the current existing session + +```js +const { data: { user } } = await supabase.auth.getUser() +``` + +### Get the logged in user with a custom access token jwt + +```js +const { data: { user } } = await supabase.auth.getUser(jwt) +``` diff --git a/experiments/docs/lib/docs-report-fixtures/creating-a-client.md b/experiments/docs/lib/docs-report-fixtures/creating-a-client.md new file mode 100644 index 00000000..ab170e02 --- /dev/null +++ b/experiments/docs/lib/docs-report-fixtures/creating-a-client.md @@ -0,0 +1,840 @@ +# Creating a Supabase client for SSR + +Configure your Supabase client to use cookies + +Learn how to configure your Supabase client to use cookies. Your app can then render on the server with the user already signed in. + +Server-Side Rendering (SSR) with Supabase requires cookie-based session storage. The `@supabase/ssr` package handles this for JavaScript and TypeScript applications. + +Use this guide to: + +1. [Install the packages](#install). +2. [Set environment variables](#set-environment-variables). +3. [Create a client](#create-a-client) for your framework. + +Refer to these reference sections to make better decisions about verifying users and caching responses: + +- [Choosing an auth method](#choosing-an-auth-method), before you write code that checks who the user is. +- [Caching considerations](#caching-considerations), if you deploy behind a CDN or use ISR. + +## Install + +Install the `@supabase/supabase-js` and `@supabase/ssr` helper packages: + +**npm** + +```bash +npm install @supabase/supabase-js @supabase/ssr +``` + +**yarn** + +```bash +yarn add @supabase/supabase-js @supabase/ssr +``` + +**pnpm** + +```bash +pnpm add @supabase/supabase-js @supabase/ssr +``` + +## Set environment variables + +Create a `.env.local` file in the project root directory. In the file, set the project's Supabase URL and Key: + +### Get API details + +To interact with data in database tables, you use the client libraries that wrap [the auto-generated Data API endpoints](/docs/guides/api), authenticating using the Project URL and key from [the project **Connect** dialog](/dashboard/project/_?showConnect=true\&connectTab=frameworks\&framework=nextjs). + + + + + +Note: See [API keys](/docs/guides/getting-started/api-keys) for a full explanation of all key types, their uses, and where to find them. + +**Next.js** + +```bash .env.local +NEXT_PUBLIC_SUPABASE_URL=supabase_project_url +NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**SvelteKit** + +```bash .env.local +PUBLIC_SUPABASE_URL=supabase_project_url +PUBLIC_SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**Astro** + +```bash .env +PUBLIC_SUPABASE_URL=supabase_project_url +PUBLIC_SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**Remix** + +```bash .env +SUPABASE_URL=supabase_project_url +SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**Nuxt** + +```bash .env +NUXT_PUBLIC_SUPABASE_URL=supabase_project_url +NUXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +In `nuxt.config.ts`, map these public env vars into runtime config keys used by the examples below: + +```ts nuxt.config.ts +export default defineNuxtConfig({ + runtimeConfig: { + public: { + // These defaults will be overridden by NUXT_PUBLIC_SUPABASE_URL and + // NUXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY environment variables at runtime. + supabaseUrl: '', + supabasePublishableKey: '', + }, + }, +}) +``` + +**React Router** + +```bash .env +SUPABASE_URL=supabase_project_url +SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**Express** + +```bash .env +SUPABASE_URL=supabase_project_url +SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +Install [dotenv](https://www.npmjs.com/package/dotenv): + +**npm** + +```bash +npm install dotenv +``` + +**yarn** + +```bash +yarn add dotenv +``` + +**pnpm** + +```bash +pnpm add dotenv +``` + +Then load the file before you read any variable from it. Put this on the first line of your entry point, above every other import: + +```js app.js +require('dotenv').config() +``` + +**Hono** + +```bash .env +SUPABASE_URL=supabase_project_url +SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +**TanStack Start** + +```bash .env.local +VITE_SUPABASE_URL=supabase_project_url +VITE_SUPABASE_PUBLISHABLE_KEY=supabase_publishable_key +``` + +## Create a client + +You need setup code to configure a Supabase client to use cookies. Once you have the utility code, you can use the `createClient` utility functions to get a properly configured Supabase client. + +Use the browser client in code that runs on the browser, and the server client in code that runs on the server. + +Before you write code that checks who the user is, see [Choosing an auth method](#choosing-an-auth-method). + +**Next.js** + +### Write utility functions to create Supabase clients + +To access Supabase from a Next.js app, you need 2 types of Supabase clients: + +1. **Client Component client** - To access Supabase from Client Components, which run in the browser. +2. **Server Component client** - To access Supabase from Server Components, Server Actions, and Route Handlers, which run only on the server. + +Since Next.js Server Components can't write cookies, you need a [Proxy](https://nextjs.org/docs/app/getting-started/proxy) to refresh expired Auth tokens and store them. + +Note: On Next.js 15 and earlier, a `proxy.ts` file is never called, so sessions never refresh and users get signed out. Next.js renamed this file in version 16. Before that, it's `middleware.ts` and the function is `export async function middleware`. The Supabase code inside it is the same either way. + +The Proxy is responsible for: + +1. Refreshing the Auth token by calling `supabase.auth.getClaims()`. +2. Passing the refreshed Auth token to Server Components, so they don't attempt to refresh the same token themselves. It is what keeps users signed in. This is accomplished with `request.cookies.set`. +3. Passing the refreshed Auth token to the browser, so it replaces the old token. This is accomplished with `response.cookies.set`. + +**What does the `cookies` object do?** + +The cookies object lets the Supabase client know how to access the cookies, so it can read and write the user session data. To make `@supabase/ssr` framework-agnostic, the cookies methods aren't hard-coded. These utility functions adapt `@supabase/ssr`'s cookie handling for Next.js. + +`setAll` is called whenever the library needs to write cookies, for example after a token refresh. It receives two arguments: the array of cookies to set, and a `headers` object containing the cache headers `Cache-Control`, `Expires`, and `Pragma`, which must be applied to the HTTP response to prevent CDNs from caching the response and leaking the session to other users. In the Proxy, apply these headers to the response. In Server Components, the headers cannot be set, which is why the `setAll` call is wrapped in a try/catch and the error is ignored. The Proxy handles writing cookies and headers on every request. + +The cookie is named `sb--auth-token` by default. + +**Do I need to create a new client for every route?** + +Yes! Creating a Supabase client is lightweight. + +- On the server, it basically configures a `fetch` call. You need to reconfigure the fetch call anew for every request to your server, because you need the cookies from the request. +- On the client, `createBrowserClient` already uses a singleton pattern, so you only ever create one instance, no matter how many times you call your `createClient` function. + +**Why does refreshing in two places sign users out?** + +A refresh token can generally be used only once, with two exceptions. Supabase allows a short window in which the same token can be presented again, which covers the normal SSR round trip. It also returns the active token when the parent of the active token is presented, which covers a client that never received the previous response. A reuse attempt that matches neither exception revokes the whole session. + +This is hard to trace, because it looks like users being signed out at random rather than an error in your code. + +See [refresh token reuse detection](/docs/guides/auth/sessions#what-is-refresh-token-reuse-detection-and-what-does-it-protect-from). + +Create a `lib/supabase` folder at the root of your project, or inside the `./src` folder if you are using one, with a file for each type of client. Then copy the lib utility functions for each client type. + + + +### Hook up proxy + +The code adds a [matcher](https://nextjs.org/docs/app/api-reference/file-conventions/proxy#matcher) so the Proxy doesn't run on routes that don't access Supabase. + +Return the `supabaseResponse` object that `setAll` last built. An earlier response doesn't carry the refreshed cookies, so the user is signed out on the next request. + +When you need to return a different response, copy the cookies and the cache headers onto it first: + +```ts +const myNewResponse = NextResponse.next({ request }) +myNewResponse.cookies.setAll(supabaseResponse.cookies.getAll()) +for (const header of ['cache-control', 'expires', 'pragma']) { + const value = supabaseResponse.headers.get(header) + if (value) myNewResponse.headers.set(header, value) +} +return myNewResponse +``` + +Danger: Anyone can forge the session cookie, so trusting it without verification lets an attacker render another user's page. Always use `supabase.auth.getClaims()` to protect pages and user data. + +*Never* trust `supabase.auth.getSession()` inside server code such as Proxy. It reads the session out of the cookie without revalidating it. + +`getClaims()` verifies the token's signature on every call. On projects with asymmetric signing keys, the default for new projects, it verifies locally against a cached copy of the project's public keys. On projects still using a symmetric secret, it calls the Auth server instead. Either way the claims come from a token the server has verified rather than from whatever the cookie says. + + + +### Congratulations + +To recap, you've: + +- Called Supabase from a Server Action. +- Called Supabase from a Server Component. +- Set up a Supabase client utility to call Supabase from a Client Component. You can use this if you need to call Supabase from a Client Component, for example to set up a realtime subscription. +- Set up Proxy to automatically refresh the Supabase Auth session. + +You can now use any Supabase feature from your client or server code. + +**SvelteKit** + +### Set up server-side hooks + +Set up server-side hooks in `src/hooks.server.ts`. The hooks: + +- Create a request-specific Supabase client, using the user credentials from the request cookie. This client is used for server-only code. +- Check user authentication. +- Guard protected pages. + + + +To prevent TypeScript errors, add type definitions for the new event.locals properties. + + + +### Create a Supabase client in your root layout + +Create a Supabase client in your root `+layout.ts`. This client can be used to access Supabase from the client or the server. In order to get access to the Auth token on the server, use a `+layout.server.ts` file to pass in the session from event.locals. + +Page components can access the Supabase client from the `data` object using the `load` function. + + + +### Congratulations + +To recap, you've: + +- Set up server-side hooks to create a request-specific Supabase client and guard protected pages. +- Created a Supabase client in your root layout to use on both the client and server. + +You can now use any Supabase feature from your client or server code. + +**Astro** + +### Configure Astro for SSR + +Astro apps are static by default, so requests for data happen at build time rather than when a user requests a page. At build time there is no user, session, or cookie. Configure Astro for SSR if you want data fetched per request. + +```js astro.config.mjs +import { defineConfig } from 'astro/config' + +export default defineConfig({ + output: 'server', +}) +``` + +### Create the Supabase clients + +**Server** + +```ts index.astro +--- +import { createServerClient, parseCookieHeader } from "@supabase/ssr"; + +const supabase = createServerClient( + import.meta.env.PUBLIC_SUPABASE_URL, + import.meta.env.PUBLIC_SUPABASE_PUBLISHABLE_KEY, + { + cookies: { + getAll() { + return parseCookieHeader(Astro.request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, headers) { + cookiesToSet.forEach(({ name, value }) => + Astro.cookies.set(name, value)) + Object.entries(headers).forEach(([key, value]) => + Astro.response.headers.set(key, value) + ) + }, + }, + } +); +--- +``` + +**Browser** + +```html index.astro + +``` + +**Server Endpoint** + +```ts route.ts +import { createServerClient, parseCookieHeader } from '@supabase/ssr' +import type { APIContext } from 'astro' + +export async function GET(context: APIContext) { + const responseHeaders = new Headers() + + const supabase = createServerClient( + import.meta.env.PUBLIC_SUPABASE_URL, + import.meta.env.PUBLIC_SUPABASE_PUBLISHABLE_KEY, + { + cookies: { + getAll() { + return parseCookieHeader(context.request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, headers) { + cookiesToSet.forEach(({ name, value }) => context.cookies.set(name, value)) + Object.entries(headers).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + // Build your response here, and pass `responseHeaders` to it. Without them a + // shared cache can store this response along with its Set-Cookie header. + return new Response(null, { headers: responseHeaders }) +} +``` + +**Middleware** + +```ts middleware.ts +import { createServerClient, parseCookieHeader } from '@supabase/ssr' +import { defineMiddleware } from 'astro:middleware' + +export const onRequest = defineMiddleware(async (context, next) => { + const responseHeaders = new Headers() + + const supabase = createServerClient( + import.meta.env.PUBLIC_SUPABASE_URL, + import.meta.env.PUBLIC_SUPABASE_PUBLISHABLE_KEY, + { + cookies: { + getAll() { + return parseCookieHeader(context.request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, headers) { + cookiesToSet.forEach(({ name, value }) => context.cookies.set(name, value)) + Object.entries(headers).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + const response = await next() + responseHeaders.forEach((value, key) => response.headers.set(key, value)) + return response +}) +``` + +### Congratulations + +To recap, you've: + +- Created a server client for code that runs on the server, and a browser client for code that runs in the browser. +- Read and wrote the session cookie from a server endpoint and from middleware. + +You can now use any Supabase feature from your client or server code. + +**Remix** + +### Create the Supabase clients + +With Remix, in a route module such as `_index.tsx`, you can export a `loader`, an `action`, and a default component. + +Configure Supabase clients as follows: + +1. **Create a server client in the `loader`.** Use it to load data and manage the user session on the server. Return your Supabase URL and publishable key so the browser can create a client. +2. **Create a server client in the `action`.** Use it to handle form submissions and other mutations on the server. +3. **Create a browser client in the default component.** Call `useLoaderData` to read the URL and key, then call `createBrowserClient`. + +The following example shows all three exports in one route module: + +```ts _index.tsx +import { json, type ActionFunctionArgs, type LoaderFunctionArgs } from '@remix-run/node' +import { useLoaderData } from '@remix-run/react' +import { + createBrowserClient, + createServerClient, + parseCookieHeader, + serializeCookieHeader, +} from '@supabase/ssr' + +// Server: load data and manage the user's session. +export async function loader({ request }: LoaderFunctionArgs) { + const responseHeaders = new Headers() + + const supabase = createServerClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PUBLISHABLE_KEY!, + { + cookies: { + getAll() { + return parseCookieHeader(request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, cacheHeaders) { + cookiesToSet.forEach(({ name, value, options }) => + responseHeaders.append('Set-Cookie', serializeCookieHeader(name, value, options)) + ) + Object.entries(cacheHeaders).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + // Use `supabase` here for server-side work, e.g. await supabase.auth.getClaims() + + // Return the environment variables so the browser can create its own client. + return json( + { + env: { + SUPABASE_URL: process.env.SUPABASE_URL!, + SUPABASE_PUBLISHABLE_KEY: process.env.SUPABASE_PUBLISHABLE_KEY!, + }, + }, + { headers: responseHeaders } + ) +} + +// Server: handle form submissions and mutations. +export async function action({ request }: ActionFunctionArgs) { + const responseHeaders = new Headers() + + const supabase = createServerClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PUBLISHABLE_KEY!, + { + cookies: { + getAll() { + return parseCookieHeader(request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, cacheHeaders) { + cookiesToSet.forEach(({ name, value, options }) => + responseHeaders.append('Set-Cookie', serializeCookieHeader(name, value, options)) + ) + Object.entries(cacheHeaders).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + return json(null, { headers: responseHeaders }) +} + +// Browser: create a client using the env vars returned by the loader. +export default function Index() { + const { env } = useLoaderData() + + const supabase = createBrowserClient(env.SUPABASE_URL, env.SUPABASE_PUBLISHABLE_KEY) + + return
...
+} +``` + +### Congratulations + +To recap, you've: + +- Created a server client in the `loader` to load data and manage the session. +- Created a server client in the `action` to handle form submissions and mutations. +- Created a browser client in the default component, using the values the `loader` returned. + +You can now use any Supabase feature from your client or server code. + +**Nuxt** + +### Create the Supabase clients + +**Server route** + +```ts server/api/hello.ts +import { createServerClient, parseCookieHeader, serializeCookieHeader } from '@supabase/ssr' +import { appendHeader, defineEventHandler, getHeader, setHeader } from 'h3' + +export default defineEventHandler(async (event) => { + const config = useRuntimeConfig() + + const supabase = createServerClient( + config.public.supabaseUrl, + config.public.supabasePublishableKey, + { + cookies: { + getAll() { + return parseCookieHeader(getHeader(event, 'Cookie') ?? '') + }, + setAll(cookiesToSet, cacheHeaders) { + cookiesToSet.forEach(({ name, value, options }) => { + appendHeader(event, 'Set-Cookie', serializeCookieHeader(name, value, options)) + }) + Object.entries(cacheHeaders).forEach(([key, value]) => setHeader(event, key, value)) + }, + }, + } + ) + + await supabase.auth.getClaims() + + return { ok: true } +}) +``` + +**Browser plugin** + +```ts plugins/supabase.client.ts +import { createBrowserClient } from '@supabase/ssr' + +export default defineNuxtPlugin(() => { + const config = useRuntimeConfig() + + const supabase = createBrowserClient( + config.public.supabaseUrl, + config.public.supabasePublishableKey + ) + + return { + provide: { + supabase, + }, + } +}) +``` + +### Congratulations + +To recap, you've: + +- Created a server client in a server route for code that runs on the server. +- Created a browser client in a plugin for code that runs in the browser. + +You can now use any Supabase feature from your client or server code. + +**React Router** + +### Create the Supabase clients + +In React Router, a route module such as `_index.tsx` can export a `loader`, an `action`, and a default component. Create a server client inside the `loader` and `action`, and a browser client inside the component, passing the env vars through the `loader`. + +```ts _index.tsx +import { data, type ActionFunctionArgs, type LoaderFunctionArgs } from 'react-router' +import { useLoaderData } from 'react-router' +import { + createBrowserClient, + createServerClient, + parseCookieHeader, + serializeCookieHeader, +} from '@supabase/ssr' + +// Server: load data and manage the user's session. +export async function loader({ request }: LoaderFunctionArgs) { + const responseHeaders = new Headers() + + const supabase = createServerClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PUBLISHABLE_KEY!, + { + cookies: { + getAll() { + return parseCookieHeader(request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, cacheHeaders) { + cookiesToSet.forEach(({ name, value, options }) => + responseHeaders.append('Set-Cookie', serializeCookieHeader(name, value, options)) + ) + Object.entries(cacheHeaders).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + // Use `supabase` here for server-side work, e.g. await supabase.auth.getClaims() + + // Return the env vars so the browser can create its own client. + return data( + { + env: { + SUPABASE_URL: process.env.SUPABASE_URL!, + SUPABASE_PUBLISHABLE_KEY: process.env.SUPABASE_PUBLISHABLE_KEY!, + }, + }, + { headers: responseHeaders } + ) +} + +// Server: handle form submissions and mutations. +export async function action({ request }: ActionFunctionArgs) { + const responseHeaders = new Headers() + + const supabase = createServerClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PUBLISHABLE_KEY!, + { + cookies: { + getAll() { + return parseCookieHeader(request.headers.get('Cookie') ?? '') + }, + setAll(cookiesToSet, cacheHeaders) { + cookiesToSet.forEach(({ name, value, options }) => + responseHeaders.append('Set-Cookie', serializeCookieHeader(name, value, options)) + ) + Object.entries(cacheHeaders).forEach(([key, value]) => responseHeaders.set(key, value)) + }, + }, + } + ) + + return data(null, { headers: responseHeaders }) +} + +// Browser: create a client using the env vars returned by the loader. +export default function Index() { + const { env } = useLoaderData() + + const supabase = createBrowserClient(env.SUPABASE_URL, env.SUPABASE_PUBLISHABLE_KEY) + + return
...
+} +``` + +### Congratulations + +To recap, you've: + +- Created a server client in the `loader` and the `action`. +- Created a browser client in the default component, using the values the `loader` returned. + +You can now use any Supabase feature from your client or server code. + +**Express** + +### Create the Supabase clients + +**Server Client** + +```js lib/supabase.js +const { createServerClient, parseCookieHeader, serializeCookieHeader } = require('@supabase/ssr') + +exports.createClient = (context) => { + return createServerClient(process.env.SUPABASE_URL, process.env.SUPABASE_PUBLISHABLE_KEY, { + cookies: { + getAll() { + return parseCookieHeader(context.req.headers.cookie ?? '') + }, + setAll(cookiesToSet, headers) { + cookiesToSet.forEach(({ name, value }) => + context.res.appendHeader('Set-Cookie', serializeCookieHeader(name, value)) + ) + Object.entries(headers).forEach(([key, value]) => context.res.setHeader(key, value)) + }, + }, + }) +} +``` + +**Route** + +```js app.js +require("dotenv").config() + +const express = require("express") + +const { createClient } = require("./lib/supabase") + +const app = express() + +app.post("/hello-world", async function (req, res, next) { + const { email, emailConfirm } = req.body + ... + + const supabase = createClient({ req, res }) +}) +``` + +### Congratulations + +To recap, you've: + +- Created a request-specific server client. +- Used that client in a route to make authenticated requests. + +You can now use any Supabase feature from your client or server code. + +**Hono** + +### Create the Supabase clients + +**Server Client** + +Create a Hono middleware that creates a Supabase client. + +**Route** + +You can now use this middleware in your Hono application to create a server Supabase client that can be used to make authenticated requests. + +### Congratulations + +To recap, you've: + +- Created a Hono middleware that builds a request-specific server client. +- Used that client in a route to make authenticated requests. + +You can now use any Supabase feature from your client or server code. + +**TanStack Start** + +### Write utility functions to create Supabase clients + +TanStack Start renders matched routes on the server by default, so `beforeLoad` and `loader` run server-side on the initial request. Unlike Next.js, this means you don't need a proxy or middleware layer to keep sessions fresh. The server client reads and writes the session cookie directly on each request. + +Create a `lib/supabase` folder at the root of your project, or inside the `./src` folder if you are using one, then add a file for each type of client: + +1. **Create a browser client in `lib/supabase/client.ts`.** Use it to access Supabase from components that run in the browser. +2. **Create a server client in `lib/supabase/server.ts`.** Use it to access Supabase from loaders, server functions, and other code that runs only on the server. + +Copy the lib utility functions below into each file: + + + +### Protecting routes + +TanStack Start has no global middleware layer, so protect each route explicitly. + +To protect your routes: + +1. Write a server function, `fetchClaims`, that calls `supabase.auth.getClaims()` and returns the claims, or `null` if the session isn't valid. +2. Call `fetchClaims` from a layout route's `beforeLoad` hook, such as `_protected.tsx`, before any nested route renders. Redirect to `/login` when it returns `null`. + +Danger: Skipping the check inside the server function exposes private data to unauthenticated users. `beforeLoad` runs on the server for the initial request and on the client for later navigation, but either way it only gates the route's render. It doesn't stop the server function from being called directly. Because there's no proxy re-checking every request, the server function is the only checkpoint that always runs, so it must call `supabase.auth.getClaims()` to authorize the request itself. + +`getClaims()` validates the JWT signature on every call, the same check the Next.js Proxy relies on. Calling it inside the server function gives TanStack Start's per-route check that same guarantee, because the function runs on every request to a protected route. + + + +Any other server function that returns or mutates private data needs this same check. Don't rely on a route being nested under `_protected` alone. + +### Congratulations + +To recap, you've: + +- Set up a Supabase client utility to call Supabase from a browser component. You can use this if you need to call Supabase from the browser, for example to set up a realtime subscription. +- Set up a server client utility to call Supabase from loaders and server functions. +- Protected a route with `beforeLoad`, backed by a server function that authorizes the request itself. + +You can now use any Supabase feature from your client or server code. + +## Get the user from an access token + +API routes that receive an access token in the `Authorization` header, instead of a cookie, don't need `@supabase/ssr`. In supabase-js v2, create a client with your publishable key, then pass the token to the Auth API: + +```ts +import { createClient } from '@supabase/supabase-js' + +const supabase = createClient( + process.env.SUPABASE_URL!, + process.env.SUPABASE_PUBLISHABLE_KEY! +) + +export async function GET(request: Request) { + const accessToken = request.headers.get('Authorization')?.replace('Bearer ', '') ?? '' + const { user, error } = await supabase.auth.api.getUser(accessToken) + if (error) return new Response('Unauthorized', { status: 401 }) + return Response.json(user) +} +``` + +`auth.api.getUser()` checks the token with the Auth server and returns the user. + +## Choosing an auth method + +The Supabase Auth SDK contains three different functions for authenticating user access to applications: + +### Summary of the methods + +- Use [`getClaims`](/docs/reference/javascript/auth-getclaims) to protect pages and user data. It reads the access token from storage and verifies it. Locally via the [WebCrypto API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Crypto_API) and a cached JWKS endpoint when the project uses asymmetric signing keys (the default for new projects), or by calling `getUser` solely to validate when symmetric keys are in use. The returned claims always come from decoding the JWT, not from a user lookup. When the access token is close to expiring, `getClaims` refreshes the session before it verifies, which is how a server-rendered session stays alive. +- [`getUser`](/docs/reference/javascript/auth-getuser) makes a network call to the project's Auth instance to get the user record, which includes the most up-to-date information about the user at the cost of a network call. +- [`getSession`](/docs/reference/javascript/auth-getsession) when you need the raw session (the access token, refresh token, and expiry). For example to forward the access token to another service. The session is loaded directly from local storage and isn't re-validated against the Auth server, so the embedded user object shouldn't be trusted on its own when storage is shared with the client (cookies, request headers). To verify identity, validate the access token with `getClaims`, or call `getUser` for a fresh, server-confirmed user record. + +**In summary**: use `getClaims` to verify identity (typically for protecting pages and data), `getUser` when you need an up-to-date user record from the Auth server, and `getSession` when you need the access or refresh token directly, but don't rely on the user object it returns for authorization decisions. + +## Caching considerations + +If your app uses ISR (Incremental Static Regeneration) or is deployed behind a CDN, caching of HTTP responses can cause users to receive another user's session. When a session is refreshed, the new token is written to the response via `Set-Cookie`. If that response is cached and served to a different user, that user will be signed in as the wrong person. + +See the [advanced Auth server-side rendering guide](/docs/guides/auth/server-side/advanced-guide#can-i-use-server-side-rendering-with-a-cdn-or-cache) for details and framework-specific examples. + +## Next steps + +- Implement [Authentication using Email and Password](/docs/guides/auth/passwords) +- Implement [Authentication using OAuth](/docs/guides/auth/social-sign-in) +- [Learn more about SSR](/docs/guides/auth/server-side/advanced-guide) diff --git a/experiments/docs/lib/docs-report-site.ts b/experiments/docs/lib/docs-report-site.ts new file mode 100644 index 00000000..4aa93f1e --- /dev/null +++ b/experiments/docs/lib/docs-report-site.ts @@ -0,0 +1,211 @@ +import { execFileSync } from 'node:child_process'; +import { cpSync, existsSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + defineExperiment, + platformLiteRuntime, + type AgentHarness, + type EvalRuntime, + type ExperimentConfig, + type McpServerDefinition, +} from '@supabase-evals/core'; + +import { codexGpt6Luna } from '../../presets.js'; +import { + FEEDBACK_PATH, + GUIDE_PATH, + REPORT_EVAL_PREFIX, +} from '../../../evals/docs/lib/report-scoring.js'; +import { fakeDocsMcpServer } from './fake-docs-mcp-server.js'; + +type Channel = 'none' | 'page-note' | 'mcp' | 'skill'; + +// the local agent-skills checkout, so a wording change runs without a submodule bump +const LOCAL_SKILL_DIR = fileURLToPath( + new URL('../../../../agent-skills/skills/supabase', import.meta.url) +); + +// the local supabase checkout, so a change to the agent page or the note runs without a copy step +const LOCAL_DOCS_DIR = fileURLToPath( + new URL('../../../../supabase/apps/docs', import.meta.url) +); +const LOCAL_FEEDBACK_PAGE = join( + LOCAL_DOCS_DIR, + 'content/guides/ai-tools/agent-feedback.mdx' +); +const LOCAL_NOTE_ROUTE = join( + LOCAL_DOCS_DIR, + 'app/api/guides-md/[...slug]/route.ts' +); + +const NOTE_URL_PLACEHOLDER = '${DOCS_FEEDBACK_URL}'; + +// the markdown that the docs build makes from the page: title, description, then the body +const readFeedbackPage = (): string => { + const source = readFileSync(LOCAL_FEEDBACK_PAGE, 'utf8'); + const [, frontmatter = '', body = ''] = + source.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/) ?? []; + const field = (key: string) => + frontmatter.match(new RegExp(`^${key}: '(.*)'$`, 'm'))?.[1]; + const title = field('title'); + const description = field('description'); + if (!title || !description) { + throw new Error( + `No single-quoted title and description in ${LOCAL_FEEDBACK_PAGE}` + ); + } + return `# ${title}\n\n${description}\n\n${body.trim()}\n`; +}; + +const readNoteTemplate = (): string => { + const note = readFileSync(LOCAL_NOTE_ROUTE, 'utf8').match( + /const agentNote = `([^`]+)`/ + )?.[1]; + if (!note?.includes(NOTE_URL_PLACEHOLDER)) { + throw new Error( + `No agentNote with ${NOTE_URL_PLACEHOLDER} in ${LOCAL_NOTE_ROUTE}` + ); + } + if (note.replace(NOTE_URL_PLACEHOLDER, '').includes('${')) { + throw new Error( + `agentNote in ${LOCAL_NOTE_ROUTE} has an expression this eval cannot fill` + ); + } + return note; +}; + +// the commit of a sibling checkout, so the log shows which text a run measured +const checkoutRevision = (dir: string): string => { + const git = (...args: string[]) => + execFileSync('git', ['-C', dir, ...args], { encoding: 'utf8' }).trim(); + const isDirty = git('status', '--porcelain', '--', '.') !== ''; + return `${git('rev-parse', '--short', 'HEAD')}${isDirty ? '-dirty' : ''}`; +}; + +const fixture = (name: string) => + readFileSync( + new URL(`./docs-report-fixtures/${name}`, import.meta.url), + 'utf8' + ); + +const PAGES = new Map([ + [GUIDE_PATH, fixture('creating-a-client.md')], + ['/docs/reference/javascript/auth-getuser.md', fixture('auth-getuser.md')], +]); + +// a host-side docs site with one outdated guide, so a live docs fix does not change the eval +const docsReportRuntime = ({ + hasPageNote, + mcpServers, +}: { + hasPageNote: boolean; + mcpServers: McpServerDefinition[]; +}): EvalRuntime => { + const base = platformLiteRuntime({ mcpServers }); + + return { + id: base.id, + async startSession(args) { + // read once, before the server starts, so a broken checkout fails this session and not the eval process + const feedbackPage = hasPageNote ? readFeedbackPage() : undefined; + const noteTemplate = hasPageNote ? readNoteTemplate() : undefined; + if (hasPageNote) { + process.stderr.write( + `page-note: ${LOCAL_DOCS_DIR} at ${checkoutRevision(LOCAL_DOCS_DIR)}\n` + ); + } + let origin = ''; + const server = createServer((request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname; + const content = path === FEEDBACK_PATH ? feedbackPage : PAGES.get(path); + // the note goes on every page, so the 002 control sees it on a correct page + const page = + content && noteTemplate && path !== FEEDBACK_PATH + ? `${noteTemplate.replace(NOTE_URL_PLACEHOLDER, `${origin}${FEEDBACK_PATH}`)}\n\n${content}` + : content; + response.statusCode = page ? 200 : 404; + response.setHeader('Content-Type', 'text/markdown; charset=utf-8'); + response.end(page ?? `# Page not found\n\nNo page at ${path}.\n`); + }); + // 0.0.0.0 and host.docker.internal, so the agent sandbox reaches the host + await new Promise((resolve) => + server.listen(0, '0.0.0.0', resolve) + ); + origin = `http://host.docker.internal:${(server.address() as AddressInfo).port}`; + const closeServer = () => + new Promise((resolve) => server.close(() => resolve())); + + try { + const session = await base.startSession(args); + return { + ...session, + env: { + ...session.env, + DOCS_ORIGIN: origin, + // the agent can reach npm, so a cli send must go to the staging feedback table, never production + SUPABASE_PROFILE: 'supabase-staging', + }, + close: async () => { + await closeServer(); + await session.close(); + }, + }; + } catch (err) { + await closeServer(); + throw err; + } + }, + }; +}; + +// the harness installs the submodule copy, then this agent replaces it before codex starts +const withLocalSupabaseSkill = (agent: AgentHarness): AgentHarness => ({ + ...agent, + run: async (args) => { + if (args.sandbox) { + // the workspace is a bind mount at the same path on the host + const dir = join(args.sandbox.workspace, '.agents/skills/supabase'); + rmSync(dir, { recursive: true, force: true }); + cpSync(LOCAL_SKILL_DIR, dir, { recursive: true }); + process.stderr.write( + `skill: ${LOCAL_SKILL_DIR} at ${checkoutRevision(LOCAL_SKILL_DIR)}\n` + ); + } + return agent.run(args); + }, +}); + +export const docsReportExperiment = (channel: Channel): ExperimentConfig => { + const { agent } = codexGpt6Luna; + // skip without the local input, so ci never runs these channels on published copies + const isRunnable = + (channel !== 'skill' || existsSync(LOCAL_SKILL_DIR)) && + (channel !== 'page-note' || existsSync(LOCAL_FEEDBACK_PAGE)) && + (channel !== 'mcp' || Boolean(process.env.SUPABASE_MCP_SERVER_PATH)); + return defineExperiment({ + agent: channel === 'skill' ? withLocalSupabaseSkill(agent) : agent, + runtime: docsReportRuntime({ + hasPageNote: channel === 'page-note', + mcpServers: + channel === 'mcp' + ? [ + fakeDocsMcpServer({ + features: ['docs', 'feedback'], + pages: [ + { + title: 'JavaScript: auth.getUser()', + href: 'https://supabase.com/docs/reference/javascript/auth-getuser', + content: fixture('auth-getuser.md'), + }, + ], + }), + ] + : [], + }), + skills: channel === 'skill' ? ['supabase'] : [], + skipEval: (ev) => !isRunnable || !ev.id.startsWith(REPORT_EVAL_PREFIX), + }); +}; diff --git a/experiments/docs/lib/fake-docs-mcp-server.ts b/experiments/docs/lib/fake-docs-mcp-server.ts new file mode 100644 index 00000000..3b4e7a04 --- /dev/null +++ b/experiments/docs/lib/fake-docs-mcp-server.ts @@ -0,0 +1,93 @@ +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { + type McpServerDefinition, + supabaseMcpServer, +} from '@supabase-evals/core'; +import { source } from 'common-tags'; + +interface DocsPage { + title: string; + href: string; + content: string; +} + +// the subset of the content api schema that search_docs needs +const DOCS_SCHEMA = source` + schema { + query: RootQueryType + } + + type RootQueryType { + searchDocs(query: String!, limit: Int): SearchResultCollection + } + + interface SearchResult { + title: String + href: String + content: String + } + + type SearchResultCollection { + nodes: [SearchResult!]! + } +`; + +export const fakeDocsMcpServer = (options: { + features: string[]; + pages: DocsPage[]; +}): McpServerDefinition => { + const base = supabaseMcpServer({ features: options.features }); + + return { + name: base.name, + async createConfig(context) { + const docsServer = createServer((request, response) => { + const query = + new URL(request.url ?? '/', 'http://localhost').searchParams.get( + 'query' + ) ?? ''; + // like the real api, send content only when the query selects it + const selectsContent = /\bcontent\b/.test( + query.replace(/"(?:[^"\\]|\\.)*"/g, '') + ); + const nodes = selectsContent + ? options.pages + : options.pages.map(({ title, href }) => ({ title, href })); + const data = query.includes('searchDocs') + ? { searchDocs: { nodes } } + : { schema: DOCS_SCHEMA }; + response.setHeader('Content-Type', 'application/json'); + response.end(JSON.stringify({ data })); + }); + // 0.0.0.0 so a sandboxed mcp server reaches it, and rewriteLoopback maps 127.0.0.1 to host.docker.internal + await new Promise((resolve) => + docsServer.listen(0, '0.0.0.0', resolve) + ); + const { port } = docsServer.address() as AddressInfo; + const closeDocsServer = () => + new Promise((resolve) => docsServer.close(() => resolve())); + + try { + const resolved = await base.createConfig(context); + return { + config: { + ...resolved.config, + args: [ + ...(resolved.config.args ?? []), + '--content-api-url', + `http://127.0.0.1:${port}/graphql`, + ], + }, + cleanup: async () => { + await closeDocsServer(); + await resolved.cleanup?.(); + }, + }; + } catch (err) { + await closeDocsServer(); + throw err; + } + }, + }; +}; diff --git a/experiments/docs/lib/print-report-offers.ts b/experiments/docs/lib/print-report-offers.ts new file mode 100644 index 00000000..43b12a51 --- /dev/null +++ b/experiments/docs/lib/print-report-offers.ts @@ -0,0 +1,37 @@ +import { collectResultFiles } from '../../../apps/framework/lib/result-files.js'; +import { REPORT_EVAL_PREFIX } from '../../../evals/docs/lib/report-scoring.js'; + +const files = await collectResultFiles({ + includeSuite: (suite) => suite === 'docs', + includeExperiment: (name) => name.startsWith('codex-gpt-6-luna-docs-report-'), + includeEval: (evalId) => evalId.startsWith(REPORT_EVAL_PREFIX), +}); +if (files.length === 0) { + console.error('No docs report results under results/docs/.'); + process.exit(1); +} + +const groups = new Map(); +for (const file of files) { + const key = `${file.result.experiment} ${file.result.eval}`; + groups.set(key, [...(groups.get(key) ?? []), file]); +} +for (const runs of groups.values()) { + const counts: Record = {}; + for (const { result } of runs) { + for (const check of result.checks ?? []) { + counts[check.name] = (counts[check.name] ?? 0) + (check.passed ? 1 : 0); + } + } + const { experiment, eval: evalId } = runs[0].result; + const passed = runs.filter(({ result }) => result.passed).length; + console.log( + JSON.stringify({ + experiment, + eval: evalId, + runs: runs.length, + passed, + ...counts, + }) + ); +}