diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 24c16b7d..33df2ff7 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -515,14 +515,6 @@ jobs: urls=$(gh pr list --head "$GITHUB_REF_NAME" --state open --json url,isCrossRepository --jq '.[] | select(.isCrossRepository == false) | .url' | paste -sd ' ' -) echo "PR_URLS=$urls" >> "$GITHUB_ENV" - - name: Upload results to Braintrust - # A Braintrust outage shouldn't block publishing results. - continue-on-error: true - env: - BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} - BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} - run: pnpm upload-braintrust --runs "${{ needs.prepare.outputs.runs }}" - - name: Upload exported results if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/evals/docs-nav/lib/navigation.test.ts b/evals/docs-nav/lib/navigation.test.ts index 9e6f658d..e6fc39cb 100644 --- a/evals/docs-nav/lib/navigation.test.ts +++ b/evals/docs-nav/lib/navigation.test.ts @@ -491,6 +491,15 @@ describe('scoreDocsNav on a tree test', () => { }); }); + it('allows opening a second-level section straight from the top', async () => { + const { navigation } = await treeNavigationOf([ + openSection('root'), + openSection('2.1'), + choosePage('2.1.1'), + ]); + expect(navigation).toMatchObject({ reached: true, noWrongTurns: true }); + }); + it('counts going back up the path as a wrong turn', async () => { const { navigation } = await treeNavigationOf([ openSection('root'), diff --git a/evals/docs-nav/lib/navigation.ts b/evals/docs-nav/lib/navigation.ts index 54666ac7..689ee969 100644 --- a/evals/docs-nav/lib/navigation.ts +++ b/evals/docs-nav/lib/navigation.ts @@ -248,10 +248,9 @@ function treePrefixes(id: string): string[] { return parts.map((_, index) => parts.slice(0, index + 1).join('.')); } -function isTreeChild(id: string, parent: string): boolean { - if (parent === TREE_ROOT) return id !== TREE_ROOT && !id.includes('.'); - const rest = id.startsWith(`${parent}.`) ? id.slice(parent.length + 1) : ''; - return rest !== '' && !rest.includes('.'); +function isTreeDescendant(id: string, parent: string): boolean { + if (parent === TREE_ROOT) return id !== TREE_ROOT; + return id.startsWith(`${parent}.`); } function isTreeCall(call: ToolCallRecord): boolean { @@ -311,7 +310,7 @@ async function scoreTreeRun( } opened.push(label(id)); const isStart = current === null && id === TREE_ROOT; - if (!isStart && (!onPath.has(id) || !isTreeChild(id, current ?? ''))) { + if (!isStart && (!onPath.has(id) || !isTreeDescendant(id, current ?? ''))) { wrongTurns.push(label(id)); } current = id; diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-revised.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-revised.experiment.ts new file mode 100644 index 00000000..3bdcf0c7 --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-revised.experiment.ts @@ -0,0 +1,9 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ + variant: 'option-1', + tree: true, + topLevels: 2, + origin: + 'https://docs-git-docs-ia-option-1-placement-revisions-supabase.vercel.app', +}); diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-subheads.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-subheads.experiment.ts new file mode 100644 index 00000000..e9ad4acb --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2-subheads.experiment.ts @@ -0,0 +1,8 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ + variant: 'option-1', + tree: true, + topLevels: 2, + origin: 'https://docs-git-docs-ia-option-1-subheads-supabase.vercel.app', +}); diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2.experiment.ts new file mode 100644 index 00000000..b3f286e9 --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-2.experiment.ts @@ -0,0 +1,7 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ + variant: 'option-1', + tree: true, + topLevels: 2, +}); diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-revised.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-revised.experiment.ts new file mode 100644 index 00000000..1803cd78 --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-revised.experiment.ts @@ -0,0 +1,8 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ + variant: 'option-1', + tree: true, + origin: + 'https://docs-git-docs-ia-option-1-placement-revisions-supabase.vercel.app', +}); diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-subheads.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-subheads.experiment.ts new file mode 100644 index 00000000..da70ed9f --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-option-1-tree-subheads.experiment.ts @@ -0,0 +1,7 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ + variant: 'option-1', + tree: true, + origin: 'https://docs-git-docs-ia-option-1-subheads-supabase.vercel.app', +}); diff --git a/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-tree-2.experiment.ts b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-tree-2.experiment.ts new file mode 100644 index 00000000..29f62663 --- /dev/null +++ b/experiments/docs-nav/claude-code-sonnet-5.5-docs-nav-tree-2.experiment.ts @@ -0,0 +1,3 @@ +import { docsNavExperiment } from './lib/docs-nav-experiment.js'; + +export default docsNavExperiment({ tree: true, topLevels: 2 }); diff --git a/experiments/docs-nav/lib/docs-nav-experiment.ts b/experiments/docs-nav/lib/docs-nav-experiment.ts index d30c009b..47112234 100644 --- a/experiments/docs-nav/lib/docs-nav-experiment.ts +++ b/experiments/docs-nav/lib/docs-nav-experiment.ts @@ -9,6 +9,9 @@ import { export type DocsVariant = 'option-1'; +const DOCS_ORIGIN = + 'https://docs-git-docs-today-nav-pre-split-baseline-supabase.vercel.app'; + const source = (file: string) => readFileSync(new URL(file, import.meta.url), 'utf8'); @@ -18,11 +21,15 @@ const TREE_NAVIGATOR_SOURCE = source('./docs-tree-navigator.mjs'); interface DocsNavOptions { variant?: DocsVariant; tree?: boolean; + origin?: string; + topLevels?: number; } function docsNavigatorMcpServer({ variant, tree, + origin = DOCS_ORIGIN, + topLevels, }: DocsNavOptions): McpServerDefinition { return { name: tree ? 'docs-tree-navigator' : 'docs-navigator', @@ -37,6 +44,8 @@ function docsNavigatorMcpServer({ ], env: { [tree ? 'DOCS_TREE_NAVIGATOR_SERVE' : 'DOCS_NAVIGATOR_SERVE']: '1', + DOCS_ORIGIN: origin, + ...(topLevels ? { DOCS_TREE_TOP_LEVELS: String(topLevels) } : {}), ...(variant ? { DOCS_IA_VARIANT: variant } : {}), }, }, diff --git a/experiments/docs-nav/lib/docs-navigator.mjs b/experiments/docs-nav/lib/docs-navigator.mjs index d4424ad0..ab33da29 100644 --- a/experiments/docs-nav/lib/docs-navigator.mjs +++ b/experiments/docs-nav/lib/docs-navigator.mjs @@ -29,8 +29,9 @@ function stripTags(html) { } export function createNavigator({ env, fetch }) { + const origin = env.DOCS_ORIGIN?.replace(/\/+$/, '') ?? SITE; const variant = env.DOCS_IA_VARIANT; - const root = `${SITE}/docs`; + const root = `${origin}/docs`; const allowed = new Set([root]); function canonicalUrl(url, base) { @@ -40,10 +41,11 @@ export function createNavigator({ env, fetch }) { } catch { return null; } - if (parsed.origin !== SITE) return null; + if (parsed.origin !== SITE && parsed.origin !== origin) return null; const path = parsed.pathname.replace(/\.md$/, '').replace(/\/+$/, ''); if (ASSET_PATTERN.test(path)) return null; - return `${SITE}${path}`; + const isDocs = path === '/docs' || path.startsWith('/docs/'); + return `${isDocs ? origin : SITE}${path}`; } function requestUrl(url) { diff --git a/experiments/docs-nav/lib/docs-tree-navigator.d.mts b/experiments/docs-nav/lib/docs-tree-navigator.d.mts index e3257c6f..291f939e 100644 --- a/experiments/docs-nav/lib/docs-tree-navigator.d.mts +++ b/experiments/docs-nav/lib/docs-tree-navigator.d.mts @@ -20,7 +20,8 @@ export declare function treeUrl( export declare function createTreeSession( tree: { name: string; root: TreeNode }, - url: string + url: string, + options?: { topLevels?: number } ): { open(id: unknown): TreeReply; choose(id: unknown): TreeReply; diff --git a/experiments/docs-nav/lib/docs-tree-navigator.mjs b/experiments/docs-nav/lib/docs-tree-navigator.mjs index 43e24c21..745f4bca 100644 --- a/experiments/docs-nav/lib/docs-tree-navigator.mjs +++ b/experiments/docs-nav/lib/docs-tree-navigator.mjs @@ -41,7 +41,7 @@ function kind(node) { return ''; } -export function createTreeSession(tree, url) { +export function createTreeSession(tree, url, { topLevels = 1 } = {}) { const visible = new Set([ROOT]); let chosen = null; @@ -64,14 +64,22 @@ export function createTreeSession(tree, url) { `${node.label} is a page, not a section. Choose it with ${CHOOSE_TOOL} if it's where you'd find the answer.` ); } - node.children.forEach((_, index) => { - visible.add(childId(id, index)); - }); + const peek = id === ROOT && topLevels >= 2; + const line = (child, childKey, indent) => + `${indent}- [${childKey}] ${child.label}${kind(child)}${child.description ? `\n${indent} ${child.description}` : ''}`; const items = node.children - .map( - (child, index) => - `- [${childId(id, index)}] ${child.label}${kind(child)}${child.description ? `\n ${child.description}` : ''}` - ) + .flatMap((child, index) => { + const key = childId(id, index); + visible.add(key); + const below = peek + ? (child.children ?? []).map((grandchild, inner) => { + const innerKey = childId(key, inner); + visible.add(innerKey); + return line(grandchild, innerKey, ' '); + }) + : []; + return [line(child, key, ''), ...below]; + }) .join('\n'); return { text: [ @@ -138,7 +146,11 @@ function serve() { if (!res.ok) throw new Error(`Couldn't load ${url}: HTTP ${res.status}`); return res.json(); }) - .then((tree) => createTreeSession(tree, url)); + .then((tree) => + createTreeSession(tree, url, { + topLevels: Number(process.env.DOCS_TREE_TOP_LEVELS ?? 1), + }) + ); const send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`); const reply = ({ isError, text }) => ({ diff --git a/experiments/docs-nav/lib/docs-tree-navigator.test.ts b/experiments/docs-nav/lib/docs-tree-navigator.test.ts index 3f3ecfa2..8fc25ae1 100644 --- a/experiments/docs-nav/lib/docs-tree-navigator.test.ts +++ b/experiments/docs-nav/lib/docs-tree-navigator.test.ts @@ -79,4 +79,13 @@ describe('createTreeSession', () => { expect(chosen.text).toContain(`Tree: ${URL}`); expect(session.choose('2').isError).toBe(true); }); + + it('shows the second level under each top-level section when asked', () => { + const session = createTreeSession(TREE, URL, { topLevels: 2 }); + const top = session.open('root').text; + expect(top).toContain(' - [1.1] Network (a page, and a section)'); + expect(top).not.toContain('SSL enforcement'); + expect(session.open('1.1').isError).toBeUndefined(); + expect(session.choose('1.2').isError).toBeUndefined(); + }); }); diff --git a/experiments/docs-nav/lib/print-report.ts b/experiments/docs-nav/lib/print-report.ts index e4c0d918..dae40c41 100644 --- a/experiments/docs-nav/lib/print-report.ts +++ b/experiments/docs-nav/lib/print-report.ts @@ -13,6 +13,16 @@ const ARMS: Record = { 'claude-code-sonnet-5.5-docs-nav-option-1': 'option-1', 'claude-code-sonnet-5.5-docs-nav-tree': 'control tree', 'claude-code-sonnet-5.5-docs-nav-option-1-tree': 'option-1 tree', + 'claude-code-sonnet-5.5-docs-nav-option-1-tree-subheads': + 'option-1 tree subheads', + 'claude-code-sonnet-5.5-docs-nav-tree-2': 'control tree 2 levels', + 'claude-code-sonnet-5.5-docs-nav-option-1-tree-2': 'option-1 tree 2 levels', + 'claude-code-sonnet-5.5-docs-nav-option-1-tree-2-subheads': + 'option-1 tree 2 levels subheads', + 'claude-code-sonnet-5.5-docs-nav-option-1-tree-revised': + 'option-1 tree revised', + 'claude-code-sonnet-5.5-docs-nav-option-1-tree-2-revised': + 'option-1 tree 2 levels revised', }; const arms: Record = Object.fromEntries(