Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 172 additions & 0 deletions .agents/scripts/instructions.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
// Instructions that name files or commands that don't exist send agents in circles. This checks
// every path, link and `npm run` script named in the agent-facing docs, and the skills' shape.

import { existsSync, lstatSync, readdirSync, readFileSync, readlinkSync } from 'node:fs'
import { basename, dirname, join, relative, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
import { git } from './lib/git.ts'
import { REPO_ROOT } from './lib/test-repo.ts'
import { AGENT_DOCS } from './lib/tree.ts'

const SKILLS_DIR = join(REPO_ROOT, '.agents/skills')
const skills = readdirSync(SKILLS_DIR).filter((name) => !name.startsWith('.'))

function markdownFiles(dir: string): string[] {
return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const path = join(dir, entry.name)
if (entry.isDirectory()) return markdownFiles(path)
return entry.name.endsWith('.md') ? [path] : []
})
}

// The same list that makes verify treat these docs as code, so a doc checked here can't be
// skipped there as docs-only.
const DOCS = [
...[...AGENT_DOCS].map((path) => join(REPO_ROOT, path)),
...markdownFiles(join(REPO_ROOT, '.agents')),
]

const tracked = git(REPO_ROOT, ['ls-files', '--cached', '--others', '--exclude-standard'])
.stdout.split('\n')
.filter(Boolean)
const scripts = Object.keys(
(
JSON.parse(readFileSync(join(REPO_ROOT, 'package.json'), 'utf8')) as {
scripts: Record<string, string>
}
).scripts,
)

const ROOTED = /^(src|scripts|docs|\.agents|\.github|\.claude|\.codex|\.cursor)\//
const FILE_NAME = /^[\w.-]+\.(md|json|ts|tsx|js|cjs|mjs|sh|yml|yaml)$/
// A path relative to the document, like `references/test-rules.md` in a skill.
const RELATIVE_FILE = /^[\w.-]+(\/[\w.-]+)+\.(md|json|ts|sh)$/
// Output and state that exist only after a build or run.
const GENERATED = /^(\.tmp|tmp|dist|node_modules)(\/|$)/

type Reference = { path: string; line?: number }

/** Paths named in backticks, with an optional `:line`. Placeholders (`<tool>`, `*`) are skipped. */
function namedPaths(text: string): Reference[] {
return [...text.matchAll(/`([^`\s]+)`/g)]
.map((match) => match[1] ?? '')
.filter((value) => !/[<>*{}]/.test(value) && !GENERATED.test(value))
.map((value) => {
const [, path = value, line] = value.match(/^(.*?):(\d+)$/) ?? []
return { path, line: line ? Number(line) : undefined }
})
.filter(({ path }) => ROOTED.test(path) || FILE_NAME.test(path) || RELATIVE_FILE.test(path))
}

// CODEBASE.md names files relative to the folder its section describes (`lib/refs.ts`,
// `users.ts` in the src/lib/ catalog), so short paths resolve against these too.
const BASES = ['', 'src/', 'src/lib/', 'src/commands/']

/** The repository path a reference names, or why it doesn't resolve. */
function resolveReference(path: string, doc: string): string | { problem: string } {
if (!ROOTED.test(path) && path.includes('/')) {
const local = relative(REPO_ROOT, resolve(dirname(doc), path))
if (existsSync(join(REPO_ROOT, local))) return local
}
if (ROOTED.test(path)) {
return existsSync(join(REPO_ROOT, path)) ? path : { problem: 'does not exist' }
}
const based = BASES.map((base) => `${base}${path}`).find((candidate) =>
existsSync(join(REPO_ROOT, candidate)),
)
if (based) return based
if (path.includes('/')) return { problem: 'does not exist' }
const matches = tracked.filter((candidate) => basename(candidate) === path)
if (matches.length === 1) return matches[0] ?? path
return {
problem: matches.length === 0 ? 'does not exist' : `is ambiguous: ${matches.join(', ')}`,
}
}

function brokenReferences(text: string, doc: string): string[] {
return namedPaths(text).flatMap(({ path, line }) => {
const resolved = resolveReference(path, doc)
if (typeof resolved !== 'string') return [`${path} ${resolved.problem}`]
if (line === undefined) return []
const full = join(REPO_ROOT, resolved)
if (lstatSync(full).isDirectory()) return [`${path}:${line} names a line in a folder`]
const lines = readFileSync(full, 'utf8').split('\n').length
return line > lines ? [`${path}:${line} is past the end (${lines} lines)`] : []
})
}

/** Backticked names in the bullet list under a heading. */
function listedUnder(text: string, heading: string): string[] {
const section = text.split(heading)[1]?.split(/\n#{1,3} /)[0] ?? ''
return [...section.matchAll(/^- `([\w-]+)`/gm)].map((match) => match[1] ?? '')
}

describe.each(DOCS.map((path) => [relative(REPO_ROOT, path), path]))('%s', (_name, path) => {
const text = readFileSync(path, 'utf8')

it('names only files and lines that exist', () => {
expect(brokenReferences(text, path)).toEqual([])
})

it('links only to files that exist', () => {
const links = [...text.matchAll(/\]\(([^)#\s]+)(#[^)]*)?\)/g)]
.map((match) => match[1] ?? '')
.filter((link) => !/^[a-z]+:/.test(link))
expect(links.filter((link) => !existsSync(resolve(dirname(path), link)))).toEqual([])
})

it('names only npm scripts that exist', () => {
const named = [...text.matchAll(/npm run (?:--silent )?([\w:-]+)/g)].map((m) => m[1] ?? '')
expect(named.filter((script) => !scripts.includes(script))).toEqual([])
})
})

describe.each(skills)('skill %s', (skill) => {
const file = join(SKILLS_DIR, skill, 'SKILL.md')
const text = readFileSync(file, 'utf8')
const frontmatter = text.match(/^---\n([\s\S]*?)\n---\n/)?.[1] ?? ''

it('has a name matching its folder and a description that says when to use it', () => {
expect(frontmatter).toMatch(new RegExp(`^name: ${skill}$`, 'm'))
const description = frontmatter.match(/^description: (.+)$/m)?.[1] ?? ''
expect(description).toContain('Use when')
expect(description.length).toBeLessThan(400)
})

it('is linked for Claude Code', () => {
const link = join(REPO_ROOT, '.claude/skills', skill)
expect(lstatSync(link).isSymbolicLink()).toBe(true)
expect(resolve(dirname(link), readlinkSync(link))).toBe(join(SKILLS_DIR, skill))
})
})

it('lists every skill in AGENTS.md, and only those', () => {
const text = readFileSync(join(REPO_ROOT, 'AGENTS.md'), 'utf8')
expect(listedUnder(text, '## Skills').toSorted()).toEqual(skills.toSorted())
})

it('names only test helpers that the fixtures, global-args or @doist/cli-core/testing export', async () => {
const skill = readFileSync(join(SKILLS_DIR, 'write-tests/SKILL.md'), 'utf8')
const section = skill.split('## Helpers')[1]?.split('\n## ')[0] ?? ''
const named = [...section.matchAll(/`(\w+)`/g)].map((match) => match[1] ?? '')
const fixtures = join(REPO_ROOT, 'src/lib/__fixtures__')
const sources = [
...readdirSync(fixtures).map((file) => join(fixtures, file)),
join(REPO_ROOT, 'src/lib/global-args.ts'),
]
const local = sources.flatMap((file) =>
[
...readFileSync(file, 'utf8').matchAll(
/^export (?:async )?(?:function|const|type) (\w+)/gm,
),
].map((match) => match[1]),
)
const shared = Object.keys(await import('@doist/cli-core/testing'))
expect(named.length).toBeGreaterThan(5)
expect(named.filter((name) => !local.includes(name) && !shared.includes(name))).toEqual([])
})

it('has no Claude Code skill link without a skill behind it', () => {
const links = readdirSync(join(REPO_ROOT, '.claude/skills'))
expect(links.toSorted()).toEqual(skills.toSorted())
})
39 changes: 34 additions & 5 deletions .agents/scripts/lib/tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,14 @@ import { rmSync, utimesSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createTempRepo, type TempRepo } from './test-repo.ts'
import { changedPaths, changedSince, classify, codeFingerprint, snapshot } from './tree.ts'
import {
changedCodePaths,
changedPaths,
changedSince,
classify,
codeFingerprint,
snapshot,
} from './tree.ts'

let repo: TempRepo

Expand Down Expand Up @@ -60,16 +67,35 @@ describe('snapshot', () => {
})

describe('codeFingerprint', () => {
it('ignores docs and files only CI reads, and nothing else', () => {
it('ignores edits to docs and files only CI reads, and nothing else', () => {
const before = codeFingerprint(snapshot(repo.dir))
repo.write('README.md', 'changed\n')
repo.write('docs/guide.md', 'new\n')
repo.write('renovate.json', '{}\n')
repo.write('.github/CODEOWNERS', '* @example\n')
expect(codeFingerprint(snapshot(repo.dir))).toBe(before)
repo.write('src/prompt.md', 'markdown the server could load\n')
repo.write('src/prompt.md', 'markdown the CLI could load\n')
expect(codeFingerprint(snapshot(repo.dir))).not.toBe(before)
})

it('changes when a doc is added or deleted, since the agent docs may name it', () => {
const before = codeFingerprint(snapshot(repo.dir))
repo.write('docs/guide.md', 'new\n')
expect(codeFingerprint(snapshot(repo.dir))).not.toBe(before)
rmSync(join(repo.dir, 'docs/guide.md'))
expect(codeFingerprint(snapshot(repo.dir))).toBe(before)
rmSync(join(repo.dir, 'README.md'))
expect(codeFingerprint(snapshot(repo.dir))).not.toBe(before)
})
})

describe('changedCodePaths', () => {
it('counts a deleted doc, but not an edited or added one', () => {
const before = snapshot(repo.dir)
repo.write('docs/guide.md', 'new\n')
expect(changedCodePaths(before, snapshot(repo.dir))).toEqual([])
rmSync(join(repo.dir, 'README.md'))
expect(changedCodePaths(before, snapshot(repo.dir))).toEqual(['README.md'])
})
})

describe('classify', () => {
Expand All @@ -78,8 +104,11 @@ describe('classify', () => {
['CHANGELOG.md', 'docs'],
['docs/SPEC.md', 'docs'],
['CONTRIBUTING.md', 'docs'],
['CODEBASE.md', 'docs'],
['docs/icon.md', 'docs'],
['AGENTS.md', 'code'],
['CODEBASE.md', 'code'],
['docs/agents.md', 'code'],
['.agents/skills/verify/SKILL.md', 'code'],
['.github/PULL_REQUEST_TEMPLATE.md', 'docs'],
['LICENSE', 'docs'],
['.github/workflows/lint.yml', 'code'],
Expand Down
35 changes: 30 additions & 5 deletions .agents/scripts/lib/tree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// made in the same second, and hashing in-process writes nothing to git's object store.

import { createHash } from 'node:crypto'
import { lstatSync, readFileSync, readlinkSync } from 'node:fs'
import { existsSync, lstatSync, readFileSync, readlinkSync } from 'node:fs'
import { join } from 'node:path'
import { git } from './git.ts'

Expand Down Expand Up @@ -44,17 +44,23 @@ export const CHECKED_WORKFLOWS = [
'.github/workflows/test.yml',
]

/** Agent instructions outside .agents/. instructions.test.ts checks every reference in them. */
export const AGENT_DOCS = new Set(['AGENTS.md', 'CODEBASE.md', 'docs/agents.md'])

/** The kind of file, as far as local checks are concerned. */
export type FileKind = 'docs' | 'unchecked' | 'code'

/**
* - docs: prose no check reads. Markdown under src/ is excluded, since it could be loaded at runtime.
* - docs: prose no check reads. Markdown under src/ is excluded, since it could be loaded at runtime,
* and so are the agent instructions, which a test reads.
* - unchecked: files only CI or GitHub reads (Renovate, other .github/ config). Local verification
* cannot vouch for them.
* - code: everything else, including config, lockfiles, the generated skill and tests.
*/
export function classify(path: string): FileKind {
if (path.startsWith('.github/workflows/')) return 'code'
// .agents/scripts/instructions.test.ts checks every reference in these.
if (AGENT_DOCS.has(path) || path.startsWith('.agents/')) return 'code'
Comment thread
scottlovegrove marked this conversation as resolved.
Resolved
// `npm run check:skill-sync` compares it with src/lib/skills/content.ts.
if (path.startsWith('skills/')) return 'code'
if (path.startsWith('.github/')) return path.endsWith('.md') ? 'docs' : 'unchecked'
Expand All @@ -64,28 +70,47 @@ export function classify(path: string): FileKind {
return 'code'
}

const present = (files: Snapshot, path: string) =>
files[path] !== undefined && files[path] !== 'missing'

/**
* Hash of every file local checks read, so editing docs, or files only CI reads, after a pass
* keeps it valid.
* keeps it valid. Docs count by name: the agent docs may name one, so adding, renaming or deleting
* a doc means checking again.
*/
export function codeFingerprint(files: Snapshot): string {
const hash = createHash('sha256')
for (const path of Object.keys(files).sort()) {
if (classify(path) === 'code') hash.update(`${path}\0${files[path]}\n`)
const kind = classify(path)
if (kind === 'code') hash.update(`${path}\0${files[path]}\n`)
else if (kind === 'docs' && present(files, path)) hash.update(`${path}\0doc\n`)
}
return hash.digest('hex')
}

/** Changed files local checks read. */
export function changedCodePaths(before: Snapshot, after: Snapshot): string[] {
return changedPaths(before, after).filter((path) => classify(path) === 'code')
return changedPaths(before, after).filter(
(path) =>
classify(path) === 'code' ||
(classify(path) === 'docs' && present(before, path) && !present(after, path)),
)
}

export function changedPaths(before: Snapshot, after: Snapshot): string[] {
const paths = new Set([...Object.keys(before), ...Object.keys(after)])
return [...paths].filter((path) => before[path] !== after[path]).sort()
}

/**
* Whether a file that differs from the default branch needs the checks: code, or a doc that was
* deleted or renamed, since the agent docs may name it.
*/
export function needsChecks(root: string, path: string): boolean {
const kind = classify(path)
return kind === 'code' || (kind === 'docs' && !existsSync(join(root, path)))
}

/** Files that differ from `base`: committed, staged, unstaged and untracked. */
export function changedSince(root: string, base: string): string[] {
const diff = git(root, ['diff', '--name-only', '-z', '--no-renames', base])
Expand Down
7 changes: 6 additions & 1 deletion .agents/scripts/lib/verify.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { existsSync, readFileSync } from 'node:fs'
import { existsSync, readFileSync, rmSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { readVerified, stateDir, writeVerified } from './state.ts'
Expand Down Expand Up @@ -173,6 +173,11 @@ describe('what counts as a change', () => {
expect(await verify([pass('lint')])).toEqual({ ok: true, recorded: true })
})

it('runs the checks when a doc is deleted, since the agent docs may name it', async () => {
rmSync(join(repo.dir, 'README.md'))
expect(await verify([pass('lint')])).toEqual({ ok: true, recorded: true })
})

it('has nothing to check for a docs-only change', async () => {
repo.write('README.md', 'changed\n')
const result = await verify([fail('lint')])
Expand Down
13 changes: 11 additions & 2 deletions .agents/scripts/lib/verify.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,14 @@ import { createWriteStream, mkdirSync, readFileSync } from 'node:fs'
import { join, relative } from 'node:path'
import { cleanGitEnv, git } from './git.ts'
import { clearVerified, stateDir, writeVerified } from './state.ts'
import { changedCodePaths, changedSince, classify, codeFingerprint, snapshot } from './tree.ts'
import {
changedCodePaths,
changedSince,
classify,
codeFingerprint,
needsChecks,
snapshot,
} from './tree.ts'

export type Step = {
name: string
Expand Down Expand Up @@ -36,7 +43,9 @@ export async function runVerify(options: VerifyOptions): Promise<VerifyResult> {
out(`No ${baseName} branch found, so checking the whole tree.`)
} else {
const changes = changedSince(root, base)
const kinds = new Set(changes.map(classify))
const kinds = new Set(
changes.map((path) => (needsChecks(root, path) ? 'code' : classify(path))),
)
if (changes.length > 0 && !kinds.has('code')) {
if (kinds.has('unchecked')) {
const unchecked = changes.filter((path) => classify(path) === 'unchecked')
Expand Down
Loading
Loading