diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c43bc00..7938d9a 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -64,15 +64,21 @@ See [Config File](/reference/config-file) for the full config format. ### Check selection -| Flag | Default | Description | -| -------------------- | ------- | ---------------------------------------- | -| `-c, --checks ` | all | Comma-separated list of check IDs to run | +| Flag | Default | Description | +| --------------------- | ------- | ----------------------------------------- | +| `-c, --checks ` | all | Comma-separated list of check IDs to run | +| `--skip-checks ` | | Comma-separated list of check IDs to skip | ```bash # Run only llms.txt checks afdocs check https://docs.example.com --checks llms-txt-exists,llms-txt-valid,llms-txt-size + +# Run all checks except one +afdocs check https://docs.example.com --skip-checks markdown-content-parity ``` +`--checks` is an include-list (only run these). `--skip-checks` is an exclude-list (run everything except these). Skipped checks appear in the report with `status: "skip"` and are excluded from scoring. + Some checks depend on others. If you include a check without its dependency, the dependent check will be skipped. See [Check dependencies](/checks/#check-dependencies) for the full list. ### Sampling diff --git a/docs/reference/config-file.md b/docs/reference/config-file.md index b375280..ddde547 100644 --- a/docs/reference/config-file.md +++ b/docs/reference/config-file.md @@ -19,6 +19,10 @@ checks: - http-status-codes - auth-gate-detection +# Optional: skip specific checks (run everything else) +# skipChecks: +# - markdown-content-parity + # Optional: override default options options: maxLinksToTest: 20 @@ -51,6 +55,17 @@ A list of check IDs to run. If omitted, all 22 checks run. Use this to focus on This is particularly useful when your docs platform doesn't support certain capabilities. For example, if you can't serve markdown, exclude the markdown-related checks so your score reflects what you can control. See [Improve Your Score](/improve-your-score#step-3-work-through-fixes-iteratively) for more on this approach. +### `skipChecks` (optional) + +A list of check IDs to skip. Unlike `checks` (which is an include-list), `skipChecks` is an exclude-list — all checks run except the ones listed here. Skipped checks appear in the report with `status: "skip"` and are excluded from scoring. + +Use this when you want to disable a specific check without having to list all the others: + +```yaml +skipChecks: + - markdown-content-parity +``` + ### `options` (optional) Override default runner options. All fields are optional: diff --git a/docs/reference/programmatic-api.md b/docs/reference/programmatic-api.md index 87acede..0456e00 100644 --- a/docs/reference/programmatic-api.md +++ b/docs/reference/programmatic-api.md @@ -32,7 +32,8 @@ Pass a second argument to configure sampling, concurrency, and thresholds: import { runChecks } from 'afdocs'; const report = await runChecks('https://docs.example.com', { - checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'], + checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'], // include-list + skipCheckIds: ['markdown-content-parity'], // exclude-list samplingStrategy: 'deterministic', maxLinksToTest: 20, maxConcurrency: 5, diff --git a/src/cli/commands/check.ts b/src/cli/commands/check.ts index a779159..713dfbc 100644 --- a/src/cli/commands/check.ts +++ b/src/cli/commands/check.ts @@ -19,6 +19,7 @@ export function registerCheckCommand(program: Command): void { .option('--config ', 'Path to config file (default: auto-discover agent-docs.config.yml)') .option('-f, --format ', 'Output format: text, json, or scorecard', 'text') .option('-c, --checks ', 'Comma-separated list of check IDs to run') + .option('--skip-checks ', 'Comma-separated list of check IDs to skip') .option('--max-concurrency ', 'Maximum concurrent requests') .option('--request-delay ', 'Delay between requests in ms') .option('--max-links ', 'Maximum links to test') @@ -109,6 +110,10 @@ export function registerCheckCommand(program: Command): void { ? (opts.checks as string).split(',').map((s) => s.trim()) : config?.checks; + const skipCheckIds = opts.skipChecks + ? (opts.skipChecks as string).split(',').map((s) => s.trim()) + : config?.skipChecks; + const format = opts.format as string; if (!FORMAT_OPTIONS.includes(format as (typeof FORMAT_OPTIONS)[number])) { process.stderr.write( @@ -196,6 +201,7 @@ export function registerCheckCommand(program: Command): void { const report = await runChecks(url, { checkIds, + skipCheckIds, maxConcurrency, requestDelay, maxLinksToTest, diff --git a/src/runner.ts b/src/runner.ts index e180e5b..b53d8a4 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -70,6 +70,7 @@ export async function runChecks( const ctx = createContext(baseUrl, options); const allChecks = getChecksSorted(); const checkIds = options?.checkIds; + const skipCheckIds = options?.skipCheckIds ?? []; const results: CheckResult[] = []; @@ -79,6 +80,20 @@ export async function runChecks( continue; } + // Emit a skip result for explicitly excluded checks without running them. + // Intentionally not stored in previousResults so dependent checks see + // "dependency never ran" and can run in standalone mode — matching the + // behaviour of checks filtered out by checkIds. + if (skipCheckIds.includes(check.id)) { + results.push({ + id: check.id, + category: check.category, + status: 'skip', + message: 'Check skipped (excluded via --skip-checks)', + }); + continue; + } + // Check dependencies — only skip if at least one dependency actually ran and none passed. // If no dependencies ran at all (e.g. filtered out via --checks), let the check handle // standalone mode itself. diff --git a/src/types.ts b/src/types.ts index 1a0e241..a5e773d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -137,6 +137,8 @@ export interface DiscoveredFile { export interface RunnerOptions extends CheckOptions { /** Only run checks matching these IDs. If empty, run all. */ checkIds?: string[]; + /** Skip checks matching these IDs, emitting a 'skip' result without running them. */ + skipCheckIds?: string[]; /** Curated page list from config or --urls. Used when samplingStrategy is 'curated'. */ curatedPages?: PageConfigEntry[]; } @@ -163,6 +165,8 @@ export interface ReportResult { export interface AgentDocsConfig { url: string; checks?: string[]; + /** Check IDs to skip, emitting a 'skip' result without running them. */ + skipChecks?: string[]; options?: Partial; /** Curated page URLs to test. Implies `samplingStrategy: 'curated'` when no strategy is set. */ pages?: PageConfigEntry[]; diff --git a/test/unit/runner.test.ts b/test/unit/runner.test.ts index 2b557cc..5c78f8b 100644 --- a/test/unit/runner.test.ts +++ b/test/unit/runner.test.ts @@ -295,6 +295,68 @@ describe('runner', () => { expect(child?.status).toBe('pass'); }); + it('skips checks listed in skipCheckIds without running them', async () => { + server.use( + http.get('http://skip-ids.local/llms.txt', () => new HttpResponse(null, { status: 404 })), + http.get( + 'http://skip-ids.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const report = await runChecks('http://skip-ids.local', { + checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'], + skipCheckIds: ['llms-txt-valid'], + requestDelay: 0, + }); + + const skipped = report.results.find((r) => r.id === 'llms-txt-valid'); + expect(skipped).toBeDefined(); + expect(skipped?.status).toBe('skip'); + expect(skipped?.message).toContain('--skip-checks'); + + // llms-txt-exists should still run (not in skipCheckIds) + const exists = report.results.find((r) => r.id === 'llms-txt-exists'); + expect(exists).toBeDefined(); + expect(exists?.status).toBe('fail'); + + // llms-txt-size depends on llms-txt-exists which failed, so it should be + // skipped due to dependency — not due to skipCheckIds + const size = report.results.find((r) => r.id === 'llms-txt-size'); + expect(size).toBeDefined(); + expect(size?.status).toBe('skip'); + expect(size?.message).toContain('dependency'); + }); + + it('skipCheckIds does not cascade-skip dependent checks', async () => { + const content = `# Test\n\n> Summary.\n\n## Links\n\n- [A](http://skip-dep.local/a): A\n`; + server.use( + http.get('http://skip-dep.local/llms.txt', () => HttpResponse.text(content)), + http.get( + 'http://skip-dep.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + // Skip llms-txt-exists; llms-txt-valid depends on it. + // Since skipCheckIds doesn't store in previousResults, llms-txt-valid + // should run in standalone mode (same as checkIds filtering). + const report = await runChecks('http://skip-dep.local', { + checkIds: ['llms-txt-exists', 'llms-txt-valid'], + skipCheckIds: ['llms-txt-exists'], + requestDelay: 0, + }); + + const exists = report.results.find((r) => r.id === 'llms-txt-exists'); + expect(exists?.status).toBe('skip'); + expect(exists?.message).toContain('--skip-checks'); + + // llms-txt-valid should run in standalone mode, not cascade-skip + const valid = report.results.find((r) => r.id === 'llms-txt-valid'); + expect(valid).toBeDefined(); + expect(valid?.message).not.toContain('dependency'); + }); + it('includes timestamp and url in report', async () => { server.use( http.get('http://meta.local/llms.txt', () => new HttpResponse(null, { status: 404 })),