Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,21 @@ See [Config File](/reference/config-file) for the full config format.

### Check selection

| Flag | Default | Description |
| -------------------- | ------- | ---------------------------------------- |
| `-c, --checks <ids>` | all | Comma-separated list of check IDs to run |
| Flag | Default | Description |
| --------------------- | ------- | ----------------------------------------- |
| `-c, --checks <ids>` | all | Comma-separated list of check IDs to run |
| `--skip-checks <ids>` | | Comma-separated list of check IDs to skip |

```bash
# Run only llms.txt checks
afdocs check https://docs.example.com --checks llms-txt-exists,llms-txt-valid,llms-txt-size

# Run all checks except one
afdocs check https://docs.example.com --skip-checks markdown-content-parity
```

`--checks` is an include-list (only run these). `--skip-checks` is an exclude-list (run everything except these). Skipped checks appear in the report with `status: "skip"` and are excluded from scoring.

Some checks depend on others. If you include a check without its dependency, the dependent check will be skipped. See [Check dependencies](/checks/#check-dependencies) for the full list.

### Sampling
Expand Down
15 changes: 15 additions & 0 deletions docs/reference/config-file.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ checks:
- http-status-codes
- auth-gate-detection

# Optional: skip specific checks (run everything else)
# skipChecks:
# - markdown-content-parity

# Optional: override default options
options:
maxLinksToTest: 20
Expand Down Expand Up @@ -51,6 +55,17 @@ A list of check IDs to run. If omitted, all 22 checks run. Use this to focus on

This is particularly useful when your docs platform doesn't support certain capabilities. For example, if you can't serve markdown, exclude the markdown-related checks so your score reflects what you can control. See [Improve Your Score](/improve-your-score#step-3-work-through-fixes-iteratively) for more on this approach.

### `skipChecks` (optional)

A list of check IDs to skip. Unlike `checks` (which is an include-list), `skipChecks` is an exclude-list — all checks run except the ones listed here. Skipped checks appear in the report with `status: "skip"` and are excluded from scoring.

Use this when you want to disable a specific check without having to list all the others:

```yaml
skipChecks:
- markdown-content-parity
```

### `options` (optional)

Override default runner options. All fields are optional:
Expand Down
3 changes: 2 additions & 1 deletion docs/reference/programmatic-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ Pass a second argument to configure sampling, concurrency, and thresholds:
import { runChecks } from 'afdocs';

const report = await runChecks('https://docs.example.com', {
checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'],
checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'], // include-list
skipCheckIds: ['markdown-content-parity'], // exclude-list
samplingStrategy: 'deterministic',
maxLinksToTest: 20,
maxConcurrency: 5,
Expand Down
6 changes: 6 additions & 0 deletions src/cli/commands/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export function registerCheckCommand(program: Command): void {
.option('--config <path>', 'Path to config file (default: auto-discover agent-docs.config.yml)')
.option('-f, --format <format>', 'Output format: text, json, or scorecard', 'text')
.option('-c, --checks <checks>', 'Comma-separated list of check IDs to run')
.option('--skip-checks <checks>', 'Comma-separated list of check IDs to skip')
.option('--max-concurrency <n>', 'Maximum concurrent requests')
.option('--request-delay <ms>', 'Delay between requests in ms')
.option('--max-links <n>', 'Maximum links to test')
Expand Down Expand Up @@ -109,6 +110,10 @@ export function registerCheckCommand(program: Command): void {
? (opts.checks as string).split(',').map((s) => s.trim())
: config?.checks;

const skipCheckIds = opts.skipChecks
? (opts.skipChecks as string).split(',').map((s) => s.trim())
: config?.skipChecks;

const format = opts.format as string;
if (!FORMAT_OPTIONS.includes(format as (typeof FORMAT_OPTIONS)[number])) {
process.stderr.write(
Expand Down Expand Up @@ -196,6 +201,7 @@ export function registerCheckCommand(program: Command): void {

const report = await runChecks(url, {
checkIds,
skipCheckIds,
maxConcurrency,
requestDelay,
maxLinksToTest,
Expand Down
15 changes: 15 additions & 0 deletions src/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export async function runChecks(
const ctx = createContext(baseUrl, options);
const allChecks = getChecksSorted();
const checkIds = options?.checkIds;
const skipCheckIds = options?.skipCheckIds ?? [];

const results: CheckResult[] = [];

Expand All @@ -79,6 +80,20 @@ export async function runChecks(
continue;
}

// Emit a skip result for explicitly excluded checks without running them.
// Intentionally not stored in previousResults so dependent checks see
// "dependency never ran" and can run in standalone mode — matching the
// behaviour of checks filtered out by checkIds.
if (skipCheckIds.includes(check.id)) {
results.push({
id: check.id,
category: check.category,
status: 'skip',
message: 'Check skipped (excluded via --skip-checks)',
});
continue;
}

// Check dependencies — only skip if at least one dependency actually ran and none passed.
// If no dependencies ran at all (e.g. filtered out via --checks), let the check handle
// standalone mode itself.
Expand Down
4 changes: 4 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ export interface DiscoveredFile {
export interface RunnerOptions extends CheckOptions {
/** Only run checks matching these IDs. If empty, run all. */
checkIds?: string[];
/** Skip checks matching these IDs, emitting a 'skip' result without running them. */
skipCheckIds?: string[];
/** Curated page list from config or --urls. Used when samplingStrategy is 'curated'. */
curatedPages?: PageConfigEntry[];
}
Expand All @@ -163,6 +165,8 @@ export interface ReportResult {
export interface AgentDocsConfig {
url: string;
checks?: string[];
/** Check IDs to skip, emitting a 'skip' result without running them. */
skipChecks?: string[];
options?: Partial<CheckOptions>;
/** Curated page URLs to test. Implies `samplingStrategy: 'curated'` when no strategy is set. */
pages?: PageConfigEntry[];
Expand Down
62 changes: 62 additions & 0 deletions test/unit/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,68 @@ describe('runner', () => {
expect(child?.status).toBe('pass');
});

it('skips checks listed in skipCheckIds without running them', async () => {
server.use(
http.get('http://skip-ids.local/llms.txt', () => new HttpResponse(null, { status: 404 })),
http.get(
'http://skip-ids.local/docs/llms.txt',
() => new HttpResponse(null, { status: 404 }),
),
);

const report = await runChecks('http://skip-ids.local', {
checkIds: ['llms-txt-exists', 'llms-txt-valid', 'llms-txt-size'],
skipCheckIds: ['llms-txt-valid'],
requestDelay: 0,
});

const skipped = report.results.find((r) => r.id === 'llms-txt-valid');
expect(skipped).toBeDefined();
expect(skipped?.status).toBe('skip');
expect(skipped?.message).toContain('--skip-checks');

// llms-txt-exists should still run (not in skipCheckIds)
const exists = report.results.find((r) => r.id === 'llms-txt-exists');
expect(exists).toBeDefined();
expect(exists?.status).toBe('fail');

// llms-txt-size depends on llms-txt-exists which failed, so it should be
// skipped due to dependency — not due to skipCheckIds
const size = report.results.find((r) => r.id === 'llms-txt-size');
expect(size).toBeDefined();
expect(size?.status).toBe('skip');
expect(size?.message).toContain('dependency');
});

it('skipCheckIds does not cascade-skip dependent checks', async () => {
const content = `# Test\n\n> Summary.\n\n## Links\n\n- [A](http://skip-dep.local/a): A\n`;
server.use(
http.get('http://skip-dep.local/llms.txt', () => HttpResponse.text(content)),
http.get(
'http://skip-dep.local/docs/llms.txt',
() => new HttpResponse(null, { status: 404 }),
),
);

// Skip llms-txt-exists; llms-txt-valid depends on it.
// Since skipCheckIds doesn't store in previousResults, llms-txt-valid
// should run in standalone mode (same as checkIds filtering).
const report = await runChecks('http://skip-dep.local', {
checkIds: ['llms-txt-exists', 'llms-txt-valid'],
skipCheckIds: ['llms-txt-exists'],
requestDelay: 0,
});

const exists = report.results.find((r) => r.id === 'llms-txt-exists');
expect(exists?.status).toBe('skip');
expect(exists?.message).toContain('--skip-checks');

// llms-txt-valid should run in standalone mode, not cascade-skip
const valid = report.results.find((r) => r.id === 'llms-txt-valid');
expect(valid).toBeDefined();
expect(valid?.message).not.toContain('dependency');
});

it('includes timestamp and url in report', async () => {
server.use(
http.get('http://meta.local/llms.txt', () => new HttpResponse(null, { status: 404 })),
Expand Down