diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e0fe36..c9bcaba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: npm ci - name: Run tests - run: go test -race ./cmd/sync-content/... + run: go test -race ./cmd/sync-content/... ./cmd/doctest/... - name: Sync content run: go run ./cmd/sync-content --org complytime --config sync-config.yaml --lock .content-lock.json --write @@ -46,3 +46,7 @@ jobs: - name: Build site run: hugo --minify --gc + + - name: Run documentation tests (informational) + run: make test-docs + continue-on-error: true diff --git a/.github/workflows/deploy-gh-pages.yml b/.github/workflows/deploy-gh-pages.yml index 035bda6..1eb59e1 100644 --- a/.github/workflows/deploy-gh-pages.yml +++ b/.github/workflows/deploy-gh-pages.yml @@ -55,7 +55,7 @@ jobs: fi - name: Run tests - run: go test -race ./cmd/sync-content/... + run: go test -race ./cmd/sync-content/... ./cmd/doctest/... - name: Sync content run: go run ./cmd/sync-content --org complytime --config sync-config.yaml --lock .content-lock.json --write @@ -65,6 +65,10 @@ jobs: - name: Build run: hugo --minify --gc + - name: Run documentation tests (informational) + run: make test-docs + continue-on-error: true + - name: Upload artifact uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 with: diff --git a/.gitignore b/.gitignore index 9102527..57b2e41 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,13 @@ hugo_stats.json node_modules/ # ─── Go ────────────────────────────────────────────────────────────── -# Compiled sync engine binary (built by CI or locally). +# Compiled tool binaries (built by CI or locally). /sync-content +cmd/sync-content/sync-content +/doctest + +# ─── Test output ───────────────────────────────────────────────────── +.test-output/ # ─── Synced content (generated by sync-content at build time) ──────── # Per-repo project pages generated by the org scan and config overlay. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..111a949 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,21 @@ +# Agent Instructions + +## Documentation with Shell Commands + +When editing documentation that contains shell commands: + +1. **Always add `{test="..."}` attributes** to fenced code blocks that contain + runnable shell commands. Use lowercase alphanumeric identifiers with hyphens. + +2. **Write a corresponding Bats test** in `tests/docs/` before fixing a snippet + (TDD for docs). The test name must match the `test` attribute value. + +3. **Run `make test-docs-coverage`** to check for untested code blocks. + +4. **Run `make test-docs`** to verify all annotated snippets pass their tests. + +## Go Code + +- Run `make check` before committing (includes `go vet`, `gofmt`, race tests, + and doc coverage). +- Follow existing patterns in `cmd/sync-content/` for test structure. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e6c819..8e16bc5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -401,6 +401,7 @@ style: fix indentation in home template - [ ] No broken links or missing images - [ ] Frontmatter includes all required fields (`title`, `description`, `weight`) - [ ] If Go code was changed: `make check` passes (`go vet`, `gofmt`, and `go test -race`) +- [ ] If documentation code blocks were changed: `make test-docs` passes - [ ] Commit messages follow conventional format - [ ] DCO sign-off is present @@ -447,7 +448,7 @@ The Makefile provides handy targets for all common sync and Go operations make sync-dry # dry-run — reads GitHub, writes nothing make sync # apply changes to disk make sync-single REPO=complytime/complyctl # single-repo dry-run -make check # go vet + fmt-check + race tests (CI equivalent) +make check # go vet + fmt-check + race tests + doc coverage (CI equivalent) make test-race # tests with race detector only ``` @@ -483,6 +484,51 @@ echo "Token set, length: ${#GITHUB_TOKEN}, prefix: ${GITHUB_TOKEN:0:4}" go run ./cmd/sync-content --org complytime --config sync-config.yaml --write ``` +### Testing Documentation + +Documentation pages with shell commands use **testable code blocks** — fenced +code blocks annotated with a `{test="..."}` attribute that links them to +automated tests. + +**Annotating a code block:** + +````markdown +```bash {test="install-complyctl"} +go install github.com/complytime/complyctl@latest +``` +```` + +The `test` value must be lowercase alphanumeric with hyphens (`[a-z0-9-]+`). +It becomes both the extracted snippet filename and the Bats test reference. +Each value must be unique within a page. + +**Writing the corresponding test:** + +Create or update a `.bats` file in `tests/docs/` matching the page name: + +```bash +# tests/docs/getting-started.bats + +@test "install-complyctl" { + run_snippet "getting-started/01-install-complyctl.bash" + assert_success +} +``` + +**Opting out a page:** Add `testable_docs: false` to the page's YAML frontmatter +to skip it entirely from extraction and coverage reporting. + +**Make targets:** + +| Target | What it does | +|--------|-------------| +| `make test-docs-extract` | Extract annotated code blocks to `.test-output/doctest-snippets` | +| `make test-docs` | Extract + run Bats tests | +| `make test-docs-coverage` | Report untested executable code blocks (warnings only) | + +Coverage warnings are non-blocking — they show which blocks could benefit from +test annotations but do not fail the build. + ### Testing Tips - Always test with **browser cache disabled** (DevTools → Network → diff --git a/Makefile b/Makefile index c7f064b..0dacccd 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # make sync-dry — dry-run content sync (reads GitHub, writes nothing) # make sync — apply content sync to disk # make dev — start Hugo dev server (after syncing content) -# make check — vet + fmt-check + race tests +# make check — vet + fmt-check + race tests + doc coverage # --------------------------------------------------------------------------- # Overridable variables @@ -22,6 +22,7 @@ REPO ?= SYNC_BIN := cmd/sync-content/sync-content SYNC_PKG := ./cmd/sync-content/... +DOCTEST_DIR ?= .test-output/doctest-snippets # Common flags passed to every sync invocation SYNC_FLAGS := --org $(ORG) --config $(CONFIG) --output $(OUTPUT) --workers $(WORKERS) --timeout $(TIMEOUT) @@ -49,24 +50,29 @@ build: ## Compile the sync-content binary go build -o $(SYNC_BIN) ./cmd/sync-content .PHONY: test -test: ## Run all Go unit tests - go test $(SYNC_PKG) +test: ## Run all tests (Go unit + doc tests + doc coverage) + go test $(SYNC_PKG) ./cmd/doctest/... + # Phase 1: doc tests and coverage are non-blocking (leading '-' ignores + # their exit status) until all executable code blocks are annotated. + # Remove the '-' prefixes in Phase 2 to make them gate `make test`. + -$(MAKE) test-docs + -$(MAKE) test-docs-coverage .PHONY: test-race test-race: ## Run Go tests with the race detector - go test -race $(SYNC_PKG) + go test -race $(SYNC_PKG) ./cmd/doctest/... .PHONY: vet vet: ## Run go vet - go vet $(SYNC_PKG) + go vet $(SYNC_PKG) ./cmd/doctest/... .PHONY: fmt fmt: ## Format Go source files with gofmt - gofmt -w cmd/sync-content/ + gofmt -w cmd/sync-content/ cmd/doctest/ .PHONY: fmt-check fmt-check: ## Check Go formatting (non-destructive) - @out=$$(gofmt -l cmd/sync-content/); \ + @out=$$(gofmt -l cmd/sync-content/ cmd/doctest/); \ if [ -n "$$out" ]; then \ echo "The following files need formatting:"; \ echo "$$out"; \ @@ -74,7 +80,10 @@ fmt-check: ## Check Go formatting (non-destructive) fi .PHONY: check -check: vet fmt-check test-race ## Run vet + fmt-check + race tests (CI equivalent) +check: vet fmt-check test-race ## Run vet + fmt-check + race tests + doc coverage (CI equivalent) + # Phase 1: coverage is non-blocking (leading '-') until all executable + # code blocks are annotated. Remove the '-' in Phase 2 to make it gate. + -$(MAKE) test-docs-coverage # --------------------------------------------------------------------------- # Content sync — uses GITHUB_TOKEN from the environment @@ -106,6 +115,24 @@ sync-single: ## Apply sync for one repo (REPO=complytime/complyctl) @if [ -z "$(REPO)" ]; then echo "Usage: make sync-single REPO=complytime/"; exit 1; fi $(MAKE) sync REPO=$(REPO) +# --------------------------------------------------------------------------- +# Documentation tests — extract, validate, and test code blocks +# --------------------------------------------------------------------------- + +.PHONY: test-docs-extract +test-docs-extract: ## Extract testable code blocks from documentation + @# The doctest tool empties DOCTEST_DIR contents itself before writing, + @# so stale snippets from previous runs never leak into the Bats tests. + @go run ./cmd/doctest extract --content-dir content/docs --output-dir $(DOCTEST_DIR) + +.PHONY: test-docs +test-docs: test-docs-extract ## Run documentation tests (Bats) + @SNIPPETS_DIR=$(DOCTEST_DIR) node_modules/.bin/bats --formatter pretty tests/docs/ + +.PHONY: test-docs-coverage +test-docs-coverage: ## Report untested code blocks in documentation + @go run ./cmd/doctest coverage --content-dir content/docs + # --------------------------------------------------------------------------- # Hugo / Node — site build and dev server # --------------------------------------------------------------------------- diff --git a/README.md b/README.md index 53fdd6d..4cd4b01 100644 --- a/README.md +++ b/README.md @@ -18,15 +18,19 @@ The site will be available at `http://localhost:1313/`. **Production build**: `npm run build` (output in `public/`). +**Documentation tests**: `make test-docs` extracts annotated code blocks and runs Bats tests against them. + ## Project Structure ``` website/ ├── cmd/sync-content/ # Go content sync tool (10 source files, package main) +├── cmd/doctest/ # Go documentation test extraction tool ├── config/_default/ # Hugo configuration (TOML) ├── content/docs/ # Markdown content (projects/ is generated by sync tool) ├── data/projects.json # Generated landing page cards (gitignored) ├── layouts/ # Custom Hugo layout overrides +├── tests/docs/ # Bats documentation tests ├── sync-config.yaml # Declarative sync configuration ├── .content-lock.json # Approved upstream SHAs per repo (committed) └── .github/workflows/ # CI, deploy, weekly content check diff --git a/cmd/doctest/extract.go b/cmd/doctest/extract.go new file mode 100644 index 0000000..5c0bcc1 --- /dev/null +++ b/cmd/doctest/extract.go @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + + goyaml "github.com/goccy/go-yaml" +) + +// gitTrackedFiles returns the set of absolute paths for files under dir that +// are tracked by git or untracked-but-not-ignored (i.e. new files not yet +// committed that aren't covered by .gitignore). This requires dir to be +// inside a git repository. +func gitTrackedFiles(dir string) (map[string]bool, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("resolving %s: %w", dir, err) + } + + // --cached: tracked files; --others: untracked; --exclude-standard: + // honour .gitignore, .git/info/exclude, and core.excludesFile. + cmd := exec.Command("git", "ls-files", "--cached", "--others", "--exclude-standard", "--", "*.md") + cmd.Dir = absDir + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git ls-files in %s: %w", absDir, err) + } + + files := make(map[string]bool) + for _, line := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") { + if line == "" { + continue + } + files[filepath.Join(absDir, line)] = true + } + return files, nil +} + +// testableLangs are languages whose untested blocks produce coverage warnings. +var testableLangs = map[string]bool{ + "bash": true, + "sh": true, + "shell": true, + "zsh": true, +} + +var testIDPattern = regexp.MustCompile(`^[a-z0-9-]+$`) + +// snippet represents a single extracted code block. +type snippet struct { + Test string `json:"test"` + File string `json:"file"` + SourceLine int `json:"source_line"` + Language string `json:"language"` +} + +// manifest represents the JSON manifest for a single page. +type manifest struct { + Page string `json:"page"` + Snippets []snippet `json:"snippets"` +} + +// codeBlock is an intermediate representation of a parsed fenced code block. +type codeBlock struct { + lang string + testName string + content []byte + line int // 1-based line number in the source file +} + +// frontmatter holds the subset of YAML frontmatter we care about. +type frontmatter struct { + TestableDocs *bool `yaml:"testable_docs"` +} + +// parseFrontmatter extracts YAML frontmatter from Markdown source. +// Returns nil (not opted-out) if no frontmatter is found. +func parseFrontmatter(source []byte) (*frontmatter, error) { + // Require an opening "---\n" delimiter so that a line like "---foo" at + // the start of the file is not mistaken for frontmatter. + if !bytes.HasPrefix(source, []byte("---\n")) { + return nil, nil + } + end := bytes.Index(source[3:], []byte("\n---")) + if end == -1 { + return nil, nil + } + yamlBytes := source[3 : end+3] + + var fm frontmatter + if err := goyaml.Unmarshal(yamlBytes, &fm); err != nil { + return nil, fmt.Errorf("parsing frontmatter: %w", err) + } + return &fm, nil +} + +// isOptedOut returns true if the page has testable_docs: false. +func isOptedOut(fm *frontmatter) bool { + return fm != nil && fm.TestableDocs != nil && !*fm.TestableDocs +} + +// parseInfoString extracts the language and test attribute from a fenced code +// block info string. Examples: +// +// "bash {test=\"install\"}" -> ("bash", "install") +// "bash" -> ("bash", "") +// "" -> ("", "") +func parseInfoString(info string) (lang string, testName string, err error) { + info = strings.TrimSpace(info) + if info == "" { + return "", "", nil + } + + // Split on '{' to separate language from attributes + langPart, attrPart, hasAttrs := strings.Cut(info, "{") + lang = strings.TrimSpace(langPart) + + if !hasAttrs { + return lang, "", nil + } + + // Re-add the '{' for parser.ParseAttributes + attrStr := "{" + attrPart + reader := text.NewReader([]byte(attrStr)) + attrs, ok := parser.ParseAttributes(reader) + if !ok { + return lang, "", nil + } + + for _, attr := range attrs { + if string(attr.Name) == "test" { + if v, ok := attr.Value.([]byte); ok { + testName = string(v) + } + break + } + } + + if testName != "" && !testIDPattern.MatchString(testName) { + return "", "", fmt.Errorf("invalid test identifier %q: must match [a-z0-9-]+", testName) + } + + return lang, testName, nil +} + +// extractBlocks parses a Markdown file and returns all fenced code blocks. +func extractBlocks(source []byte) ([]codeBlock, error) { + md := goldmark.New() + reader := text.NewReader(source) + doc := md.Parser().Parse(reader) + + var blocks []codeBlock + err := ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering || node.Kind() != ast.KindFencedCodeBlock { + return ast.WalkContinue, nil + } + fcb := node.(*ast.FencedCodeBlock) + + // Get the full info string + var info string + if fcb.Info != nil { + info = string(fcb.Info.Segment.Value(source)) + } + + lang, testName, err := parseInfoString(info) + if err != nil { + return ast.WalkStop, fmt.Errorf("line %d: %w", lineNumber(source, node), err) + } + + // Collect code content + var content []byte + for i := 0; i < fcb.Lines().Len(); i++ { + line := fcb.Lines().At(i) + content = append(content, line.Value(source)...) + } + + blocks = append(blocks, codeBlock{ + lang: lang, + testName: testName, + content: content, + line: lineNumber(source, node), + }) + + return ast.WalkContinue, nil + }) + + return blocks, err +} + +// lineNumber computes the 1-based line number for an AST node's position. +func lineNumber(source []byte, node ast.Node) int { + // Use the first line of the code block to find the position, + // then scan backwards to find the fence line. + // The node itself doesn't track the fence line, but we can use + // the text segment positions. + pos := 0 + if fcb, ok := node.(*ast.FencedCodeBlock); ok && fcb.Lines().Len() > 0 { + seg := fcb.Lines().At(0) + pos = seg.Start + } + // Count newlines before pos to get line number, then subtract 1 + // for the fence line itself. + line := 1 + for i := 0; i < pos && i < len(source); i++ { + if source[i] == '\n' { + line++ + } + } + // The fence line (```) is one line before the first content line + if line > 1 { + line-- + } + return line +} + +// langExtension returns the file extension for a language identifier. +func langExtension(lang string) string { + switch lang { + case "bash", "sh", "shell", "zsh": + return lang + case "yaml", "yml": + return "yaml" + case "json": + return "json" + case "toml": + return "toml" + case "go": + return "go" + case "python", "py": + return "py" + default: + if lang == "" { + return "txt" + } + return lang + } +} + +// pageSlug computes the output directory name from a file path relative to +// the content directory. For "getting-started/_index.md" -> "getting-started". +// For "guides/advanced/_index.md" -> "guides-advanced". +// For "_index.md" at the root -> "root". +func pageSlug(relPath string) string { + dir := filepath.Dir(relPath) + base := filepath.Base(relPath) + name := strings.TrimSuffix(base, filepath.Ext(base)) + + if name == "_index" || name == "index" { + // Section pages: use directory path + if dir == "." || dir == "" { + return "root" + } + return strings.ReplaceAll(filepath.ToSlash(dir), "/", "-") + } + + // Named pages: include filename stem to avoid collisions + if dir == "." || dir == "" { + return name + } + return strings.ReplaceAll(filepath.ToSlash(dir), "/", "-") + "-" + name +} + +// cleanDirContents removes all entries inside dir without removing dir itself, +// preserving the directory's inode for any process holding a handle to it. +// If dir does not exist it is created. +func cleanDirContents(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return os.MkdirAll(dir, 0o755) + } + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + +// dirsOverlap reports whether a and b resolve to the same directory, or +// whether one is an ancestor of the other. +func dirsOverlap(a, b string) (bool, error) { + absA, err := filepath.Abs(a) + if err != nil { + return false, fmt.Errorf("resolving %s: %w", a, err) + } + absB, err := filepath.Abs(b) + if err != nil { + return false, fmt.Errorf("resolving %s: %w", b, err) + } + rel, err := filepath.Rel(absA, absB) + if err != nil { + return false, nil // different volumes/roots: cannot overlap + } + if rel == "." { + return true, nil // identical directories + } + if !strings.HasPrefix(rel, "..") { + return true, nil // b is inside a + } + // rel starts with "..": b is outside a, unless every segment is "..", + // in which case a is inside b (b is an ancestor of a). + for _, part := range strings.Split(rel, string(filepath.Separator)) { + if part != ".." { + return false, nil + } + } + return true, nil +} + +// runExtract walks the content directory, extracts annotated code blocks, +// and writes them to the output directory. +func runExtract(contentDir, outputDir string) error { + // cleanDirContents below recursively deletes everything under + // outputDir; reject any configuration where outputDir equals, contains, + // or is contained by contentDir before touching the filesystem, so a + // misconfigured --output-dir cannot destroy tracked documentation. + if overlap, err := dirsOverlap(contentDir, outputDir); err != nil { + return err + } else if overlap { + return fmt.Errorf("--content-dir %s and --output-dir %s overlap: refusing to delete content", contentDir, outputDir) + } + + tracked, err := gitTrackedFiles(contentDir) + if err != nil { + return err + } + // Empty stale output from previous runs so renamed or deleted blocks + // don't leave orphaned snippets that Bats would test against. Only the + // directory contents are removed, not the directory itself, so any + // process watching or cd'd into it keeps a valid handle. + if err := cleanDirContents(outputDir); err != nil { + return fmt.Errorf("cleaning output dir %s: %w", outputDir, err) + } + return filepath.WalkDir(contentDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // Skip symlinks to avoid following them out of the content tree (gosec G122). + if d.Type()&fs.ModeSymlink != 0 { + return nil + } + if d.IsDir() || filepath.Ext(path) != ".md" { + return nil + } + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolving %s: %w", path, err) + } + if !tracked[absPath] { + return nil + } + + source, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + // Check frontmatter opt-out + fm, err := parseFrontmatter(source) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + if isOptedOut(fm) { + return nil + } + + blocks, err := extractBlocks(source) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + + // Filter to annotated blocks only + var annotated []codeBlock + for _, b := range blocks { + if b.testName != "" { + annotated = append(annotated, b) + } + } + if len(annotated) == 0 { + return nil + } + + // Check for duplicate test names + seen := make(map[string]int) // testName -> line number + for _, b := range annotated { + if prevLine, ok := seen[b.testName]; ok { + return fmt.Errorf("%s: duplicate test name %q at lines %d and %d", + path, b.testName, prevLine, b.line) + } + seen[b.testName] = b.line + } + + // Compute output paths and write + relPath, err := filepath.Rel(contentDir, path) + if err != nil { + return fmt.Errorf("computing relative path for %s: %w", path, err) + } + slug := pageSlug(relPath) + pageDir := filepath.Join(outputDir, slug) + if err := os.MkdirAll(pageDir, 0o755); err != nil { + return fmt.Errorf("creating directory %s: %w", pageDir, err) + } + + var snippets []snippet + for i, b := range annotated { + ext := langExtension(b.lang) + filename := fmt.Sprintf("%02d-%s.%s", i+1, b.testName, ext) + outPath := filepath.Join(pageDir, filename) + + if err := os.WriteFile(outPath, b.content, 0o600); err != nil { + return fmt.Errorf("writing %s: %w", outPath, err) + } + + snippets = append(snippets, snippet{ + Test: b.testName, + File: filename, + SourceLine: b.line, + Language: b.lang, + }) + } + + // Write manifest + m := manifest{ + Page: relPath, + Snippets: snippets, + } + manifestData, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("marshaling manifest for %s: %w", path, err) + } + manifestPath := filepath.Join(pageDir, "manifest.json") + if err := os.WriteFile(manifestPath, append(manifestData, '\n'), 0o600); err != nil { + return fmt.Errorf("writing %s: %w", manifestPath, err) + } + + return nil + }) +} + +// coverageEntry represents an untested executable code block. +type coverageEntry struct { + File string + Line int + Language string +} + +// runCoverage walks the content directory and reports untested executable blocks. +func runCoverage(contentDir string) error { + tracked, err := gitTrackedFiles(contentDir) + if err != nil { + return err + } + + var entries []coverageEntry + + err = filepath.WalkDir(contentDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // Skip symlinks to avoid following them out of the content tree (gosec G122). + if d.Type()&fs.ModeSymlink != 0 { + return nil + } + if d.IsDir() || filepath.Ext(path) != ".md" { + return nil + } + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolving %s: %w", path, err) + } + if !tracked[absPath] { + return nil + } + + source, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + fm, err := parseFrontmatter(source) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + if isOptedOut(fm) { + return nil + } + + blocks, err := extractBlocks(source) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + + for _, b := range blocks { + if b.testName != "" { + continue // annotated = tested + } + if !testableLangs[b.lang] { + continue // non-testable or unknown language + } + entries = append(entries, coverageEntry{ + File: path, + Line: b.line, + Language: b.lang, + }) + } + + return nil + }) + if err != nil { + return err + } + + if len(entries) > 0 { + fmt.Printf("Untested executable code blocks (%d):\n", len(entries)) + for _, e := range entries { + fmt.Printf(" %s:%d [%s]\n", e.File, e.Line, e.Language) + } + return fmt.Errorf("%d untested executable code block(s) found", len(entries)) + } + + fmt.Println("All executable code blocks are tested.") + return nil +} diff --git a/cmd/doctest/extract_test.go b/cmd/doctest/extract_test.go new file mode 100644 index 0000000..8013729 --- /dev/null +++ b/cmd/doctest/extract_test.go @@ -0,0 +1,709 @@ +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestParseInfoStringBasic(t *testing.T) { + tests := []struct { + name string + info string + wantLang string + wantTest string + wantErr bool + }{ + {"empty", "", "", "", false}, + {"lang only", "bash", "bash", "", false}, + {"lang with test", `bash {test="install"}`, "bash", "install", false}, + {"no space before brace", `bash{test="install"}`, "bash", "install", false}, + {"attrs without test key", `bash {.highlight}`, "bash", "", false}, + {"test with hyphens and numbers", `sh {test="my-test-01"}`, "sh", "my-test-01", false}, + {"invalid test id uppercase", `bash {test="Install"}`, "", "", true}, + {"invalid test id spaces", `bash {test="my test"}`, "", "", true}, + {"invalid test id underscore", `bash {test="my_test"}`, "", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lang, testName, err := parseInfoString(tt.info) + if (err != nil) != tt.wantErr { + t.Fatalf("parseInfoString(%q) error = %v, wantErr = %v", tt.info, err, tt.wantErr) + } + if err != nil { + return + } + if lang != tt.wantLang { + t.Errorf("lang = %q, want %q", lang, tt.wantLang) + } + if testName != tt.wantTest { + t.Errorf("testName = %q, want %q", testName, tt.wantTest) + } + }) + } +} + +func TestParseFrontmatter(t *testing.T) { + tests := []struct { + name string + source string + wantOptOut bool + wantNil bool + }{ + {"no frontmatter", "# Hello\nworld", false, true}, + {"empty frontmatter", "---\n---\n# Hello", false, false}, + {"opted out", "---\ntestable_docs: false\n---\n# Hello", true, false}, + {"opted in explicitly", "---\ntestable_docs: true\n---\n# Hello", false, false}, + {"no testable_docs key", "---\ntitle: Test\n---\n# Hello", false, false}, + {"dash-prefixed non-frontmatter", "---foo\nbar", false, true}, + {"dash-prefixed heading followed by a real delimiter", "---foo\nbaz: 1\n---\n# hello", false, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fm, err := parseFrontmatter([]byte(tt.source)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.wantNil && fm != nil && fm.TestableDocs != nil { + t.Fatal("expected nil TestableDocs") + } + if got := isOptedOut(fm); got != tt.wantOptOut { + t.Errorf("isOptedOut = %v, want %v", got, tt.wantOptOut) + } + }) + } +} + +func TestExtractBlocksBasic(t *testing.T) { + source := []byte("---\ntitle: Test\n---\n\n# Hello\n\n```bash {test=\"install\"}\necho hello\n```\n") + blocks, err := extractBlocks(source) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(blocks) != 1 { + t.Fatalf("got %d blocks, want 1", len(blocks)) + } + b := blocks[0] + if b.lang != "bash" { + t.Errorf("lang = %q, want %q", b.lang, "bash") + } + if b.testName != "install" { + t.Errorf("testName = %q, want %q", b.testName, "install") + } + if strings.TrimSpace(string(b.content)) != "echo hello" { + t.Errorf("content = %q, want %q", string(b.content), "echo hello\n") + } + // Fence line is line 7: frontmatter (1-3), blank (4), heading (5), + // blank (6), then the ```bash fence (7). + if b.line != 7 { + t.Errorf("line = %d, want 7", b.line) + } +} + +func TestExtractBlocksOrdering(t *testing.T) { + source := []byte("```bash {test=\"first\"}\necho 1\n```\n\nsome text\n\n```bash {test=\"second\"}\necho 2\n```\n") + blocks, err := extractBlocks(source) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var annotated []codeBlock + for _, b := range blocks { + if b.testName != "" { + annotated = append(annotated, b) + } + } + if len(annotated) != 2 { + t.Fatalf("got %d annotated blocks, want 2", len(annotated)) + } + if annotated[0].testName != "first" { + t.Errorf("first block testName = %q, want %q", annotated[0].testName, "first") + } + if annotated[1].testName != "second" { + t.Errorf("second block testName = %q, want %q", annotated[1].testName, "second") + } +} + +func TestExtractBlocksNoInfoString(t *testing.T) { + source := []byte("```\necho hello\n```\n") + blocks, err := extractBlocks(source) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(blocks) != 1 { + t.Fatalf("got %d blocks, want 1", len(blocks)) + } + if blocks[0].lang != "" { + t.Errorf("lang = %q, want empty", blocks[0].lang) + } + if blocks[0].testName != "" { + t.Errorf("testName = %q, want empty", blocks[0].testName) + } +} + +func TestExtractBlocksNonTestableLanguage(t *testing.T) { + source := []byte("```yaml {test=\"my-config\"}\nkey: value\n```\n") + blocks, err := extractBlocks(source) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(blocks) != 1 { + t.Fatalf("got %d blocks, want 1", len(blocks)) + } + if blocks[0].lang != "yaml" { + t.Errorf("lang = %q, want %q", blocks[0].lang, "yaml") + } + if blocks[0].testName != "my-config" { + t.Errorf("testName = %q, want %q", blocks[0].testName, "my-config") + } +} + +func TestPageSlug(t *testing.T) { + tests := []struct { + relPath string + want string + }{ + {"getting-started/_index.md", "getting-started"}, + {"guides/advanced/_index.md", "guides-advanced"}, + {"_index.md", "root"}, + {"overview.md", "overview"}, + {"guides/quickstart.md", "guides-quickstart"}, + } + for _, tt := range tests { + t.Run(tt.relPath, func(t *testing.T) { + if got := pageSlug(tt.relPath); got != tt.want { + t.Errorf("pageSlug(%q) = %q, want %q", tt.relPath, got, tt.want) + } + }) + } +} + +func TestLangExtension(t *testing.T) { + tests := []struct { + lang string + want string + }{ + {"bash", "bash"}, + {"sh", "sh"}, + {"yaml", "yaml"}, + {"python", "py"}, + {"go", "go"}, + {"", "txt"}, + {"rust", "rust"}, + } + for _, tt := range tests { + t.Run(tt.lang, func(t *testing.T) { + if got := langExtension(tt.lang); got != tt.want { + t.Errorf("langExtension(%q) = %q, want %q", tt.lang, got, tt.want) + } + }) + } +} + +func TestRunExtractBasic(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + md := "---\ntitle: Test\n---\n\n```bash {test=\"hello\"}\necho hello\n```\n" + subDir := filepath.Join(contentDir, "getting-started") + if err := os.MkdirAll(subDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(subDir, "_index.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/getting-started/_index.md") + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + // Check snippet file + snippetPath := filepath.Join(outputDir, "getting-started", "01-hello.bash") + data, err := os.ReadFile(snippetPath) + if err != nil { + t.Fatalf("reading snippet: %v", err) + } + if strings.TrimSpace(string(data)) != "echo hello" { + t.Errorf("snippet content = %q, want %q", string(data), "echo hello\n") + } + + // Check manifest + manifestPath := filepath.Join(outputDir, "getting-started", "manifest.json") + manifestData, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("reading manifest: %v", err) + } + var m manifest + if err := json.Unmarshal(manifestData, &m); err != nil { + t.Fatalf("parsing manifest: %v", err) + } + if len(m.Snippets) != 1 { + t.Fatalf("manifest has %d snippets, want 1", len(m.Snippets)) + } + if m.Snippets[0].Test != "hello" { + t.Errorf("snippet test = %q, want %q", m.Snippets[0].Test, "hello") + } + if m.Snippets[0].Language != "bash" { + t.Errorf("snippet language = %q, want %q", m.Snippets[0].Language, "bash") + } +} + +func TestRunExtractRejectsOverlappingDirs(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + outputDir string + wantOverlap bool + }{ + {"identical dirs", contentDir, true}, + {"output is ancestor of content", repoDir, true}, + {"output is descendant of content", filepath.Join(contentDir, "snippets"), true}, + {"sibling dir sharing a name prefix", contentDir + "-backup", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := runExtract(contentDir, tt.outputDir) + if tt.wantOverlap { + if err == nil { + t.Fatal("expected error for overlapping content/output dirs, got nil") + } + if !strings.Contains(err.Error(), "overlap") { + t.Errorf("error = %q, want it to mention directory overlap", err.Error()) + } + return + } + if err != nil { + t.Fatalf("expected no error for non-overlapping sibling dirs, got: %v", err) + } + }) + } +} + +func TestRunExtractDuplicateError(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + md := "```bash {test=\"dupe\"}\necho 1\n```\n\n```bash {test=\"dupe\"}\necho 2\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + err := runExtract(contentDir, outputDir) + if err == nil { + t.Fatal("expected error for duplicate test names, got nil") + } + if !strings.Contains(err.Error(), "duplicate test name") { + t.Errorf("error = %q, want it to contain %q", err.Error(), "duplicate test name") + } +} + +func TestRunExtractFrontmatterOptOut(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + md := "---\ntestable_docs: false\n---\n\n```bash {test=\"hello\"}\necho hello\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + // Output directory should have no subdirectories + entries, err := os.ReadDir(outputDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("output dir has %d entries, want 0 (page should be skipped)", len(entries)) + } +} + +func TestRunExtractOrdering(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + md := "```bash {test=\"alpha\"}\necho a\n```\n\n```bash {test=\"beta\"}\necho b\n```\n\n```bash {test=\"gamma\"}\necho c\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + expected := []string{"01-alpha.bash", "02-beta.bash", "03-gamma.bash"} + pageDir := filepath.Join(outputDir, "test") + for _, name := range expected { + if _, err := os.Stat(filepath.Join(pageDir, name)); err != nil { + t.Errorf("expected file %s not found: %v", name, err) + } + } +} + +func TestRunExtractRemovesStaleSnippets(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + md := "```bash {test=\"current\"}\necho current\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + // Pre-populate outputDir with a stale snippet and manifest left over from + // a prior run whose source block was since renamed or deleted. + staleDir := filepath.Join(outputDir, "test") + if err := os.MkdirAll(staleDir, 0o755); err != nil { + t.Fatal(err) + } + staleFile := filepath.Join(staleDir, "01-removed-test.bash") + if err := os.WriteFile(staleFile, []byte("echo stale\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + if _, err := os.Stat(staleFile); !os.IsNotExist(err) { + t.Errorf("stale snippet %s should have been removed, stat err = %v", staleFile, err) + } + if _, err := os.Stat(filepath.Join(staleDir, "01-current.bash")); err != nil { + t.Errorf("expected current snippet to exist: %v", err) + } +} + +func TestRunExtractSkipsSymlinks(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + outputDir := t.TempDir() + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + + // A target file outside contentDir that a malicious/misconfigured + // symlink could otherwise cause WalkDir to follow into the tree. + outsideDir := t.TempDir() + outsideFile := filepath.Join(outsideDir, "outside.md") + if err := os.WriteFile(outsideFile, []byte("```bash {test=\"outside\"}\necho outside\n```\n"), 0o644); err != nil { + t.Fatal(err) + } + + symlinkPath := filepath.Join(contentDir, "escape.md") + if err := os.Symlink(outsideFile, symlinkPath); err != nil { + t.Skipf("symlinks not supported on this filesystem: %v", err) + } + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + if _, err := os.Stat(filepath.Join(outputDir, "escape", "01-outside.bash")); err == nil { + t.Error("symlinked file outside contentDir should not have been followed/extracted, but it was") + } +} + +func TestRunCoverageSkipsSymlinks(t *testing.T) { + dir := t.TempDir() + initGitRepo(t, dir) + contentDir := filepath.Join(dir, "content") + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + + outsideDir := t.TempDir() + outsideFile := filepath.Join(outsideDir, "outside.md") + if err := os.WriteFile(outsideFile, []byte("```bash\necho untested\n```\n"), 0o644); err != nil { + t.Fatal(err) + } + + symlinkPath := filepath.Join(contentDir, "escape.md") + if err := os.Symlink(outsideFile, symlinkPath); err != nil { + t.Skipf("symlinks not supported on this filesystem: %v", err) + } + + // The symlinked file's untested block must not be followed/reported; + // if it were, coverage would fail with 1 untested block. + if err := runCoverage(contentDir); err != nil { + t.Fatalf("runCoverage should not follow symlinks out of content tree, got: %v", err) + } +} + +func TestRunCoverageReport(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + // One tested, one untested bash block, one yaml block (not testable) + md := "```bash {test=\"tested\"}\necho tested\n```\n\n```bash\necho untested\n```\n\n```yaml\nkey: value\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + err := runCoverage(contentDir) + if err == nil { + t.Fatal("expected error for untested blocks, got nil") + } + if !strings.Contains(err.Error(), "1 untested") { + t.Fatalf("unexpected error message: %v", err) + } +} + +func TestRunCoverageAllTested(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + // Every executable block is annotated; the yaml block is not testable. + md := "```bash {test=\"tested\"}\necho tested\n```\n\n```yaml\nkey: value\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + if err := runCoverage(contentDir); err != nil { + t.Fatalf("runCoverage should pass when all executable blocks are tested, got: %v", err) + } +} + +func TestRunCoverageOptOut(t *testing.T) { + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + md := "---\ntestable_docs: false\n---\n\n```bash\necho untested\n```\n" + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + + err := runCoverage(contentDir) + if err != nil { + t.Fatalf("runCoverage error: %v", err) + } +} + +// initGitRepo initialises a git repository in dir with isolated config +// so that user/system gitconfig cannot interfere with tests. +func initGitRepo(t *testing.T, dir string) { + t.Helper() + env := append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + ) + for _, args := range [][]string{ + {"init"}, + {"config", "user.email", "test@test"}, + {"config", "user.name", "test"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } +} + +// gitAdd stages and commits files in the repo at dir. +func gitAdd(t *testing.T, dir string, files ...string) { + t.Helper() + env := append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + ) + args := append([]string{"add"}, files...) + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git add: %v\n%s", err, out) + } + cmd = exec.Command("git", "commit", "-m", "test") + cmd.Dir = dir + cmd.Env = env + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("git commit: %v\n%s", err, out) + } +} + +func TestGitTrackedFiles(t *testing.T) { + dir := t.TempDir() + initGitRepo(t, dir) + + // Create tracked file + tracked := filepath.Join(dir, "docs", "guide.md") + if err := os.MkdirAll(filepath.Dir(tracked), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tracked, []byte("# Guide"), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, dir, "docs/guide.md") + + // Create gitignored file + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("ignored/\n"), 0o644); err != nil { + t.Fatal(err) + } + ignoredDir := filepath.Join(dir, "ignored") + if err := os.MkdirAll(ignoredDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(ignoredDir, "secret.md"), []byte("# Secret"), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, dir, ".gitignore") + + // Create untracked-but-not-ignored file (should be included) + untracked := filepath.Join(dir, "docs", "new.md") + if err := os.WriteFile(untracked, []byte("# New"), 0o644); err != nil { + t.Fatal(err) + } + + files, err := gitTrackedFiles(dir) + if err != nil { + t.Fatalf("gitTrackedFiles: %v", err) + } + + if !files[filepath.Join(dir, "docs", "guide.md")] { + t.Error("tracked file docs/guide.md should be in set") + } + if !files[filepath.Join(dir, "docs", "new.md")] { + t.Error("untracked-but-not-ignored file docs/new.md should be in set") + } + if files[filepath.Join(dir, "ignored", "secret.md")] { + t.Error("gitignored file ignored/secret.md should NOT be in set") + } +} + +func TestRunExtractSkipsGitignored(t *testing.T) { + dir := t.TempDir() + initGitRepo(t, dir) + + contentDir := filepath.Join(dir, "content") + outputDir := t.TempDir() + + // Create tracked file with annotated block + trackedDir := filepath.Join(contentDir, "guide") + if err := os.MkdirAll(trackedDir, 0o755); err != nil { + t.Fatal(err) + } + md := "```bash {test=\"tracked\"}\necho tracked\n```\n" + if err := os.WriteFile(filepath.Join(trackedDir, "_index.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + + // Create gitignored file with annotated block + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("content/generated/\n"), 0o644); err != nil { + t.Fatal(err) + } + ignoredDir := filepath.Join(contentDir, "generated") + if err := os.MkdirAll(ignoredDir, 0o755); err != nil { + t.Fatal(err) + } + ignoredMd := "```bash {test=\"ignored\"}\necho ignored\n```\n" + if err := os.WriteFile(filepath.Join(ignoredDir, "_index.md"), []byte(ignoredMd), 0o644); err != nil { + t.Fatal(err) + } + + gitAdd(t, dir, "content/guide/_index.md", ".gitignore") + + if err := runExtract(contentDir, outputDir); err != nil { + t.Fatalf("runExtract error: %v", err) + } + + // Tracked file should produce output + if _, err := os.Stat(filepath.Join(outputDir, "guide", "01-tracked.bash")); err != nil { + t.Errorf("expected output for tracked file: %v", err) + } + + // Gitignored file should NOT produce output + if _, err := os.Stat(filepath.Join(outputDir, "generated", "01-ignored.bash")); err == nil { + t.Error("gitignored file should not produce output, but it did") + } +} + +func TestRunCoverageSkipsGitignored(t *testing.T) { + dir := t.TempDir() + initGitRepo(t, dir) + + contentDir := filepath.Join(dir, "content") + + // Create gitignored file with untested block + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("content/generated/\n"), 0o644); err != nil { + t.Fatal(err) + } + ignoredDir := filepath.Join(contentDir, "generated") + if err := os.MkdirAll(ignoredDir, 0o755); err != nil { + t.Fatal(err) + } + md := "```bash\necho untested\n```\n" + if err := os.WriteFile(filepath.Join(ignoredDir, "_index.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + + gitAdd(t, dir, ".gitignore") + + // Coverage should pass since the only untested block is gitignored + err := runCoverage(contentDir) + if err != nil { + t.Fatalf("runCoverage should pass (gitignored file), got: %v", err) + } +} diff --git a/cmd/doctest/main.go b/cmd/doctest/main.go new file mode 100644 index 0000000..e23aa5b --- /dev/null +++ b/cmd/doctest/main.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Command doctest extracts annotated code blocks from Markdown documentation +// and reports test coverage gaps. +// +// Fenced code blocks with a {test=""} attribute in the info string are +// extracted to individual files. Hand-written Bats tests then verify them. +// +// Usage: +// +// doctest extract --content-dir content/docs --output-dir .test-output/doctest-snippets +// doctest coverage --content-dir content/docs +package main + +import ( + "flag" + "fmt" + "log/slog" + "os" +) + +func main() { os.Exit(run(os.Args[1:])) } + +// run executes the doctest CLI with the given arguments (excluding the program +// name) and returns a process exit code. It takes args explicitly and uses +// flag.ContinueOnError so that all exit paths are unit-testable without +// mutating global state or terminating the process. +func run(args []string) int { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: doctest [flags]") + return 1 + } + + subcmd := args[0] + switch subcmd { + case "extract": + fs := flag.NewFlagSet("extract", flag.ContinueOnError) + contentDir := fs.String("content-dir", "", "Root directory of Markdown content (required)") + outputDir := fs.String("output-dir", "", "Directory for extracted snippets (required)") + if err := fs.Parse(args[1:]); err != nil { + return 1 + } + if *contentDir == "" || *outputDir == "" { + fmt.Fprintln(os.Stderr, "extract: --content-dir and --output-dir are required") + return 1 + } + if err := runExtract(*contentDir, *outputDir); err != nil { + slog.Error("extract failed", "error", err) + return 1 + } + case "coverage": + fs := flag.NewFlagSet("coverage", flag.ContinueOnError) + contentDir := fs.String("content-dir", "", "Root directory of Markdown content (required)") + if err := fs.Parse(args[1:]); err != nil { + return 1 + } + if *contentDir == "" { + fmt.Fprintln(os.Stderr, "coverage: --content-dir is required") + return 1 + } + if err := runCoverage(*contentDir); err != nil { + return 1 // coverage report already printed to stdout + } + default: + fmt.Fprintf(os.Stderr, "unknown subcommand: %s\nusage: doctest [flags]\n", subcmd) + return 1 + } + return 0 +} diff --git a/cmd/doctest/main_test.go b/cmd/doctest/main_test.go new file mode 100644 index 0000000..13ed585 --- /dev/null +++ b/cmd/doctest/main_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// initRepoWithBlock creates an isolated git repo containing a content dir with +// a single tracked Markdown file and returns the content directory path. +func initRepoWithBlock(t *testing.T, md string) string { + t.Helper() + repoDir := t.TempDir() + initGitRepo(t, repoDir) + contentDir := filepath.Join(repoDir, "content") + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(contentDir, "test.md"), []byte(md), 0o644); err != nil { + t.Fatal(err) + } + gitAdd(t, repoDir, "content/test.md") + return contentDir +} + +func TestRunExitCodes(t *testing.T) { + // Content with one untested executable block so coverage fails. + untestedContent := initRepoWithBlock(t, "```bash\necho hi\n```\n") + // Content fully annotated so coverage passes. + testedContent := initRepoWithBlock(t, "```bash {test=\"ok\"}\necho hi\n```\n") + + outputDir := t.TempDir() + + tests := []struct { + name string + args []string + want int + }{ + {"no args", nil, 1}, + {"unknown subcommand", []string{"bogus"}, 1}, + {"extract missing flags", []string{"extract"}, 1}, + {"extract unknown flag", []string{"extract", "--nope"}, 1}, + {"coverage missing flag", []string{"coverage"}, 1}, + {"extract success", []string{"extract", "--content-dir", testedContent, "--output-dir", outputDir}, 0}, + {"coverage failure", []string{"coverage", "--content-dir", untestedContent}, 1}, + {"coverage success", []string{"coverage", "--content-dir", testedContent}, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := run(tt.args); got != tt.want { + t.Errorf("run(%v) = %d, want %d", tt.args, got, tt.want) + } + }) + } +} + +func TestRunExtractError(t *testing.T) { + // content-dir outside any git repo makes gitTrackedFiles fail. + nonRepo := t.TempDir() + env := append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null") + // Ensure the temp dir is not inside a repo by checking git status fails. + cmd := exec.Command("git", "rev-parse", "--is-inside-work-tree") + cmd.Dir = nonRepo + cmd.Env = env + if err := cmd.Run(); err == nil { + t.Skip("temp dir unexpectedly inside a git repo; skipping") + } + + if got := run([]string{"extract", "--content-dir", nonRepo, "--output-dir", t.TempDir()}); got != 1 { + t.Errorf("run extract on non-repo = %d, want 1", got) + } +} diff --git a/go.mod b/go.mod index 05e8853..36fd49a 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/complytime/website go 1.25.12 -require github.com/goccy/go-yaml v1.19.2 +require ( + github.com/goccy/go-yaml v1.19.2 + github.com/yuin/goldmark v1.8.2 +) diff --git a/go.sum b/go.sum index bd88ba6..c1e624b 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= diff --git a/package-lock.json b/package-lock.json index 96e84f5..73e77cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,9 @@ "devDependencies": { "@changesets/changelog-github": "^0.7.0", "@changesets/cli": "^2.31.1", + "bats": "^1.13.0", + "bats-assert": "^2.2.4", + "bats-support": "^0.3.0", "playwright": "^1.62.0", "prettier": "^3.9.6", "vite": "^8.1.4" @@ -2656,6 +2659,37 @@ "integrity": "sha512-EsuNWmfcFXWZOe0txKXsllYOC7bDpoaVLc4HHHlYKB/roymlZs+FBdLUU6rx2yPpnJZhulwheKdPjqr2k0+NGQ==", "license": "MIT" }, + "node_modules/bats": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/bats/-/bats-1.13.0.tgz", + "integrity": "sha512-giSYKGTOcPZyJDbfbTtzAedLcNWdjCLbXYU3/MwPnjyvDXzu6Dgw8d2M+8jHhZXSmsCMSQqCp+YBsJ603UO4vQ==", + "dev": true, + "license": "MIT", + "bin": { + "bats": "bin/bats" + } + }, + "node_modules/bats-assert": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/bats-assert/-/bats-assert-2.2.4.tgz", + "integrity": "sha512-EcaY4Z+Tbz1c7pnC1SrVSq0epr7tLwFpz6qt7KUW9K8uSw8V12DTfH9d2HxZWvBEATaCuMsZ7KoZMFiSQPRoXw==", + "dev": true, + "license": "CC0-1.0", + "peerDependencies": { + "bats": "0.4 || ^1", + "bats-support": "^0.3" + } + }, + "node_modules/bats-support": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/bats-support/-/bats-support-0.3.0.tgz", + "integrity": "sha512-z+2WzXbI4OZgLnynydqH8GpI3+DcOtepO66PlK47SfEzTkiuV9hxn9eIQX+uLVFbt2Oqoc7Ky3TJ/N83lqD+cg==", + "dev": true, + "license": "CC0-1.0", + "peerDependencies": { + "bats": "0.4 || ^1" + } + }, "node_modules/better-path-resolve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/better-path-resolve/-/better-path-resolve-1.0.0.tgz", diff --git a/package.json b/package.json index e615d84..e5bb5d5 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,9 @@ "devDependencies": { "@changesets/changelog-github": "^0.7.0", "@changesets/cli": "^2.31.1", + "bats": "^1.13.0", + "bats-assert": "^2.2.4", + "bats-support": "^0.3.0", "playwright": "^1.62.0", "prettier": "^3.9.6", "vite": "^8.1.4" diff --git a/specs/015-testable-documentation/plan.md b/specs/015-testable-documentation/plan.md new file mode 100644 index 0000000..d98173b --- /dev/null +++ b/specs/015-testable-documentation/plan.md @@ -0,0 +1,786 @@ +# Testable Documentation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build testable documentation infrastructure so fenced code blocks in Markdown can be extracted, tested with Bats, and coverage-reported. + +**Architecture:** A Go extraction tool (`cmd/doctest/`) parses Markdown with goldmark, finds fenced code blocks annotated with `{test="..."}`, writes them to disk. Hand-written Bats tests run those snippets. Coverage reporting flags untested executable blocks. Everything integrates into existing Makefile targets and CI workflows. + +**Tech Stack:** Go 1.25+ (goldmark, go-yaml), Bats-core + bats-support + bats-assert (git submodules), Hugo 0.155.1 extended (existing), GNU Make. + +**Spec:** `specs/015-testable-documentation/spec.md` + +--- + +## File Map + +### New files + +| Path | Purpose | +| ------ | --------- | +| `cmd/doctest/main.go` | CLI entry point: `extract` and `coverage` subcommands | +| `cmd/doctest/extract.go` | Markdown walker, attribute parser, snippet writer, manifest generator | +| `cmd/doctest/extract_test.go` | Table-driven unit tests for extraction and coverage | +| `tests/docs/setup_suite.bash` | Bats suite setup: sets `SNIPPETS_DIR` | +| `tests/docs/helpers/bash.bash` | `run_snippet()` helper function | +| `tests/docs/getting-started.bats` | Skeleton Bats test file (placeholder for future snippet tests) | +| `tests/libs/bats-core/` | Git submodule | +| `tests/libs/bats-support/` | Git submodule | +| `tests/libs/bats-assert/` | Git submodule | + +### Modified files + +| Path | Change | +| ------ | -------- | +| `go.mod` | Add `github.com/yuin/goldmark` dependency | +| `go.sum` | Updated by `go mod tidy` | +| `Makefile` | Add `test-docs-extract`, `test-docs`, `test-docs-coverage` targets; update `check` | +| `.gitignore` | Add `/doctest` binary and `.test-output/` directory | +| `.github/workflows/ci.yml` | Add `checkout submodules`, doc tests step after Hugo build | +| `.github/workflows/deploy-gh-pages.yml` | Add `checkout submodules`, doc tests step before artifact upload | +| `CONTRIBUTING.md` | New "Testing Documentation" section | +| `README.md` | Add `cmd/doctest/` and `tests/docs/` to project structure; mention `make test-docs` | + +--- + +## Task Dependency Graph + +```text +Task 1 (Go extractor core + coverage) + └─> Task 2 (Go extractor tests) + └─> Task 3 (Bats submodules + harness) + └─> Task 4 (Makefile targets) + └─> Task 5 (CI integration) + └─> Task 6 (Documentation updates) + └─> Task 7 (.gitignore + housekeeping) + └─> Task 8 (End-to-end verification) +``` + +Tasks 3 and 7 can be parallelized with adjacent tasks if using subagent-driven-development (they have no code dependencies on each other beyond ordering). + +--- + +### Task 1: Go Extraction Tool — Core Implementation + +**Files:** +- Create: `cmd/doctest/main.go` +- Create: `cmd/doctest/extract.go` +- Modify: `go.mod` (add goldmark dependency) + +**Context:** The existing `cmd/sync-content/` tool is a `package main` with all files in one directory. Follow the same pattern. The project already uses `github.com/goccy/go-yaml` for YAML parsing. Use goldmark (`github.com/yuin/goldmark`) for Markdown AST parsing — the same parser Hugo uses internally. + +- [ ] **Step 1: Add goldmark dependency** + +```bash +go get github.com/yuin/goldmark@latest +go mod tidy +``` + +Verify `go.mod` now contains `github.com/yuin/goldmark`. + +- [ ] **Step 2: Create `cmd/doctest/main.go`** + +Create `cmd/doctest/main.go` with the CLI entry point: + +See the implemented source in [`cmd/doctest/main.go`](../../cmd/doctest/main.go) for the authoritative version; the design intent is described in the prose above. + +- [ ] **Step 3: Create `cmd/doctest/extract.go`** + +Create `cmd/doctest/extract.go` with the extraction and coverage logic: + +See the implemented source in [`cmd/doctest/extract.go`](../../cmd/doctest/extract.go) for the authoritative version; the design intent is described in the prose above. + +- [ ] **Step 4: Verify compilation** + +```bash +go build ./cmd/doctest/ +``` + +Expected: no errors, no output. A `doctest` binary appears in the workspace root. + +```bash +rm -f doctest +``` + +- [ ] **Step 5: Smoke test against existing content** + +```bash +go run ./cmd/doctest extract --content-dir content/docs --output-dir .test-output/doctest-snippets +``` + +Expected: completes with exit 0. Since no blocks have `{test="..."}` yet, `.test-output/doctest-snippets` should be empty or not created. + +```bash +go run ./cmd/doctest coverage --content-dir content/docs +``` + +Expected: lists untested bash/sh blocks from `getting-started/_index.md`. + +- [ ] **Step 6: Commit** + +```bash +git add cmd/doctest/main.go cmd/doctest/extract.go go.mod go.sum +git commit -m "feat: add doctest extraction tool + +Go tool using goldmark to parse Markdown AST and extract fenced code +blocks annotated with {test=\"...\"} attributes. Supports extract and +coverage subcommands. + +Part of testable documentation infrastructure (spec 015)." +``` + +--- + +### Task 2: Go Extraction Tool — Unit Tests + +**Files:** +- Create: `cmd/doctest/extract_test.go` + +**Context:** Follow the existing test patterns from `cmd/sync-content/path_test.go` — table-driven tests, `t.TempDir()` for filesystem tests, `testing` package only (no external test framework). The file under test is `cmd/doctest/extract.go` which exports `parseFrontmatter`, `isOptedOut`, `parseInfoString`, `extractBlocks`, `pageSlug`, `lineNumber`, `langExtension`, `runExtract`, `runCoverage`. + +- [ ] **Step 1: Create `cmd/doctest/extract_test.go`** + +See the implemented source in [`cmd/doctest/extract_test.go`](../../cmd/doctest/extract_test.go) for the authoritative version; the design intent is described in the prose above. + +- [ ] **Step 2: Run tests to verify they pass** + +```bash +go test -v ./cmd/doctest/... +``` + +Expected: all tests pass. + +- [ ] **Step 3: Run tests with race detector** + +```bash +go test -race ./cmd/doctest/... +``` + +Expected: passes with zero race warnings. + +- [ ] **Step 4: Commit** + +```bash +git add cmd/doctest/extract_test.go +git commit -m "test: add unit tests for doctest extraction tool + +Table-driven tests covering info string parsing, frontmatter opt-out, +block extraction, ordering, duplicate detection, page slug derivation, +language extensions, manifest generation, and coverage reporting." +``` + +--- + +### Task 3: Bats Test Harness Setup + +**Files:** +- Create: `tests/docs/setup_suite.bash` +- Create: `tests/docs/helpers/bash.bash` +- Create: `tests/docs/getting-started.bats` +- Create: `tests/libs/` (git submodules) +- Modify: `.gitmodules` (created by `git submodule add`) + +**Context:** Bats-core, bats-support, and bats-assert are installed as git submodules. This is the standard distribution pattern — no brew/npm dependency. The `tests/` directory does not exist yet. The project has no `.gitmodules` file yet. + +- [ ] **Step 1: Create directory structure** + +```bash +mkdir -p tests/docs/helpers tests/libs +``` + +- [ ] **Step 2: Add Bats git submodules** + +```bash +git submodule add https://github.com/bats-core/bats-core.git tests/libs/bats-core +git submodule add https://github.com/bats-core/bats-support.git tests/libs/bats-support +git submodule add https://github.com/bats-core/bats-assert.git tests/libs/bats-assert +``` + +This creates `.gitmodules` and clones the repos into `tests/libs/`. + +- [ ] **Step 3: Create `tests/docs/setup_suite.bash`** + +```bash +# SPDX-License-Identifier: Apache-2.0 + +# Suite-level setup for documentation tests. +# Sets the default snippets directory. Individual .bats files load +# their own libraries in setup(). + +export SNIPPETS_DIR="${SNIPPETS_DIR:-.test-output/doctest-snippets}" +``` + +- [ ] **Step 4: Create `tests/docs/helpers/bash.bash`** + +```bash +# SPDX-License-Identifier: Apache-2.0 + +# Helper for running extracted bash snippets. +# Usage in a @test block: +# run_snippet "getting-started/01-install-complyctl.bash" + +run_snippet() { + local snippet="$SNIPPETS_DIR/$1" + [[ -f "$snippet" ]] || { echo "Snippet not found: $snippet" >&2; return 1; } + run bash "$snippet" +} +``` + +- [ ] **Step 5: Create `tests/docs/getting-started.bats`** + +```bash +# SPDX-License-Identifier: Apache-2.0 + +# Documentation tests for content/docs/getting-started/_index.md +# +# Tests will be added here as code blocks in the getting started guide +# are annotated with {test="..."} attributes. Each @test name must +# match a test attribute value in the Markdown source. + +setup() { + load 'helpers/bash' + load '../libs/bats-support/load' + load '../libs/bats-assert/load' +} + +# Placeholder: add @test blocks as snippets are annotated. +# Example: +# +# @test "install-complyctl" { +# run_snippet "getting-started/01-install-complyctl.bash" +# assert_success +# } +``` + +- [ ] **Step 6: Verify Bats runs (expect no tests)** + +```bash +tests/libs/bats-core/bin/bats tests/docs/ +``` + +Expected: `0 tests, 0 failures` or similar output indicating no test functions found. The skeleton file has no `@test` blocks, so Bats should report zero tests. + +- [ ] **Step 7: Commit** + +```bash +git add .gitmodules tests/libs/bats-core tests/libs/bats-support tests/libs/bats-assert +git add tests/docs/setup_suite.bash tests/docs/helpers/bash.bash tests/docs/getting-started.bats +git commit -m "feat: add Bats test harness for documentation tests + +Install bats-core, bats-support, and bats-assert as git submodules. +Set up test directory structure with helpers and a skeleton test file +for the getting-started guide. + +Part of testable documentation infrastructure (spec 015)." +``` + +--- + +### Task 4: Makefile Targets + +**Files:** +- Modify: `Makefile` + +**Context:** The Makefile is 153 lines. Sections are separated by comment headers. The `check` target is on line 77. New doc test targets go after the "Content sync" section (after line 108). The convention uses `##` for help comments, `.PHONY` before each target, and `@` prefix for quiet execution. + +- [ ] **Step 1: Add documentation test section to Makefile** + +Insert after line 108 (after the `sync-single` target), before the "Hugo / Node" section: + + +```makefile +# --------------------------------------------------------------------------- +# Documentation tests — extract, validate, and test code blocks +# --------------------------------------------------------------------------- + +.PHONY: test-docs-extract +test-docs-extract: ## Extract testable code blocks from documentation + @go run ./cmd/doctest extract --content-dir content/docs --output-dir .test-output/doctest-snippets + +.PHONY: test-docs +test-docs: test-docs-extract ## Run documentation tests (Bats) + @tests/libs/bats-core/bin/bats tests/docs/ + +.PHONY: test-docs-coverage +test-docs-coverage: ## Report untested code blocks in documentation + @go run ./cmd/doctest coverage --content-dir content/docs +``` + + +- [ ] **Step 2: Update `check` meta-target** + +Change line 77 from: + +```makefile +check: vet fmt-check test-race ## Run vet + fmt-check + race tests (CI equivalent) +``` + +to: + +```makefile +check: vet fmt-check test-race test-docs-coverage ## Run vet + fmt-check + race tests + doc coverage (CI equivalent) +``` + +Also update the quick reference comment at the top of the Makefile (line 11) to mention doc tests: + +```makefile +# make check — vet + fmt-check + race tests + doc coverage +``` + +- [ ] **Step 3: Update Go targets to include doctest** + +The existing `test` and `test-race` targets only test `./cmd/sync-content/...`. Update them to also test `./cmd/doctest/...`: + +Change `test` (line 53): + +```makefile +test: ## Run all Go unit tests + go test $(SYNC_PKG) ./cmd/doctest/... +``` + + +Change `test-race` (line 57): + +```makefile +test-race: ## Run Go tests with the race detector + go test -race $(SYNC_PKG) ./cmd/doctest/... +``` + + +Also update `vet` (line 61) and `fmt`/`fmt-check` to cover the new package: + +Change `vet`: + +```makefile +vet: ## Run go vet + go vet $(SYNC_PKG) ./cmd/doctest/... +``` + + +Change `fmt`: + +```makefile +fmt: ## Format Go source files with gofmt + gofmt -w cmd/sync-content/ cmd/doctest/ +``` + + +Change `fmt-check`: + +```makefile +fmt-check: ## Check Go formatting (non-destructive) + @out=$$(gofmt -l cmd/sync-content/ cmd/doctest/); \ + if [ -n "$$out" ]; then \ + echo "The following files need formatting:"; \ + echo "$$out"; \ + exit 1; \ + fi +``` + + +- [ ] **Step 4: Verify targets work** + +```bash +make help +``` + +Expected: new targets `test-docs-extract`, `test-docs`, `test-docs-coverage` appear in help output. + +```bash +make test-docs-extract +``` + +Expected: exit 0 (no annotated blocks to extract yet). + +```bash +make test-docs-coverage +``` + +Expected: lists untested bash blocks from getting-started page. + +```bash +make test +``` + +Expected: runs both sync-content and doctest Go tests. + +```bash +make check +``` + +Expected: runs vet, fmt-check, race tests, and doc coverage. + +- [ ] **Step 5: Commit** + +```bash +git add Makefile +git commit -m "feat: add Makefile targets for documentation testing + +Add test-docs-extract, test-docs, and test-docs-coverage targets. +Include doctest package in existing Go test/vet/fmt targets. +Add test-docs-coverage to check meta-target." +``` + +--- + +### Task 5: CI Integration + +**Files:** +- Modify: `.github/workflows/ci.yml` +- Modify: `.github/workflows/deploy-gh-pages.yml` + +**Context:** Both workflows use `actions/checkout` with SHA-pinned versions and `persist-credentials: false`. Neither currently checks out submodules. The Bats submodules in `tests/libs/` must be available for `make test-docs` to work. The `ci.yml` checkout is at line 16-18, `deploy-gh-pages.yml` at line 22-24. + +- [ ] **Step 1: Update `ci.yml` — add submodule checkout and doc tests** + +In `.github/workflows/ci.yml`, update the checkout step to include submodules, and add a doc tests step after the Hugo build: + +Update the Checkout step (lines 16-18) to: +```yaml + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + submodules: true +``` + +Add after the "Build site" step (after line 48): +```yaml + - name: Run documentation tests + run: make test-docs +``` + +- [ ] **Step 2: Update `deploy-gh-pages.yml` — add submodule checkout and doc tests** + +In `.github/workflows/deploy-gh-pages.yml`, update the checkout step to include submodules, and add a doc tests step after the Hugo build: + +Update the Checkout step (lines 22-24) to: +```yaml + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + submodules: true +``` + +Add after the "Build" step (after line 66): +```yaml + - name: Run documentation tests + run: make test-docs +``` + +- [ ] **Step 3: Also update Go test step in `ci.yml` to include doctest** + +Update line 40 from: +```yaml + - name: Run tests + run: go test -race ./cmd/sync-content/... +``` + +to: +```yaml + - name: Run tests + run: go test -race ./cmd/sync-content/... ./cmd/doctest/... +``` + +And in `deploy-gh-pages.yml`, update line 57-58 from: +```yaml + - name: Run tests + run: go test -race ./cmd/sync-content/... +``` + +to: +```yaml + - name: Run tests + run: go test -race ./cmd/sync-content/... ./cmd/doctest/... +``` + +- [ ] **Step 4: Verify YAML validity** + +```bash +python3 -c "import yaml; yaml.safe_load(open('.github/workflows/ci.yml')); print('ci.yml OK')" +python3 -c "import yaml; yaml.safe_load(open('.github/workflows/deploy-gh-pages.yml')); print('deploy-gh-pages.yml OK')" +``` + +Expected: both print OK. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/ci.yml .github/workflows/deploy-gh-pages.yml +git commit -m "ci: add documentation test steps to CI workflows + +Check out git submodules (bats-core, bats-support, bats-assert) and +run make test-docs in both ci.yml and deploy-gh-pages.yml. Include +cmd/doctest in Go race-test steps." +``` + +--- + +### Task 6: Documentation Updates + +**Files:** +- Modify: `CONTRIBUTING.md` +- Modify: `README.md` +- Create: `AGENTS.md` (if the project wants one; otherwise skip) + +**Context:** `CONTRIBUTING.md` is 537 lines with a clear section structure. The spec calls for a new "Testing Documentation" section after existing testing sections. `README.md` is 49 lines. The project has no `AGENTS.md` file (only `.agents/skills/.gitkeep`). + +- [ ] **Step 1: Add "Testing Documentation" section to CONTRIBUTING.md** + +Insert after the "Testing the Sync Tool" section (after line 476, before the "### Testing Tips" section). Add: + +```markdown +### Testing Documentation + +Documentation pages with shell commands use **testable code blocks** — fenced +code blocks annotated with a `{test="..."}` attribute that links them to +automated tests. + +**Annotating a code block:** + + +````markdown +```bash {test="install-complyctl"} +go install github.com/complytime/complyctl@latest +``` +```` + + + +The `test` value must be lowercase alphanumeric with hyphens (`[a-z0-9-]+`). +It becomes both the extracted snippet filename and the Bats test reference. +Each value must be unique within a page. + +**Writing the corresponding test:** + +Create or update a `.bats` file in `tests/docs/` matching the page name: + +```bash +# tests/docs/getting-started.bats + +@test "install-complyctl" { + run_snippet "getting-started/01-install-complyctl.bash" + assert_success +} +``` + +**Opting out a page:** Add `testable_docs: false` to the page's YAML frontmatter +to skip it entirely from extraction and coverage reporting. + +**Make targets:** + +| Target | What it does | +|--------|-------------| +| `make test-docs-extract` | Extract annotated code blocks to `.test-output/doctest-snippets` | +| `make test-docs` | Extract + run Bats tests | +| `make test-docs-coverage` | Report untested executable code blocks (warnings only) | + +Coverage warnings are non-blocking — they show which blocks could benefit from +test annotations but do not fail the build. +``` + +- [ ] **Step 2: Update CONTRIBUTING.md table of contents** + +Add "Testing Documentation" to the table of contents under "Common Tasks": + +After the line ` - [Add Images](#add-images)` (line 23), the existing TOC does not have a "Testing Documentation" entry. Find the "Development Workflow" section entry and add the new entry in the appropriate location. Insert under the existing testing items at the right nesting level. + +Actually, looking at the TOC structure, "Testing the Sync Tool" is under "Development Workflow". Add "Testing Documentation" after it: + +Find the line: +``` +- [Troubleshooting](#troubleshooting) +``` + +And add before it (under Development Workflow): +``` + - [Testing Documentation](#testing-documentation) +``` + +Wait — looking more carefully, the TOC items for Development Workflow sub-sections are not listed in the TOC. The "Testing the Sync Tool" section is not in the TOC either. So just add the section content and it will be discoverable by scrolling or heading search. + +- [ ] **Step 3: Update CONTRIBUTING.md PR checklist** + +Add a checklist item for documentation testing. Find the PR checklist section (around line 399) and add: + +```markdown +- [ ] If documentation code blocks were changed: `make test-docs` passes +``` + +- [ ] **Step 4: Update README.md project structure** + +In `README.md`, update the project structure tree (lines 23-33) to include the new directories: + +``` +website/ +├── cmd/sync-content/ # Go content sync tool (10 source files, package main) +├── cmd/doctest/ # Go documentation test extraction tool +├── config/_default/ # Hugo configuration (TOML) +├── content/docs/ # Markdown content (projects/ is generated by sync tool) +├── data/projects.json # Generated landing page cards (gitignored) +├── layouts/ # Custom Hugo layout overrides +├── tests/docs/ # Bats documentation tests +├── tests/libs/ # Bats test libraries (git submodules) +├── sync-config.yaml # Declarative sync configuration +├── .content-lock.json # Approved upstream SHAs per repo (committed) +└── .github/workflows/ # CI, deploy, weekly content check +``` + +- [ ] **Step 5: Update README.md quick start / development info** + +After the "Production build" line (line 19), add: + +```markdown +**Documentation tests**: `make test-docs` extracts annotated code blocks and runs Bats tests against them. +``` + +- [ ] **Step 6: Create AGENTS.md** + +Create `AGENTS.md` in the project root with guidance for AI agents: + +```markdown +# Agent Instructions + +## Documentation with Shell Commands + +When editing documentation that contains shell commands: + +1. **Always add `{test="..."}` attributes** to fenced code blocks that contain + runnable shell commands. Use lowercase alphanumeric identifiers with hyphens. + +2. **Write a corresponding Bats test** in `tests/docs/` before fixing a snippet + (TDD for docs). The test name must match the `test` attribute value. + +3. **Run `make test-docs-coverage`** to check for untested code blocks. + +4. **Run `make test-docs`** to verify all annotated snippets pass their tests. + +## Go Code + +- Run `make check` before committing (includes `go vet`, `gofmt`, race tests, + and doc coverage). +- Follow existing patterns in `cmd/sync-content/` for test structure. +``` + +- [ ] **Step 7: Commit** + +```bash +git add CONTRIBUTING.md README.md AGENTS.md +git commit -m "docs: add testable documentation workflow to contributor guides + +Add Testing Documentation section to CONTRIBUTING.md with annotation +convention, Bats test examples, and make targets. Update README.md +project structure. Create AGENTS.md with documentation testing guidance." +``` + +--- + +### Task 7: .gitignore and Housekeeping + +**Files:** +- Modify: `.gitignore` + +**Context:** The `.gitignore` is 60 lines. Extracted snippets go to `.test-output/doctest-snippets`, an in-repo directory that must be gitignored so generated snippets never get committed. The `doctest` binary (if built locally) should also be ignored, similar to the existing `/sync-content` ignore on line 13. + +- [ ] **Step 1: Add doctest binary to .gitignore** + +After the existing line `/sync-content` (line 13), add: + +```gitignore +/doctest +``` + +Also add a test output directory entry: + +```gitignore +# ─── Test output ───────────────────────────────────────────────────── +.test-output/ +``` + +- [ ] **Step 2: Commit** + +```bash +git add .gitignore +git commit -m "chore: gitignore doctest binary and test output directory" +``` + +--- + +### Task 8: End-to-End Verification + +**Files:** None created or modified. This task verifies the full pipeline. + +**Context:** At this point all code is written and committed. This task runs through the complete workflow to verify everything works together. + +- [ ] **Step 1: Run Go tests for doctest** + +```bash +go test -v ./cmd/doctest/... +``` + +Expected: all tests pass. + +- [ ] **Step 2: Run Go tests with race detector** + +```bash +go test -race ./cmd/doctest/... +``` + +Expected: passes with zero data race warnings. + +- [ ] **Step 3: Run full check** + +```bash +make check +``` + +Expected: vet, fmt-check, race tests (sync-content + doctest), and doc coverage all pass. Coverage reports untested bash blocks in getting-started page. + +- [ ] **Step 4: Run extraction** + +```bash +make test-docs-extract +``` + +Expected: exit 0. `.test-output/doctest-snippets` should be empty (no annotated blocks yet). + +- [ ] **Step 5: Run doc tests** + +```bash +make test-docs +``` + +Expected: exit 0. Bats reports 0 tests (skeleton file has no `@test` blocks). + +- [ ] **Step 6: Run coverage** + +```bash +make test-docs-coverage +``` + +Expected: lists untested bash/sh blocks from getting-started page with file, line, language. + +- [ ] **Step 7: Verify Hugo still renders correctly** + +```bash +hugo --minify --gc 2>&1 | head -5 +``` + +Expected: Hugo builds successfully. The `{test="..."}` attributes don't exist on any blocks yet, but verify Hugo configuration is still valid. + +- [ ] **Step 8: Verify Go formatting** + +```bash +make fmt-check +``` + +Expected: no unformatted files. + +- [ ] **Step 9: Review all commits** + +```bash +git log --oneline feat/testable-documentaiton ^main +``` + +Expected: 7 commits (Tasks 1-7) in logical order with conventional commit messages. diff --git a/specs/015-testable-documentation/spec.md b/specs/015-testable-documentation/spec.md index a699b32..56308b4 100644 --- a/specs/015-testable-documentation/spec.md +++ b/specs/015-testable-documentation/spec.md @@ -19,7 +19,7 @@ This feature adds testable documentation infrastructure: authors annotate fenced ### In Scope | ID | Capability | -|----|-----------| +| ---- | ----------- | | IS-001 | Markdown authoring convention: `{test=""}` attribute on fenced code blocks in the info string | | IS-002 | Go extraction tool (`cmd/doctest/`) with `extract` and `coverage` subcommands | | IS-003 | Goldmark-based AST parsing of fenced code blocks with `parser.ParseAttributes()` for info-string attribute extraction | @@ -28,7 +28,7 @@ This feature adds testable documentation infrastructure: authors annotate fenced | IS-006 | `manifest.json` per page mapping test names to source `file:line` for traceability | | IS-007 | Coverage reporting: list untested executable-language blocks with file, line, and language | | IS-008 | Duplicate `test` value detection within a page (extractor errors on duplicates) | -| IS-009 | Bats test harness with git submodules for bats-core, bats-support, bats-assert | +| IS-009 | Bats test harness with bats-core, bats-support, bats-assert (npm devDependencies) | | IS-010 | Helper pattern for language-specific snippet execution (`tests/docs/helpers/bash.bash`) | | IS-011 | Makefile targets: `test-docs-extract`, `test-docs`, `test-docs-coverage` | | IS-012 | CI integration: `make test-docs` step in `ci.yml` and `deploy-gh-pages.yml` | @@ -41,13 +41,13 @@ This feature adds testable documentation infrastructure: authors annotate fenced - Testing synced/generated content (`content/docs/projects/*/`) — future goal - Auto-generating Bats test files from extracted snippets - Block-level opt-out attribute (e.g., `{skip=true}`) — use non-executable language instead -- Strict coverage enforcement in CI (Phase 2, one-line change when ready) +- Strict coverage enforcement in CI (Phase 2 — remove `-` prefix and `continue-on-error: true` when ready) - Non-shell language test execution (Go, Python helpers are future additions) ### Edge Cases | Case | Expected Behavior | -|------|-------------------| +| ------ | ------------------- | | Code block with no info string | Silently skipped — no language, not testable | | Code block with language but no `{...}` | Flagged in coverage report if language is testable; silently skipped if non-testable | | Code block with `{test="..."}` but non-testable language | Extracted and written to output dir (enables future expansion); not flagged in coverage | @@ -79,10 +79,13 @@ go install github.com/complytime/complyctl@latest **Language classification:** + | Category | Languages | |----------|-----------| | Testable (flagged if untested) | `bash`, `sh`, `shell`, `zsh` | | Non-testable (silently skipped) | `text`, `plaintext`, `console`, `yaml`, `toml`, `json`, `xml`, `csv`, `markdown`, `go`, `python`, and any language without a helper | + + **Ordering:** Blocks within a page are extracted in document order with a numeric prefix (`01-install-complyctl.bash`, `02-run-scan.bash`) to preserve sequencing for tests that build on prior state. @@ -96,7 +99,7 @@ go install github.com/complytime/complyctl@latest ### CLI Interface -``` +```text doctest extract --content-dir --output-dir doctest coverage --content-dir ``` @@ -112,7 +115,7 @@ doctest coverage --content-dir **`coverage` subcommand:** - Same file walk and parse as `extract`. - Reports untested executable-language blocks: one line per block with file path, line number, and language. -- Exit 0 always (warnings only). Future `--strict` flag exits 1 when untested blocks exist. +- Exit 1 when untested executable-language blocks exist. Exit 0 when all blocks are covered. ### Implementation Details @@ -124,7 +127,7 @@ doctest coverage --content-dir ### Output Structure -``` +```text / └── getting-started/ ├── 01-install-complyctl.bash @@ -152,7 +155,7 @@ doctest coverage --content-dir Unit tests following the existing `cmd/sync-content/` pattern — table-driven tests with temp dirs. Coverage: | Test | What it verifies | -|------|-----------------| +| ------ | ----------------- | | `TestExtractBasic` | Single annotated block extracted with correct filename and content | | `TestExtractOrdering` | Multiple blocks get sequential numeric prefixes in document order | | `TestExtractDuplicateError` | Duplicate `test` values within a page produce an error | @@ -169,20 +172,19 @@ Unit tests following the existing `cmd/sync-content/` pattern — table-driven t ### Installation -Bats-core, bats-support, and bats-assert installed as git submodules in `tests/libs/`: +Bats-core, bats-support, and bats-assert installed as npm devDependencies: -``` -tests/libs/ -├── bats-core/ # github.com/bats-core/bats-core -├── bats-support/ # github.com/bats-core/bats-support -└── bats-assert/ # github.com/bats-core/bats-assert +```json +"bats": "^1.13.0", +"bats-support": "^0.3.0", +"bats-assert": "^2.2.4" ``` -This is the standard Bats distribution pattern — no npm/brew dependency, works identically in CI and local development. +This leverages the project's existing npm/Node.js toolchain — `npm install` is already a prerequisite for Hugo/Thulite development. ### Directory Layout -``` +```text tests/docs/ ├── setup_suite.bash # shared setup: set SNIPPETS_DIR, load libraries ├── helpers/ @@ -193,7 +195,7 @@ tests/docs/ ### `setup_suite.bash` ```bash -export SNIPPETS_DIR="${SNIPPETS_DIR:-/tmp/doctest-snippets}" +export SNIPPETS_DIR="${SNIPPETS_DIR:-.test-output/doctest-snippets}" ``` Suite-level setup sets the snippets directory. Library loading happens per-file in `setup()` — this is the standard Bats pattern where each `.bats` file declares its own dependencies. @@ -204,7 +206,7 @@ Suite-level setup sets the snippets directory. Library loading happens per-file run_snippet() { local snippet="$SNIPPETS_DIR/$1" [[ -f "$snippet" ]] || { echo "Snippet not found: $snippet" >&2; return 1; } - run bash "$snippet" + run bash -- "$snippet" } ``` @@ -215,8 +217,8 @@ run_snippet() { setup() { load 'helpers/bash' - load '../libs/bats-support/load' - load '../libs/bats-assert/load' + load '../../node_modules/bats-support/load' + load '../../node_modules/bats-assert/load' } @test "install-complyctl" { @@ -241,6 +243,7 @@ setup() { New section after existing "Content sync" section: + ```makefile # --------------------------------------------------------------------------- # Documentation tests — extract, validate, and test code blocks @@ -249,14 +252,15 @@ New section after existing "Content sync" section: .PHONY: test-docs-extract test-docs test-docs-coverage test-docs-extract: ## Extract testable code blocks from documentation - @go run ./cmd/doctest extract --content-dir content/docs --output-dir /tmp/doctest-snippets + @go run ./cmd/doctest extract --content-dir content/docs --output-dir .test-output/doctest-snippets test-docs: test-docs-extract ## Run documentation tests (Bats) - @tests/libs/bats-core/bin/bats tests/docs/ + @node_modules/.bin/bats --formatter pretty tests/docs/ test-docs-coverage: ## Report untested code blocks in documentation @go run ./cmd/doctest coverage --content-dir content/docs ``` + **`check` meta-target update:** @@ -264,7 +268,7 @@ test-docs-coverage: ## Report untested code blocks in documentation check: vet fmt-check test-race test-docs-coverage ``` -Coverage runs as a warning — non-zero exit doesn't fail the build. +Coverage exits non-zero when untested blocks exist. The `-` prefix in Make ignores the exit code so `check` does not fail the build (Phase 1). Remove the prefix to enforce (Phase 2). ## CI Integration @@ -272,32 +276,34 @@ Coverage runs as a warning — non-zero exit doesn't fail the build. **`ci.yml`** — add step after Hugo build: ```yaml -- name: Run documentation tests +- name: Run documentation tests (informational) run: make test-docs + continue-on-error: true ``` **`deploy-gh-pages.yml`** — add step before deploy: ```yaml -- name: Run documentation tests +- name: Run documentation tests (informational) run: make test-docs + continue-on-error: true ``` -`make test-docs` depends on `test-docs-extract`, so ordering is automatic. Documentation with broken snippets blocks both PR checks and deployment. +`make test-docs` depends on `test-docs-extract`, so ordering is automatic. The `continue-on-error: true` step attribute makes doc test failures non-blocking in CI (Phase 1). Remove it to enforce (Phase 2). ## Coverage Enforcement Model ### Phase 1 (This Implementation) -- `make test-docs-coverage` prints untested blocks to stdout. Always exits 0. -- Included in `check` meta-target — developers see warnings during normal workflow. -- `make test-docs` (extract + Bats) runs as a blocking CI step — test failures break the build. +- `doctest coverage` exits non-zero when untested executable blocks exist. +- `make test-docs-coverage` and `make test-docs` use `-` prefix (Make) or `continue-on-error: true` (CI) to run without failing the build. +- Included in `check` and `test` meta-targets — developers see warnings during normal workflow. - Coverage gaps are visible but non-blocking. ### Phase 2 (Future) -- `doctest coverage --strict` exits 1 if any executable-language block lacks `test="..."`. -- Flip CI to `--strict` once coverage is solid across all pages. -- One-line change when the team is ready. +- Remove `-` prefix from Makefile `test-docs` / `test-docs-coverage` calls. +- Remove `continue-on-error: true` from CI workflow steps. +- Coverage failures then block builds and PRs. ### Opt-Out Mechanism @@ -330,12 +336,12 @@ New section "Testing Documentation" after the existing testing section: ## Success Criteria | ID | Criterion | -|----|-----------| +| ---- | ----------- | | SC-001 | `go test ./cmd/doctest/...` passes all unit tests | | SC-002 | `go test -race ./cmd/doctest/...` passes with zero data race warnings | | SC-003 | `make test-docs-extract` produces snippet files from annotated Markdown blocks | | SC-004 | `make test-docs` runs Bats tests against extracted snippets | -| SC-005 | `make test-docs-coverage` reports untested executable blocks without failing | +| SC-005 | `make test-docs-coverage` reports untested executable blocks (exits non-zero when gaps exist, non-blocking in CI/check via `-` prefix) | | SC-006 | `make check` includes coverage reporting | | SC-007 | CI pipelines (`ci.yml`, `deploy-gh-pages.yml`) run `make test-docs` | | SC-008 | CONTRIBUTING.md documents the testable-docs workflow | diff --git a/tests/docs/getting-started.bats b/tests/docs/getting-started.bats new file mode 100644 index 0000000..ddd4fac --- /dev/null +++ b/tests/docs/getting-started.bats @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Documentation tests for content/docs/getting-started/_index.md +# +# Tests will be added here as code blocks in the getting started guide +# are annotated with {test="..."} attributes. Each @test name must +# match a test attribute value in the Markdown source. + +setup() { + load 'helpers/bash' + load '../../node_modules/bats-support/load' + load '../../node_modules/bats-assert/load' +} + +# Placeholder: add @test blocks as snippets are annotated (Phase 2). +# The end-to-end harness wiring is already validated in harness.bats. +# Example: +# +# @test "install-complyctl" { +# run_snippet "getting-started/01-install-complyctl.bash" +# assert_success +# } diff --git a/tests/docs/harness.bats b/tests/docs/harness.bats new file mode 100644 index 0000000..6896584 --- /dev/null +++ b/tests/docs/harness.bats @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 + +# End-to-end smoke test for the documentation test harness itself. +# +# This validates the run_snippet helper and the SNIPPETS_DIR wiring without +# depending on any doc-page annotations (those arrive in Phase 2). It creates +# a throwaway snippet, runs it, and asserts the harness reports success and +# failure correctly. + +setup() { + load 'helpers/bash' + load '../../node_modules/bats-support/load' + load '../../node_modules/bats-assert/load' + + HARNESS_TMP="$(mktemp -d)" + export SNIPPETS_DIR="$HARNESS_TMP" +} + +teardown() { + rm -rf "$HARNESS_TMP" +} + +@test "run_snippet succeeds for a passing snippet" { + mkdir -p "$SNIPPETS_DIR/harness" + printf 'echo hello\n' > "$SNIPPETS_DIR/harness/01-pass.bash" + + run_snippet "harness/01-pass.bash" + assert_success + assert_output "hello" +} + +@test "run_snippet fails for a failing snippet" { + mkdir -p "$SNIPPETS_DIR/harness" + printf 'exit 3\n' > "$SNIPPETS_DIR/harness/02-fail.bash" + + # Wrap in `run` so the trace-on-failure diagnostic (emitted on fd 3 for a + # failing snippet) is captured rather than printed, keeping suite output + # clean; run_snippet returns the snippet's own exit status. + run run_snippet "harness/02-fail.bash" + assert_failure 3 +} + +@test "run_snippet reports missing snippet" { + run run_snippet "harness/does-not-exist.bash" + assert_failure + assert_output --partial "Snippet not found" +} + +@test "snippet_origin resolves source file:line from the manifest" { + mkdir -p "$SNIPPETS_DIR/getting-started" + printf 'echo hi\n' > "$SNIPPETS_DIR/getting-started/01-install.bash" + cat > "$SNIPPETS_DIR/getting-started/manifest.json" <<'JSON' +{ + "page": "getting-started/_index.md", + "snippets": [ + { "test": "install", "file": "01-install.bash", "source_line": 49, "language": "bash" } + ] +} +JSON + + run snippet_origin "getting-started/01-install.bash" + assert_success + assert_output "getting-started/_index.md:49" +} + +@test "snippet_origin falls back to the snippet path without a manifest" { + run snippet_origin "harness/01-pass.bash" + assert_success + assert_output "$SNIPPETS_DIR/harness/01-pass.bash" +} diff --git a/tests/docs/helpers/bash.bash b/tests/docs/helpers/bash.bash new file mode 100644 index 0000000..06faa9b --- /dev/null +++ b/tests/docs/helpers/bash.bash @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Helper for running extracted bash snippets. +# Usage in a @test block: +# run_snippet "getting-started/01-install-complyctl.bash" + +# snippet_origin resolves an extracted snippet (e.g. "getting-started/01-x.bash") +# back to the documentation file and line it came from, using the per-page +# manifest.json the extractor writes alongside the snippets. Prints +# ":" on success. Falls back to the raw snippet path when no +# manifest exists (e.g. the harness's own throwaway self-tests) or when jq is +# unavailable. +snippet_origin() { + local rel="$1" + local slug="${rel%/*}" + local base="${rel##*/}" + local manifest="$SNIPPETS_DIR/$slug/manifest.json" + + if [[ "$slug" != "$rel" ]] && [[ -f "$manifest" ]] && command -v jq >/dev/null 2>&1; then + local page line + page="$(jq -r '.page' "$manifest" 2>/dev/null)" + line="$(jq -r --arg f "$base" \ + '.snippets[] | select(.file == $f) | .source_line' \ + "$manifest" 2>/dev/null)" + if [[ -n "$page" && "$page" != "null" && -n "$line" && "$line" != "null" ]]; then + echo "$page:$line" + return 0 + fi + fi + echo "$SNIPPETS_DIR/$rel" +} + +run_snippet() { + local snippet="$SNIPPETS_DIR/$1" + [[ -f "$snippet" ]] || { echo "Snippet not found: $snippet" >&2; return 1; } + run bash -- "$snippet" + # Surface the originating documentation source (file:line) only when the + # snippet failed, so a red test is easy to trace back to the doc it came + # from. Emit on stderr (not fd 3): Bats hides stderr for passing tests and + # for `run`-wrapped calls, but prints it as failure context for a genuinely + # failing test, keeping normal suite output clean. Return the snippet's own + # exit status so callers using `run run_snippet ...` capture it. + # $status/$output are set by Bats' `run` above. + # shellcheck disable=SC2154 + if [[ "$status" -ne 0 ]]; then + echo "# source: $(snippet_origin "$1")" >&2 + fi + # shellcheck disable=SC2154 + return "$status" +} diff --git a/tests/docs/setup_suite.bash b/tests/docs/setup_suite.bash new file mode 100644 index 0000000..26f14bb --- /dev/null +++ b/tests/docs/setup_suite.bash @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Suite-level setup for documentation tests. +# Sets the default snippets directory. Individual .bats files load +# their own libraries in setup(). + +setup_suite() { + export SNIPPETS_DIR="${SNIPPETS_DIR:-.test-output/doctest-snippets}" +}