diff --git a/.performance/ratchets.json b/.performance/ratchets.json index 73c0562771f..26060937b0f 100644 --- a/.performance/ratchets.json +++ b/.performance/ratchets.json @@ -160,6 +160,54 @@ "field": "spawns", "ceiling": 1, "goal": "With powershell_format_end_of_turn at its default (false), the Stop row costs only the node launcher: the --require-true gate closes before bash starts. Stop fires on every turn end, so this is what every turn costs a user who never opted in. Measured 1 on WSL2 Ubuntu with bash 5.3.9, dash as sh, strace 6.19; the ratchet step prints the current versions. Correlation: unproven: not measured" + }, + { + "name": "skill-stack-fences", + "command": "python3 scripts/skill-stack-fences.py", + "field": "fences", + "goal": "Shipped skill and agent bodies carry no ecosystem-specific examples (docs/plugin-philosophy.md, \"Hardcoded consumer specifics\"): no file gains a code fence tagged for one language, stylesheet or framework, and existing ones are cut toward 0. The scope and the excluded shell, data, query and pre-compute tags are in scripts/skill-stack-fences.py; every other tag counts. A fence a declared-ecosystem surface or an executed adapter keeps (ADR 0101) raises its file ceiling in the PR that adds it, with the reason in the PR body. Measured 174 fences in 39 files at f552369db7. Correlation: unproven: a content-shape count, not a runtime cost", + "ceilings": { + "plugins/ai-briefing/skills/generate/reference/build-pipeline.md": 2, + "plugins/architecture/skills/improve/research/deepening/html-report.md": 3, + "plugins/code-tidying/skills/audit-dead-code/context/adjudication.md": 2, + "plugins/education/skills/teach/context/lessons.md": 1, + "plugins/harness-ops/skills/audit-skill-visibility/reference/listing-scorer.md": 2, + "plugins/harness-ops/skills/audit-skill-visibility/reference/usage-counters.md": 1, + "plugins/harness-ops/skills/inventory/reference/extraction.md": 4, + "plugins/kindle-dedrm/skills/manage/reference/versions.md": 1, + "plugins/knowledge/skills/course-digest/reference/adapters/discovery-checklist.md": 5, + "plugins/knowledge/skills/course-digest/reference/adapters/dometrain.md": 4, + "plugins/knowledge/skills/course-digest/reference/adapters/teachable.md": 1, + "plugins/knowledge/skills/course-digest/reference/screenshot-strategy.md": 1, + "plugins/mutation-testing/skills/audit/context/suppression.md": 1, + "plugins/mutation-testing/skills/principles/reference/theory.md": 1, + "plugins/performance/skills/snapshot/SKILL.md": 1, + "plugins/planning/skills/design/reference/type-discipline/typescript.md": 5, + "plugins/playwright/skills/demo-video/reference/replay-script.md": 1, + "plugins/playwright/skills/playwright/reference/running-code.md": 3, + "plugins/playwright/skills/playwright/reference/test-generation.md": 2, + "plugins/tdd/skills/principles/reference/anti-patterns-khorikov.md": 11, + "plugins/tdd/skills/principles/reference/classical-vs-london-khorikov.md": 2, + "plugins/tdd/skills/principles/reference/code-coverage-khorikov.md": 4, + "plugins/tdd/skills/principles/reference/integration-testing-khorikov.md": 5, + "plugins/tdd/skills/principles/reference/observable-behavior-khorikov.md": 6, + "plugins/tdd/skills/principles/reference/test-design.md": 10, + "plugins/tdd/skills/principles/reference/test-doubles.md": 5, + "plugins/tdd/skills/principles/reference/testable-architecture-khorikov.md": 6, + "plugins/tdd/skills/principles/reference/testing-styles-khorikov.md": 9, + "plugins/testing/skills/write/context/write.md": 1, + "plugins/user-interface/skills/audit/reference/launch-scope.md": 2, + "plugins/user-interface/skills/write-css/reference/foundations.md": 6, + "plugins/user-interface/skills/write-css/reference/interaction.md": 14, + "plugins/user-interface/skills/write-css/reference/layout.md": 8, + "plugins/user-interface/skills/write-css/reference/motion.md": 10, + "plugins/user-interface/skills/write-css/reference/scroll-and-viewport.md": 15, + "plugins/user-interface/skills/write-css/reference/show-and-hide.md": 7, + "plugins/user-interface/skills/write-css/reference/spacing-and-shape.md": 4, + "plugins/user-interface/skills/write-css/reference/text-and-media.md": 5, + "plugins/visualization/skills/visualize/context/code-shapes.md": 3 + } } - ] + ], + "break_glass": false } diff --git a/docs/adr/0101-resolve-ecosystem-specifics-from-the-consumer-repository.md b/docs/adr/0101-resolve-ecosystem-specifics-from-the-consumer-repository.md index c9db2074e56..d7ad5f97d58 100644 --- a/docs/adr/0101-resolve-ecosystem-specifics-from-the-consumer-repository.md +++ b/docs/adr/0101-resolve-ecosystem-specifics-from-the-consumer-repository.md @@ -1,6 +1,7 @@ # Resolve ecosystem specifics from the consumer repository -- Status: accepted +- Status: accepted; amended 2026-11-16 (source citations exclude one ecosystem's tool documentation + and examples, see the amendment below) - Date: 2026-11-13 - Refs: [#15703](https://github.com/melodic-software/claude-code-plugins/issues/15703) (spec), [#15706](https://github.com/melodic-software/claude-code-plugins/issues/15706); owner decision of @@ -44,7 +45,7 @@ a rule for what may stay shipped, and an end to the bundled fallback. | Executed adapter | code a script runs that parses or drives a named tool | kept; it names each file or value it cannot read, and accepts a value it has no adapter for, reporting that no shipped adapter covers it | | Security or safety floor | a check that holds whatever the consumer chose, such as a secret pattern or a guard on a destructive command | kept; removing or narrowing one is the owner's decision each time | | Declared-ecosystem surface | a plugin or skill whose purpose is one ecosystem, declaring that narrower scope at the coupling site | kept, as [ADR 0060](0060-abstract-provider-details-in-skills-behind-consumer-conventions-and-adapters.md) keeps declared-provider surfaces | - | Source citation | a documentation URL or name cited for a reader to consult, not a choice the skill makes | kept | + | Source citation | a documentation URL or name cited for a reader to consult, not a choice the skill makes; not a link to one ecosystem's tool documentation, nor an example written for one stack | kept | The named exceptions are the last four classes. A model-read specific in a generic skill outside them is a review finding: [`docs/review/skills-and-agents.md`](../review/skills-and-agents.md). @@ -94,3 +95,22 @@ a rule for what may stay shipped, and an end to the bundled fallback. - A consumer reports that an unresolved skip blocks work a shipped default used to do, and no record, statement or inference could have resolved it. - A class of shipped content that fits none of the five classes above. + +## Amendment (2026-11-16): source citations exclude ecosystem-specific links and examples + +The Source citation row kept any documentation URL or name a skill cites for a reader. The owner +decided on 2026-11-16 that shipped skills carry no ecosystem-specific examples: no code sample +written for one language, framework or tool, and no link to one tool's documentation, in a skill +body or reference file. Such examples change with the stack they show, every change becomes our +maintenance, and a sample reads as a choice the skill made for the consumer. The skill states its +rule in ecosystem-agnostic terms and resolves the specifics at run time from the consumer's own +rules and conventions, with `/discovery:research` where they do not settle it; detail that must +persist goes into the consumer's configuration, or into a repository-specific skill the skill +generates. The rule, its reason and what stays are in the plugin philosophy's +[Hardcoded consumer specifics](../plugin-philosophy.md#hardcoded-consumer-specifics). + +The row now excludes those links and examples. The other four classes are unchanged: executed +adapters and declared-ecosystem surfaces keep what they hold. The `skill-stack-fences` row of +`.performance/ratchets.json` (`scripts/skill-stack-fences.py`) holds the count of stack-tagged +code fences in skill and agent bodies to its value on 2026-11-16, per file, so a new one fails CI +unless its pull request raises that file's ceiling and says why. diff --git a/scripts/skill-stack-fences.py b/scripts/skill-stack-fences.py new file mode 100755 index 00000000000..135adff7e6d --- /dev/null +++ b/scripts/skill-stack-fences.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Count the stack-specific code fences in shipped skill and agent bodies. + + skill-stack-fences.py + +Run from the repository root. Prints one ` fences=` line per file +with a count above 0, sorted by path, which the per-file `skill-stack-fences` +row of .performance/ratchets.json reads. Shipped skills carry no +ecosystem-specific examples (docs/plugin-philosophy.md, "Hardcoded consumer +specifics"); the ratchet stops the count of fences that show one stack from +growing while the existing ones are cut. + +The scope: tracked `.md` files under `plugins//skills/` and +`plugins//agents/`, except those under a directory named evals, tests, +test, __tests__, fixtures, testdata or vendor (eval inputs, test data, and +upstream copies a sync replaces). + +A fence counts when its info string's first word, lowercased with `{`, `.` and +`}` stripped, is not in EXCLUDED_TAGS: shells the agent runs commands through, +data and prose formats a skill reads or emits, diagrams, the skill pre-compute +block (`!`), and the queries and patterns skills run against their own data +(sql, regex). Every other tag counts, so a new language is counted without an +edit here. An untagged fence does not count. Fences follow CommonMark: an +opening run of three or more backticks or tildes at any indent, closed by a +run of the same character at least as long with nothing after it; a fence +inside another fence is content and does not count. An opener may follow any +mix of blockquote markers (`>`), list markers (`-`, `*`, `+`, `1.`, `1)`) and +their indentation. A closer may carry only indentation and exactly the +opener's number of `>` markers. A line with fewer `>` markers than the opener +ends its blockquote and so the fence, and is then read as a normal line that +may open a fence. Any other line, a list marker included, is content of the +open fence. + +Exit 0 with the lines printed; 2 when git cannot list files or no file is in +scope, because a count of nothing would pass any ceiling. +""" + +from __future__ import annotations + +import re +import subprocess +import sys + +MIN_PYTHON = (3, 9) +AGENT_SHELLS = "bash sh shell zsh console powershell pwsh ps1 batch bat cmd" +DATA_AND_PROSE = ( + "text txt plaintext markdown md json jsonc jsonl yaml yml toml ini csv diff" +) +DIAGRAMS_AND_CONFIG = "mermaid gitconfig" +PRECOMPUTE_BLOCK = "!" +OWN_DATA_QUERIES = "sql regex" +EXCLUDED_TAGS = frozenset( + " ".join( + ( + AGENT_SHELLS, + DATA_AND_PROSE, + DIAGRAMS_AND_CONFIG, + PRECOMPUTE_BLOCK, + OWN_DATA_QUERIES, + ) + ).split() +) +SKIPPED_DIRS = frozenset( + {"evals", "tests", "test", "__tests__", "fixtures", "testdata", "vendor"} +) +CONTAINER_PREFIX = r"(?:\s*(?:>|[-*+](?=\s)|\d{1,9}[.)](?=\s)))*\s*" +OPENING = re.compile(rf"^({CONTAINER_PREFIX})(`{{3,}}|~{{3,}})(.*)$") +CLOSING = re.compile(r"^([\s>]*)(`{3,}|~{3,})\s*$") +QUOTE_MARKERS = re.compile(r"[\s>]*") + + +def fail(message: str) -> int: + print(f"skill-stack-fences: {message}", file=sys.stderr) + return 2 + + +def in_scope(path: str) -> bool: + parts = path.split("/") + return ( + len(parts) >= 4 + and parts[0] == "plugins" + and parts[2] in ("skills", "agents") + and path.endswith(".md") + and not SKIPPED_DIRS.intersection(parts[3:-1]) + ) + + +def tag_of(info: str) -> str: + words = info.split() + return words[0].strip("{}.").lower() if words else "" + + +def count_fences(text: str) -> int: + count = 0 + fence = "" + quote_depth = 0 + for line in text.splitlines(): + if fence: + if QUOTE_MARKERS.match(line).group(0).count(">") < quote_depth: + fence = "" + else: + closing = CLOSING.match(line) + if ( + closing + and closing.group(1).count(">") == quote_depth + and closing.group(2)[0] == fence[0] + and len(closing.group(2)) >= len(fence) + ): + fence = "" + continue + match = OPENING.match(line) + if not match: + continue + prefix, marker, info = match.groups() + if marker[0] == "`" and "`" in info: + continue + fence = marker + quote_depth = prefix.count(">") + tag = tag_of(info) + if tag and tag not in EXCLUDED_TAGS: + count += 1 + return count + + +def main() -> int: + listed = subprocess.run( + ["git", "ls-files", "-z", "--", "plugins/"], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + if listed.returncode != 0: + return fail(f"git ls-files failed: {listed.stderr.strip()}") + paths = sorted(p for p in listed.stdout.split("\0") if p and in_scope(p)) + if not paths: + return fail("no skill or agent markdown in scope") + for path in paths: + with open(path, encoding="utf-8", errors="replace") as handle: + count = count_fences(handle.read()) + if count: + print(f"{path} fences={count}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/skill-stack-fences.test.sh b/scripts/skill-stack-fences.test.sh new file mode 100755 index 00000000000..9a85b2b2e35 --- /dev/null +++ b/scripts/skill-stack-fences.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=lib/python-probe.sh +. "$SCRIPT_DIR/lib/python-probe.sh" + +PYTHON="" +python_probe::require_to PYTHON "$SCRIPT_DIR/skill-stack-fences.py" + +"$PYTHON" "$SCRIPT_DIR/test_skill_stack_fences.py" -v diff --git a/scripts/test_skill_stack_fences.py b/scripts/test_skill_stack_fences.py new file mode 100755 index 00000000000..6b5ac549896 --- /dev/null +++ b/scripts/test_skill_stack_fences.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Tests for skill-stack-fences.py, the counter behind the skill-stack-fences ratchet. + +Each case builds a throwaway git repository and runs the counter at its command +line, so every expected value below comes from the fixture and the rules in the +counter's docstring, not from running the counter. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +for _leaked_git_var in ("GIT_DIR", "GIT_WORK_TREE", "GIT_CONFIG", "GIT_INDEX_FILE"): + os.environ.pop(_leaked_git_var, None) + +SCRIPT = Path(__file__).resolve().parent / "skill-stack-fences.py" + +SKILL = "plugins/demo/skills/run/SKILL.md" + + +class SkillStackFencesTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.repo = Path(self._tmp.name) / "repo" + self.repo.mkdir() + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def write(self, rel: str, text: str) -> None: + path = self.repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def stage(self) -> None: + subprocess.run(["git", "-C", str(self.repo), "add", "-A"], check=True) + + def run_counter(self, cwd: Path | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT)], + cwd=cwd or self.repo, + capture_output=True, + text=True, + check=False, + ) + + def counts(self) -> dict[str, int]: + result = self.run_counter() + self.assertEqual(result.returncode, 0, result.stderr) + out: dict[str, int] = {} + for line in result.stdout.splitlines(): + path, token = line.rsplit(" ", 1) + field, value = token.split("=") + self.assertEqual(field, "fences") + out[path] = int(value) + return out + + def test_counts_stack_tags_and_skips_shell_data_and_untagged(self) -> None: + self.write( + SKILL, + "```csharp\nvar x = 1;\n```\n" + "```ts\nconst x = 1;\n```\n" + "```rust\nlet x = 1;\n```\n" + "```bash\nls\n```\n" + "```powershell\nGet-Item .\n```\n" + "```json\n{}\n```\n" + "```yaml\na: 1\n```\n" + "```text\nplain\n```\n" + "```markdown\n# h\n```\n" + "```sql\nselect 1;\n```\n" + "```regex\na+\n```\n" + "```!\ngit status\n```\n" + "```\nuntagged\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 3}) + + def test_tag_is_first_word_case_insensitive_with_attributes(self) -> None: + self.write( + SKILL, + "```Python title=x.py\nx = 1\n```\n" + "``` js\nx\n```\n" + "```{.go}\nx\n```\n" + "```Bash\nls\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 3}) + + def test_list_indented_and_tilde_fences_count(self) -> None: + self.write( + SKILL, + "1. Step\n\n ```java\n class A {}\n ```\n\n" + "~~~kotlin\nval a = 1\n~~~\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 2}) + + def test_blockquote_fences_count(self) -> None: + self.write( + SKILL, + "> ```python\n> x = 1\n> ```\n" + ">> ```ts\n>> x\n>> ```\n" + "> - ```go\n> x\n> ```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 3}) + + def test_fences_right_after_a_list_marker_count(self) -> None: + self.write( + SKILL, + "- ```python\n x = 1\n ```\n" + "* ```java\n x\n ```\n" + "1. ```csharp\n x\n ```\n" + "2) ```ruby\n x\n ```\n" + "- ```bash\n ls\n ```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 4}) + + def test_list_item_opener_does_not_hide_the_next_block(self) -> None: + self.write( + SKILL, + "- ```python\n x = 1\n ```\n\n```csharp\nvar x;\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 2}) + + def test_blockquote_closer_does_not_close_a_top_level_fence(self) -> None: + self.write( + SKILL, + "```markdown\n> ```python\n> x = 1\n> ```\n```\n\n```python\nx = 1\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 1}) + + def test_blockquote_fence_ends_with_its_blockquote(self) -> None: + self.write( + SKILL, + "> ```python\n> x = 1\n```\n> ```\n\n```go\nx\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 1}) + + def test_unclosed_blockquote_fence_does_not_swallow_a_later_fence(self) -> None: + self.write( + SKILL, + "> ```python\n> x = 1\n\n```python\ny = 2\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 2}) + + def test_fences_inside_another_fence_are_content(self) -> None: + self.write( + SKILL, + "````markdown\n```python\nx = 1\n```\n````\n" + "~~~text\n```csharp\nvar x;\n```\n~~~\n" + "```python\nprint()\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 1}) + + def test_closing_fence_needs_same_char_and_at_least_opening_length(self) -> None: + self.write( + SKILL, + "````text\n```\n~~~~\n```csharp\n````\n```python\nx\n```\n", + ) + self.stage() + self.assertEqual(self.counts(), {SKILL: 1}) + + def test_scope_is_tracked_skill_and_agent_markdown_outside_test_and_vendor_dirs( + self, + ) -> None: + fence = "```python\nx = 1\n```\n" + agent = "plugins/demo/agents/worker.md" + nested = "plugins/demo/skills/run/reference/deep/notes.md" + self.write(SKILL, fence) + self.write(agent, fence + fence) + self.write(nested, fence) + for rel in ( + "plugins/demo/skills/run/evals/case/prompt.md", + "plugins/demo/skills/run/tests/fixtures/a.md", + "plugins/demo/skills/run/vendor/SKILL.md", + "plugins/demo/skills/run/scripts/notes.txt", + "plugins/demo/reference/shared.md", + "plugins/demo/README.md", + "docs/guide.md", + ): + self.write(rel, fence) + self.stage() + self.write("plugins/demo/skills/run/untracked.md", fence) + self.assertEqual(self.counts(), {SKILL: 1, agent: 2, nested: 1}) + + def test_output_is_sorted_and_omits_files_at_zero(self) -> None: + fence = "```go\nx\n```\n" + self.write("plugins/b/skills/s/SKILL.md", fence) + self.write("plugins/a/skills/s/SKILL.md", fence) + self.write("plugins/c/skills/s/SKILL.md", "```bash\nls\n```\n") + self.stage() + result = self.run_counter() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.splitlines(), + [ + "plugins/a/skills/s/SKILL.md fences=1", + "plugins/b/skills/s/SKILL.md fences=1", + ], + ) + + def test_exit_2_when_no_file_is_in_scope(self) -> None: + self.write("docs/guide.md", "```python\nx\n```\n") + self.stage() + result = self.run_counter() + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + + def test_exit_2_outside_a_git_repository(self) -> None: + outside = Path(self._tmp.name) / "plain" + outside.mkdir() + result = self.run_counter(cwd=outside) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main()