From 9c041c569fa9a92abbf7336746c99a716e859899 Mon Sep 17 00:00:00 2001 From: Martin Zeman Date: Thu, 9 Jul 2026 23:13:23 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20ForgeSkill=20=E2=80=94=20skill=20author?= =?UTF-8?q?ing=20with=20eval=20loop,=20adopted=20from=20anthropics/skills?= =?UTF-8?q?=20skill-creator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + skills/BuildSkill/ClaudeSkill.md | 10 +- skills/BuildSkill/CreateWorkflow.md | 1 + skills/BuildSkill/DynamicContextInjection.md | 48 + skills/BuildSkill/SKILL.md | 5 + skills/BuildSkill/UserConfigSchema.md | 52 + skills/BuildSkill/ValidateWorkflow.md | 9 +- .../DescriptionOptimization.md.yaml | 32 + .../ForgeSkill/.provenance/EvalLoop.md.yaml | 32 + skills/ForgeSkill/.provenance/SKILL.yaml | 32 + skills/ForgeSkill/ClaudeSkill.md | 84 ++ skills/ForgeSkill/CliToolIntegration.md | 6 + skills/ForgeSkill/CreateWorkflow.md | 54 + skills/ForgeSkill/DescriptionOptimization.md | 54 + skills/ForgeSkill/DynamicContextInjection.md | 48 + skills/ForgeSkill/EvalLoop.md | 119 ++ skills/ForgeSkill/MultiProviderRouting.md | 16 + skills/ForgeSkill/PlatformAgnostic.md | 7 + skills/ForgeSkill/SKILL.md | 65 + skills/ForgeSkill/SkillInstallation.md | 39 + skills/ForgeSkill/SkillStructure.md | 97 ++ skills/ForgeSkill/UserConfigSchema.md | 52 + skills/ForgeSkill/ValidateWorkflow.md | 39 + .../agents/.provenance/analyzer.md.yaml | 32 + .../agents/.provenance/comparator.md.yaml | 32 + .../agents/.provenance/grader.md.yaml | 32 + skills/ForgeSkill/agents/analyzer.md | 274 ++++ skills/ForgeSkill/agents/comparator.md | 202 +++ skills/ForgeSkill/agents/grader.md | 223 +++ .../assets/.provenance/eval_review.html.yaml | 32 + skills/ForgeSkill/assets/eval_review.html | 146 ++ .../.provenance/generate_review.py.yaml | 32 + .../eval-viewer/.provenance/viewer.html.yaml | 32 + .../ForgeSkill/eval-viewer/generate_review.py | 471 ++++++ skills/ForgeSkill/eval-viewer/viewer.html | 1325 +++++++++++++++++ .../references/.provenance/schemas.md.yaml | 32 + skills/ForgeSkill/references/schemas.md | 430 ++++++ .../scripts/.provenance/__init__.py.yaml | 32 + .../.provenance/aggregate_benchmark.py.yaml | 32 + .../.provenance/generate_report.py.yaml | 32 + .../.provenance/improve_description.py.yaml | 32 + .../scripts/.provenance/package_skill.py.yaml | 32 + .../.provenance/quick_validate.py.yaml | 32 + .../scripts/.provenance/run_eval.py.yaml | 32 + .../scripts/.provenance/run_loop.py.yaml | 32 + .../scripts/.provenance/utils.py.yaml | 32 + skills/ForgeSkill/scripts/__init__.py | 0 .../ForgeSkill/scripts/aggregate_benchmark.py | 401 +++++ skills/ForgeSkill/scripts/generate_report.py | 326 ++++ .../ForgeSkill/scripts/improve_description.py | 247 +++ skills/ForgeSkill/scripts/package_skill.py | 136 ++ skills/ForgeSkill/scripts/quick_validate.py | 102 ++ skills/ForgeSkill/scripts/run_eval.py | 310 ++++ skills/ForgeSkill/scripts/run_loop.py | 328 ++++ skills/ForgeSkill/scripts/utils.py | 47 + 55 files changed, 6373 insertions(+), 9 deletions(-) create mode 100644 skills/BuildSkill/DynamicContextInjection.md create mode 100644 skills/BuildSkill/UserConfigSchema.md create mode 100644 skills/ForgeSkill/.provenance/DescriptionOptimization.md.yaml create mode 100644 skills/ForgeSkill/.provenance/EvalLoop.md.yaml create mode 100644 skills/ForgeSkill/.provenance/SKILL.yaml create mode 100644 skills/ForgeSkill/ClaudeSkill.md create mode 100644 skills/ForgeSkill/CliToolIntegration.md create mode 100644 skills/ForgeSkill/CreateWorkflow.md create mode 100644 skills/ForgeSkill/DescriptionOptimization.md create mode 100644 skills/ForgeSkill/DynamicContextInjection.md create mode 100644 skills/ForgeSkill/EvalLoop.md create mode 100644 skills/ForgeSkill/MultiProviderRouting.md create mode 100644 skills/ForgeSkill/PlatformAgnostic.md create mode 100644 skills/ForgeSkill/SKILL.md create mode 100644 skills/ForgeSkill/SkillInstallation.md create mode 100644 skills/ForgeSkill/SkillStructure.md create mode 100644 skills/ForgeSkill/UserConfigSchema.md create mode 100644 skills/ForgeSkill/ValidateWorkflow.md create mode 100644 skills/ForgeSkill/agents/.provenance/analyzer.md.yaml create mode 100644 skills/ForgeSkill/agents/.provenance/comparator.md.yaml create mode 100644 skills/ForgeSkill/agents/.provenance/grader.md.yaml create mode 100644 skills/ForgeSkill/agents/analyzer.md create mode 100644 skills/ForgeSkill/agents/comparator.md create mode 100644 skills/ForgeSkill/agents/grader.md create mode 100644 skills/ForgeSkill/assets/.provenance/eval_review.html.yaml create mode 100644 skills/ForgeSkill/assets/eval_review.html create mode 100644 skills/ForgeSkill/eval-viewer/.provenance/generate_review.py.yaml create mode 100644 skills/ForgeSkill/eval-viewer/.provenance/viewer.html.yaml create mode 100644 skills/ForgeSkill/eval-viewer/generate_review.py create mode 100644 skills/ForgeSkill/eval-viewer/viewer.html create mode 100644 skills/ForgeSkill/references/.provenance/schemas.md.yaml create mode 100644 skills/ForgeSkill/references/schemas.md create mode 100644 skills/ForgeSkill/scripts/.provenance/__init__.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/aggregate_benchmark.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/generate_report.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/improve_description.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/package_skill.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/quick_validate.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/run_eval.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/run_loop.py.yaml create mode 100644 skills/ForgeSkill/scripts/.provenance/utils.py.yaml create mode 100644 skills/ForgeSkill/scripts/__init__.py create mode 100755 skills/ForgeSkill/scripts/aggregate_benchmark.py create mode 100755 skills/ForgeSkill/scripts/generate_report.py create mode 100755 skills/ForgeSkill/scripts/improve_description.py create mode 100755 skills/ForgeSkill/scripts/package_skill.py create mode 100755 skills/ForgeSkill/scripts/quick_validate.py create mode 100755 skills/ForgeSkill/scripts/run_eval.py create mode 100755 skills/ForgeSkill/scripts/run_loop.py create mode 100644 skills/ForgeSkill/scripts/utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index be3c388..809bba9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). - `skills/GuardRails/DestructiveCommandGuard.md` companion (dcg block anatomy and workarounds), `skills/IncrementalEdits/` (revise files as reviewable hunks), `skills/SessionSearch/LocalCheckpoints.md`; updates to SettingsMaintenance, SystemCheck, Wtf, ProvenanceAudit, and the ProvenanceAuditor agent - `skills/InitProject/` — `project` / `atlas` shell functions that scaffold a workshop project spine (git + jj colocate + entire + forge hooks, private/public/assets flow, `.tlp` manifest, vault-mounted `.claude/`) with on-demand satellites (vault folder-note, `~/Data` domain-path mirror granted to agent sessions, private remote). Config in `~/.config/forge/project.yaml` with `FORGE_*` per-shell overrides; `project init` renders `CLAUDE.md.tmpl` into each project's brief - `.githooks/` gains `commit-msg`, `prepare-commit-msg`, `post-commit`, `post-rewrite`, and the pre-entire `pre-push` variant, completing the commit-gate set that `make install` wires via `core.hooksPath`; `BACKLOG.md` tracks module-level follow-ups +- `skills/ForgeSkill/` — BuildSkill's router and companions merged with the eval machinery adopted from [anthropics/skills](https://github.com/anthropics/skills) skill-creator (grader/comparator/analyzer agents, eval and description-optimization scripts, browser viewer), pinned per-file by SLSA provenance sidecars (Apache-2.0 under EUPL-1.2). New companions `EvalLoop.md` and `DescriptionOptimization.md` route the test-improve-iterate loop and trigger optimization. BuildSkill remains until the artifact-merge analysis lands - `skills/BashConventions/` + `BashPatterns.md` companion — Bash pitfalls (BSD vs GNU, `set -euo pipefail` traps, subprocess env) relocated from forge-dev - `skills/VersionControl/GitWorktrees.md` — companion covering parallel feature work via `git worktree`, with SLSA provenance (adapted from [davila7/claude-code-templates](https://github.com/davila7/claude-code-templates) MIT under EUPL-1.2) - `VersionControl` now serves as the canonical home for git conventions and platform-specific repo governance (absorbing the retired forge-dev `Git` skill). forge-dev's `GitHub` skill remains distinct — it covers CI, Actions, releases, and operational `gh` workflows diff --git a/skills/BuildSkill/ClaudeSkill.md b/skills/BuildSkill/ClaudeSkill.md index 934c6a6..3685785 100644 --- a/skills/BuildSkill/ClaudeSkill.md +++ b/skills/BuildSkill/ClaudeSkill.md @@ -79,12 +79,6 @@ Claude Code discovers skills through `plugin.json`: Every directory listed is scanned for `*/SKILL.md` files. Skills in later directories override earlier ones of the same name (last wins). -## Skill Load Hooks +## Dynamic context injection (`!`) -Skills can execute shell commands on load using `!` backtick lines at the end of SKILL.md: - -```markdown -!`some-command arg1 arg2` -``` - -These run when the skill is invoked. Use for loading additional context, checking prerequisites, or initializing state. +A Claude skill can open with live machine state via `` !`` `` lines in the SKILL.md body, the output runs and is substituted before Claude sees the content. This is a first-class authoring concern with its own guide: **[@DynamicContextInjection.md](DynamicContextInjection.md)**. Reach for it whenever a skill's job starts with orienting on current state. diff --git a/skills/BuildSkill/CreateWorkflow.md b/skills/BuildSkill/CreateWorkflow.md index bd495dc..3207e68 100644 --- a/skills/BuildSkill/CreateWorkflow.md +++ b/skills/BuildSkill/CreateWorkflow.md @@ -16,6 +16,7 @@ Follow the structure from [SkillStructure.md](SkillStructure.md). - [ ] Frontmatter has `name:` and `description:` with `USE WHEN` - [ ] Description is single-line, under 1024 characters - [ ] Body starts with `# SkillName` heading +- [ ] **For a Claude skill, decide what live state to inject.** Ask what current machine state would orient the model on load (branch, tool status, the names of things), and open the body with a `!` injection of it. Default to injecting unless there is a reason not to; see [DynamicContextInjection.md](DynamicContextInjection.md) - [ ] Clear step-by-step instructions (numbered steps for sequential operations) - [ ] If wrapping a CLI tool: usage examples, intent-to-flag mapping, output format (see [CliToolIntegration.md](CliToolIntegration.md)) - [ ] Constraints section with boundary conditions diff --git a/skills/BuildSkill/DynamicContextInjection.md b/skills/BuildSkill/DynamicContextInjection.md new file mode 100644 index 0000000..8baaf26 --- /dev/null +++ b/skills/BuildSkill/DynamicContextInjection.md @@ -0,0 +1,48 @@ +# Dynamic context injection (`!`) + +A Claude Code skill can open with **live machine state** instead of stale prose. `` !`` `` lines in the SKILL.md body run when the skill is invoked, and their output replaces the placeholder before Claude sees the content ("Inject dynamic context", a Claude Code extension to the Agent Skills standard). + +When authoring a Claude skill, treat this as a first-class step, not an afterthought: **ask what live state would orient the model on load, and inject it.** A skill that opens with the actual situation (the current branch and diff, a tool's auth status, the names of things that exist right now) beats one that only describes how to go find it. Default to injecting unless there is a reason not to. + +```markdown +--- +name: MySkill +description: ... +allowed-tools: Bash(git status *) Bash(git diff *) +--- + +# MySkill + +Current branch and changes: + +!`git status --short 2>/dev/null || echo "(not a git repo)"` +``` + +Each `` !`` `` runs once, before the rendered SKILL.md is sent to Claude; the output replaces the placeholder inline. Substitution is single-pass: injected output is not re-scanned for further placeholders. + +## Hard constraints (verified by running it, not just the docs) + +- **SKILL.md body only.** `!` executes only in the SKILL.md body, never in `@`-companion files (those load as plain text). Put every injected command in SKILL.md and keep companions as reference prose. +- **`allowed-tools` is required.** List the Bash scopes the injected commands need in the SKILL.md frontmatter, e.g. `allowed-tools: Bash(pass *) Bash(git *)`. It is a frontmatter field in SKILL.md itself (space-, comma-, or list-separated); there is no sidecar location. (Note: `forge assemble` currently strips it for the Claude provider, forge-cli#69, until fixed, a forge-deployed skill loses the field.) +- **No shell expansions.** The injection rejects any command containing `$(...)`, `${...}`, or backticks with a `Contains expansion` error. Keep injected commands simple and static; a guarded summary that needs command substitution will not run, inject the raw command output instead. +- **No built-in error handling.** A failing command does not degrade gracefully; it breaks or blanks the injection. Self-guard every command so a missing tool, logged-out session, or empty result cannot break skill load: + + ```markdown + !`pass-cli vault list 2>/dev/null || echo "(proton pass: not logged in)"` + ``` + +- **Claude Code only.** `!`, `@`, and `$ARGUMENTS` are Claude Code extensions, not part of the portable Agent Skills standard. In Codex / Gemini / opencode the `!` lines render as inert literal text. Use injection in skills you accept as Claude-first; it degrades to harmless text elsewhere. + +## Substitutions available alongside `!` + +`$ARGUMENTS`, `$ARGUMENTS[N]`, `$N`, `$name` (named args), and `${CLAUDE_SESSION_ID}` / `${CLAUDE_EFFORT}` / `${CLAUDE_SKILL_DIR}` are substituted in the SKILL.md body the same way. + +## What to inject, and what never to + +Inject **read-only, fast, structural state** that orients Claude: the current branch, a file listing, a tool's status, the names of things. Never inject: + +- **Secret values.** For a credential skill, inject the *map* (entry names, vault list, auth status), never the *territory*. `` !`pass ls` `` is fine (entry names); `` !`pass show ` `` is forbidden, it would dump a live secret into the transcript and context. +- **Slow or interactive commands.** Injection blocks skill load; a command that prompts, hangs, or takes seconds makes the skill feel broken. +- **Mutating commands.** Injection should observe, not change state. + +The litmus test: if the command's output appearing verbatim in the session transcript would be a problem, do not inject it. diff --git a/skills/BuildSkill/SKILL.md b/skills/BuildSkill/SKILL.md index c0fb386..8337c3c 100644 --- a/skills/BuildSkill/SKILL.md +++ b/skills/BuildSkill/SKILL.md @@ -20,10 +20,12 @@ Create and validate skills following forge conventions. Skills are markdown file | Topic | Companion | | ----------------------------------------------------------- | -------------------------------------------------------- | | SKILL.md structure, frontmatter, body layout, naming | [@SkillStructure.md](SkillStructure.md) | +| **Dynamic context injection (`!`): open a Claude skill with live state** | [@DynamicContextInjection.md](DynamicContextInjection.md) | | Multi-provider routing via `defaults.yaml` | [@MultiProviderRouting.md](MultiProviderRouting.md) | | Wrapping a CLI tool in a skill | [@CliToolIntegration.md](CliToolIntegration.md) | | Platform-agnostic writing — no placeholders or `/` prefix | [@PlatformAgnostic.md](PlatformAgnostic.md) | | User-config schema for AI-first artifacts (autoMode mirror) | [@UserConfigSchema.md](UserConfigSchema.md) | +| Claude-only features: `@` refs, skill discovery, `allowed-tools` | [@ClaudeSkill.md](ClaudeSkill.md) | | When to author a per-skill INSTALL.md | [@SkillInstallation.md](SkillInstallation.md) | ## Red Flags @@ -36,6 +38,9 @@ Create and validate skills following forge conventions. Skills are markdown file | "Leave a stub section as a placeholder" | Skill bodies are plain prose. Delete empty sections, don't scaffold them. | | "Inline every example in the SKILL.md" | SKILL.md should stay slim. Move static reference material to companion files. | | "Skill directory can have any name" | Directory name must match the `name:` frontmatter field exactly. | +| "Put a `!` injection in a companion file" | `!` runs only in the SKILL.md body; in a companion it renders as literal text. See [@ClaudeSkill.md](ClaudeSkill.md). | +| "Inject a secret value with `!` (e.g. `pass show`)" | Injection lands in the transcript. Inject structure/status (names, vault list), never secret values. | +| "It ran in my sandbox, so the probe is done" | Your sandbox has tooling (uv, personal paths, aliases) the target machine lacks — resolve interpreters at preflight and assume a clean environment. | ## Constraints diff --git a/skills/BuildSkill/UserConfigSchema.md b/skills/BuildSkill/UserConfigSchema.md new file mode 100644 index 0000000..d827e64 --- /dev/null +++ b/skills/BuildSkill/UserConfigSchema.md @@ -0,0 +1,52 @@ +# User Config Schema (autoMode mirror) + +When an artifact needs per-user runtime data, follow the [UserConfig](../../rules/UserConfig.md) rule — one file per artifact at `~/.config/forge/.{ext}`. When the config is intended to be read by an AI in the loop (skill, agent, or hook with model access), shape it after [Claude Code's `autoMode`][AM]: natural-language entries with a four-tier precedence model and a `$defaults` splice token. + +## Why mirror autoMode + +The pattern is already familiar to anyone configuring Claude Code. Entries are prose — descriptions a human (or AI) would naturally write — not regex or tool-pattern grammars. The `$defaults` token gives a splice-or-replace toggle for built-in defaults shipped with the artifact source. Users extend the built-ins by adding entries; they take full ownership by omitting `"$defaults"`. + +## Shape + +Top-level keys are tiers with strict precedence: `hard_deny` > `soft_deny` > `allow` > `environment`. Each value is an array of prose strings. + +```yaml +environment: + - "$defaults" + - "" + +allow: + - "$defaults" + - "" + +soft_deny: + - "$defaults" + - "" + +hard_deny: + - "$defaults" + - "" +``` + +## Tier semantics + +- `hard_deny` blocks unconditionally. No `allow` exception or user intent applies. +- `soft_deny` blocks next. `allow` exceptions and explicit user intent can override. +- `allow` overrides matching `soft_deny` entries. +- `environment` provides context: trusted infrastructure, identities, repo ownership. + +Setting any tier without `"$defaults"` replaces the entire built-in list for that tier. Default entries are spliced at the position of the token, so custom entries can go before or after them. + +## When NOT to use this pattern + +Deterministic consumers (shell scripts, pre-commit hooks, CI gates without model access) can't interpret prose. Ship a sibling artifact for those — same `~/.config/forge/` directory, different filename — with a flat regex list or other machine-readable structure. Don't try to mix prose and regex in one file; the consumer types diverge. + +Example pairing: `~/.config/forge/forensic.yaml` (prose, read by ForensicAgent) and `~/.config/forge/danger-strings` (regex, read by the pre-commit hook). + +## Discovery and inspection + +The artifact source documents its built-in `$defaults` inline (in `SKILL.md` body or the agent body) so users can read what they inherit before extending. A `forge config` subcommand printing the effective merged config is recommended but not required. + +## Reference + +[AM]: https://code.claude.com/docs/en/auto-mode-config "Claude Code: Configure auto mode" diff --git a/skills/BuildSkill/ValidateWorkflow.md b/skills/BuildSkill/ValidateWorkflow.md index 9c209f2..da6d761 100644 --- a/skills/BuildSkill/ValidateWorkflow.md +++ b/skills/BuildSkill/ValidateWorkflow.md @@ -25,7 +25,14 @@ Read the SKILL.md file. - [ ] Intent-to-flag mapping table (if tool has flags) - [ ] Output format described -## Step 5: Report +## Step 5: Check content quality + +- [ ] Inputs and outputs stated explicitly (what the skill consumes, what artifact it produces) +- [ ] Guardrails pause rather than auto-escalate — no silent jump from read-only analysis to executing targets or installing tooling +- [ ] No authoring-environment fingerprints: stdlib-only probes use a preflight-resolved `python3`, not `uv run python`; no personal paths, package managers, or aliases the target machine may lack +- [ ] Deliverable is an artifact (table, file, verdict schema), not prose + +## Step 6: Report **COMPLIANT** — all checks pass. diff --git a/skills/ForgeSkill/.provenance/DescriptionOptimization.md.yaml b/skills/ForgeSkill/.provenance/DescriptionOptimization.md.yaml new file mode 100644 index 0000000..142f919 --- /dev/null +++ b/skills/ForgeSkill/.provenance/DescriptionOptimization.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/DescriptionOptimization.md + digest: + sha256: f6bd7a7d74423620cf792c87d3add8055949f13ffd6ea2d83d9fd58e0df48e3d + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + digest: + sha256: dcd4803e61e913e6fc27294184cd3a71f09f5e924ff20c8a9a20173e7b3c2bcf + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/.provenance/EvalLoop.md.yaml b/skills/ForgeSkill/.provenance/EvalLoop.md.yaml new file mode 100644 index 0000000..c1b3a65 --- /dev/null +++ b/skills/ForgeSkill/.provenance/EvalLoop.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/EvalLoop.md + digest: + sha256: 135a0897e42e517d5d32959a3e7083e2c274e49393fe0d41fcea717334d295ae + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + digest: + sha256: dcd4803e61e913e6fc27294184cd3a71f09f5e924ff20c8a9a20173e7b3c2bcf + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/.provenance/SKILL.yaml b/skills/ForgeSkill/.provenance/SKILL.yaml new file mode 100644 index 0000000..65908da --- /dev/null +++ b/skills/ForgeSkill/.provenance/SKILL.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/SKILL.md + digest: + sha256: 92201e60243b40081f9e4853fbac262e3a04321e372626a6b145c671ed42792b + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/SKILL.md + digest: + sha256: dcd4803e61e913e6fc27294184cd3a71f09f5e924ff20c8a9a20173e7b3c2bcf + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/ClaudeSkill.md b/skills/ForgeSkill/ClaudeSkill.md new file mode 100644 index 0000000..3685785 --- /dev/null +++ b/skills/ForgeSkill/ClaudeSkill.md @@ -0,0 +1,84 @@ +# Claude Code Skill Conventions + +Claude Code-specific features for skills. This supplements the generic skill conventions in SKILL.md. + +## @ File References + +The `@` symbol in SKILL.md (and CLAUDE.md) files includes another file's content into the AI's context. This is the primary mechanism for composing instructions from multiple files. + +```markdown +--- +name: MySkill +description: Does something useful... +--- + +@conventions.md +@tools-reference.md + +# MySkill +... +``` + +Both referenced files get expanded inline before the AI sees the skill. Resolution is always relative to the SKILL.md file's directory. + +### Resolution Rules + +| SKILL.md location | `@companion.md` resolves to | +|---------------------------------------|-----------------------------------------| +| `skills/MySkill/SKILL.md` | `skills/MySkill/companion.md` | +| `~/.claude/skills/MySkill/SKILL.md` | `~/.claude/skills/MySkill/companion.md` | + +### Directory References + +`@src/components/` shows the file listing of that directory -- useful for giving the AI a structural overview without reading every file. + +### When to Use @ References + +- **Any content the skill needs** -- conventions, reference tables, shared config, tool catalogs +- **Content shared across multiple skills** in the same module +- **Provider-specific content** that only applies to Claude Code +- **Large reference material** that would bloat the main SKILL.md + +Keep inline when the section is short (<20 lines) and tightly coupled to the skill body. + +## CLAUDE.md @ References + +CLAUDE.md files (global or project-level) use the same `@` mechanism: + +| Level | File | Applies to | +|---------------|-------------------------------|--------------------------------| +| Global | `~/.claude/CLAUDE.md` | Every project, every session | +| Project | `/CLAUDE.md` | All sessions in this project | +| Project-local | `/.claude/CLAUDE.md` | All sessions (gitignored) | + +### When to Extract from CLAUDE.md + +Extract a section into a separate `@`-referenced file when: +- **It's domain-specific** -- tool catalogs, API references, module docs +- **It's reusable across projects** -- coding conventions, commit rules +- **It exceeds ~50 lines** -- large blocks dilute surrounding instructions +- **It changes independently** -- tool docs update on different cadence than project rules + +### Naming Conventions + +| Pattern | Use for | +|-----------------------|--------------------------------------| +| `RTK.md`, `TOOLS.md` | Uppercase -- tool/system references | +| `conventions.md` | Lowercase -- style and practice docs | +| `.md` | Module-specific reference | + +## Skill Discovery + +Claude Code discovers skills through `plugin.json`: + +```json +{ + "skills": ["./skills"] +} +``` + +Every directory listed is scanned for `*/SKILL.md` files. Skills in later directories override earlier ones of the same name (last wins). + +## Dynamic context injection (`!`) + +A Claude skill can open with live machine state via `` !`` `` lines in the SKILL.md body, the output runs and is substituted before Claude sees the content. This is a first-class authoring concern with its own guide: **[@DynamicContextInjection.md](DynamicContextInjection.md)**. Reach for it whenever a skill's job starts with orienting on current state. diff --git a/skills/ForgeSkill/CliToolIntegration.md b/skills/ForgeSkill/CliToolIntegration.md new file mode 100644 index 0000000..4dc0033 --- /dev/null +++ b/skills/ForgeSkill/CliToolIntegration.md @@ -0,0 +1,6 @@ +When a skill wraps a CLI tool (Rust binary, shell script), include: + +1. **Tool location** — where the binary lives +2. **Usage examples** — concrete `bash` blocks showing invocation +3. **Intent-to-flag mapping** — table translating natural language to CLI flags +4. **Output format** — what the tool returns (JSONL, plain text, etc.) diff --git a/skills/ForgeSkill/CreateWorkflow.md b/skills/ForgeSkill/CreateWorkflow.md new file mode 100644 index 0000000..3207e68 --- /dev/null +++ b/skills/ForgeSkill/CreateWorkflow.md @@ -0,0 +1,54 @@ +## Step 1: Understand the request + +Determine: +1. What does this skill do? +2. What should trigger it? (intent phrases for `USE WHEN`) +3. Does it wrap a CLI tool, or is it purely procedural? +4. Which module should it live in? + +If the user hasn't specified, ask using AskUserQuestion. + +## Step 2: Write the SKILL.md + +Follow the structure from [SkillStructure.md](SkillStructure.md). + +**Checklist while writing:** +- [ ] Frontmatter has `name:` and `description:` with `USE WHEN` +- [ ] Description is single-line, under 1024 characters +- [ ] Body starts with `# SkillName` heading +- [ ] **For a Claude skill, decide what live state to inject.** Ask what current machine state would orient the model on load (branch, tool status, the names of things), and open the body with a `!` injection of it. Default to injecting unless there is a reason not to; see [DynamicContextInjection.md](DynamicContextInjection.md) +- [ ] Clear step-by-step instructions (numbered steps for sequential operations) +- [ ] If wrapping a CLI tool: usage examples, intent-to-flag mapping, output format (see [CliToolIntegration.md](CliToolIntegration.md)) +- [ ] Constraints section with boundary conditions +- [ ] No unnecessary complexity — minimum needed for the task +- [ ] Skill listed in module's `defaults.yaml` under each target provider (see [MultiProviderRouting.md](MultiProviderRouting.md)) +- [ ] If locale-specific (e.g., Czech tax): description mixes English action phrases ("record transaction", "validate balance") with backticked native terms (`účetní deník`, `bilance`). Avoid diacritic-stripped czenglish (`podvojne ucetnictvi`) — matches neither natural English nor natural Czech queries + +## Step 3: Create the skill directory and file + +```sh +mkdir -p skills/SkillName +``` + +Write the SKILL.md using the Write tool. + +## Step 4: Register + +For Claude Code: ensure the skill's parent directory is listed in `plugin.json` under `skills`. + +For other providers: run `make install` from the module's Makefile. + +## Step 5: Verify + +1. Test invocation: does the description trigger correctly? +2. Review: does the procedure work end-to-end? +3. Dispatch the **SkillReviewer** agent on the new `SKILL.md` (and any companion files). It catches trigger weaknesses, czenglish descriptions, broken cross-references, body bloat, and convention drift that self-review misses. Apply confirmed fixes before declaring done. + +## Step 6: Pressure test + +Apply TDD to the skill itself — write a scenario where the skill should apply but might be rationalized away, then verify it holds. + +1. **Write a pressure scenario** — describe a situation where someone would think "this skill doesn't apply here" but it actually does. Example for a debugging skill: "The fix seems obvious, I'll just change it." +2. **Test the trigger** — does the description match this scenario? Would the AI load this skill? +3. **Test the procedure** — does following the skill's steps produce the right outcome in this scenario? +4. **Tighten** — if the skill would be bypassed, improve the description's USE WHEN triggers or add entries to the Red Flags table. diff --git a/skills/ForgeSkill/DescriptionOptimization.md b/skills/ForgeSkill/DescriptionOptimization.md new file mode 100644 index 0000000..ba0b961 --- /dev/null +++ b/skills/ForgeSkill/DescriptionOptimization.md @@ -0,0 +1,54 @@ +# Description Optimization + +The `description` field is the primary triggering mechanism: Claude sees only name + description when deciding whether to consult a skill. Claude tends to undertrigger, so descriptions should be a little pushy, naming both what the skill does and the concrete contexts that should invoke it, even when the user doesn't use the skill's own vocabulary. + +Triggering mechanics worth knowing: Claude only consults skills for tasks it can't easily handle directly. Simple one-step queries ("read this PDF") may not trigger a skill even with a perfect description match; complex, multi-step, or specialized queries trigger reliably. Eval queries must be substantive enough that consulting a skill would actually help. + +## Step 1: Generate trigger eval queries + +Create 20 queries, a mix of should-trigger and should-not-trigger, saved as JSON: + +```json +[ + {"query": "the user prompt", "should_trigger": true}, + {"query": "another prompt", "should_trigger": false} +] +``` + +Queries must be realistic: concrete and specific, with file paths, personal context, column names, company names, typos, casual speech, mixed lengths. Favor edge cases over clear-cut ones; the user signs off before the run. + +- **Should-trigger (8-10)**: different phrasings of the same intent, formal and casual; cases where the user never names the skill or file type but clearly needs it; uncommon use cases; cases where this skill competes with another but should win. +- **Should-not-trigger (8-10)**: near-misses that share keywords or concepts but need something different: adjacent domains, ambiguous phrasing a naive keyword match would catch, contexts where another tool is more appropriate. Obviously irrelevant negatives test nothing. + +## Step 2: Review with the user + +Render the eval set for review using the bundled template: + +1. Read `assets/eval_review.html` +2. Replace `__EVAL_DATA_PLACEHOLDER__` (the JSON array, unquoted), `__SKILL_NAME_PLACEHOLDER__`, `__SKILL_DESCRIPTION_PLACEHOLDER__` +3. Write to a temp file and open it in the browser +4. The user edits queries, toggles should-trigger, and clicks "Export Eval Set" +5. The export lands in `~/Downloads/eval_set.json`; take the most recent if there are duplicates + +Bad eval queries produce bad descriptions; this review step is load-bearing. + +## Step 3: Run the optimization loop + +Warn the user it takes a while, then run in the background from this skill's directory: + +```bash +python -m scripts.run_loop \ + --eval-set \ + --skill-path \ + --model \ + --max-iterations 5 \ + --verbose +``` + +Use the model ID powering the current session so the triggering test matches what the user experiences. The loop splits 60% train / 40% held-out test, measures the current description (each query 3 times), proposes improvements from the failures, and re-evaluates up to 5 iterations. It reports per-iteration results and returns JSON with `best_description`, selected by test score to avoid overfitting. Tail the output periodically to report progress. + +Requires the `claude` CLI (`claude -p`); Claude Code only. + +## Step 4: Apply the result + +Update the skill's frontmatter with `best_description`, show the user before/after, and report the scores. For forge skills, keep the `USE WHEN` clause convention when merging the optimized text. diff --git a/skills/ForgeSkill/DynamicContextInjection.md b/skills/ForgeSkill/DynamicContextInjection.md new file mode 100644 index 0000000..8baaf26 --- /dev/null +++ b/skills/ForgeSkill/DynamicContextInjection.md @@ -0,0 +1,48 @@ +# Dynamic context injection (`!`) + +A Claude Code skill can open with **live machine state** instead of stale prose. `` !`` `` lines in the SKILL.md body run when the skill is invoked, and their output replaces the placeholder before Claude sees the content ("Inject dynamic context", a Claude Code extension to the Agent Skills standard). + +When authoring a Claude skill, treat this as a first-class step, not an afterthought: **ask what live state would orient the model on load, and inject it.** A skill that opens with the actual situation (the current branch and diff, a tool's auth status, the names of things that exist right now) beats one that only describes how to go find it. Default to injecting unless there is a reason not to. + +```markdown +--- +name: MySkill +description: ... +allowed-tools: Bash(git status *) Bash(git diff *) +--- + +# MySkill + +Current branch and changes: + +!`git status --short 2>/dev/null || echo "(not a git repo)"` +``` + +Each `` !`` `` runs once, before the rendered SKILL.md is sent to Claude; the output replaces the placeholder inline. Substitution is single-pass: injected output is not re-scanned for further placeholders. + +## Hard constraints (verified by running it, not just the docs) + +- **SKILL.md body only.** `!` executes only in the SKILL.md body, never in `@`-companion files (those load as plain text). Put every injected command in SKILL.md and keep companions as reference prose. +- **`allowed-tools` is required.** List the Bash scopes the injected commands need in the SKILL.md frontmatter, e.g. `allowed-tools: Bash(pass *) Bash(git *)`. It is a frontmatter field in SKILL.md itself (space-, comma-, or list-separated); there is no sidecar location. (Note: `forge assemble` currently strips it for the Claude provider, forge-cli#69, until fixed, a forge-deployed skill loses the field.) +- **No shell expansions.** The injection rejects any command containing `$(...)`, `${...}`, or backticks with a `Contains expansion` error. Keep injected commands simple and static; a guarded summary that needs command substitution will not run, inject the raw command output instead. +- **No built-in error handling.** A failing command does not degrade gracefully; it breaks or blanks the injection. Self-guard every command so a missing tool, logged-out session, or empty result cannot break skill load: + + ```markdown + !`pass-cli vault list 2>/dev/null || echo "(proton pass: not logged in)"` + ``` + +- **Claude Code only.** `!`, `@`, and `$ARGUMENTS` are Claude Code extensions, not part of the portable Agent Skills standard. In Codex / Gemini / opencode the `!` lines render as inert literal text. Use injection in skills you accept as Claude-first; it degrades to harmless text elsewhere. + +## Substitutions available alongside `!` + +`$ARGUMENTS`, `$ARGUMENTS[N]`, `$N`, `$name` (named args), and `${CLAUDE_SESSION_ID}` / `${CLAUDE_EFFORT}` / `${CLAUDE_SKILL_DIR}` are substituted in the SKILL.md body the same way. + +## What to inject, and what never to + +Inject **read-only, fast, structural state** that orients Claude: the current branch, a file listing, a tool's status, the names of things. Never inject: + +- **Secret values.** For a credential skill, inject the *map* (entry names, vault list, auth status), never the *territory*. `` !`pass ls` `` is fine (entry names); `` !`pass show ` `` is forbidden, it would dump a live secret into the transcript and context. +- **Slow or interactive commands.** Injection blocks skill load; a command that prompts, hangs, or takes seconds makes the skill feel broken. +- **Mutating commands.** Injection should observe, not change state. + +The litmus test: if the command's output appearing verbatim in the session transcript would be a problem, do not inject it. diff --git a/skills/ForgeSkill/EvalLoop.md b/skills/ForgeSkill/EvalLoop.md new file mode 100644 index 0000000..3d83c98 --- /dev/null +++ b/skills/ForgeSkill/EvalLoop.md @@ -0,0 +1,119 @@ +# Eval Loop + +Run test cases against a skill, review results with the user, improve, repeat. This is one continuous sequence; don't stop partway through. + +Results live in `-workspace/` as a sibling of the skill directory, organized by iteration (`iteration-1/`, `iteration-2/`), with one directory per test case named `eval--`. The aggregator discovers only `eval-*` directories, and each config directory must contain `run-/` subdirectories holding `outputs/`, `grading.json`, and `timing.json`; files placed directly in the config directory are ignored. Create directories as you go, not upfront. + +## Write test cases + +Draft 2-3 realistic test prompts, the kind a real user would type. Confirm them with the user before running. Save to `evals/evals.json` (prompts only; assertions come later): + +```json +{ + "skill_name": "ExampleSkill", + "evals": [ + { + "id": 1, + "prompt": "User's task prompt", + "expected_output": "Description of expected result", + "files": [] + } + ] +} +``` + +Full schema, including the `assertions` field: [references/schemas.md](references/schemas.md). + +Skills with objectively verifiable outputs (file transforms, data extraction, code generation, fixed workflows) benefit from assertions. Subjective skills (writing style, design) are better judged qualitatively; don't force assertions onto human-judgment outputs. + +## Step 1: Spawn all runs in the same turn + +For each test case, spawn two subagents in the same turn, one with the skill and one baseline, so everything finishes together. + +With-skill run prompt: + +``` +Execute this task: +- Skill path: +- Task: +- Input files: +- Save outputs to: /iteration-/eval--/with_skill/run-1/outputs/ +- Outputs to save: +``` + +Baseline run, same prompt: + +- New skill: no skill at all; save to `without_skill/outputs/`. +- Improving an existing skill: snapshot first (`cp -r /skill-snapshot/`), point the baseline at the snapshot, save to `old_skill/outputs/`. + +Write an `eval_metadata.json` per test case (assertions may be empty for now): + +```json +{ + "eval_id": 0, + "eval_name": "descriptive-name-here", + "prompt": "The user's task prompt", + "assertions": [] +} +``` + +## Step 2: Draft assertions while runs are in progress + +Good assertions are objectively verifiable and have descriptive names that read clearly in the benchmark viewer. Update `eval_metadata.json` and `evals/evals.json` once drafted, and explain to the user what they'll see in the viewer. + +## Step 3: Capture timing as each run completes + +Each subagent completion notification carries `total_tokens` and `duration_ms`. Save immediately to `timing.json` in the run directory; the notification is the only place this data exists. Process notifications as they arrive, never batch. + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3 +} +``` + +## Step 4: Grade, aggregate, launch the viewer + +1. **Grade each run.** Spawn a grader subagent (instructions: [agents/grader.md](agents/grader.md)) evaluating each assertion against the outputs; save `grading.json` in each `run-/` directory. The expectations array MUST use the fields `text`, `passed`, `evidence`, and the file MUST carry a `summary` object (`passed`, `failed`, `total`, `pass_rate`); the aggregator and viewer depend on these exact names. Check programmatically verifiable assertions with a script, not by eyeballing. +2. **Aggregate.** From this skill's directory: `python -m scripts.aggregate_benchmark /iteration-N --skill-name `. Produces `benchmark.json` and `benchmark.md` (pass rate, time, tokens, mean, stddev, delta). Put each with_skill version before its baseline counterpart. +3. **Analyst pass.** Read the benchmark data for patterns the aggregates hide: non-discriminating assertions, high-variance evals, time/token tradeoffs. See the "Analyzing Benchmark Results" section of [agents/analyzer.md](agents/analyzer.md). +4. **Launch the viewer** before doing your own revision pass; get outputs in front of the human first: + + ```bash + nohup python /eval-viewer/generate_review.py \ + /iteration-N \ + --skill-name "" \ + --benchmark /iteration-N/benchmark.json \ + > /dev/null 2>&1 & + VIEWER_PID=$! + ``` + + For iteration 2+, add `--previous-workspace /iteration-`. Headless or no-display environments: pass `--static ` to write standalone HTML instead of serving; feedback then downloads as `feedback.json`, which you copy into the workspace. Always use `generate_review.py`; never hand-write viewer HTML. + +5. **Tell the user** the viewer is open: the Outputs tab collects per-case feedback, the Benchmark tab shows the quantitative comparison. + +## Step 5: Read the feedback + +When the user says they're done, read `feedback.json`. Empty feedback on a case means it was fine; focus on cases with specific complaints. Kill the viewer: `kill $VIEWER_PID 2>/dev/null`. + +## Improve the skill + +- **Generalize from the feedback.** The skill must work across many prompts, not just the test set. Avoid fiddly overfit patches and rigid MUSTs; if an issue is stubborn, try a different framing or working pattern instead. +- **Keep the prompt lean.** Read the transcripts, not just outputs. If the skill makes the model do unproductive work, cut the part causing it and re-measure. +- **Explain the why.** Reframe all-caps imperatives as reasoning the model can apply; that generalizes better than rote instruction. +- **Bundle repeated work.** If every test run independently wrote the same helper script, move that script into `scripts/` and reference it from the skill. + +Then rerun everything into `iteration-/` (baselines included), launch the viewer with `--previous-workspace`, and read the new feedback. Stop when the user is happy, the feedback is all empty, or progress stalls. + +## Blind comparison (optional) + +For a rigorous "is the new version actually better?" answer: give two outputs to an independent agent without revealing which is which, let it judge, then analyze why the winner won. Instructions: [agents/comparator.md](agents/comparator.md) and [agents/analyzer.md](agents/analyzer.md). Requires subagents; the human review loop is usually sufficient. + +## Harness adaptations + +- **No subagents** (e.g. claude.ai): run test cases yourself, one at a time, skill instructions in context. Skip baselines and quantitative benchmarking; present outputs inline and gather feedback conversationally. +- **No browser or display**: skip the viewer server, use `--static`, present file outputs by path. +- **Description optimization** requires `claude -p` (Claude Code only): see [DescriptionOptimization.md](DescriptionOptimization.md). +- **Packaging for claude.ai upload**: `python -m scripts.package_skill ` produces a `.skill` file; works anywhere with Python. Not needed for forge modules, which deploy via `forge install`. +- **Updating an existing skill**: preserve the original `name` and directory name unchanged; copy read-only installs to a writable location before editing. diff --git a/skills/ForgeSkill/MultiProviderRouting.md b/skills/ForgeSkill/MultiProviderRouting.md new file mode 100644 index 0000000..204de73 --- /dev/null +++ b/skills/ForgeSkill/MultiProviderRouting.md @@ -0,0 +1,16 @@ +Provider routing is controlled by the module's `defaults.yaml`, not by individual SKILL.yaml files. `forge install` reads provider-keyed allowlists to decide which skills deploy where: + +```yaml +# defaults.yaml +skills: + claude: + SkillName: + gemini: + SkillName: + codex: + SkillName: + opencode: + SkillName: +``` + +Skills listed under a provider key are installed for that provider. Skills omitted from a provider's list are skipped. This allows Claude-only skills (e.g., those using TeamCreate or agent teams) to be excluded from Gemini/Codex/OpenCode without per-skill configuration. diff --git a/skills/ForgeSkill/PlatformAgnostic.md b/skills/ForgeSkill/PlatformAgnostic.md new file mode 100644 index 0000000..376ca87 --- /dev/null +++ b/skills/ForgeSkill/PlatformAgnostic.md @@ -0,0 +1,7 @@ +Skills and agent definitions are instructions, not templates. Write them as plain prose an AI can follow — no scaffolding artifacts. + +Forbidden: `{{placeholders}}`, `[REPLACE THIS]`, ``, stub headings with no content, empty tables generated by a scaffold. If a section has nothing to say, delete it. + +Keep YAML frontmatter and `@` file includes — those are structural. But never reference skills with `/` prefix inside definitions. Use the skill name directly (e.g. "MemoryCapture", not "/MemoryCapture"). The slash is user-facing invocation syntax, not an internal identifier. + +A skill or agent that deploys to every provider must not assume one provider. Don't hardcode a single tool's agent identifier, CLI binary, or command syntax (`--agent claude-code`, `claude --resume`) when the artifact also runs under Codex, Gemini, or OpenCode. Parameterize the agent (`--agent `, listing the supported tools) and name actions neutrally ("the agent's resume command", not `claude --resume`). Documenting a tool's own factual default (a CLI flag that defaults to `claude-code`) is fine; that is the tool's behavior, not your assumption. diff --git a/skills/ForgeSkill/SKILL.md b/skills/ForgeSkill/SKILL.md new file mode 100644 index 0000000..c9ef3d7 --- /dev/null +++ b/skills/ForgeSkill/SKILL.md @@ -0,0 +1,65 @@ +--- +name: ForgeSkill +version: 0.1.0 +description: "Create, validate, evaluate, and iterate skills for forge modules. USE WHEN create skill, new skill, write skill, validate skill, check skill, skill structure, skill conventions, test a skill, run skill evals, benchmark a skill, skill not triggering, optimize skill description. Not for adopting community skills (AdoptArtifact) or shipping artifacts downstream (PublishArtifact)." +upstream: https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md +--- + +# ForgeSkill + +Create, validate, evaluate, and iterate skills following forge conventions. Skills are markdown files (`SKILL.md`) with YAML frontmatter that teach AI coding tools new capabilities. Load only the companion relevant to the current task. + +The evaluation machinery (grader/comparator/analyzer prompts, eval scripts, browser viewer) lives in `agents/`, `scripts/`, `references/`, `eval-viewer/`, and `assets/`. Scripts run with `python -m scripts.` from this skill's directory (`${CLAUDE_SKILL_DIR}` when deployed). The files under `agents/` are worker prompt templates this skill feeds to generic subagents during the eval loop, not standalone agent definitions; harness-discoverable agents belong in the module-level `agents/` directory. + +## Workflow Routing + +| Workflow | Trigger | Companion | +| ------------------ | ------------------------------------------------------------ | --------------------------------------------------------- | +| Create | "create a skill", "new skill", "write a skill" | [@CreateWorkflow.md](CreateWorkflow.md) | +| Validate | "validate skill", "check skill structure" | [@ValidateWorkflow.md](ValidateWorkflow.md) | +| Evaluate | "test this skill", "run skill evals", "benchmark the skill" | [@EvalLoop.md](EvalLoop.md) | +| Optimize triggering | "skill doesn't trigger", "improve the skill description" | [@DescriptionOptimization.md](DescriptionOptimization.md) | + +## Topics + +| Topic | Companion | +| ----------------------------------------------------------- | -------------------------------------------------------- | +| SKILL.md structure, frontmatter, body layout, naming | [@SkillStructure.md](SkillStructure.md) | +| **Dynamic context injection (`!`): open a Claude skill with live state** | [@DynamicContextInjection.md](DynamicContextInjection.md) | +| Multi-provider routing via `defaults.yaml` | [@MultiProviderRouting.md](MultiProviderRouting.md) | +| Wrapping a CLI tool in a skill | [@CliToolIntegration.md](CliToolIntegration.md) | +| Platform-agnostic writing — no placeholders or `/` prefix | [@PlatformAgnostic.md](PlatformAgnostic.md) | +| User-config schema for AI-first artifacts (autoMode mirror) | [@UserConfigSchema.md](UserConfigSchema.md) | +| Claude-only features: `@` refs, skill discovery, `allowed-tools` | [@ClaudeSkill.md](ClaudeSkill.md) | +| When to author a per-skill INSTALL.md | [@SkillInstallation.md](SkillInstallation.md) | +| Eval JSON structures (evals.json, grading.json, benchmark.json) | [@references/schemas.md](references/schemas.md) | + +## Red Flags + +| Thought | Reality | +| -------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| "Put argument-hint in SKILL.md frontmatter" | Obsidian Linter reformats frontmatter. Provider-specific fields go in SKILL.yaml. | +| "Use `/SkillName` inside a skill body" | Slashes are user-facing invocation syntax, not internal references. | +| "Skip the `USE WHEN` clause" | Claude uses it to route. Missing trigger = skill never fires. | +| "Leave a stub section as a placeholder" | Skill bodies are plain prose. Delete empty sections, don't scaffold them. | +| "Inline every example in the SKILL.md" | SKILL.md should stay slim. Move static reference material to companion files. | +| "Skill directory can have any name" | Directory name must match the `name:` frontmatter field exactly. | +| "Put a `!` injection in a companion file" | `!` runs only in the SKILL.md body; in a companion it renders as literal text. See [@ClaudeSkill.md](ClaudeSkill.md). | +| "Inject a secret value with `!` (e.g. `pass show`)" | Injection lands in the transcript. Inject structure/status (names, vault list), never secret values. | +| "It ran in my sandbox, so the probe is done" | Your sandbox has tooling (uv, personal paths, aliases) the target machine lacks — resolve interpreters at preflight and assume a clean environment. | +| "I'll write custom HTML to show eval results" | `eval-viewer/generate_review.py` already renders outputs and benchmarks. Use it. | +| "The skill passed its three evals, ship it" | A handful of examples overfits. Generalize the fix; don't patch the skill to the test set. | + +## Constraints + +- Every skill MUST have `name:` and `description:` in frontmatter +- Description MUST include `USE WHEN` trigger phrases +- PascalCase for multi-word skill names, natural case for single words +- Skill directory name must match the `name:` field +- Prefer one SKILL.md per skill — extract reference material into companion files when body exceeds ~150 lines or contains dense static data +- Eval artifacts land in `-workspace/` as a sibling of the skill directory, never inside the skill + +## Sources + +- +- diff --git a/skills/ForgeSkill/SkillInstallation.md b/skills/ForgeSkill/SkillInstallation.md new file mode 100644 index 0000000..b5e2ce7 --- /dev/null +++ b/skills/ForgeSkill/SkillInstallation.md @@ -0,0 +1,39 @@ +# Skill Installation (INSTALL.md) + +## When to include + +A per-skill INSTALL.md is required when the skill needs user actions the plugin system cannot automate: + +- Creating a user-config file (`~/.config/forge/.{ext}`) +- Authenticating with an external service (API tokens, OAuth flows) +- Installing a tool unique to that skill (not a module-wide shared prerequisite) + +Skills that work after `make install` or plugin add need no INSTALL.md. Hooks auto-discovered via `hooks/hooks.json` need no INSTALL.md. + +## What does NOT belong in per-skill INSTALL.md + +- **Shared prerequisites** (gitleaks, yq, jq) belong in the module-level INSTALL.md at the repo root +- **Hook wiring** belongs in `hooks/hooks.json` (auto-discovered by the plugin system) +- **Behavioral guidance** belongs in SKILL.md + +## Shape + +Same Mintlify standard as the repo-level INSTALL.md. Required elements: H1 title, blockquote summary, conversational opening, OBJECTIVE, DONE WHEN (measurable), TODO checklist, Steps with shell commands, EXECUTE NOW closing. + +Template at `templates/init/INSTALL.md` in [forge-cli][TEMPLATE]. + +## Boundary + +| Content type | Lives in | +|---|---| +| "When committing, follow these rules" | SKILL.md | +| "Run this command to set up the skill" | INSTALL.md | +| "Install gitleaks" (used by multiple skills) | Module INSTALL.md | +| Hook script that fires on PreToolUse | `hooks/` + `hooks.json` | + +## Related + +- [InstallInstructions](../../rules/InstallInstructions.md) — the rule establishing this convention +- [UserConfigSchema](UserConfigSchema.md) — when the config file uses the autoMode-mirror pattern + +[TEMPLATE]: https://github.com/N4M3Z/forge-cli/blob/main/templates/init/INSTALL.md diff --git a/skills/ForgeSkill/SkillStructure.md b/skills/ForgeSkill/SkillStructure.md new file mode 100644 index 0000000..1aee4ae --- /dev/null +++ b/skills/ForgeSkill/SkillStructure.md @@ -0,0 +1,97 @@ +A skill is a directory under `skills/` containing `SKILL.md` as the entrypoint ([Claude Code docs][CCDOCS]). + +`SKILL.md` carries YAML frontmatter (`name`, `description`, and optionally `argument-hint`, `allowed-tools`, `model`, `effort`, `context`, `hooks`, `paths`, `shell`) plus the workflow body. Companion files (templates, examples, reference material) live alongside. Skills are lazy-loaded: `SKILL.md` is only injected into context when the user invokes the skill or the AI matches the description. Companion files are loaded on demand when the AI decides it needs them during execution. + +## Forge additions beyond the native spec + +| File | Purpose | +| ------------- | ---------------------------------------------- | +| `SKILL.yaml` | Sidecar for reference URLs and provider hints | +| `user/` | Qualifier directory, flattened at assembly | +| `@` includes | Companion file references, resolved by forge | + +**`@` includes vs plain references**: use `@File.md` only for companions that should be auto-injected alongside SKILL.md on every invocation. For optional or variant companions (e.g. a multi-mode skill where only one mode is loaded per run), use plain filename references like `` `File.md` `` and let the AI load on demand. Over-use of `@` wastes tokens on unused companions. Never mix the forms. + +```markdown +GOOD — always-loaded reference (forge inlines content into SKILL.md at parse time) +@SkillStructure.md + +GOOD — load-on-demand reference (AI reads via Read tool when the workflow needs it) +See [`Linting.md`](Linting.md) for the full lint pipeline. + +GOOD — same companion table, different intent +| Workflow | Companion | +| -------- | ------------------- | +| Create | @CreateWorkflow.md | <- always inline +| Validate | `ValidateWorkflow.md` | <- load on demand + +BAD — `@` inside a markdown link reads as "auto-inject", looks like a regular link, behaves like neither +[@File.md](File.md) + +BAD — `@` on a multi-mode router; every variant inlines on every invocation, defeating routing +@FantasyMode.md +@SciFiMode.md +@NoirMode.md +``` + +## SKILL.md frontmatter + +```yaml +--- +name: SkillName +description: What it does. USE WHEN trigger phrase one, trigger phrase two, or trigger phrase three. +--- +``` + +**Frontmatter rules:** +- `name:` — PascalCase for multi-word (`VaultOperations`, `DailyPlan`), natural casing for single words (`Log`, `Draft`, `Init`) +- `version:` — semantic version (required for module skills, optional for personal/vault skills) +- `description:` — single line, under 1024 characters, includes `USE WHEN` with intent-based triggers joined by commas/OR +- Optional: `argument-hint:` for skills invoked with `/SkillName ` (e.g., `"[natural language description]"`) +- No separate `triggers:` or `workflows:` arrays in YAML + +## Body structure + +```markdown +# SkillName + +Brief description of what the skill does. + +## Instructions (or ## Usage) + +Step-by-step procedure. Use plain numbered lists for sequential operations. + +1. First action +2. Second action +3. Third action + +## Constraints + +- Boundary conditions and rules +- What NOT to do +``` + +**Instruction format**: Use plain numbered lists (1, 2, 3) — not labeled steps (`### Step 1:`, `### Phase 2:`, `### Step M1:`). Headings within Instructions are for separating modes or major sections, not for individual steps. + +**For skills with multiple workflows:** use a `## Workflow Routing` table pointing at companion files. Keep SKILL.md focused on flow and routing, not static data. Extract reference material (schema templates, configuration examples, lookup tables) into companion files. + +## Where skills live + +| Location | Purpose | +| -------------------- | ------------------------------------- | +| `skills/SkillName/` | Module skills (shipped with a module) | +| User vault workspace | Personal/experimental skills | + +All parent directories must be registered in `plugin.json` under the `skills` array for Claude Code discovery. Other providers (Gemini, Codex, OpenCode) use `make install` from the module's Makefile. + +## Naming conventions + +| Component | Convention | Examples | +| ----------------- | ----------------- | --------------------------------------------- | +| Skill directory | PascalCase | `BuildSkill`, `DailyPlan`, `VaultOperations` | +| Single-word skill | Natural case | `Log`, `Draft`, `Init`, `Update` | +| SKILL.md | Always `SKILL.md` | — | + +**Naming around variants**: when a skill could plausibly spawn siblings (e.g. `StyleCzech` may want `Fantasy`, `Scifi`, `Noir`), don't bake the variant into the skill name. Name the skill for its stable scope and push variants into companion files (`Fantasy.md`, `Scifi.md`). Prefer `StyleCzech` with `Fantasy.md` over `StyleCzechFantasy`. Apply this only when variants are plausible — a truly single-purpose skill stays named for its purpose. + +[CCDOCS]: https://code.claude.com/docs/en/skills diff --git a/skills/ForgeSkill/UserConfigSchema.md b/skills/ForgeSkill/UserConfigSchema.md new file mode 100644 index 0000000..d827e64 --- /dev/null +++ b/skills/ForgeSkill/UserConfigSchema.md @@ -0,0 +1,52 @@ +# User Config Schema (autoMode mirror) + +When an artifact needs per-user runtime data, follow the [UserConfig](../../rules/UserConfig.md) rule — one file per artifact at `~/.config/forge/.{ext}`. When the config is intended to be read by an AI in the loop (skill, agent, or hook with model access), shape it after [Claude Code's `autoMode`][AM]: natural-language entries with a four-tier precedence model and a `$defaults` splice token. + +## Why mirror autoMode + +The pattern is already familiar to anyone configuring Claude Code. Entries are prose — descriptions a human (or AI) would naturally write — not regex or tool-pattern grammars. The `$defaults` token gives a splice-or-replace toggle for built-in defaults shipped with the artifact source. Users extend the built-ins by adding entries; they take full ownership by omitting `"$defaults"`. + +## Shape + +Top-level keys are tiers with strict precedence: `hard_deny` > `soft_deny` > `allow` > `environment`. Each value is an array of prose strings. + +```yaml +environment: + - "$defaults" + - "" + +allow: + - "$defaults" + - "" + +soft_deny: + - "$defaults" + - "" + +hard_deny: + - "$defaults" + - "" +``` + +## Tier semantics + +- `hard_deny` blocks unconditionally. No `allow` exception or user intent applies. +- `soft_deny` blocks next. `allow` exceptions and explicit user intent can override. +- `allow` overrides matching `soft_deny` entries. +- `environment` provides context: trusted infrastructure, identities, repo ownership. + +Setting any tier without `"$defaults"` replaces the entire built-in list for that tier. Default entries are spliced at the position of the token, so custom entries can go before or after them. + +## When NOT to use this pattern + +Deterministic consumers (shell scripts, pre-commit hooks, CI gates without model access) can't interpret prose. Ship a sibling artifact for those — same `~/.config/forge/` directory, different filename — with a flat regex list or other machine-readable structure. Don't try to mix prose and regex in one file; the consumer types diverge. + +Example pairing: `~/.config/forge/forensic.yaml` (prose, read by ForensicAgent) and `~/.config/forge/danger-strings` (regex, read by the pre-commit hook). + +## Discovery and inspection + +The artifact source documents its built-in `$defaults` inline (in `SKILL.md` body or the agent body) so users can read what they inherit before extending. A `forge config` subcommand printing the effective merged config is recommended but not required. + +## Reference + +[AM]: https://code.claude.com/docs/en/auto-mode-config "Claude Code: Configure auto mode" diff --git a/skills/ForgeSkill/ValidateWorkflow.md b/skills/ForgeSkill/ValidateWorkflow.md new file mode 100644 index 0000000..da6d761 --- /dev/null +++ b/skills/ForgeSkill/ValidateWorkflow.md @@ -0,0 +1,39 @@ +## Step 1: Read the target skill + +Read the SKILL.md file. + +## Step 2: Check frontmatter + +- [ ] `name:` present and uses correct casing +- [ ] `description:` is single-line with `USE WHEN` clause +- [ ] `description:` is under 1024 characters +- [ ] No deprecated fields (`triggers:`, `workflows:` arrays) +- [ ] Optional fields (`argument-hint:`, `version:`) are correctly formatted + +## Step 3: Check body structure + +- [ ] Starts with `# SkillName` heading (matches `name:` frontmatter) +- [ ] Has clear instructions (numbered steps, usage section, or workflow routing) +- [ ] If multiple workflows: `## Workflow Routing` table present +- [ ] Constraints or rules section for boundary conditions +- [ ] No unnecessary sections or boilerplate + +## Step 4: Check CLI tool integration (if applicable) + +- [ ] Tool path is documented +- [ ] Usage examples with `bash` blocks +- [ ] Intent-to-flag mapping table (if tool has flags) +- [ ] Output format described + +## Step 5: Check content quality + +- [ ] Inputs and outputs stated explicitly (what the skill consumes, what artifact it produces) +- [ ] Guardrails pause rather than auto-escalate — no silent jump from read-only analysis to executing targets or installing tooling +- [ ] No authoring-environment fingerprints: stdlib-only probes use a preflight-resolved `python3`, not `uv run python`; no personal paths, package managers, or aliases the target machine may lack +- [ ] Deliverable is an artifact (table, file, verdict schema), not prose + +## Step 6: Report + +**COMPLIANT** — all checks pass. + +**NON-COMPLIANT** — list failures with specific fixes. Offer to fix automatically. diff --git a/skills/ForgeSkill/agents/.provenance/analyzer.md.yaml b/skills/ForgeSkill/agents/.provenance/analyzer.md.yaml new file mode 100644 index 0000000..00602c9 --- /dev/null +++ b/skills/ForgeSkill/agents/.provenance/analyzer.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/agents/analyzer.md + digest: + sha256: bf68f4cac5a56c673a928c2e6d619586c5b93ea364026ab37547772cb45a663a + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/analyzer.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/analyzer.md + digest: + sha256: bf68f4cac5a56c673a928c2e6d619586c5b93ea364026ab37547772cb45a663a + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/agents/.provenance/comparator.md.yaml b/skills/ForgeSkill/agents/.provenance/comparator.md.yaml new file mode 100644 index 0000000..80611e4 --- /dev/null +++ b/skills/ForgeSkill/agents/.provenance/comparator.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/agents/comparator.md + digest: + sha256: fe1fc9787c495d864c5d6eada47396478572325fde1b33a96d78bf4b849b7a3e + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/comparator.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/comparator.md + digest: + sha256: fe1fc9787c495d864c5d6eada47396478572325fde1b33a96d78bf4b849b7a3e + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/agents/.provenance/grader.md.yaml b/skills/ForgeSkill/agents/.provenance/grader.md.yaml new file mode 100644 index 0000000..0866d76 --- /dev/null +++ b/skills/ForgeSkill/agents/.provenance/grader.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/agents/grader.md + digest: + sha256: 57134da0c1a4eea33fbd74a1c9c44aa814f07d6bc64de303edb586f941e5d21a + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/grader.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/agents/grader.md + digest: + sha256: 57134da0c1a4eea33fbd74a1c9c44aa814f07d6bc64de303edb586f941e5d21a + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/agents/analyzer.md b/skills/ForgeSkill/agents/analyzer.md new file mode 100644 index 0000000..14e41d6 --- /dev/null +++ b/skills/ForgeSkill/agents/analyzer.md @@ -0,0 +1,274 @@ +# Post-hoc Analyzer Agent + +Analyze blind comparison results to understand WHY the winner won and generate improvement suggestions. + +## Role + +After the blind comparator determines a winner, the Post-hoc Analyzer "unblids" the results by examining the skills and transcripts. The goal is to extract actionable insights: what made the winner better, and how can the loser be improved? + +## Inputs + +You receive these parameters in your prompt: + +- **winner**: "A" or "B" (from blind comparison) +- **winner_skill_path**: Path to the skill that produced the winning output +- **winner_transcript_path**: Path to the execution transcript for the winner +- **loser_skill_path**: Path to the skill that produced the losing output +- **loser_transcript_path**: Path to the execution transcript for the loser +- **comparison_result_path**: Path to the blind comparator's output JSON +- **output_path**: Where to save the analysis results + +## Process + +### Step 1: Read Comparison Result + +1. Read the blind comparator's output at comparison_result_path +2. Note the winning side (A or B), the reasoning, and any scores +3. Understand what the comparator valued in the winning output + +### Step 2: Read Both Skills + +1. Read the winner skill's SKILL.md and key referenced files +2. Read the loser skill's SKILL.md and key referenced files +3. Identify structural differences: + - Instructions clarity and specificity + - Script/tool usage patterns + - Example coverage + - Edge case handling + +### Step 3: Read Both Transcripts + +1. Read the winner's transcript +2. Read the loser's transcript +3. Compare execution patterns: + - How closely did each follow their skill's instructions? + - What tools were used differently? + - Where did the loser diverge from optimal behavior? + - Did either encounter errors or make recovery attempts? + +### Step 4: Analyze Instruction Following + +For each transcript, evaluate: +- Did the agent follow the skill's explicit instructions? +- Did the agent use the skill's provided tools/scripts? +- Were there missed opportunities to leverage skill content? +- Did the agent add unnecessary steps not in the skill? + +Score instruction following 1-10 and note specific issues. + +### Step 5: Identify Winner Strengths + +Determine what made the winner better: +- Clearer instructions that led to better behavior? +- Better scripts/tools that produced better output? +- More comprehensive examples that guided edge cases? +- Better error handling guidance? + +Be specific. Quote from skills/transcripts where relevant. + +### Step 6: Identify Loser Weaknesses + +Determine what held the loser back: +- Ambiguous instructions that led to suboptimal choices? +- Missing tools/scripts that forced workarounds? +- Gaps in edge case coverage? +- Poor error handling that caused failures? + +### Step 7: Generate Improvement Suggestions + +Based on the analysis, produce actionable suggestions for improving the loser skill: +- Specific instruction changes to make +- Tools/scripts to add or modify +- Examples to include +- Edge cases to address + +Prioritize by impact. Focus on changes that would have changed the outcome. + +### Step 8: Write Analysis Results + +Save structured analysis to `{output_path}`. + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors", + "Explicit guidance on fallback behavior when OCR fails" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise and made errors", + "No guidance on OCR failure, agent gave up instead of trying alternatives" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": [ + "Minor: skipped optional logging step" + ] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3", + "Missed the 'always validate output' instruction" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps: 1) Extract text, 2) Identify sections, 3) Format per template", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + }, + { + "priority": "high", + "category": "tools", + "suggestion": "Add validate_output.py script similar to winner skill's validation approach", + "expected_impact": "Would catch formatting errors before final output" + }, + { + "priority": "medium", + "category": "error_handling", + "suggestion": "Add fallback instructions: 'If OCR fails, try: 1) different resolution, 2) image preprocessing, 3) manual extraction'", + "expected_impact": "Would prevent early failure on difficult documents" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script -> Fixed 2 issues -> Produced output", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods -> No validation -> Output had errors" + } +} +``` + +## Guidelines + +- **Be specific**: Quote from skills and transcripts, don't just say "instructions were unclear" +- **Be actionable**: Suggestions should be concrete changes, not vague advice +- **Focus on skill improvements**: The goal is to improve the losing skill, not critique the agent +- **Prioritize by impact**: Which changes would most likely have changed the outcome? +- **Consider causation**: Did the skill weakness actually cause the worse output, or is it incidental? +- **Stay objective**: Analyze what happened, don't editorialize +- **Think about generalization**: Would this improvement help on other evals too? + +## Categories for Suggestions + +Use these categories to organize improvement suggestions: + +| Category | Description | +|----------|-------------| +| `instructions` | Changes to the skill's prose instructions | +| `tools` | Scripts, templates, or utilities to add/modify | +| `examples` | Example inputs/outputs to include | +| `error_handling` | Guidance for handling failures | +| `structure` | Reorganization of skill content | +| `references` | External docs or resources to add | + +## Priority Levels + +- **high**: Would likely change the outcome of this comparison +- **medium**: Would improve quality but may not change win/loss +- **low**: Nice to have, marginal improvement + +--- + +# Analyzing Benchmark Results + +When analyzing benchmark results, the analyzer's purpose is to **surface patterns and anomalies** across multiple runs, not suggest skill improvements. + +## Role + +Review all benchmark run results and generate freeform notes that help the user understand skill performance. Focus on patterns that wouldn't be visible from aggregate metrics alone. + +## Inputs + +You receive these parameters in your prompt: + +- **benchmark_data_path**: Path to the in-progress benchmark.json with all run results +- **skill_path**: Path to the skill being benchmarked +- **output_path**: Where to save the notes (as JSON array of strings) + +## Process + +### Step 1: Read Benchmark Data + +1. Read the benchmark.json containing all run results +2. Note the configurations tested (with_skill, without_skill) +3. Understand the run_summary aggregates already calculated + +### Step 2: Analyze Per-Assertion Patterns + +For each expectation across all runs: +- Does it **always pass** in both configurations? (may not differentiate skill value) +- Does it **always fail** in both configurations? (may be broken or beyond capability) +- Does it **always pass with skill but fail without**? (skill clearly adds value here) +- Does it **always fail with skill but pass without**? (skill may be hurting) +- Is it **highly variable**? (flaky expectation or non-deterministic behavior) + +### Step 3: Analyze Cross-Eval Patterns + +Look for patterns across evals: +- Are certain eval types consistently harder/easier? +- Do some evals show high variance while others are stable? +- Are there surprising results that contradict expectations? + +### Step 4: Analyze Metrics Patterns + +Look at time_seconds, tokens, tool_calls: +- Does the skill significantly increase execution time? +- Is there high variance in resource usage? +- Are there outlier runs that skew the aggregates? + +### Step 5: Generate Notes + +Write freeform observations as a list of strings. Each note should: +- State a specific observation +- Be grounded in the data (not speculation) +- Help the user understand something the aggregate metrics don't show + +Examples: +- "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value" +- "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure that may be flaky" +- "Without-skill runs consistently fail on table extraction expectations (0% pass rate)" +- "Skill adds 13s average execution time but improves pass rate by 50%" +- "Token usage is 80% higher with skill, primarily due to script output parsing" +- "All 3 without-skill runs for eval 1 produced empty output" + +### Step 6: Write Notes + +Save notes to `{output_path}` as a JSON array of strings: + +```json +[ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" +] +``` + +## Guidelines + +**DO:** +- Report what you observe in the data +- Be specific about which evals, expectations, or runs you're referring to +- Note patterns that aggregate metrics would hide +- Provide context that helps interpret the numbers + +**DO NOT:** +- Suggest improvements to the skill (that's for the improvement step, not benchmarking) +- Make subjective quality judgments ("the output was good/bad") +- Speculate about causes without evidence +- Repeat information already in the run_summary aggregates diff --git a/skills/ForgeSkill/agents/comparator.md b/skills/ForgeSkill/agents/comparator.md new file mode 100644 index 0000000..80e00eb --- /dev/null +++ b/skills/ForgeSkill/agents/comparator.md @@ -0,0 +1,202 @@ +# Blind Comparator Agent + +Compare two outputs WITHOUT knowing which skill produced them. + +## Role + +The Blind Comparator judges which output better accomplishes the eval task. You receive two outputs labeled A and B, but you do NOT know which skill produced which. This prevents bias toward a particular skill or approach. + +Your judgment is based purely on output quality and task completion. + +## Inputs + +You receive these parameters in your prompt: + +- **output_a_path**: Path to the first output file or directory +- **output_b_path**: Path to the second output file or directory +- **eval_prompt**: The original task/prompt that was executed +- **expectations**: List of expectations to check (optional - may be empty) + +## Process + +### Step 1: Read Both Outputs + +1. Examine output A (file or directory) +2. Examine output B (file or directory) +3. Note the type, structure, and content of each +4. If outputs are directories, examine all relevant files inside + +### Step 2: Understand the Task + +1. Read the eval_prompt carefully +2. Identify what the task requires: + - What should be produced? + - What qualities matter (accuracy, completeness, format)? + - What would distinguish a good output from a poor one? + +### Step 3: Generate Evaluation Rubric + +Based on the task, generate a rubric with two dimensions: + +**Content Rubric** (what the output contains): +| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) | +|-----------|----------|----------------|---------------| +| Correctness | Major errors | Minor errors | Fully correct | +| Completeness | Missing key elements | Mostly complete | All elements present | +| Accuracy | Significant inaccuracies | Minor inaccuracies | Accurate throughout | + +**Structure Rubric** (how the output is organized): +| Criterion | 1 (Poor) | 3 (Acceptable) | 5 (Excellent) | +|-----------|----------|----------------|---------------| +| Organization | Disorganized | Reasonably organized | Clear, logical structure | +| Formatting | Inconsistent/broken | Mostly consistent | Professional, polished | +| Usability | Difficult to use | Usable with effort | Easy to use | + +Adapt criteria to the specific task. For example: +- PDF form → "Field alignment", "Text readability", "Data placement" +- Document → "Section structure", "Heading hierarchy", "Paragraph flow" +- Data output → "Schema correctness", "Data types", "Completeness" + +### Step 4: Evaluate Each Output Against the Rubric + +For each output (A and B): + +1. **Score each criterion** on the rubric (1-5 scale) +2. **Calculate dimension totals**: Content score, Structure score +3. **Calculate overall score**: Average of dimension scores, scaled to 1-10 + +### Step 5: Check Assertions (if provided) + +If expectations are provided: + +1. Check each expectation against output A +2. Check each expectation against output B +3. Count pass rates for each output +4. Use expectation scores as secondary evidence (not the primary decision factor) + +### Step 6: Determine the Winner + +Compare A and B based on (in priority order): + +1. **Primary**: Overall rubric score (content + structure) +2. **Secondary**: Assertion pass rates (if applicable) +3. **Tiebreaker**: If truly equal, declare a TIE + +Be decisive - ties should be rare. One output is usually better, even if marginally. + +### Step 7: Write Comparison Results + +Save results to a JSON file at the path specified (or `comparison.json` if not specified). + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true}, + {"text": "Output includes date", "passed": true}, + {"text": "Format is PDF", "passed": true}, + {"text": "Contains signature", "passed": false}, + {"text": "Readable text", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true}, + {"text": "Output includes date", "passed": false}, + {"text": "Format is PDF", "passed": true}, + {"text": "Contains signature", "passed": false}, + {"text": "Readable text", "passed": true} + ] + } + } +} +``` + +If no expectations were provided, omit the `expectation_results` field entirely. + +## Field Descriptions + +- **winner**: "A", "B", or "TIE" +- **reasoning**: Clear explanation of why the winner was chosen (or why it's a tie) +- **rubric**: Structured rubric evaluation for each output + - **content**: Scores for content criteria (correctness, completeness, accuracy) + - **structure**: Scores for structure criteria (organization, formatting, usability) + - **content_score**: Average of content criteria (1-5) + - **structure_score**: Average of structure criteria (1-5) + - **overall_score**: Combined score scaled to 1-10 +- **output_quality**: Summary quality assessment + - **score**: 1-10 rating (should match rubric overall_score) + - **strengths**: List of positive aspects + - **weaknesses**: List of issues or shortcomings +- **expectation_results**: (Only if expectations provided) + - **passed**: Number of expectations that passed + - **total**: Total number of expectations + - **pass_rate**: Fraction passed (0.0 to 1.0) + - **details**: Individual expectation results + +## Guidelines + +- **Stay blind**: DO NOT try to infer which skill produced which output. Judge purely on output quality. +- **Be specific**: Cite specific examples when explaining strengths and weaknesses. +- **Be decisive**: Choose a winner unless outputs are genuinely equivalent. +- **Output quality first**: Assertion scores are secondary to overall task completion. +- **Be objective**: Don't favor outputs based on style preferences; focus on correctness and completeness. +- **Explain your reasoning**: The reasoning field should make it clear why you chose the winner. +- **Handle edge cases**: If both outputs fail, pick the one that fails less badly. If both are excellent, pick the one that's marginally better. diff --git a/skills/ForgeSkill/agents/grader.md b/skills/ForgeSkill/agents/grader.md new file mode 100644 index 0000000..558ab05 --- /dev/null +++ b/skills/ForgeSkill/agents/grader.md @@ -0,0 +1,223 @@ +# Grader Agent + +Evaluate expectations against an execution transcript and outputs. + +## Role + +The Grader reviews a transcript and output files, then determines whether each expectation passes or fails. Provide clear evidence for each judgment. + +You have two jobs: grade the outputs, and critique the evals themselves. A passing grade on a weak assertion is worse than useless — it creates false confidence. When you notice an assertion that's trivially satisfied, or an important outcome that no assertion checks, say so. + +## Inputs + +You receive these parameters in your prompt: + +- **expectations**: List of expectations to evaluate (strings) +- **transcript_path**: Path to the execution transcript (markdown file) +- **outputs_dir**: Directory containing output files from execution + +## Process + +### Step 1: Read the Transcript + +1. Read the transcript file completely +2. Note the eval prompt, execution steps, and final result +3. Identify any issues or errors documented + +### Step 2: Examine Output Files + +1. List files in outputs_dir +2. Read/examine each file relevant to the expectations. If outputs aren't plain text, use the inspection tools provided in your prompt — don't rely solely on what the transcript says the executor produced. +3. Note contents, structure, and quality + +### Step 3: Evaluate Each Assertion + +For each expectation: + +1. **Search for evidence** in the transcript and outputs +2. **Determine verdict**: + - **PASS**: Clear evidence the expectation is true AND the evidence reflects genuine task completion, not just surface-level compliance + - **FAIL**: No evidence, or evidence contradicts the expectation, or the evidence is superficial (e.g., correct filename but empty/wrong content) +3. **Cite the evidence**: Quote the specific text or describe what you found + +### Step 4: Extract and Verify Claims + +Beyond the predefined expectations, extract implicit claims from the outputs and verify them: + +1. **Extract claims** from the transcript and outputs: + - Factual statements ("The form has 12 fields") + - Process claims ("Used pypdf to fill the form") + - Quality claims ("All fields were filled correctly") + +2. **Verify each claim**: + - **Factual claims**: Can be checked against the outputs or external sources + - **Process claims**: Can be verified from the transcript + - **Quality claims**: Evaluate whether the claim is justified + +3. **Flag unverifiable claims**: Note claims that cannot be verified with available information + +This catches issues that predefined expectations might miss. + +### Step 5: Read User Notes + +If `{outputs_dir}/user_notes.md` exists: +1. Read it and note any uncertainties or issues flagged by the executor +2. Include relevant concerns in the grading output +3. These may reveal problems even when expectations pass + +### Step 6: Critique the Evals + +After grading, consider whether the evals themselves could be improved. Only surface suggestions when there's a clear gap. + +Good suggestions test meaningful outcomes — assertions that are hard to satisfy without actually doing the work correctly. Think about what makes an assertion *discriminating*: it passes when the skill genuinely succeeds and fails when it doesn't. + +Suggestions worth raising: +- An assertion that passed but would also pass for a clearly wrong output (e.g., checking filename existence but not file content) +- An important outcome you observed — good or bad — that no assertion covers at all +- An assertion that can't actually be verified from the available outputs + +Keep the bar high. The goal is to flag things the eval author would say "good catch" about, not to nitpick every assertion. + +### Step 7: Write Grading Results + +Save results to `{outputs_dir}/../grading.json` (sibling to outputs_dir). + +## Grading Criteria + +**PASS when**: +- The transcript or outputs clearly demonstrate the expectation is true +- Specific evidence can be cited +- The evidence reflects genuine substance, not just surface compliance (e.g., a file exists AND contains correct content, not just the right filename) + +**FAIL when**: +- No evidence found for the expectation +- Evidence contradicts the expectation +- The expectation cannot be verified from available information +- The evidence is superficial — the assertion is technically satisfied but the underlying task outcome is wrong or incomplete +- The output appears to meet the assertion by coincidence rather than by actually doing the work + +**When uncertain**: The burden of proof to pass is on the expectation. + +### Step 8: Read Executor Metrics and Timing + +1. If `{outputs_dir}/metrics.json` exists, read it and include in grading output +2. If `{outputs_dir}/../timing.json` exists, read it and include timing data + +## Output Format + +Write a JSON file with this structure: + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + }, + { + "text": "The assistant used the skill's OCR script", + "passed": true, + "evidence": "Transcript Step 2 shows: 'Tool: Bash - python ocr_script.py image.png'" + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + }, + { + "claim": "All required fields were populated", + "type": "quality", + "verified": false, + "evidence": "Reference section was left blank despite data being available" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass — consider checking it appears as the primary contact with matching phone and email from the input" + }, + { + "reason": "No assertion checks whether the extracted phone numbers match the input — I observed incorrect numbers in the output that went uncaught" + } + ], + "overall": "Assertions check presence but not correctness. Consider adding content verification." + } +} +``` + +## Field Descriptions + +- **expectations**: Array of graded expectations + - **text**: The original expectation text + - **passed**: Boolean - true if expectation passes + - **evidence**: Specific quote or description supporting the verdict +- **summary**: Aggregate statistics + - **passed**: Count of passed expectations + - **failed**: Count of failed expectations + - **total**: Total expectations evaluated + - **pass_rate**: Fraction passed (0.0 to 1.0) +- **execution_metrics**: Copied from executor's metrics.json (if available) + - **output_chars**: Total character count of output files (proxy for tokens) + - **transcript_chars**: Character count of transcript +- **timing**: Wall clock timing from timing.json (if available) + - **executor_duration_seconds**: Time spent in executor subagent + - **total_duration_seconds**: Total elapsed time for the run +- **claims**: Extracted and verified claims from the output + - **claim**: The statement being verified + - **type**: "factual", "process", or "quality" + - **verified**: Boolean - whether the claim holds + - **evidence**: Supporting or contradicting evidence +- **user_notes_summary**: Issues flagged by the executor + - **uncertainties**: Things the executor wasn't sure about + - **needs_review**: Items requiring human attention + - **workarounds**: Places where the skill didn't work as expected +- **eval_feedback**: Improvement suggestions for the evals (only when warranted) + - **suggestions**: List of concrete suggestions, each with a `reason` and optionally an `assertion` it relates to + - **overall**: Brief assessment — can be "No suggestions, evals look solid" if nothing to flag + +## Guidelines + +- **Be objective**: Base verdicts on evidence, not assumptions +- **Be specific**: Quote the exact text that supports your verdict +- **Be thorough**: Check both transcript and output files +- **Be consistent**: Apply the same standard to each expectation +- **Explain failures**: Make it clear why evidence was insufficient +- **No partial credit**: Each expectation is pass or fail, not partial diff --git a/skills/ForgeSkill/assets/.provenance/eval_review.html.yaml b/skills/ForgeSkill/assets/.provenance/eval_review.html.yaml new file mode 100644 index 0000000..4848eda --- /dev/null +++ b/skills/ForgeSkill/assets/.provenance/eval_review.html.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/assets/eval_review.html + digest: + sha256: ce477dcc74dc1c0d1d3352646a79167b5a63634e936b1019160025065974e452 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/assets/eval_review.html + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/assets/eval_review.html + digest: + sha256: ce477dcc74dc1c0d1d3352646a79167b5a63634e936b1019160025065974e452 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/assets/eval_review.html b/skills/ForgeSkill/assets/eval_review.html new file mode 100644 index 0000000..938ff32 --- /dev/null +++ b/skills/ForgeSkill/assets/eval_review.html @@ -0,0 +1,146 @@ + + + + + + Eval Set Review - __SKILL_NAME_PLACEHOLDER__ + + + + + + +

Eval Set Review: __SKILL_NAME_PLACEHOLDER__

+

Current description: __SKILL_DESCRIPTION_PLACEHOLDER__

+ +
+ + +
+ + + + + + + + + + +
QueryShould TriggerActions
+ +

+ + + + diff --git a/skills/ForgeSkill/eval-viewer/.provenance/generate_review.py.yaml b/skills/ForgeSkill/eval-viewer/.provenance/generate_review.py.yaml new file mode 100644 index 0000000..5758d45 --- /dev/null +++ b/skills/ForgeSkill/eval-viewer/.provenance/generate_review.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/eval-viewer/generate_review.py + digest: + sha256: 71eef1a5e8b7ade005c97968ea7bb81a6018a8c013a6e812dc68ffa2047d28dd + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/eval-viewer/generate_review.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/eval-viewer/generate_review.py + digest: + sha256: fc9d1b9243fe5ab6012ebd579bd76d0035de1b79fd3b969de114defab26478fb + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/eval-viewer/.provenance/viewer.html.yaml b/skills/ForgeSkill/eval-viewer/.provenance/viewer.html.yaml new file mode 100644 index 0000000..524ce12 --- /dev/null +++ b/skills/ForgeSkill/eval-viewer/.provenance/viewer.html.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/eval-viewer/viewer.html + digest: + sha256: a53213426ee1100441d701a3a0d49cda7a842f992d2c36463f4d3cc0258575fa + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/eval-viewer/viewer.html + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/eval-viewer/viewer.html + digest: + sha256: a53213426ee1100441d701a3a0d49cda7a842f992d2c36463f4d3cc0258575fa + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/eval-viewer/generate_review.py b/skills/ForgeSkill/eval-viewer/generate_review.py new file mode 100644 index 0000000..26e2c19 --- /dev/null +++ b/skills/ForgeSkill/eval-viewer/generate_review.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Generate and serve a review page for eval results. + +Reads the workspace directory, discovers runs (directories with outputs/), +embeds all output data into a self-contained HTML page, and serves it via +a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace. + +Usage: + python generate_review.py [--port PORT] [--skill-name NAME] + python generate_review.py --previous-feedback /path/to/old/feedback.json + +No dependencies beyond the Python stdlib are required. +""" + +import argparse +import base64 +import json +import mimetypes +import os +import re +import signal +import subprocess +import sys +import time +import webbrowser +from functools import partial +from http.server import HTTPServer, BaseHTTPRequestHandler +from pathlib import Path + +# Files to exclude from output listings +METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"} + +# Extensions we render as inline text +TEXT_EXTENSIONS = { + ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx", + ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs", + ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml", +} + +# Extensions we render as inline images +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} + +# MIME type overrides for common types +MIME_OVERRIDES = { + ".svg": "image/svg+xml", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", +} + + +def get_mime_type(path: Path) -> str: + ext = path.suffix.lower() + if ext in MIME_OVERRIDES: + return MIME_OVERRIDES[ext] + mime, _ = mimetypes.guess_type(str(path)) + return mime or "application/octet-stream" + + +def find_runs(workspace: Path) -> list[dict]: + """Recursively find directories that contain an outputs/ subdirectory.""" + runs: list[dict] = [] + _find_runs_recursive(workspace, workspace, runs) + runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"])) + return runs + + +def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None: + if not current.is_dir(): + return + + outputs_dir = current / "outputs" + if outputs_dir.is_dir(): + run = build_run(root, current) + if run: + runs.append(run) + return + + skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"} + for child in sorted(current.iterdir()): + if child.is_dir() and child.name not in skip: + _find_runs_recursive(root, child, runs) + + +def build_run(root: Path, run_dir: Path) -> dict | None: + """Build a run dict with prompt, outputs, and grading data.""" + prompt = "" + eval_id = None + + # Try eval_metadata.json + for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]: + if candidate.exists(): + try: + metadata = json.loads(candidate.read_text()) + prompt = metadata.get("prompt", "") + eval_id = metadata.get("eval_id") + except (json.JSONDecodeError, OSError): + pass + if prompt: + break + + # Fall back to transcript.md + if not prompt: + for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]: + if candidate.exists(): + try: + text = candidate.read_text() + match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text) + if match: + prompt = match.group(1).strip() + except OSError: + pass + if prompt: + break + + if not prompt: + prompt = "(No prompt found)" + + run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-") + + # Collect output files + outputs_dir = run_dir / "outputs" + output_files: list[dict] = [] + if outputs_dir.is_dir(): + for f in sorted(outputs_dir.iterdir()): + if f.is_file() and f.name not in METADATA_FILES: + output_files.append(embed_file(f)) + + # Load grading if present + grading = None + for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]: + if candidate.exists(): + try: + grading = json.loads(candidate.read_text()) + except (json.JSONDecodeError, OSError): + pass + if grading: + break + + return { + "id": run_id, + "prompt": prompt, + "eval_id": eval_id, + "outputs": output_files, + "grading": grading, + } + + +def embed_file(path: Path) -> dict: + """Read a file and return an embedded representation.""" + ext = path.suffix.lower() + mime = get_mime_type(path) + + if ext in TEXT_EXTENSIONS: + try: + content = path.read_text(errors="replace") + except OSError: + content = "(Error reading file)" + return { + "name": path.name, + "type": "text", + "content": content, + } + elif ext in IMAGE_EXTENSIONS: + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "image", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".pdf": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "pdf", + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".xlsx": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "xlsx", + "data_b64": b64, + } + else: + # Binary / unknown — base64 download link + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "binary", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + + +def load_previous_iteration(workspace: Path) -> dict[str, dict]: + """Load previous iteration's feedback and outputs. + + Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}. + """ + result: dict[str, dict] = {} + + # Load feedback + feedback_map: dict[str, str] = {} + feedback_path = workspace / "feedback.json" + if feedback_path.exists(): + try: + data = json.loads(feedback_path.read_text()) + feedback_map = { + r["run_id"]: r["feedback"] + for r in data.get("reviews", []) + if r.get("feedback", "").strip() + } + except (json.JSONDecodeError, OSError, KeyError): + pass + + # Load runs (to get outputs) + prev_runs = find_runs(workspace) + for run in prev_runs: + result[run["id"]] = { + "feedback": feedback_map.get(run["id"], ""), + "outputs": run.get("outputs", []), + } + + # Also add feedback for run_ids that had feedback but no matching run + for run_id, fb in feedback_map.items(): + if run_id not in result: + result[run_id] = {"feedback": fb, "outputs": []} + + return result + + +def generate_html( + runs: list[dict], + skill_name: str, + previous: dict[str, dict] | None = None, + benchmark: dict | None = None, +) -> str: + """Generate the complete standalone HTML page with embedded data.""" + template_path = Path(__file__).parent / "viewer.html" + template = template_path.read_text() + + # Build previous_feedback and previous_outputs maps for the template + previous_feedback: dict[str, str] = {} + previous_outputs: dict[str, list[dict]] = {} + if previous: + for run_id, data in previous.items(): + if data.get("feedback"): + previous_feedback[run_id] = data["feedback"] + if data.get("outputs"): + previous_outputs[run_id] = data["outputs"] + + embedded = { + "skill_name": skill_name, + "runs": runs, + "previous_feedback": previous_feedback, + "previous_outputs": previous_outputs, + } + if benchmark: + embedded["benchmark"] = benchmark + + data_json = json.dumps(embedded) + + return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};") + + +# --------------------------------------------------------------------------- +# HTTP server (stdlib only, zero dependencies) +# --------------------------------------------------------------------------- + +def _kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + for pid_str in result.stdout.strip().split("\n"): + if pid_str.strip(): + try: + os.kill(int(pid_str.strip()), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + if result.stdout.strip(): + time.sleep(0.5) + except subprocess.TimeoutExpired: + pass + except FileNotFoundError: + print("Note: lsof not found, cannot check if port is in use", file=sys.stderr) + +class ReviewHandler(BaseHTTPRequestHandler): + """Serves the review HTML and handles feedback saves. + + Regenerates the HTML on each page load so that refreshing the browser + picks up new eval outputs without restarting the server. + """ + + def __init__( + self, + workspace: Path, + skill_name: str, + feedback_path: Path, + previous: dict[str, dict], + benchmark_path: Path | None, + *args, + **kwargs, + ): + self.workspace = workspace + self.skill_name = skill_name + self.feedback_path = feedback_path + self.previous = previous + self.benchmark_path = benchmark_path + super().__init__(*args, **kwargs) + + def do_GET(self) -> None: + if self.path == "/" or self.path == "/index.html": + # Regenerate HTML on each request (re-scans workspace for new outputs) + runs = find_runs(self.workspace) + benchmark = None + if self.benchmark_path and self.benchmark_path.exists(): + try: + benchmark = json.loads(self.benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + html = generate_html(runs, self.skill_name, self.previous, benchmark) + content = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + elif self.path == "/api/feedback": + data = b"{}" + if self.feedback_path.exists(): + data = self.feedback_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + else: + self.send_error(404) + + def do_POST(self) -> None: + if self.path == "/api/feedback": + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + if not isinstance(data, dict) or "reviews" not in data: + raise ValueError("Expected JSON object with 'reviews' key") + self.feedback_path.write_text(json.dumps(data, indent=2) + "\n") + resp = b'{"ok":true}' + self.send_response(200) + except (json.JSONDecodeError, OSError, ValueError) as e: + resp = json.dumps({"error": str(e)}).encode() + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + self.wfile.write(resp) + else: + self.send_error(404) + + def log_message(self, format: str, *args: object) -> None: + # Suppress request logging to keep terminal clean + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate and serve eval review") + parser.add_argument("workspace", type=Path, help="Path to workspace directory") + parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)") + parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header") + parser.add_argument( + "--previous-workspace", type=Path, default=None, + help="Path to previous iteration's workspace (shows old outputs and feedback as context)", + ) + parser.add_argument( + "--benchmark", type=Path, default=None, + help="Path to benchmark.json to show in the Benchmark tab", + ) + parser.add_argument( + "--static", "-s", type=Path, default=None, + help="Write standalone HTML to this path instead of starting a server", + ) + args = parser.parse_args() + + workspace = args.workspace.resolve() + if not workspace.is_dir(): + print(f"Error: {workspace} is not a directory", file=sys.stderr) + sys.exit(1) + + runs = find_runs(workspace) + if not runs: + print(f"No runs found in {workspace}", file=sys.stderr) + sys.exit(1) + + skill_name = args.skill_name or workspace.name.replace("-workspace", "") + feedback_path = workspace / "feedback.json" + + previous: dict[str, dict] = {} + if args.previous_workspace: + previous = load_previous_iteration(args.previous_workspace.resolve()) + + benchmark_path = args.benchmark.resolve() if args.benchmark else None + benchmark = None + if benchmark_path and benchmark_path.exists(): + try: + benchmark = json.loads(benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + + if args.static: + html = generate_html(runs, skill_name, previous, benchmark) + args.static.parent.mkdir(parents=True, exist_ok=True) + args.static.write_text(html) + print(f"\n Static viewer written to: {args.static}\n") + sys.exit(0) + + # Kill any existing process on the target port + port = args.port + _kill_port(port) + handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path) + try: + server = HTTPServer(("127.0.0.1", port), handler) + except OSError: + # Port still in use after kill attempt — find a free one + server = HTTPServer(("127.0.0.1", 0), handler) + port = server.server_address[1] + + url = f"http://localhost:{port}" + print("\n Eval Viewer") + print(" ─────────────────────────────────") + print(f" URL: {url}") + print(f" Workspace: {workspace}") + print(f" Feedback: {feedback_path}") + if previous: + print(f" Previous: {args.previous_workspace} ({len(previous)} runs)") + if benchmark_path: + print(f" Benchmark: {benchmark_path}") + print("\n Press Ctrl+C to stop.\n") + + webbrowser.open(url) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopped.") + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/eval-viewer/viewer.html b/skills/ForgeSkill/eval-viewer/viewer.html new file mode 100644 index 0000000..6d8e963 --- /dev/null +++ b/skills/ForgeSkill/eval-viewer/viewer.html @@ -0,0 +1,1325 @@ + + + + + + Eval Review + + + + + + + +
+
+
+

Eval Review:

+
Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.
+
+
+
+ + + + + +
+
+ +
+
Prompt
+
+
+
+
+ + +
+
Output
+
+
No output files found
+
+
+ + + + + + + + +
+
Your Feedback
+
+ + + +
+
+
+ + +
+ + +
+
+
No benchmark data available. Run a benchmark to see quantitative results here.
+
+
+
+ + +
+
+

Review Complete

+

Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.

+
+ +
+
+
+ + +
+ + + + diff --git a/skills/ForgeSkill/references/.provenance/schemas.md.yaml b/skills/ForgeSkill/references/.provenance/schemas.md.yaml new file mode 100644 index 0000000..6a0eb6f --- /dev/null +++ b/skills/ForgeSkill/references/.provenance/schemas.md.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/references/schemas.md + digest: + sha256: 8e8876180a8989b406a4d3edddf875b04cdfd5805cc8616686d552b11ce4455f + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/references/schemas.md + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/references/schemas.md + digest: + sha256: 8e8876180a8989b406a4d3edddf875b04cdfd5805cc8616686d552b11ce4455f + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/references/schemas.md b/skills/ForgeSkill/references/schemas.md new file mode 100644 index 0000000..b6eeaa2 --- /dev/null +++ b/skills/ForgeSkill/references/schemas.md @@ -0,0 +1,430 @@ +# JSON Schemas + +This document defines the JSON schemas used by skill-creator. + +--- + +## evals.json + +Defines the evals for a skill. Located at `evals/evals.json` within the skill directory. + +```json +{ + "skill_name": "example-skill", + "evals": [ + { + "id": 1, + "prompt": "User's example prompt", + "expected_output": "Description of expected result", + "files": ["evals/files/sample1.pdf"], + "expectations": [ + "The output includes X", + "The skill used script Y" + ] + } + ] +} +``` + +**Fields:** +- `skill_name`: Name matching the skill's frontmatter +- `evals[].id`: Unique integer identifier +- `evals[].prompt`: The task to execute +- `evals[].expected_output`: Human-readable description of success +- `evals[].files`: Optional list of input file paths (relative to skill root) +- `evals[].expectations`: List of verifiable statements + +--- + +## history.json + +Tracks version progression in Improve mode. Located at workspace root. + +```json +{ + "started_at": "2026-01-15T10:30:00Z", + "skill_name": "pdf", + "current_best": "v2", + "iterations": [ + { + "version": "v0", + "parent": null, + "expectation_pass_rate": 0.65, + "grading_result": "baseline", + "is_current_best": false + }, + { + "version": "v1", + "parent": "v0", + "expectation_pass_rate": 0.75, + "grading_result": "won", + "is_current_best": false + }, + { + "version": "v2", + "parent": "v1", + "expectation_pass_rate": 0.85, + "grading_result": "won", + "is_current_best": true + } + ] +} +``` + +**Fields:** +- `started_at`: ISO timestamp of when improvement started +- `skill_name`: Name of the skill being improved +- `current_best`: Version identifier of the best performer +- `iterations[].version`: Version identifier (v0, v1, ...) +- `iterations[].parent`: Parent version this was derived from +- `iterations[].expectation_pass_rate`: Pass rate from grading +- `iterations[].grading_result`: "baseline", "won", "lost", or "tie" +- `iterations[].is_current_best`: Whether this is the current best version + +--- + +## grading.json + +Output from the grader agent. Located at `/grading.json`. + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass" + } + ], + "overall": "Assertions check presence but not correctness." + } +} +``` + +**Fields:** +- `expectations[]`: Graded expectations with evidence +- `summary`: Aggregate pass/fail counts +- `execution_metrics`: Tool usage and output size (from executor's metrics.json) +- `timing`: Wall clock timing (from timing.json) +- `claims`: Extracted and verified claims from the output +- `user_notes_summary`: Issues flagged by the executor +- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising + +--- + +## metrics.json + +Output from the executor agent. Located at `/outputs/metrics.json`. + +```json +{ + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8, + "Edit": 1, + "Glob": 2, + "Grep": 0 + }, + "total_tool_calls": 18, + "total_steps": 6, + "files_created": ["filled_form.pdf", "field_values.json"], + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 +} +``` + +**Fields:** +- `tool_calls`: Count per tool type +- `total_tool_calls`: Sum of all tool calls +- `total_steps`: Number of major execution steps +- `files_created`: List of output files created +- `errors_encountered`: Number of errors during execution +- `output_chars`: Total character count of output files +- `transcript_chars`: Character count of transcript + +--- + +## timing.json + +Wall clock timing for a run. Located at `/timing.json`. + +**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact. + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3, + "executor_start": "2026-01-15T10:30:00Z", + "executor_end": "2026-01-15T10:32:45Z", + "executor_duration_seconds": 165.0, + "grader_start": "2026-01-15T10:32:46Z", + "grader_end": "2026-01-15T10:33:12Z", + "grader_duration_seconds": 26.0 +} +``` + +--- + +## benchmark.json + +Output from Benchmark mode. Located at `benchmarks//benchmark.json`. + +```json +{ + "metadata": { + "skill_name": "pdf", + "skill_path": "/path/to/pdf", + "executor_model": "claude-sonnet-4-20250514", + "analyzer_model": "most-capable-model", + "timestamp": "2026-01-15T10:30:00Z", + "evals_run": [1, 2, 3], + "runs_per_configuration": 3 + }, + + "runs": [ + { + "eval_id": 1, + "eval_name": "Ocean", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.85, + "passed": 6, + "failed": 1, + "total": 7, + "time_seconds": 42.5, + "tokens": 3800, + "tool_calls": 18, + "errors": 0 + }, + "expectations": [ + {"text": "...", "passed": true, "evidence": "..."} + ], + "notes": [ + "Used 2023 data, may be stale", + "Fell back to text overlay for non-fillable fields" + ] + } + ], + + "run_summary": { + "with_skill": { + "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90}, + "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0}, + "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100} + }, + "without_skill": { + "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45}, + "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0}, + "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500} + }, + "delta": { + "pass_rate": "+0.50", + "time_seconds": "+13.0", + "tokens": "+1700" + } + }, + + "notes": [ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" + ] +} +``` + +**Fields:** +- `metadata`: Information about the benchmark run + - `skill_name`: Name of the skill + - `timestamp`: When the benchmark was run + - `evals_run`: List of eval names or IDs + - `runs_per_configuration`: Number of runs per config (e.g. 3) +- `runs[]`: Individual run results + - `eval_id`: Numeric eval identifier + - `eval_name`: Human-readable eval name (used as section header in the viewer) + - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding) + - `run_number`: Integer run number (1, 2, 3...) + - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors` +- `run_summary`: Statistical aggregates per configuration + - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields + - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"` +- `notes`: Freeform observations from the analyzer + +**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually. + +--- + +## comparison.json + +Output from blind comparator. Located at `/comparison-N.json`. + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true} + ] + } + } +} +``` + +--- + +## analysis.json + +Output from post-hoc analyzer. Located at `/analysis.json`. + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": ["Minor: skipped optional logging step"] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods" + } +} +``` diff --git a/skills/ForgeSkill/scripts/.provenance/__init__.py.yaml b/skills/ForgeSkill/scripts/.provenance/__init__.py.yaml new file mode 100644 index 0000000..cd4b11d --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/__init__.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/__init__.py + digest: + sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/__init__.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/__init__.py + digest: + sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/aggregate_benchmark.py.yaml b/skills/ForgeSkill/scripts/.provenance/aggregate_benchmark.py.yaml new file mode 100644 index 0000000..ae8bf5d --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/aggregate_benchmark.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/aggregate_benchmark.py + digest: + sha256: 611193feb2263e0dae80f3dfc62c0467bf84d52e14ee491adcf99449ffaec6dd + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/aggregate_benchmark.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/aggregate_benchmark.py + digest: + sha256: 123ef128ea5ccc01a4b1ac212ef5567f21e9c13d3d240609780beeb3200c49aa + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/generate_report.py.yaml b/skills/ForgeSkill/scripts/.provenance/generate_report.py.yaml new file mode 100644 index 0000000..ebd2385 --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/generate_report.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/generate_report.py + digest: + sha256: 1ca34689d20bd882103f254392b3f26a190a15a405b667342eabf2c3d62633a3 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/generate_report.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/generate_report.py + digest: + sha256: 13df7118a3c50c83c4c3250a606d5f2b20b25a3d44cbc392b3d669ec75281453 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/improve_description.py.yaml b/skills/ForgeSkill/scripts/.provenance/improve_description.py.yaml new file mode 100644 index 0000000..9a52d7d --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/improve_description.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/improve_description.py + digest: + sha256: 686f346cd682687f16350aa509982f8c7dbc6d9b3bb8a3ad044d565ea21aef24 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/improve_description.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/improve_description.py + digest: + sha256: 87d864570220b699fac52da309d2d6efdb060647bfebc74f768128e646accf80 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/package_skill.py.yaml b/skills/ForgeSkill/scripts/.provenance/package_skill.py.yaml new file mode 100644 index 0000000..f1bb621 --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/package_skill.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/package_skill.py + digest: + sha256: 1a33059b0db1ef73375d46d513e5ea81369d2e8838c970597b0d52ddef8d1c0f + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/package_skill.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/package_skill.py + digest: + sha256: 1a33059b0db1ef73375d46d513e5ea81369d2e8838c970597b0d52ddef8d1c0f + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/quick_validate.py.yaml b/skills/ForgeSkill/scripts/.provenance/quick_validate.py.yaml new file mode 100644 index 0000000..472d543 --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/quick_validate.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/quick_validate.py + digest: + sha256: ac0f7db8fcf99755c12d05975cd0f33ac599a77094b828e0ff29d6b413ca5bd0 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/quick_validate.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/quick_validate.py + digest: + sha256: 67cf5703402013936c8fb75ad6a1afecd8841d45cc5e606b634eb05825fde365 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/run_eval.py.yaml b/skills/ForgeSkill/scripts/.provenance/run_eval.py.yaml new file mode 100644 index 0000000..0440c97 --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/run_eval.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/run_eval.py + digest: + sha256: 43e3b8f80dbf69c343967ba77e268fae991d9fa3ed68b32a0ff02532cd48657f + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/run_eval.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/run_eval.py + digest: + sha256: 43e3b8f80dbf69c343967ba77e268fae991d9fa3ed68b32a0ff02532cd48657f + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/run_loop.py.yaml b/skills/ForgeSkill/scripts/.provenance/run_loop.py.yaml new file mode 100644 index 0000000..33db09e --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/run_loop.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/run_loop.py + digest: + sha256: 472d58cb5cc4a6b34094dedc0279eda8f2bad3e20afe01ca37162a0c98865d53 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/run_loop.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/run_loop.py + digest: + sha256: 7bd6f674203168520517eec94c55f493c0d154339b061b4d7c0f0dad187d0f21 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/.provenance/utils.py.yaml b/skills/ForgeSkill/scripts/.provenance/utils.py.yaml new file mode 100644 index 0000000..29b170f --- /dev/null +++ b/skills/ForgeSkill/scripts/.provenance/utils.py.yaml @@ -0,0 +1,32 @@ +provenance: + _type: https://in-toto.io/Statement/v1 + subject: + - name: skills/ForgeSkill/scripts/utils.py + digest: + sha256: 3af8ae62c40c73ab712207436a0d9a981e845f25c5a7040229eb189cc8e45bb1 + predicateType: https://slsa.dev/provenance/v1 + predicate: + buildDefinition: + buildType: https://forge-cli/adopt/v1 + externalParameters: + upstream_url: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/utils.py + resolvedDependencies: + - name: upstream + uri: https://github.com/anthropics/skills/blob/b9e19e6f44773509fbdd7001d77ff41a49a486c1/skills/skill-creator/scripts/utils.py + digest: + sha256: 3af8ae62c40c73ab712207436a0d9a981e845f25c5a7040229eb189cc8e45bb1 + - name: AdoptArtifact + uri: forge-core/skills/AdoptArtifact/SKILL.md + digest: + sha256: 722d5e82ebc093749d170b4b5c650ca523e21a52a6079bfabd912613379c1c02 + - name: RefinePrompt + uri: forge-core/skills/RefinePrompt/SKILL.md + digest: + sha256: d04d2610148eb9ec2589cd521f1214b6a11d033600335c0295f2fc7a472ca87c + runDetails: + builder: + id: forge-cli + version: + forge: 0.3.2 + metadata: + startedOn: 2026-07-08T01:26:42Z diff --git a/skills/ForgeSkill/scripts/__init__.py b/skills/ForgeSkill/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/skills/ForgeSkill/scripts/aggregate_benchmark.py b/skills/ForgeSkill/scripts/aggregate_benchmark.py new file mode 100755 index 0000000..32c3560 --- /dev/null +++ b/skills/ForgeSkill/scripts/aggregate_benchmark.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Aggregate individual run results into benchmark summary statistics. + +Reads grading.json files from run directories and produces: +- run_summary with mean, stddev, min, max for each metric +- delta between with_skill and without_skill configurations + +Usage: + python aggregate_benchmark.py + +Example: + python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/ + +The script supports two directory layouts: + + Workspace layout (from skill-creator iterations): + / + └── eval-N/ + ├── with_skill/ + │ ├── run-1/grading.json + │ └── run-2/grading.json + └── without_skill/ + ├── run-1/grading.json + └── run-2/grading.json + + Legacy layout (with runs/ subdirectory): + / + └── runs/ + └── eval-N/ + ├── with_skill/ + │ └── run-1/grading.json + └── without_skill/ + └── run-1/grading.json +""" + +import argparse +import json +import math +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def calculate_stats(values: list[float]) -> dict: + """Calculate mean, stddev, min, max for a list of values.""" + if not values: + return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0} + + n = len(values) + mean = sum(values) / n + + if n > 1: + variance = sum((x - mean) ** 2 for x in values) / (n - 1) + stddev = math.sqrt(variance) + else: + stddev = 0.0 + + return { + "mean": round(mean, 4), + "stddev": round(stddev, 4), + "min": round(min(values), 4), + "max": round(max(values), 4) + } + + +def load_run_results(benchmark_dir: Path) -> dict: + """ + Load all run results from a benchmark directory. + + Returns dict keyed by config name (e.g. "with_skill"/"without_skill", + or "new_skill"/"old_skill"), each containing a list of run results. + """ + # Support both layouts: eval dirs directly under benchmark_dir, or under runs/ + runs_dir = benchmark_dir / "runs" + if runs_dir.exists(): + search_dir = runs_dir + elif list(benchmark_dir.glob("eval-*")): + search_dir = benchmark_dir + else: + print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}") + return {} + + results: dict[str, list] = {} + + for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))): + metadata_path = eval_dir / "eval_metadata.json" + if metadata_path.exists(): + try: + with open(metadata_path) as mf: + eval_id = json.load(mf).get("eval_id", eval_idx) + except (json.JSONDecodeError, OSError): + eval_id = eval_idx + else: + try: + eval_id = int(eval_dir.name.split("-")[1]) + except ValueError: + eval_id = eval_idx + + # Discover config directories dynamically rather than hardcoding names + for config_dir in sorted(eval_dir.iterdir()): + if not config_dir.is_dir(): + continue + # Skip non-config directories (inputs, outputs, etc.) + if not list(config_dir.glob("run-*")): + continue + config = config_dir.name + if config not in results: + results[config] = [] + + for run_dir in sorted(config_dir.glob("run-*")): + run_number = int(run_dir.name.split("-")[1]) + grading_file = run_dir / "grading.json" + + if not grading_file.exists(): + print(f"Warning: grading.json not found in {run_dir}") + continue + + try: + with open(grading_file) as f: + grading = json.load(f) + except json.JSONDecodeError as e: + print(f"Warning: Invalid JSON in {grading_file}: {e}") + continue + + # Extract metrics + result = { + "eval_id": eval_id, + "run_number": run_number, + "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0), + "passed": grading.get("summary", {}).get("passed", 0), + "failed": grading.get("summary", {}).get("failed", 0), + "total": grading.get("summary", {}).get("total", 0), + } + + # Extract timing — check grading.json first, then sibling timing.json + timing = grading.get("timing", {}) + result["time_seconds"] = timing.get("total_duration_seconds", 0.0) + timing_file = run_dir / "timing.json" + if result["time_seconds"] == 0.0 and timing_file.exists(): + try: + with open(timing_file) as tf: + timing_data = json.load(tf) + result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0) + result["tokens"] = timing_data.get("total_tokens", 0) + except json.JSONDecodeError: + pass + + # Extract metrics if available + metrics = grading.get("execution_metrics", {}) + result["tool_calls"] = metrics.get("total_tool_calls", 0) + if not result.get("tokens"): + result["tokens"] = metrics.get("output_chars", 0) + result["errors"] = metrics.get("errors_encountered", 0) + + # Extract expectations — viewer requires fields: text, passed, evidence + raw_expectations = grading.get("expectations", []) + for exp in raw_expectations: + if "text" not in exp or "passed" not in exp: + print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}") + result["expectations"] = raw_expectations + + # Extract notes from user_notes_summary + notes_summary = grading.get("user_notes_summary", {}) + notes = [] + notes.extend(notes_summary.get("uncertainties", [])) + notes.extend(notes_summary.get("needs_review", [])) + notes.extend(notes_summary.get("workarounds", [])) + result["notes"] = notes + + results[config].append(result) + + return results + + +def aggregate_results(results: dict) -> dict: + """ + Aggregate run results into summary statistics. + + Returns run_summary with stats for each configuration and delta. + """ + run_summary = {} + configs = list(results.keys()) + + for config in configs: + runs = results.get(config, []) + + if not runs: + run_summary[config] = { + "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0} + } + continue + + pass_rates = [r["pass_rate"] for r in runs] + times = [r["time_seconds"] for r in runs] + tokens = [r.get("tokens", 0) for r in runs] + + run_summary[config] = { + "pass_rate": calculate_stats(pass_rates), + "time_seconds": calculate_stats(times), + "tokens": calculate_stats(tokens) + } + + # Calculate delta between the first two configs (if two exist) + if len(configs) >= 2: + primary = run_summary.get(configs[0], {}) + baseline = run_summary.get(configs[1], {}) + else: + primary = run_summary.get(configs[0], {}) if configs else {} + baseline = {} + + delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0) + delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0) + delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0) + + run_summary["delta"] = { + "pass_rate": f"{delta_pass_rate:+.2f}", + "time_seconds": f"{delta_time:+.1f}", + "tokens": f"{delta_tokens:+.0f}" + } + + return run_summary + + +def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict: + """ + Generate complete benchmark.json from run results. + """ + results = load_run_results(benchmark_dir) + run_summary = aggregate_results(results) + + # Build runs array for benchmark.json + runs = [] + for config in results: + for result in results[config]: + runs.append({ + "eval_id": result["eval_id"], + "configuration": config, + "run_number": result["run_number"], + "result": { + "pass_rate": result["pass_rate"], + "passed": result["passed"], + "failed": result["failed"], + "total": result["total"], + "time_seconds": result["time_seconds"], + "tokens": result.get("tokens", 0), + "tool_calls": result.get("tool_calls", 0), + "errors": result.get("errors", 0) + }, + "expectations": result["expectations"], + "notes": result["notes"] + }) + + # Determine eval IDs from results + eval_ids = sorted(set( + r["eval_id"] + for config in results.values() + for r in config + )) + + benchmark = { + "metadata": { + "skill_name": skill_name or "", + "skill_path": skill_path or "", + "executor_model": "", + "analyzer_model": "", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evals_run": eval_ids, + "runs_per_configuration": 3 + }, + "runs": runs, + "run_summary": run_summary, + "notes": [] # To be filled by analyzer + } + + return benchmark + + +def generate_markdown(benchmark: dict) -> str: + """Generate human-readable benchmark.md from benchmark data.""" + metadata = benchmark["metadata"] + run_summary = benchmark["run_summary"] + + # Determine config names (excluding "delta") + configs = [k for k in run_summary if k != "delta"] + config_a = configs[0] if len(configs) >= 1 else "config_a" + config_b = configs[1] if len(configs) >= 2 else "config_b" + label_a = config_a.replace("_", " ").title() + label_b = config_b.replace("_", " ").title() + + lines = [ + f"# Skill Benchmark: {metadata['skill_name']}", + "", + f"**Model**: {metadata['executor_model']}", + f"**Date**: {metadata['timestamp']}", + f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)", + "", + "## Summary", + "", + f"| Metric | {label_a} | {label_b} | Delta |", + "|--------|------------|---------------|-------|", + ] + + a_summary = run_summary.get(config_a, {}) + b_summary = run_summary.get(config_b, {}) + delta = run_summary.get("delta", {}) + + # Format pass rate + a_pr = a_summary.get("pass_rate", {}) + b_pr = b_summary.get("pass_rate", {}) + lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |") + + # Format time + a_time = a_summary.get("time_seconds", {}) + b_time = b_summary.get("time_seconds", {}) + lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |") + + # Format tokens + a_tokens = a_summary.get("tokens", {}) + b_tokens = b_summary.get("tokens", {}) + lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |") + + # Notes section + if benchmark.get("notes"): + lines.extend([ + "", + "## Notes", + "" + ]) + for note in benchmark["notes"]: + lines.append(f"- {note}") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate benchmark run results into summary statistics" + ) + parser.add_argument( + "benchmark_dir", + type=Path, + help="Path to the benchmark directory" + ) + parser.add_argument( + "--skill-name", + default="", + help="Name of the skill being benchmarked" + ) + parser.add_argument( + "--skill-path", + default="", + help="Path to the skill being benchmarked" + ) + parser.add_argument( + "--output", "-o", + type=Path, + help="Output path for benchmark.json (default: /benchmark.json)" + ) + + args = parser.parse_args() + + if not args.benchmark_dir.exists(): + print(f"Directory not found: {args.benchmark_dir}") + sys.exit(1) + + # Generate benchmark + benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path) + + # Determine output paths + output_json = args.output or (args.benchmark_dir / "benchmark.json") + output_md = output_json.with_suffix(".md") + + # Write benchmark.json + with open(output_json, "w") as f: + json.dump(benchmark, f, indent=2) + print(f"Generated: {output_json}") + + # Write benchmark.md + markdown = generate_markdown(benchmark) + with open(output_md, "w") as f: + f.write(markdown) + print(f"Generated: {output_md}") + + # Print summary + run_summary = benchmark["run_summary"] + configs = [k for k in run_summary if k != "delta"] + delta = run_summary.get("delta", {}) + + print("\nSummary:") + for config in configs: + pr = run_summary[config]["pass_rate"]["mean"] + label = config.replace("_", " ").title() + print(f" {label}: {pr*100:.1f}% pass rate") + print(f" Delta: {delta.get('pass_rate', '—')}") + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/generate_report.py b/skills/ForgeSkill/scripts/generate_report.py new file mode 100755 index 0000000..ea4a861 --- /dev/null +++ b/skills/ForgeSkill/scripts/generate_report.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Generate an HTML report from run_loop.py output. + +Takes the JSON output from run_loop.py and generates a visual HTML report +showing each description attempt with check/x for each test case. +Distinguishes between train and test queries. +""" + +import argparse +import html +import json +import sys +from pathlib import Path + + +def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str: + """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag.""" + history = data.get("history", []) + data.get("holdout", 0) + title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else "" + + # Get all unique queries from train and test sets, with should_trigger info + train_queries: list[dict] = [] + test_queries: list[dict] = [] + if history: + for r in history[0].get("train_results", history[0].get("results", [])): + train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + if history[0].get("test_results"): + for r in history[0].get("test_results", []): + test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + + refresh_tag = ' \n' if auto_refresh else "" + + html_parts = [""" + + + +""" + refresh_tag + """ """ + title_prefix + """Skill Description Optimization + + + + + + +

""" + title_prefix + """Skill Description Optimization

+
+ Optimizing your skill's description. This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill. +
+"""] + + # Summary section + best_test_score = data.get('best_test_score') + data.get('best_train_score') + html_parts.append(f""" +
+

Original: {html.escape(data.get('original_description', 'N/A'))}

+

Best: {html.escape(data.get('best_description', 'N/A'))}

+

Best Score: {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}

+

Iterations: {data.get('iterations_run', 0)} | Train: {data.get('train_size', '?')} | Test: {data.get('test_size', '?')}

+
+""") + + # Legend + html_parts.append(""" +
+ Query columns: + Should trigger + Should NOT trigger + Train + Test +
+""") + + # Table header + html_parts.append(""" +
+ + + + + + + +""") + + # Add column headers for train queries + for qinfo in train_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + # Add column headers for test queries (different color) + for qinfo in test_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + html_parts.append(""" + + +""") + + # Find best iteration for highlighting + if test_queries: + best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration") + else: + best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration") + + # Add rows for each iteration + for h in history: + iteration = h.get("iteration", "?") + h.get("train_passed", h.get("passed", 0)) + h.get("train_total", h.get("total", 0)) + h.get("test_passed") + h.get("test_total") + description = h.get("description", "") + train_results = h.get("train_results", h.get("results", [])) + test_results = h.get("test_results", []) + + # Create lookups for results by query + train_by_query = {r["query"]: r for r in train_results} + test_by_query = {r["query"]: r for r in test_results} if test_results else {} + + # Compute aggregate correct/total runs across all retries + def aggregate_runs(results: list[dict]) -> tuple[int, int]: + correct = 0 + total = 0 + for r in results: + runs = r.get("runs", 0) + triggers = r.get("triggers", 0) + total += runs + if r.get("should_trigger", True): + correct += triggers + else: + correct += runs - triggers + return correct, total + + train_correct, train_runs = aggregate_runs(train_results) + test_correct, test_runs = aggregate_runs(test_results) + + # Determine score classes + def score_class(correct: int, total: int) -> str: + if total > 0: + ratio = correct / total + if ratio >= 0.8: + return "score-good" + elif ratio >= 0.5: + return "score-ok" + return "score-bad" + + train_class = score_class(train_correct, train_runs) + test_class = score_class(test_correct, test_runs) + + row_class = "best-row" if iteration == best_iter else "" + + html_parts.append(f""" + + + + +""") + + # Add result for each train query + for qinfo in train_queries: + r = train_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + # Add result for each test query (with different background) + for qinfo in test_queries: + r = test_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + html_parts.append(" \n") + + html_parts.append(""" +
IterTrainTestDescription{html.escape(qinfo["query"])}{html.escape(qinfo["query"])}
{iteration}{train_correct}/{train_runs}{test_correct}/{test_runs}{html.escape(description)}{icon}{triggers}/{runs}{icon}{triggers}/{runs}
+
+""") + + html_parts.append(""" + + +""") + + return "".join(html_parts) + + +def main(): + parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output") + parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)") + parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)") + parser.add_argument("--skill-name", default="", help="Skill name to include in the report title") + args = parser.parse_args() + + if args.input == "-": + data = json.load(sys.stdin) + else: + data = json.loads(Path(args.input).read_text()) + + html_output = generate_html(data, skill_name=args.skill_name) + + if args.output: + Path(args.output).write_text(html_output) + print(f"Report written to {args.output}", file=sys.stderr) + else: + print(html_output) + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/improve_description.py b/skills/ForgeSkill/scripts/improve_description.py new file mode 100755 index 0000000..9a3ad7e --- /dev/null +++ b/skills/ForgeSkill/scripts/improve_description.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Improve a skill description based on eval results. + +Takes eval results (from run_eval.py) and generates an improved description +by calling `claude -p` as a subprocess (same auth pattern as run_eval.py — +uses the session's Claude Code auth, no separate ANTHROPIC_API_KEY needed). +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def _call_claude(prompt: str, model: str | None, timeout: int = 300) -> str: + """Run `claude -p` with the prompt on stdin and return the text response. + + Prompt goes over stdin (not argv) because it embeds the full SKILL.md + body and can easily exceed comfortable argv length. + """ + cmd = ["claude", "-p", "--output-format", "text"] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. Same pattern as run_eval.py. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + result = subprocess.run( + cmd, + input=prompt, + capture_output=True, + text=True, + env=env, + timeout=timeout, + ) + if result.returncode != 0: + raise RuntimeError( + f"claude -p exited {result.returncode}\nstderr: {result.stderr}" + ) + return result.stdout + + +def improve_description( + skill_name: str, + skill_content: str, + current_description: str, + eval_results: dict, + history: list[dict], + model: str, + test_results: dict | None = None, + log_dir: Path | None = None, + iteration: int | None = None, +) -> str: + """Call Claude to improve the description based on eval results.""" + failed_triggers = [ + r for r in eval_results["results"] + if r["should_trigger"] and not r["pass"] + ] + false_triggers = [ + r for r in eval_results["results"] + if not r["should_trigger"] and not r["pass"] + ] + + # Build scores summary + train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}" + if test_results: + test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}" + scores_summary = f"Train: {train_score}, Test: {test_score}" + else: + scores_summary = f"Train: {train_score}" + + prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples. + +The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones. + +Here's the current description: + +"{current_description}" + + +Current scores ({scores_summary}): + +""" + if failed_triggers: + prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n" + for r in failed_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if false_triggers: + prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n" + for r in false_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if history: + prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n" + for h in history: + train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}" + test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None + score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "") + prompt += f'\n' + prompt += f'Description: "{h["description"]}"\n' + if "results" in h: + prompt += "Train results:\n" + for r in h["results"]: + status = "PASS" if r["pass"] else "FAIL" + prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n' + if h.get("note"): + prompt += f'Note: {h["note"]}\n' + prompt += "\n\n" + + prompt += f""" + +Skill content (for context on what the skill does): + +{skill_content} + + +Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold: + +1. Avoid overfitting +2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description. + +Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. There is a hard limit of 1024 characters — descriptions over that will be truncated, so stay comfortably under it. + +Here are some tips that we've found to work well in writing these descriptions: +- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does" +- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works. +- The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable. +- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings. + +I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end. + +Please respond with only the new description text in tags, nothing else.""" + + text = _call_claude(prompt, model) + + match = re.search(r"(.*?)", text, re.DOTALL) + description = match.group(1).strip().strip('"') if match else text.strip().strip('"') + + transcript: dict = { + "iteration": iteration, + "prompt": prompt, + "response": text, + "parsed_description": description, + "char_count": len(description), + "over_limit": len(description) > 1024, + } + + # Safety net: the prompt already states the 1024-char hard limit, but if + # the model blew past it anyway, make one fresh single-turn call that + # quotes the too-long version and asks for a shorter rewrite. (The old + # SDK path did this as a true multi-turn; `claude -p` is one-shot, so we + # inline the prior output into the new prompt instead.) + if len(description) > 1024: + shorten_prompt = ( + f"{prompt}\n\n" + f"---\n\n" + f"A previous attempt produced this description, which at " + f"{len(description)} characters is over the 1024-character hard limit:\n\n" + f'"{description}"\n\n' + f"Rewrite it to be under 1024 characters while keeping the most " + f"important trigger words and intent coverage. Respond with only " + f"the new description in tags." + ) + shorten_text = _call_claude(shorten_prompt, model) + match = re.search(r"(.*?)", shorten_text, re.DOTALL) + shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"') + + transcript["rewrite_prompt"] = shorten_prompt + transcript["rewrite_response"] = shorten_text + transcript["rewrite_description"] = shortened + transcript["rewrite_char_count"] = len(shortened) + description = shortened + + transcript["final_description"] = description + + if log_dir: + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json" + log_file.write_text(json.dumps(transcript, indent=2)) + + return description + + +def main(): + parser = argparse.ArgumentParser(description="Improve a skill description based on eval results") + parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr") + args = parser.parse_args() + + skill_path = Path(args.skill_path) + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + eval_results = json.loads(Path(args.eval_results).read_text()) + history = [] + if args.history: + history = json.loads(Path(args.history).read_text()) + + name, _, content = parse_skill_md(skill_path) + current_description = eval_results["description"] + + if args.verbose: + print(f"Current: {current_description}", file=sys.stderr) + print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr) + + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=eval_results, + history=history, + model=args.model, + ) + + if args.verbose: + print(f"Improved: {new_description}", file=sys.stderr) + + # Output as JSON with both the new description and updated history + output = { + "description": new_description, + "history": history + [{ + "description": current_description, + "passed": eval_results["summary"]["passed"], + "failed": eval_results["summary"]["failed"], + "total": eval_results["summary"]["total"], + "results": eval_results["results"], + }], + } + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/package_skill.py b/skills/ForgeSkill/scripts/package_skill.py new file mode 100755 index 0000000..f48eac4 --- /dev/null +++ b/skills/ForgeSkill/scripts/package_skill.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Skill Packager - Creates a distributable .skill file of a skill folder + +Usage: + python utils/package_skill.py [output-directory] + +Example: + python utils/package_skill.py skills/public/my-skill + python utils/package_skill.py skills/public/my-skill ./dist +""" + +import fnmatch +import sys +import zipfile +from pathlib import Path +from scripts.quick_validate import validate_skill + +# Patterns to exclude when packaging skills. +EXCLUDE_DIRS = {"__pycache__", "node_modules"} +EXCLUDE_GLOBS = {"*.pyc"} +EXCLUDE_FILES = {".DS_Store"} +# Directories excluded only at the skill root (not when nested deeper). +ROOT_EXCLUDE_DIRS = {"evals"} + + +def should_exclude(rel_path: Path) -> bool: + """Check if a path should be excluded from packaging.""" + parts = rel_path.parts + if any(part in EXCLUDE_DIRS for part in parts): + return True + # rel_path is relative to skill_path.parent, so parts[0] is the skill + # folder name and parts[1] (if present) is the first subdir. + if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS: + return True + name = rel_path.name + if name in EXCLUDE_FILES: + return True + return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS) + + +def package_skill(skill_path, output_dir=None): + """ + Package a skill folder into a .skill file. + + Args: + skill_path: Path to the skill folder + output_dir: Optional output directory for the .skill file (defaults to current directory) + + Returns: + Path to the created .skill file, or None if error + """ + skill_path = Path(skill_path).resolve() + + # Validate skill folder exists + if not skill_path.exists(): + print(f"❌ Error: Skill folder not found: {skill_path}") + return None + + if not skill_path.is_dir(): + print(f"❌ Error: Path is not a directory: {skill_path}") + return None + + # Validate SKILL.md exists + skill_md = skill_path / "SKILL.md" + if not skill_md.exists(): + print(f"❌ Error: SKILL.md not found in {skill_path}") + return None + + # Run validation before packaging + print("🔍 Validating skill...") + valid, message = validate_skill(skill_path) + if not valid: + print(f"❌ Validation failed: {message}") + print(" Please fix the validation errors before packaging.") + return None + print(f"✅ {message}\n") + + # Determine output location + skill_name = skill_path.name + if output_dir: + output_path = Path(output_dir).resolve() + output_path.mkdir(parents=True, exist_ok=True) + else: + output_path = Path.cwd() + + skill_filename = output_path / f"{skill_name}.skill" + + # Create the .skill file (zip format) + try: + with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Walk through the skill directory, excluding build artifacts + for file_path in skill_path.rglob('*'): + if not file_path.is_file(): + continue + arcname = file_path.relative_to(skill_path.parent) + if should_exclude(arcname): + print(f" Skipped: {arcname}") + continue + zipf.write(file_path, arcname) + print(f" Added: {arcname}") + + print(f"\n✅ Successfully packaged skill to: {skill_filename}") + return skill_filename + + except Exception as e: + print(f"❌ Error creating .skill file: {e}") + return None + + +def main(): + if len(sys.argv) < 2: + print("Usage: python utils/package_skill.py [output-directory]") + print("\nExample:") + print(" python utils/package_skill.py skills/public/my-skill") + print(" python utils/package_skill.py skills/public/my-skill ./dist") + sys.exit(1) + + skill_path = sys.argv[1] + output_dir = sys.argv[2] if len(sys.argv) > 2 else None + + print(f"📦 Packaging skill: {skill_path}") + if output_dir: + print(f" Output directory: {output_dir}") + print() + + result = package_skill(skill_path, output_dir) + + if result: + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/quick_validate.py b/skills/ForgeSkill/scripts/quick_validate.py new file mode 100755 index 0000000..cf1169b --- /dev/null +++ b/skills/ForgeSkill/scripts/quick_validate.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Quick validation script for skills - minimal version +""" + +import sys +import re +import yaml +from pathlib import Path + +def validate_skill(skill_path): + """Basic validation of a skill""" + skill_path = Path(skill_path) + + # Check SKILL.md exists + skill_md = skill_path / 'SKILL.md' + if not skill_md.exists(): + return False, "SKILL.md not found" + + # Read and validate frontmatter + content = skill_md.read_text() + if not content.startswith('---'): + return False, "No YAML frontmatter found" + + # Extract frontmatter + match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL) + if not match: + return False, "Invalid frontmatter format" + + frontmatter_text = match.group(1) + + # Parse YAML frontmatter + try: + frontmatter = yaml.safe_load(frontmatter_text) + if not isinstance(frontmatter, dict): + return False, "Frontmatter must be a YAML dictionary" + except yaml.YAMLError as e: + return False, f"Invalid YAML in frontmatter: {e}" + + # Define allowed properties + ALLOWED_PROPERTIES = {'name', 'description', 'license', 'allowed-tools', 'metadata', 'compatibility'} + + # Check for unexpected properties (excluding nested keys under metadata) + unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES + if unexpected_keys: + return False, ( + f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. " + f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}" + ) + + # Check required fields + if 'name' not in frontmatter: + return False, "Missing 'name' in frontmatter" + if 'description' not in frontmatter: + return False, "Missing 'description' in frontmatter" + + # Extract name for validation + name = frontmatter.get('name', '') + if not isinstance(name, str): + return False, f"Name must be a string, got {type(name).__name__}" + name = name.strip() + if name: + # Check naming convention (kebab-case: lowercase with hyphens) + if not re.match(r'^[a-z0-9-]+$', name): + return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)" + if name.startswith('-') or name.endswith('-') or '--' in name: + return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Check name length (max 64 characters per spec) + if len(name) > 64: + return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters." + + # Extract and validate description + description = frontmatter.get('description', '') + if not isinstance(description, str): + return False, f"Description must be a string, got {type(description).__name__}" + description = description.strip() + if description: + # Check for angle brackets + if '<' in description or '>' in description: + return False, "Description cannot contain angle brackets (< or >)" + # Check description length (max 1024 characters per spec) + if len(description) > 1024: + return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters." + + # Validate compatibility field if present (optional) + compatibility = frontmatter.get('compatibility', '') + if compatibility: + if not isinstance(compatibility, str): + return False, f"Compatibility must be a string, got {type(compatibility).__name__}" + if len(compatibility) > 500: + return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters." + + return True, "Skill is valid!" + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python quick_validate.py ") + sys.exit(1) + + valid, message = validate_skill(sys.argv[1]) + print(message) + sys.exit(0 if valid else 1) \ No newline at end of file diff --git a/skills/ForgeSkill/scripts/run_eval.py b/skills/ForgeSkill/scripts/run_eval.py new file mode 100755 index 0000000..e58c70b --- /dev/null +++ b/skills/ForgeSkill/scripts/run_eval.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Run trigger evaluation for a skill description. + +Tests whether a skill's description causes Claude to trigger (read the skill) +for a set of queries. Outputs results as JSON. +""" + +import argparse +import json +import os +import select +import subprocess +import sys +import time +import uuid +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def find_project_root() -> Path: + """Find the project root by walking up from cwd looking for .claude/. + + Mimics how Claude Code discovers its project root, so the command file + we create ends up where claude -p will look for it. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + if (parent / ".claude").is_dir(): + return parent + return current + + +def run_single_query( + query: str, + skill_name: str, + skill_description: str, + timeout: int, + project_root: str, + model: str | None = None, +) -> bool: + """Run a single query and return whether the skill was triggered. + + Creates a command file in .claude/commands/ so it appears in Claude's + available_skills list, then runs `claude -p` with the raw query. + Uses --include-partial-messages to detect triggering early from + stream events (content_block_start) rather than waiting for the + full assistant message, which only arrives after tool execution. + """ + unique_id = uuid.uuid4().hex[:8] + clean_name = f"{skill_name}-skill-{unique_id}" + project_commands_dir = Path(project_root) / ".claude" / "commands" + command_file = project_commands_dir / f"{clean_name}.md" + + try: + project_commands_dir.mkdir(parents=True, exist_ok=True) + # Use YAML block scalar to avoid breaking on quotes in description + indented_desc = "\n ".join(skill_description.split("\n")) + command_content = ( + f"---\n" + f"description: |\n" + f" {indented_desc}\n" + f"---\n\n" + f"# {skill_name}\n\n" + f"This skill handles: {skill_description}\n" + ) + command_file.write_text(command_content) + + cmd = [ + "claude", + "-p", query, + "--output-format", "stream-json", + "--verbose", + "--include-partial-messages", + ] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=project_root, + env=env, + ) + + triggered = False + start_time = time.time() + buffer = "" + # Track state for stream event detection + pending_tool_name = None + accumulated_json = "" + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + remaining = process.stdout.read() + if remaining: + buffer += remaining.decode("utf-8", errors="replace") + break + + ready, _, _ = select.select([process.stdout], [], [], 1.0) + if not ready: + continue + + chunk = os.read(process.stdout.fileno(), 8192) + if not chunk: + break + buffer += chunk.decode("utf-8", errors="replace") + + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + if not line: + continue + + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + # Early detection via stream events + if event.get("type") == "stream_event": + se = event.get("event", {}) + se_type = se.get("type", "") + + if se_type == "content_block_start": + cb = se.get("content_block", {}) + if cb.get("type") == "tool_use": + tool_name = cb.get("name", "") + if tool_name in ("Skill", "Read"): + pending_tool_name = tool_name + accumulated_json = "" + else: + return False + + elif se_type == "content_block_delta" and pending_tool_name: + delta = se.get("delta", {}) + if delta.get("type") == "input_json_delta": + accumulated_json += delta.get("partial_json", "") + if clean_name in accumulated_json: + return True + + elif se_type in ("content_block_stop", "message_stop"): + if pending_tool_name: + return clean_name in accumulated_json + if se_type == "message_stop": + return False + + # Fallback: full assistant message + elif event.get("type") == "assistant": + message = event.get("message", {}) + for content_item in message.get("content", []): + if content_item.get("type") != "tool_use": + continue + tool_name = content_item.get("name", "") + tool_input = content_item.get("input", {}) + if tool_name == "Skill" and clean_name in tool_input.get("skill", ""): + triggered = True + elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""): + triggered = True + return triggered + + elif event.get("type") == "result": + return triggered + finally: + # Clean up process on any exit path (return, exception, timeout) + if process.poll() is None: + process.kill() + process.wait() + + return triggered + finally: + if command_file.exists(): + command_file.unlink() + + +def run_eval( + eval_set: list[dict], + skill_name: str, + description: str, + num_workers: int, + timeout: int, + project_root: Path, + runs_per_query: int = 1, + trigger_threshold: float = 0.5, + model: str | None = None, +) -> dict: + """Run the full eval set and return results.""" + results = [] + + with ProcessPoolExecutor(max_workers=num_workers) as executor: + future_to_info = {} + for item in eval_set: + for run_idx in range(runs_per_query): + future = executor.submit( + run_single_query, + item["query"], + skill_name, + description, + timeout, + str(project_root), + model, + ) + future_to_info[future] = (item, run_idx) + + query_triggers: dict[str, list[bool]] = {} + query_items: dict[str, dict] = {} + for future in as_completed(future_to_info): + item, _ = future_to_info[future] + query = item["query"] + query_items[query] = item + if query not in query_triggers: + query_triggers[query] = [] + try: + query_triggers[query].append(future.result()) + except Exception as e: + print(f"Warning: query failed: {e}", file=sys.stderr) + query_triggers[query].append(False) + + for query, triggers in query_triggers.items(): + item = query_items[query] + trigger_rate = sum(triggers) / len(triggers) + should_trigger = item["should_trigger"] + if should_trigger: + did_pass = trigger_rate >= trigger_threshold + else: + did_pass = trigger_rate < trigger_threshold + results.append({ + "query": query, + "should_trigger": should_trigger, + "trigger_rate": trigger_rate, + "triggers": sum(triggers), + "runs": len(triggers), + "pass": did_pass, + }) + + passed = sum(1 for r in results if r["pass"]) + total = len(results) + + return { + "skill_name": skill_name, + "description": description, + "results": results, + "summary": { + "total": total, + "passed": passed, + "failed": total - passed, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override description to test") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, original_description, content = parse_skill_md(skill_path) + description = args.description or original_description + project_root = find_project_root() + + if args.verbose: + print(f"Evaluating: {description}", file=sys.stderr) + + output = run_eval( + eval_set=eval_set, + skill_name=name, + description=description, + num_workers=args.num_workers, + timeout=args.timeout, + project_root=project_root, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + model=args.model, + ) + + if args.verbose: + summary = output["summary"] + print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr) + for r in output["results"]: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr) + + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/run_loop.py b/skills/ForgeSkill/scripts/run_loop.py new file mode 100755 index 0000000..5a9f979 --- /dev/null +++ b/skills/ForgeSkill/scripts/run_loop.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Run the eval + improve loop until all pass or max iterations reached. + +Combines run_eval.py and improve_description.py in a loop, tracking history +and returning the best description found. Supports train/test split to prevent +overfitting. +""" + +import argparse +import json +import random +import sys +import tempfile +import time +import webbrowser +from pathlib import Path + +from scripts.generate_report import generate_html +from scripts.improve_description import improve_description +from scripts.run_eval import find_project_root, run_eval +from scripts.utils import parse_skill_md + + +def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]: + """Split eval set into train and test sets, stratified by should_trigger.""" + random.seed(seed) + + # Separate by should_trigger + trigger = [e for e in eval_set if e["should_trigger"]] + no_trigger = [e for e in eval_set if not e["should_trigger"]] + + # Shuffle each group + random.shuffle(trigger) + random.shuffle(no_trigger) + + # Calculate split points + n_trigger_test = max(1, int(len(trigger) * holdout)) + n_no_trigger_test = max(1, int(len(no_trigger) * holdout)) + + # Split + test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test] + train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:] + + return train_set, test_set + + +def run_loop( + eval_set: list[dict], + skill_path: Path, + description_override: str | None, + num_workers: int, + timeout: int, + max_iterations: int, + runs_per_query: int, + trigger_threshold: float, + holdout: float, + model: str, + verbose: bool, + live_report_path: Path | None = None, + log_dir: Path | None = None, +) -> dict: + """Run the eval + improvement loop.""" + project_root = find_project_root() + name, original_description, content = parse_skill_md(skill_path) + current_description = description_override or original_description + + # Split into train/test if holdout > 0 + if holdout > 0: + train_set, test_set = split_eval_set(eval_set, holdout) + if verbose: + print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr) + else: + train_set = eval_set + test_set = [] + + history = [] + exit_reason = "unknown" + + for iteration in range(1, max_iterations + 1): + if verbose: + print(f"\n{'='*60}", file=sys.stderr) + print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr) + print(f"Description: {current_description}", file=sys.stderr) + print(f"{'='*60}", file=sys.stderr) + + # Evaluate train + test together in one batch for parallelism + all_queries = train_set + test_set + t0 = time.time() + all_results = run_eval( + eval_set=all_queries, + skill_name=name, + description=current_description, + num_workers=num_workers, + timeout=timeout, + project_root=project_root, + runs_per_query=runs_per_query, + trigger_threshold=trigger_threshold, + model=model, + ) + eval_elapsed = time.time() - t0 + + # Split results back into train/test by matching queries + train_queries_set = {q["query"] for q in train_set} + train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set] + test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set] + + train_passed = sum(1 for r in train_result_list if r["pass"]) + train_total = len(train_result_list) + train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total} + train_results = {"results": train_result_list, "summary": train_summary} + + if test_set: + test_passed = sum(1 for r in test_result_list if r["pass"]) + test_total = len(test_result_list) + test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total} + test_results = {"results": test_result_list, "summary": test_summary} + else: + test_results = None + test_summary = None + + history.append({ + "iteration": iteration, + "description": current_description, + "train_passed": train_summary["passed"], + "train_failed": train_summary["failed"], + "train_total": train_summary["total"], + "train_results": train_results["results"], + "test_passed": test_summary["passed"] if test_summary else None, + "test_failed": test_summary["failed"] if test_summary else None, + "test_total": test_summary["total"] if test_summary else None, + "test_results": test_results["results"] if test_results else None, + # For backward compat with report generator + "passed": train_summary["passed"], + "failed": train_summary["failed"], + "total": train_summary["total"], + "results": train_results["results"], + }) + + # Write live report if path provided + if live_report_path: + partial_output = { + "original_description": original_description, + "best_description": current_description, + "best_score": "in progress", + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name)) + + if verbose: + def print_eval_stats(label, results, elapsed): + pos = [r for r in results if r["should_trigger"]] + neg = [r for r in results if not r["should_trigger"]] + tp = sum(r["triggers"] for r in pos) + pos_runs = sum(r["runs"] for r in pos) + fn = pos_runs - tp + fp = sum(r["triggers"] for r in neg) + neg_runs = sum(r["runs"] for r in neg) + tn = neg_runs - fp + total = tp + tn + fp + fn + precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0 + accuracy = (tp + tn) / total if total > 0 else 0.0 + print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr) + + print_eval_stats("Train", train_results["results"], eval_elapsed) + if test_summary: + print_eval_stats("Test ", test_results["results"], 0) + + if train_summary["failed"] == 0: + exit_reason = f"all_passed (iteration {iteration})" + if verbose: + print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr) + break + + if iteration == max_iterations: + exit_reason = f"max_iterations ({max_iterations})" + if verbose: + print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr) + break + + # Improve the description based on train results + if verbose: + print("\nImproving description...", file=sys.stderr) + + t0 = time.time() + # Strip test scores from history so improvement model can't see them + blinded_history = [ + {k: v for k, v in h.items() if not k.startswith("test_")} + for h in history + ] + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=train_results, + history=blinded_history, + model=model, + log_dir=log_dir, + iteration=iteration, + ) + improve_elapsed = time.time() - t0 + + if verbose: + print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr) + + current_description = new_description + + # Find the best iteration by TEST score (or train if no test set) + if test_set: + best = max(history, key=lambda h: h["test_passed"] or 0) + best_score = f"{best['test_passed']}/{best['test_total']}" + else: + best = max(history, key=lambda h: h["train_passed"]) + best_score = f"{best['train_passed']}/{best['train_total']}" + + if verbose: + print(f"\nExit reason: {exit_reason}", file=sys.stderr) + print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr) + + return { + "exit_reason": exit_reason, + "original_description": original_description, + "best_description": best["description"], + "best_score": best_score, + "best_train_score": f"{best['train_passed']}/{best['train_total']}", + "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None, + "final_description": current_description, + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run eval + improve loop") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override starting description") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)") + parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, _, _ = parse_skill_md(skill_path) + + # Set up live report path + if args.report != "none": + if args.report == "auto": + timestamp = time.strftime("%Y%m%d_%H%M%S") + live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html" + else: + live_report_path = Path(args.report) + # Open the report immediately so the user can watch + live_report_path.write_text("

Starting optimization loop...

") + webbrowser.open(str(live_report_path)) + else: + live_report_path = None + + # Determine output directory (create before run_loop so logs can be written) + if args.results_dir: + timestamp = time.strftime("%Y-%m-%d_%H%M%S") + results_dir = Path(args.results_dir) / timestamp + results_dir.mkdir(parents=True, exist_ok=True) + else: + results_dir = None + + log_dir = results_dir / "logs" if results_dir else None + + output = run_loop( + eval_set=eval_set, + skill_path=skill_path, + description_override=args.description, + num_workers=args.num_workers, + timeout=args.timeout, + max_iterations=args.max_iterations, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + holdout=args.holdout, + model=args.model, + verbose=args.verbose, + live_report_path=live_report_path, + log_dir=log_dir, + ) + + # Save JSON output + json_output = json.dumps(output, indent=2) + print(json_output) + if results_dir: + (results_dir / "results.json").write_text(json_output) + + # Write final HTML report (without auto-refresh) + if live_report_path: + live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name)) + print(f"\nReport: {live_report_path}", file=sys.stderr) + + if results_dir and live_report_path: + (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name)) + + if results_dir: + print(f"Results saved to: {results_dir}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/ForgeSkill/scripts/utils.py b/skills/ForgeSkill/scripts/utils.py new file mode 100644 index 0000000..51b6a07 --- /dev/null +++ b/skills/ForgeSkill/scripts/utils.py @@ -0,0 +1,47 @@ +"""Shared utilities for skill-creator scripts.""" + +from pathlib import Path + + + +def parse_skill_md(skill_path: Path) -> tuple[str, str, str]: + """Parse a SKILL.md file, returning (name, description, full_content).""" + content = (skill_path / "SKILL.md").read_text() + lines = content.split("\n") + + if lines[0].strip() != "---": + raise ValueError("SKILL.md missing frontmatter (no opening ---)") + + end_idx = None + for i, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + end_idx = i + break + + if end_idx is None: + raise ValueError("SKILL.md missing frontmatter (no closing ---)") + + name = "" + description = "" + frontmatter_lines = lines[1:end_idx] + i = 0 + while i < len(frontmatter_lines): + line = frontmatter_lines[i] + if line.startswith("name:"): + name = line[len("name:"):].strip().strip('"').strip("'") + elif line.startswith("description:"): + value = line[len("description:"):].strip() + # Handle YAML multiline indicators (>, |, >-, |-) + if value in (">", "|", ">-", "|-"): + continuation_lines: list[str] = [] + i += 1 + while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith(" ") or frontmatter_lines[i].startswith("\t")): + continuation_lines.append(frontmatter_lines[i].strip()) + i += 1 + description = " ".join(continuation_lines) + continue + else: + description = value.strip('"').strip("'") + i += 1 + + return name, description, content