-
Notifications
You must be signed in to change notification settings - Fork 9
fix: harden forge-global and always-on-guidance skills for compression resilience #59
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -511,6 +511,7 @@ func TestWorkerPrompt_HardenedStructure(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
|
|
||
| func TestForgeMD_StructuralHardening(t *testing.T) { | ||
| // Read forge.md from embedded content. | ||
| data, err := content.ReadFile("content/commands/forge.md") | ||
|
|
@@ -903,6 +904,162 @@ func TestForgeCoordinationSkill_StructuralHardening(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
| func TestAlwaysOnGuidance_StructuralHardening(t *testing.T) { | ||
| data, err := content.ReadFile("content/skills/always-on-guidance/SKILL.md") | ||
| if err != nil { | ||
| t.Fatalf("read embedded always-on-guidance/SKILL.md: %v", err) | ||
| } | ||
| text := string(data) | ||
|
|
||
| // (1) Critical Safety section exists and appears before Tool Usage Discipline. | ||
| safetyIdx := strings.Index(text, "## Critical Safety") | ||
| if safetyIdx < 0 { | ||
| t.Error("always-on-guidance: missing '## Critical Safety' section") | ||
| } | ||
| toolUsageIdx := strings.Index(text, "## Tool Usage Discipline") | ||
| if toolUsageIdx < 0 { | ||
| t.Error("always-on-guidance: missing '## Tool Usage Discipline' section") | ||
| } | ||
| if safetyIdx >= 0 && toolUsageIdx >= 0 && safetyIdx >= toolUsageIdx { | ||
| t.Error("always-on-guidance: '## Critical Safety' must appear before '## Tool Usage Discipline'") | ||
| } | ||
|
|
||
| // (2) Force push rule uses RFC 2119 uppercase keyword (DR-002). | ||
| if !strings.Contains(text, "NEVER force push") { | ||
| t.Error("always-on-guidance: force push rule must use RFC 2119 keyword 'NEVER'") | ||
| } | ||
|
|
||
| // (3) hivemind_find is the first item in Tool Usage Discipline section. | ||
| if toolUsageIdx >= 0 { | ||
| afterToolUsage := text[toolUsageIdx:] | ||
| firstDashIdx := strings.Index(afterToolUsage, "\n- ") | ||
| if firstDashIdx < 0 { | ||
| t.Error("always-on-guidance: no list items in Tool Usage Discipline") | ||
| } else { | ||
| // Extract the first list item line. | ||
| firstItemStart := firstDashIdx + 3 // skip "\n- " | ||
| firstItemEnd := strings.Index(afterToolUsage[firstItemStart:], "\n") | ||
| if firstItemEnd < 0 { | ||
| firstItemEnd = len(afterToolUsage) - firstItemStart | ||
| } | ||
| firstItem := afterToolUsage[firstItemStart : firstItemStart+firstItemEnd] | ||
| if !strings.Contains(firstItem, "hivemind_find") { | ||
| t.Errorf("always-on-guidance: first Tool Usage item should mention hivemind_find, got %q", firstItem) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // (4) Code Quality split into sub-headers. | ||
| for _, sub := range []string{"### Structure", "### Clarity"} { | ||
| if !strings.Contains(text, sub) { | ||
| t.Errorf("always-on-guidance: missing Code Quality sub-header %q", sub) | ||
| } | ||
| } | ||
|
|
||
| // (5) Testing split into sub-headers. | ||
| for _, sub := range []string{"### Test Infrastructure", "### Test Practice"} { | ||
| if !strings.Contains(text, sub) { | ||
| t.Errorf("always-on-guidance: missing Testing sub-header %q", sub) | ||
| } | ||
| } | ||
|
|
||
| // (6) Error Handling split into sub-headers. | ||
| for _, sub := range []string{"### Error Propagation", "### Error Coverage"} { | ||
| if !strings.Contains(text, sub) { | ||
| t.Errorf("always-on-guidance: missing Error Handling sub-header %q", sub) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestForgeGlobal_StructuralHardening(t *testing.T) { | ||
| data, err := content.ReadFile("content/skills/forge-global/SKILL.md") | ||
| if err != nil { | ||
| t.Fatalf("read embedded forge-global/SKILL.md: %v", err) | ||
| } | ||
| text := string(data) | ||
|
|
||
| // (1) Decision table format with Signal/Forge/Skip columns. | ||
| if !strings.Contains(text, "| Signal | Forge | Skip |") { | ||
| t.Error("forge-global: missing decision table header (Signal/Forge/Skip)") | ||
| } | ||
|
|
||
| // (2) All 6 original criteria present in decision table. | ||
| criteria := []string{ | ||
| "File count", | ||
| "Task structure", | ||
| "Work type", | ||
| "3+ files", | ||
| "single-file change", | ||
| "parallelize", | ||
| } | ||
| for _, c := range criteria { | ||
| if !strings.Contains(text, c) { | ||
| t.Errorf("forge-global: decision table missing criterion %q", c) | ||
| } | ||
| } | ||
|
|
||
| // (3) Temporal ordering markers in File Reservation Protocol. | ||
| for _, marker := range []string{"FIRST,", "THEN,", "FINALLY,"} { | ||
| if !strings.Contains(text, marker) { | ||
| t.Errorf("forge-global: missing temporal marker %q in File Reservation Protocol", marker) | ||
| } | ||
| } | ||
|
|
||
| // (4) TTL inlined in step 1 with specific value. | ||
| if !strings.Contains(text, "ttl_seconds=300") { | ||
| t.Error("forge-global: ttl_seconds=300 must be inlined in reservation step") | ||
| } | ||
|
|
||
| // (5) TTL explanation parenthetical present (5-minute auto-release). | ||
| if !strings.Contains(text, "(5-minute auto-release)") { | ||
| t.Error("forge-global: missing '(5-minute auto-release)' explanation for TTL") | ||
| } | ||
|
|
||
| // (6) No standalone TTL bullet (old format removed). | ||
| lines := strings.Split(text, "\n") | ||
| for _, line := range lines { | ||
| trimmed := strings.TrimSpace(line) | ||
| // Old format was a standalone step like "3. Set `ttl_seconds` to auto-release..." | ||
| if strings.HasPrefix(trimmed, "3.") && strings.Contains(trimmed, "ttl_seconds") && strings.Contains(trimmed, "auto-release") { | ||
| t.Error("forge-global: standalone TTL step should be removed (inlined into step 1)") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestSkillFiles_DriftDetection(t *testing.T) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MEDIUM Also |
||
| // TC-007: Verify embedded skill files match the .opencode/ scaffolded copies. | ||
| // This detects drift where one copy is updated but the other is not. | ||
| // | ||
| // Find the repo root by walking up from the test working directory | ||
| // until we find go.mod. | ||
| repoRoot := findRepoRoot(t) | ||
|
|
||
| skills := []string{ | ||
| "always-on-guidance", | ||
| "forge-global", | ||
| } | ||
|
|
||
| for _, skill := range skills { | ||
| embeddedPath := filepath.Join("content", "skills", skill, "SKILL.md") | ||
| embedded, err := content.ReadFile(embeddedPath) | ||
| if err != nil { | ||
| t.Fatalf("read embedded %s: %v", embeddedPath, err) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. agentkit_test.go:1046 and 1052: Both ReadFile error handlers use t.Fatalf, which stops the entire test on the first skill that fails to read. If both skills have issues, only the first is reported. For a 2-element loop this is marginal, but t.Errorf + continue would be more informative. The drift comparison itself at line 1055 correctly uses t.Errorf. |
||
| } | ||
|
|
||
| scaffoldedPath := filepath.Join(repoRoot, ".opencode", "skills", skill, "SKILL.md") | ||
| scaffolded, err := os.ReadFile(scaffoldedPath) | ||
| if err != nil { | ||
| t.Fatalf("read scaffolded %s: %v", scaffoldedPath, err) | ||
| } | ||
|
|
||
| if string(embedded) != string(scaffolded) { | ||
| t.Errorf("drift detected: embedded %s differs from scaffolded .opencode/skills/%s/SKILL.md", embeddedPath, skill) | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
||
|
|
||
| func TestSkillTemplates_HaveNameField(t *testing.T) { | ||
| // Walk the embedded content filesystem and verify every SKILL.md | ||
| // has a "name: <directory-name>" field in its YAML frontmatter. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| schema: unbound-force | ||
| created: 2026-08-02 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
MEDIUM
The test iterates all lines looking for one that starts with "3." AND contains "ttl_seconds" AND contains "auto-release". The current forge-global SKILL.md step 3 reads:
3. FINALLY, coordinator can emergency release if workers fail: comms_release_all()This line starts with "3." but does not contain "ttl_seconds", so the triple-AND condition can never be true. The test passes trivially — it's testing for the absence of something that's already structurally impossible given how the file was rewritten. It would still pass if someone reintroduced a 4. Set ttl_seconds... step (the old format used step 3, but a future author might use step 4). The assertion is coupled to the old numbering, not to the actual invariant ("no standalone TTL step exists").
A more robust check:
This fires regardless of step number and regardless of whether the line is a numbered step at all.
Also
MEDIUM
The conditional at agentkit_test.go:1023:
if strings.HasPrefix(trimmed, "3.") && strings.Contains(trimmed, "ttl_seconds") && strings.Contains(trimmed, "auto-release") {Tracing the input: text comes from content.ReadFile("content/skills/forge-global/SKILL.md"). The file's line starting with "3." is "3. FINALLY, coordinator can emergency release if workers fail: comms_release_all()". This line does not contain "ttl_seconds", so the branch body (the t.Error call) is dead code. The test would need a future regression that coincidentally uses "3." as the step number AND reintroduces both "ttl_seconds" and "auto-release" in the same line to ever trigger.