Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions .claude/skills/verify-open-pstack/features/formal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,22 @@ function observed(record:SessionRecord, verdict:string):boolean {
return record.tools.some(t=>!t.parent && JSON.stringify(t.input).includes('verification/check.mjs') && t.result?.includes(`"verdict":"${verdict}"`));
}

/** The commit the skill's `Sync commit:` record names. The fixture prompts ask for that line, as a plain line, a list item, a bold label or a table cell; words before the label do not count, and a value that is not a bare hash (a revision such as `<hash>^` or `<hash>..HEAD`) counts as none. Records that name different commits are ambiguous and count as none. */
export function syncCommit(skill:string):string|undefined {
const values=[...skill.matchAll(/^[\s|*-]*sync commit\**\s*[|:]([^\n]*)/gim)]
.map(m=>/^[\s*`]*([0-9a-f]{7,40})(?=[`*]*(?:[\s|,;)]|\.(?!\.)|$))/i.exec(m[1]!)?.[1]?.toLowerCase());
if(values.some(v=>v===undefined)) return undefined;
const found=values as string[];
const longest=found.reduce((a,b)=>b.length>a.length?b:a,'');
return longest && found.every(v=>longest.startsWith(v)) ? longest : undefined;
}
/** Whether the skill's sync record names this commit, abbreviated or in full. */
export function namesCommit(skill:string,head:string):boolean { const token=syncCommit(skill); return token!==undefined && head.startsWith(token); }

export function maintainFormal():Case {
return {
id:'stale-model',
claudePermission:'bypassPermissions',
async prepare(f) {
await git(f,'init','-q','-b','main');
await put(f,'capacity.mjs',ATOMIC);await put(f,'replay.mjs',REPLAY);
Expand All @@ -72,7 +85,7 @@ export function maintainFormal():Case {
const head=await commit(f,'Yield during capacity reservation',['capacity.mjs']);
await writeFile(`${f.dir}.formal-baseline.json`,JSON.stringify({head,product:sha256(SPLIT),checker:sha256(CHECKER)}));
},
prompt:f=>request(f,'maintain-formal-verification-skill','Maintain .claude/skills/verify-capacity/SKILL.md. A change to reserve landed after its sync commit. The existing checker is an exhaustive bounded-state interpreter; keep verification/check.mjs and its two-client, capacity-one bounds unchanged. Update the model from the current code, run the check before and after, replay any counterexample with bun replay.mjs, and report whether the old pass was stale and whether the current code has a bug.'),
prompt:f=>request(f,'maintain-formal-verification-skill','Maintain .claude/skills/verify-capacity/SKILL.md. A change to reserve landed after its sync commit. The existing checker is an exhaustive bounded-state interpreter; keep verification/check.mjs and its two-client, capacity-one bounds unchanged. Update the model from the current code, run the check before and after, replay any counterexample with bun replay.mjs, record the new sync commit on its own line as "Sync commit: <full commit hash>" once the model describes the current code, and report whether the old pass was stale and whether the current code has a bug.'),
async assert(record,f) {
loaded(record,'maintain-formal-verification-skill');
const baseline=JSON.parse(await readFile(`${f.dir}.formal-baseline.json`,'utf8'));
Expand All @@ -84,7 +97,7 @@ export function maintainFormal():Case {
const replay=record.tools.some(t=>JSON.stringify(t.input).includes('replay.mjs') && t.result?.includes('"used":2'));
if(!replay) throw new Error('formal-counterexample-not-replayed');
const skill=await readFile(join(f.dir,'.claude/skills/verify-capacity/SKILL.md'),'utf8');
if(!skill.includes(baseline.head)) throw new Error('formal-sync-commit-not-updated');
if(!namesCommit(skill,baseline.head)) throw new Error('formal-sync-commit-not-updated');
const result=await readFile(join(f.dir,'result.md'),'utf8');
if(!/bug|oversubscri/i.test(result) || !/stale|drift/i.test(result)) throw new Error('formal-finding-not-reported');
// The captain independently runs the unmodified checker on the maintained model.
Expand All @@ -98,6 +111,7 @@ export function maintainFormal():Case {
export function createFormal():Case {
return {
id:'create-and-mutate',
claudePermission:'bypassPermissions',
async prepare(f) {
await git(f,'init','-q','-b','main');
await put(f,'capacity.mjs',ATOMIC);await put(f,'replay.mjs',REPLAY);
Expand All @@ -107,7 +121,7 @@ export function createFormal():Case {
const head=await commit(f,'Atomic capacity module and bounded verification tool',['capacity.mjs','replay.mjs','verification/check.mjs','verification/model.json','README.md']);
await writeFile(`${f.dir}.formal-baseline.json`,JSON.stringify({head,product:sha256(ATOMIC),checker:sha256(CHECKER)}));
},
prompt:f=>request(f,'create-formal-verification-skill',`Create .claude/skills/verify-capacity/SKILL.md for reserve's capacity invariant. Use the bundled exhaustive bounded model checker (bun verification/check.mjs); keep its interpreter and two-client, capacity-one scope unchanged. Confirm the model matches capacity.mjs. The only authorized product edit is the skill's disposable mutation test: split reserve's check and increment with an await, reflect that split in the model, observe the failure and replay it, then restore both code and model. Record the checked commit, coverage, assumptions, bounds, command and how to distinguish model drift from a bug. Follow the generated skill and report the evidence. The scratch mutation is explicitly permitted by this fixture.`).replace('or change product code.', 'or change product code except for the explicitly permitted scratch mutation.'),
prompt:f=>request(f,'create-formal-verification-skill',`Create .claude/skills/verify-capacity/SKILL.md for reserve's capacity invariant. Use the bundled exhaustive bounded model checker (bun verification/check.mjs); keep its interpreter and two-client, capacity-one scope unchanged. Confirm the model matches capacity.mjs. The only authorized product edit is the skill's disposable mutation test: split reserve's check and increment with an await, reflect that split in the model, observe the failure and replay it, then restore both code and model. Record the checked commit on its own line as "Sync commit: <full commit hash>", then the coverage, assumptions, bounds, command and how to distinguish model drift from a bug. Follow the generated skill and report the evidence. The scratch mutation is explicitly permitted by this fixture.`).replace('or change product code.', 'or change product code except for the explicitly permitted scratch mutation.'),
async assert(record,f) {
loaded(record,'create-formal-verification-skill');
const baseline=JSON.parse(await readFile(`${f.dir}.formal-baseline.json`,'utf8'));
Expand All @@ -118,7 +132,7 @@ export function createFormal():Case {
if(!observed(record,'pass') || !observed(record,'bug')) throw new Error('formal-mutation-not-observed');
if(!record.tools.some(t=>JSON.stringify(t.input).includes('replay.mjs') && t.result?.includes('"used":2'))) throw new Error('formal-mutation-not-replayed');
const path=join(f.dir,'.claude/skills/verify-capacity/SKILL.md'), skill=await readFile(path,'utf8');
if(!/^---\n[\s\S]*name:\s*verify-capacity[\s\S]*description:/m.test(skill) || !skill.includes(baseline.head) || !skill.includes('capacity.mjs') || !skill.includes('verification/check.mjs') || !/bound|two|2/i.test(skill)) throw new Error('formal-skill-contract-incomplete');
if(!/^---\n[\s\S]*name:\s*verify-capacity[\s\S]*description:/m.test(skill) || !namesCommit(skill,baseline.head) || !skill.includes('capacity.mjs') || !skill.includes('verification/check.mjs') || !/bound|two|2/i.test(skill)) throw new Error('formal-skill-contract-incomplete');
const output=await f.run(['bun','verification/check.mjs'],{cwd:f.dir});
if(!output.includes('"verdict":"pass"')) throw new Error('formal-final-check-failed');
return {assertions:['skill-loaded','project-skill-created','coverage-and-sync-recorded','mutation-caught','mutation-replayed','code-and-model-restored','bounded-check-passes'],files:[path,join(f.dir,'result.md'),join(f.dir,'verification/model.json')]};
Expand Down
2 changes: 1 addition & 1 deletion .claude/skills/verify-open-pstack/features/registry.json
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,6 @@
".gitignore", ".mergify.yml", "docs/**", "assets/**",
"plugins/pstack/skills/poteto-mode/scripts/bootstrap.test.ts",
"plugins/pstack/skills/poteto-mode/scripts/check-plan.test.ts",
"plugins/pstack/skills/poteto-mode/scripts/resolve-codex-transcript.test.ts", "scripts/release.py", "scripts/release_test.py"
"plugins/pstack/skills/poteto-mode/scripts/resolve-codex-transcript.test.ts", "scripts/release.py", "scripts/release_test.py", "scripts/sync-port.sh", "scripts/front-door.py", "scripts/cloud-setup.sh", "tests/cloud-setup-test.sh", "tests/sync-port-test.sh"
]
}
46 changes: 44 additions & 2 deletions .claude/skills/verify-open-pstack/scripts/formal.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import { afterEach, expect, test } from 'bun:test';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { command } from './io.ts';
import { createFormal, maintainFormal } from '../features/formal.ts';
import { createFormal, maintainFormal, namesCommit, syncCommit } from '../features/formal.ts';
import type { Fixture } from '../features/recipes.ts';
import type { SessionRecord } from './types.ts';
const roots:string[]=[];
Expand All @@ -30,3 +30,45 @@ test('creation begins with passing code but requires mutation evidence',async()=
expect(await command(['bun','verification/check.mjs'],{cwd:f.dir})).toContain('"verdict":"pass"');
await expect(c.assert(record('create-formal-verification-skill'),f)).rejects.toThrow('mutation-not-observed');
});
test('the sync record is the Sync commit line, compared whole; ambiguity and prose count as none',()=>{
const head='efb41f2f3838c0902016a9a97bf7e101d4b36e97', old='761782b83ad658da668937afdf9281ef2e45af56';
expect(syncCommit('| Sync commit | `efb41f2` |')).toBe('efb41f2');
expect(syncCommit(`Sync commit: ${head}`)).toBe(head);
expect(syncCommit(`- **Sync commit**: ${head}`)).toBe(head);
expect(syncCommit(`Sync commit: ${head.slice(0,7)}\n\n| Sync commit | ${head} |`)).toBe(head);
expect(syncCommit(`Previous sync commit: ${old}\n\nSync commit: ${head}`)).toBe(head);
expect(syncCommit(`Last confirmed code/model sync commit: ${head}`)).toBeUndefined();
expect(syncCommit(`Compare with git diff ${head}..HEAD`)).toBeUndefined();
expect(syncCommit(`Sync commit: pending\n\nRun git diff ${head}..HEAD`)).toBeUndefined();
expect(syncCommit(`Update the sync commit after checking ${head}.`)).toBeUndefined();
expect(syncCommit(`Sync commit candidate: ${head}`)).toBeUndefined();
expect(syncCommit(`Sync commit has not been updated; counterexample reproduced at: ${head}`)).toBeUndefined();
expect(syncCommit(`Sync commit: ${old}\n\nSync commit: ${head}`)).toBeUndefined();
expect(syncCommit(`Sync commit: ${head}^`)).toBeUndefined();
expect(syncCommit(`Sync commit: ${head}~1`)).toBeUndefined();
expect(syncCommit(`Sync commit: ${head}..HEAD`)).toBeUndefined();
expect(syncCommit(`Sync commit: ${head}:capacity.mjs`)).toBeUndefined();
expect(syncCommit(`Sync commit: \`${head.slice(0,7)}\`. At this commit the model was read against reserve.`)).toBe(head.slice(0,7));
expect(syncCommit(`Sync commit: ${head}, confirmed 2026-11-01`)).toBe(head);
expect(namesCommit('| Sync commit | `efb41f20` |',head)).toBe(false);
expect(namesCommit('| Sync commit | `0000000` |',head)).toBe(false);
expect(namesCommit(`Sync commit: ${head.slice(0,7)}\n\nHistory: the stale model previously described ${old}.`,head)).toBe(true);
expect(namesCommit(`Sync commit: ${old}\n\nCounterexample reproduced on HEAD ${head.slice(0,7)}.`,head)).toBe(false);
});
test('maintenance proves the sync moved by the skill\'s own record, whatever else it mentions',async()=>{
const f=await fixture(),c=maintainFormal();await c.prepare!(f);
const path=join(f.dir,'.claude/skills/verify-capacity/SKILL.md');
const stale=/Sync commit: ([0-9a-f]{40})/.exec(await readFile(path,'utf8'))![1]!;
const baseline=JSON.parse(await readFile(`${f.dir}.formal-baseline.json`,'utf8'));
await writeFile(join(f.dir,'verification/model.json'),JSON.stringify({clients:2,limit:1,splitReservation:true}));
await writeFile(join(f.dir,'result.md'),'A real bug: the stale model passed while the split reserve oversubscribes.');
const tool=(command:string,result:string)=>({name:'Bash',id:command,input:{command},parent:null,result,failed:false});
const done={...record('maintain-formal-verification-skill'),tools:[tool('bun verification/check.mjs','{"verdict":"pass"}'),tool('bun verification/check.mjs','{"verdict":"bug"}'),tool('bun replay.mjs','{"used":2,"limit":1}')]};
const skill=(body:string)=>writeFile(path,`---\nname: verify-capacity\ndescription: x\n---\n${body}\n`);
await skill(`Sync commit: ${stale}\n\nCounterexample reproduced on HEAD ${baseline.head.slice(0,7)}.`);
await expect(c.assert(done,f)).rejects.toThrow('sync-commit-not-updated');
await skill(`Sync commit: pending\n\nRun git diff ${baseline.head}..HEAD -- capacity.mjs`);
await expect(c.assert(done,f)).rejects.toThrow('sync-commit-not-updated');
await skill(`Sync commit: ${baseline.head.slice(0,7)}\n\nHistory: the stale model previously described ${stale}.`);
await expect(c.assert(done,f)).resolves.toBeDefined();
});
9 changes: 7 additions & 2 deletions .claude/skills/verify-open-pstack/scripts/harness.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { copyFile, mkdir, readdir, readFile, realpath, stat } from 'node:fs/promises';
import { basename, join } from 'node:path';
import { CLAUDE_TOOLS, recipe, type Fixture } from '../features/recipes.ts';
import { CLAUDE_TOOLS, recipe, type Case, type Fixture } from '../features/recipes.ts';
import { requiredFeatures, requiredHarnesses } from './core.ts';
import { command, isolatedEnv, retainedFile, save, session, treeHash, type Command, type Session } from './io.ts';
import { doctor } from './doctor.ts';
Expand All @@ -11,6 +11,11 @@ import { publishable } from './verify.ts';
import { REPO, type Driver, type Harness, type Installation, type Observation, type Receipt, type SessionRecord } from './types.ts';

const setupFeature = (feature: string): boolean => feature === 'setup' || feature === 'skill-invocation:setup-pstack';
/** The routed launcher fixes each harness's isolation, so only a case that needs its own harness's override cannot be routed. */
export function unroutable(feature: string, harness: Harness, cases: Case[]): boolean {
return feature === 'runner' || setupFeature(feature) || feature.startsWith('agent-preload:') ||
cases.some(c => harness === 'codex' ? c.codexSandbox !== undefined : c.claudePermission !== undefined);
}
export function launch(harness: Harness, workspace: string, options: { cwd: string; addDirs: string[]; codexSandbox?: 'danger-full-access'; claudePermission?: 'bypassPermissions' }): string[] {
const addDirs = options.addDirs.flatMap(dir => ['--add-dir', dir]);
return harness === 'claude'
Expand Down Expand Up @@ -155,7 +160,7 @@ export class MacDriver implements Driver {
for (const { installation, features } of plans) {
if (!this.routing.models[installation.harness]) throw new Error(`native-model-missing:${installation.harness}`);
for (const { feature, cases } of features) {
if (feature === 'runner' || setupFeature(feature) || feature.startsWith('agent-preload:') || cases.some(c => c.codexSandbox || c.claudePermission)) throw new Error(`unsupported-routed-recipe:${feature}`);
if (unroutable(feature, installation.harness, cases)) throw new Error(`unsupported-routed-recipe:${feature}`);
}
}
}
Expand Down
10 changes: 10 additions & 0 deletions .claude/skills/verify-open-pstack/scripts/routing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { parse } from './cli.ts';
import { routedSession } from './routing.ts';
import { unroutable } from './harness.ts';
import { createFormal, maintainFormal } from '../features/formal.ts';
import type { Session } from './io.ts';
const roots:string[]=[];
afterEach(async()=>{for(const root of roots.splice(0)) await rm(root,{recursive:true,force:true});});
Expand All @@ -30,3 +32,11 @@ test('native proof rejects an incomplete admission receipt',async()=>{
if(terminal) expect((await proof).evidence).toHaveLength(3);else await expect(proof).rejects.toThrow('not-successfully-completed');
}
});
test('routed eligibility follows the selected harness',()=>{
const formal=[createFormal(),maintainFormal()];
expect(unroutable('skill-invocation:create-formal-verification-skill','claude',formal)).toBe(true);
expect(unroutable('skill-invocation:create-formal-verification-skill','codex',formal)).toBe(false);
expect(unroutable('skill-invocation:maintain-formal-verification-skill','codex',formal)).toBe(false);
expect(unroutable('skill-invocation:poteto-mode','claude',[])).toBe(false);
expect(unroutable('setup','codex',[])).toBe(true);
});
Loading
Loading