Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions evals/sent-feedback.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,5 @@ cases:
expect: ambiguous
rationale: Use sent-messaging for investigation and sent-feedback for the authorized report.
- query: I need tomorrow's scheduled send stopped, but there's no cancel tool and my last two workarounds failed. What can I do?
expect: ambiguous
rationale: sent-messaging owns the task; report the missing tool only if the user then asks.
expect: no_trigger
rationale: sent-messaging owns the task; the user asked for help, not for a report, and feedback is never solicited.
25 changes: 24 additions & 1 deletion scripts/repository_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,28 @@ def load_claude_overlay(root: Path, metadata: RepositoryMetadata) -> ClaudeOverl
"""Load the Claude-only tool withholding and text edits applied to claude-plugins/sent."""
import yaml

class UniqueKeyLoader(yaml.SafeLoader):
"""PyYAML keeps the last of two equal keys; here that would drop a whole block of edits unnoticed."""

def construct_mapping(self, node, deep=False): # type: ignore[override]
seen: set[Any] = set()
for key_node, _ in node.value:
# Merge keys (`<<: *anchor`) are flattened by the parent and have no constructor here;
# a non-scalar key is malformed and is left for the parent to report.
if key_node.tag == "tag:yaml.org,2002:merge" or not isinstance(key_node, yaml.ScalarNode):
continue
key = self.construct_object(key_node, deep=deep)
Comment on lines +161 to +168

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 (optional) Maintainers who use a YAML merge key (<<: *anchor) in overlay.yaml will now get a load failure; the base yaml.safe_load accepted it. The duplicate check calls construct_object on every raw key node before flatten_mapping runs, which includes the << node tagged tag:yaml.org,2002:merge. SafeLoader has no constructor for that tag, so loading fails with "could not determine a constructor for the tag", which does not point to the merge key. Fix: skip key nodes tagged tag:yaml.org,2002:merge in the duplicate scan and check only explicit keys. Merged keys that explicit keys override are legal YAML and must still load. [also at: scripts/repository_metadata.py:170 - nit: A malformed overlay with a non-scalar mapping key now crashes with a raw TypeError traceback instead of a clean MetadataError.]

Why this was flagged

The trigger is an overlay.yaml that uses a merge key, for example an edit entry written as - <<: *common plus find:. No such entry exists today, so this is conditional on a maintainer adding one. UniqueKeyLoader.construct_mapping (repository_metadata.py:163-164) loops over node.value and calls self.construct_object(key_node) for every key. It does this before super().construct_mapping runs flatten_mapping, the step that removes merge-key nodes. The << scalar resolves to tag:yaml.org,2002:merge. SafeConstructor registers no constructor for that tag, so construct_undefined raises a ConstructorError. repository_metadata.py:176-177 turns that into MetadataError "could not read adapter-sources/claude/overlay.yaml: ... could not determine a constructor for the tag 'tag:yaml.org,2002:merge'". On the base, yaml.safe_load flattens merge keys and loads the file. After this change, generation, validation and test_contracts all fail on a valid overlay, with an error that does not mention a merge key.

Verification: Trigger: a maintainer adds a YAML merge key (<<: *anchor) anywhere in adapter-sources/claude/overlay.yaml. The current file has none. In scripts/repository_metadata.py:161-170, construct_object runs on every key before return super().construct_mapping(node, deep=deep), and only the super call runs self.flatten_mapping(node). Base yaml.safe_load flattened merge keys and accepted the same file.

if key in seen:
raise yaml.constructor.ConstructorError(
"while constructing a mapping", node.start_mark,
f"found duplicate key {key!r}", key_node.start_mark,
)
seen.add(key)
return super().construct_mapping(node, deep=deep)

path = root / CLAUDE_OVERLAY
try:
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
raw = yaml.load(path.read_text(encoding="utf-8"), Loader=UniqueKeyLoader)
except (OSError, yaml.YAMLError) as exc:
raise MetadataError(f"could not read {CLAUDE_OVERLAY}: {exc}") from exc
if not isinstance(raw, dict) or set(raw) != {"withheld_tools", "edits"}:
Expand Down Expand Up @@ -189,5 +208,9 @@ def load_claude_overlay(root: Path, metadata: RepositoryMetadata) -> ClaudeOverl
or not isinstance(entry["replace"], str)):
raise MetadataError(f"{CLAUDE_OVERLAY}: {relative}#{index} needs string find and replace")
pairs.append((entry["find"], entry["replace"]))
# Two spellings of one path (`./skills/x` and `skills/x`) normalise to the same key, and the
# second would silently replace the first.
if str(target) in edits:
raise MetadataError(f"{CLAUDE_OVERLAY}: edit path {relative!r} collides with an earlier entry for {target}")
edits[str(target)] = tuple(pairs)
return ClaudeOverlay(withheld_tools=dict(withheld), edits=edits)
58 changes: 58 additions & 0 deletions scripts/test_validation_gates.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,11 +307,69 @@ def change(overlay: dict) -> None:
edit_claude_overlay(root, change)


def claude_withheld_tool_after_prohibition(root: Path) -> None:
"""The line already says "do not call"; a later clause instructs the call anyway."""
def change(overlay: dict) -> None:
overlay["edits"]["skills/sent-compliance/SKILL.md"][0]["replace"] += (
" Otherwise call `sender_profiles.create` with the plan.")
edit_claude_overlay(root, change)


def claude_withheld_tool_in_readme(root: Path) -> None:
"""README.md ships with the adapter but sat outside the old skills-only scan."""
def change(overlay: dict) -> None:
overlay["edits"]["README.md"][0]["replace"] += " Call `sender_profiles.delete` to remove one."
edit_claude_overlay(root, change)


def claude_withheld_tool_bare_negation(root: Path) -> None:
"""A bare "not" or "never" in the sentence is not a do-not-call instruction."""
def change(overlay: dict) -> None:
overlay["edits"]["README.md"][0]["replace"] += (
" If no profile exists, it is not optional: create one with `sender_profiles.create`.")
edit_claude_overlay(root, change)


def claude_withheld_tool_negated_filler(root: Path) -> None:
""""Do not skip calling" instructs the call; the negation must sit on the verb itself."""
def change(overlay: dict) -> None:
overlay["edits"]["README.md"][0]["replace"] += " Do not skip calling `sender_profiles.create`."
edit_claude_overlay(root, change)


def claude_withheld_tool_in_command(root: Path) -> None:
"""Slash commands ship with the adapter and are agent instructions too; the gate reads the shipped file."""
path = root / "claude-plugins" / "sent" / ".claude" / "commands" / "sent.md"
with path.open("a", encoding="utf-8") as handle:
handle.write("\nThen call `sender_profiles.create` with the plan.\n")


def duplicate_claude_overlay_path(root: Path) -> None:
"""PyYAML would keep only the second block and drop the first without a word."""
path = root / "adapter-sources/claude/overlay.yaml"
with path.open("a", encoding="utf-8") as handle:
handle.write("\n skills/sent/SKILL.md:\n - find: anything\n replace: anything else\n")


def colliding_claude_overlay_path(root: Path) -> None:
"""A second spelling of an existing path normalises onto it and would replace its edits."""
path = root / "adapter-sources/claude/overlay.yaml"
with path.open("a", encoding="utf-8") as handle:
handle.write("\n ./skills/sent/SKILL.md:\n - find: anything\n replace: anything else\n")


CASES: tuple[tuple[str, Mutation, str], ...] = (
("stale Claude overlay edit", stale_claude_overlay_find, "matched 0 times"),
("unknown Claude withheld tool", unknown_claude_withheld_tool, "is not in the canonical public surface"),
("Claude retained-tool confirmation", removed_claude_confirmation, "Claude adapter sent-profile-provisioning"),
("Claude withheld-tool instruction", claude_withheld_tool_instruction, "outside a do-not-call instruction"),
("Claude withheld-tool after a prohibition", claude_withheld_tool_after_prohibition, "outside a do-not-call instruction"),
("Claude withheld-tool in README", claude_withheld_tool_in_readme, "outside a do-not-call instruction"),
("Claude withheld-tool under a bare negation", claude_withheld_tool_bare_negation, "outside a do-not-call instruction"),
("Claude withheld-tool under a negated filler verb", claude_withheld_tool_negated_filler, "outside a do-not-call instruction"),
("Claude withheld-tool in a command", claude_withheld_tool_in_command, "outside a do-not-call instruction"),
("duplicate Claude overlay path", duplicate_claude_overlay_path, "found duplicate key"),
("colliding Claude overlay path", colliding_claude_overlay_path, "collides with an earlier entry"),
("ambient credential reads", ambient_credential_read, "public gate rejected ambient credential read"),
("Claude privacy policy listing", missing_claude_privacy_url, "Claude privacyPolicyUrl must match"),
("independent mutation confirmation policy", disabled_profile_confirmation, "independent safety policy"),
Expand Down
40 changes: 35 additions & 5 deletions scripts/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -687,12 +687,42 @@ def validate_claude_adapter(validation: Validation) -> None:
and isinstance(description, str) and 1 <= len(description) <= 1024,
f"{skill_file.relative_to(ROOT)}: Claude overlay broke skill frontmatter",
)
for path in sorted((CLAUDE_ADAPTER / "skills").rglob("*.md")):
# Every Markdown and YAML file the adapter ships is agent-facing text: skills and their
# references, the slash commands copied from adapter-sources, the README, and the agents/*.yaml
# metadata. The overlay can target any of them.
for path in sorted(p for p in CLAUDE_ADAPTER.rglob("*") if p.is_file() and p.suffix in {".md", ".yaml", ".yml"}):
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
if any(tool in line for tool in overlay.withheld_tools) and "do not call" not in line.lower():
validation.errors.append(
f"{path.relative_to(ROOT)}:{number}: Claude skill names a withheld tool outside a do-not-call instruction"
)
for tool in sorted(overlay.withheld_tools):
if not withheld_tool_mentions_are_prohibitions(line, tool):
validation.errors.append(
f"{path.relative_to(ROOT)}:{number}: Claude adapter names withheld tool {tool} outside a do-not-call instruction"
)


WITHHELD_SENTENCE_BOUNDARY = re.compile(r"(?<=[.;!?])\s+")
WITHHELD_CALL_VERBS = r"(?:call|calls|calling|run|runs|running|invoke|invokes|use|uses|execute|executes)"
# The negation must sit directly on the verb: "do not skip calling `x`" is an instruction to call.
WITHHELD_DO_NOT_CALL = re.compile(rf"\b(?:do not|don't|never)\s+{WITHHELD_CALL_VERBS}\b", re.IGNORECASE)
WITHHELD_CALL_VERB = re.compile(rf"\b{WITHHELD_CALL_VERBS}\s*`?\s*$", re.IGNORECASE)
WITHHELD_NEGATED_CALL = re.compile(rf"\b(?:do not|don't|never)\s+{WITHHELD_CALL_VERBS}\s*`?\s*$", re.IGNORECASE)


def withheld_tool_mentions_are_prohibitions(line: str, tool: str) -> bool:
"""True when every mention of ``tool`` on ``line`` is governed by a do-not-call instruction.

Sharing a line with the phrase is not enough: "do not call `x`; otherwise call `x` with the plan"
instructs the call in its second clause. Each mention must sit in a sentence that contains an
explicit do-not-call phrase (a bare "not" or "never" does not count: "it is not optional, create
one with `x`"), and must not itself be the direct object of an un-negated call verb.
"""
for sentence in WITHHELD_SENTENCE_BOUNDARY.split(line):
for match in re.finditer(re.escape(tool), sentence):
if not WITHHELD_DO_NOT_CALL.search(sentence):
return False
preceding = " ".join(sentence[:match.start()].split()[-4:])
if WITHHELD_CALL_VERB.search(preceding) and not WITHHELD_NEGATED_CALL.search(preceding):
return False
return True


def validate_documentation(validation: Validation) -> None:
Expand Down