Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
### Fixed

- **repo run: uptake of a root-level file** — a bare `lint` at the start of a
command no longer counts as a run of the arm's `lint` file, since a bare name
resolves through `PATH`; it counts as `./lint`, under the workspace path, or
after an interpreter.
2 changes: 1 addition & 1 deletion dokimi/docs/repo-evaluation-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an
`mcp_servers` list what it added (sorted); `commands` counts
command events that name an added file as a workspace path (`tools/fmt`,
`./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`; a file
with no `/` only as `./name`, under the workspace path, first in a command or after an interpreter, so
with no `/` only as `./name`, under the workspace path or after an interpreter (a bare `name` resolves through `PATH`), so
`npm run lint` does not name a `lint` file), `reads` the read events of one (a
read of an added script counts as use), `mcp_calls` the calls of the servers' tools
(`mcp__<server>__<tool>`), none of them a call the harness denied; `used`
Expand Down
10 changes: 5 additions & 5 deletions dokimi/dokimi/repos/uptake.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
- **commands**: command events whose text names one of those files, as a path
relative to the workspace (``tools/fmt``, ``./tools/fmt``, ``bash tools/fmt``)
or under the workspace's absolute path. A file at the workspace root is named only as ``./name``, under the
absolute path, as the first word of a command (or of a ``;``/``&&``/``|`` part) or after a shell or
interpreter (``sh run.sh``): ``npm run lint`` does not name a ``lint`` file. A command run as ``cd tools && ./fmt``
absolute path or after a shell or interpreter (``sh run.sh``): a bare ``lint`` resolves through ``PATH``, not to
the workspace file, and ``npm run lint`` does not name a ``lint`` file. A command run as ``cd tools && ./fmt``
is not counted;
- **reads**: read events of one of those files (reading a script's text counts: it is the agent engaging with
what the arm added, not only running it);
Expand Down Expand Up @@ -92,8 +92,8 @@ def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: l

def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None:
"""One of ``files`` as a whole word of a command: relative (optionally ``./``) or under ``workdir``.
A file with no ``/`` is a common word, so it must be ``./``-prefixed, under ``workdir``, first in a command
or run by an interpreter."""
A file with no ``/`` is a common word, and bare it resolves through ``PATH``, so it must be ``./``-prefixed,
under ``workdir`` or run by an interpreter."""
if not files:
return None
root = re.escape(workdir.rstrip("/") + "/")
Expand All @@ -106,7 +106,7 @@ def _path_pattern(files: list[str], workdir: str) -> re.Pattern[str] | None:
parts.append(rf"(?:^|(?<=[\s'\"`=;&|(<>])|(?<=\./)|(?<={root}))(?:{paths}){end}")
if flat:
names = "|".join(re.escape(f) for f in flat)
parts.append(rf"(?:^\s*(?:\./)?|[;&|(]\s*(?:\./)?|(?:^|(?<=\s)){_INTERPRETERS}\s+|(?<=[\s'\"`=;&|(<>])\./|(?<={root}))"
parts.append(rf"(?:^\s*\./|(?:^|(?<=\s)){_INTERPRETERS}\s+|(?<=[\s'\"`=;&|(<>])\./|(?<={root}))"
rf"(?:{names}){end}")
return re.compile("|".join(parts))

Expand Down
7 changes: 4 additions & 3 deletions dokimi/tests/test_repo_uptake.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,13 +235,14 @@ def test_compare_counts_the_uptake_of_the_models_it_compares(setup, monkeypatch)
assert m2["uptake"] == {arm: {"cells": 2, "used": 0, "unmeasured": 0}} and _used_none(m2)


@pytest.mark.parametrize("text", ["npm run lint", "docker run img", "git log -- build", "cat lint", "make build"])
@pytest.mark.parametrize("text", ["npm run lint", "docker run img", "git log -- build", "cat lint", "make build",
"lint x", "cd d && lint", "run", "x | lint", "(lint)", ";lint"])
def test_a_root_level_added_file_is_not_named_by_a_common_word_of_a_command(text):
for name in ("lint", "run", "build"):
assert arm_uptake([_cmd(text)], [name], [], WS)["commands"] == (1 if text.split()[0] == name else 0)
assert arm_uptake([_cmd(text)], [name], [], WS)["commands"] == 0


@pytest.mark.parametrize("text", ["./lint x", "lint x", f"{WS}/lint", "cd d && lint", "x; ./lint", "sh lint", "bash ./lint a",
@pytest.mark.parametrize("text", ["./lint x", f"{WS}/lint", "x; ./lint", "sh lint", "bash ./lint a",
"python3 lint"])
def test_a_root_level_added_file_is_named_by_a_path_or_as_a_command(text):
assert arm_uptake([_cmd(text)], ["lint"], [], WS)["commands"] == 1
Expand Down