diff --git a/dokimi/changelog.d/20261018_120000_napetrov_d3-host-requirements.md b/dokimi/changelog.d/20261018_120000_napetrov_d3-host-requirements.md index 60954eda..cfabda61 100644 --- a/dokimi/changelog.d/20261018_120000_napetrov_d3-host-requirements.md +++ b/dokimi/changelog.d/20261018_120000_napetrov_d3-host-requirements.md @@ -19,5 +19,4 @@ and flags match in any case. A `/proc/cpuinfo` the probe could not read is a probe failure, not a hardware miss. A failed probe writes no `unsupported.json`, so `repo report` counts the cells missing. On a host miss `tasks check` still prepares the workspace and probes isolation - (and runs the oracle when only CPUs are short); it skips the null check. `--warm` does not - judge the host. + (and runs the oracle when only CPUs are short); it skips the null check. diff --git a/dokimi/changelog.d/20261022_120000_napetrov_host-under-warm.md b/dokimi/changelog.d/20261022_120000_napetrov_host-under-warm.md new file mode 100644 index 00000000..68f1c7b3 --- /dev/null +++ b/dokimi/changelog.d/20261022_120000_napetrov_host-under-warm.md @@ -0,0 +1,7 @@ +### Fixed + +- **`repo tasks check --warm` judges task `host` requirements** — before, `--warm` skipped the + host verdict, so on a host that misses a task's requirements the untouched-tree null run + could pass and the task was reported `broken` instead of `unsupported`, and + `--allow-unsupported` could not pass it. The probe is the same container view (affinity, + cgroup CPU quota) with or without network; `repo run` already judged it. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 99d57849..5a58157e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -452,7 +452,8 @@ cells is left unrun; a cell that already has a `result.json` keeps it. Each task runs with its own network (`image.network` plus its `egress`). `tasks check --warm` runs the same null and oracle steps with network and -writable caches, to fill the caches before offline runs (§2). +writable caches, to fill the caches before offline runs (§2); it judges task +`host` requirements as an offline check does. ### 4.4 `repo_static.v1` (written by `static` to `{out}`) diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index c9d543b5..3e3a1a78 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -19,7 +19,7 @@ ``unsupported`` until the proxy exists, and so is a task whose ``host`` requirements (:mod:`.host`) the evaluation's containers do not meet: its bug cannot show here, so null and oracle would say nothing. ``--warm`` runs null -and oracle with network so the caches fill before offline runs (§2). +and oracle with network so the caches fill before offline runs (§2), and judges the host the same way. """ from __future__ import annotations @@ -66,7 +66,7 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, A task the host cannot show still has its workspace prepared and isolation probed, and its oracle run when only CPUs are short; only the null check, which depends on the hardware, is skipped. ``--warm`` - does not judge the host. + judges the host the same way: network changes neither the CPU view nor the hardware. ``host`` is :func:`.host.probe_host`'s facts, probed once per check; without it a task with ``host`` requirements probes for itself.""" @@ -77,29 +77,14 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, except ContainerError as exc: res.update(verdict="unsupported", unsupported_by="egress", problems=[str(exc)]) return res - # --warm has network and a different image state, as for egress: the host is not judged there. - reason, unsupported_by, facts = (None, None, None) if warm else _host_reason(ctx, task, work_dir, host) + # The probe sees what every container of this runner sees (affinity, cgroup quota), with or without + # --warm's network: an unmet host requirement is unsupported either way, never a passing null run. + reason, unsupported_by, facts = _host_reason(ctx, task, work_dir, host) # Only what the hardware decides is skipped: the null check (it passes where the bug cannot show) and, # unless the miss is CPUs alone, the oracle. Preparing the workspace and isolation hold anywhere. run_oracle = not reason or (unsupported_by == "host" and shortfall_kinds(task, facts) <= {"cpus"}) - for kind in ("null", "oracle"): - rd = work_dir / f"{task.id}__{kind}" - try: - _prepare(ctx, task, rd) - if reason and kind == "null" or kind == "oracle" and not run_oracle: - continue - if kind == "oracle": - run = run_entry(ctx, "oracle", task, rd, net, container_name(rd.name, "oracle")) - res["oracle_run"] = {"rc": run.returncode, "timed_out": run.timed_out, "wall_s": run.wall_s} - if not run.ok: - res["problems"].append(f"oracle {'timed out' if run.timed_out else f'exited {run.returncode}'}" - f"; see {rd / 'oracle.log'}") - continue - res[kind] = outcome(task, rd, run_entry(ctx, "grade", task, rd, net, container_name(rd.name, "grade"))) - except (PrepError, ContainerError) as exc: - res["problems"].append(f"{kind}: {exc}") - continue - res["problems"] += _verdict_problems(task, kind, res[kind], rd) + for kind, run_it in (("null", not reason), ("oracle", run_oracle)): + _check_step(ctx, task, work_dir / f"{task.id}__{kind}", kind, run_it, net, res) if not res["problems"]: res["isolation"] = isolation_probe(ctx, [task], work_dir / f"{task.id}__null", guidance_names) res["problems"] += [f"isolation: {p}" for p in res["isolation"]] @@ -118,6 +103,27 @@ def _host_reason(ctx: EvalContext, task: Task, work_dir: Path, return (*unmet(task, host), host) +def _check_step(ctx: EvalContext, task: Task, rd: Path, kind: str, run_it: bool, net: str, + res: dict[str, Any]) -> None: + """Prepare ``rd`` and, when ``run_it``, run the null or oracle step into ``res``.""" + try: + _prepare(ctx, task, rd) + if not run_it: + return + if kind == "oracle": + run = run_entry(ctx, "oracle", task, rd, net, container_name(rd.name, "oracle")) + res["oracle_run"] = {"rc": run.returncode, "timed_out": run.timed_out, "wall_s": run.wall_s} + if not run.ok: + res["problems"].append(f"oracle {'timed out' if run.timed_out else f'exited {run.returncode}'}" + f"; see {rd / 'oracle.log'}") + return + res[kind] = outcome(task, rd, run_entry(ctx, "grade", task, rd, net, container_name(rd.name, "grade"))) + except (PrepError, ContainerError) as exc: + res["problems"].append(f"{kind}: {exc}") + return + res["problems"] += _verdict_problems(task, kind, res[kind], rd) + + def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: if rd.exists(): shutil.rmtree(rd) diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index e592e0d0..4a960630 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -33,7 +33,7 @@ from dokimi.repos.manifest import parse_manifest from dokimi.repos.report import format_report, load_cells, load_unsupported, summarize from dokimi.repos.runner import UNSUPPORTED_FILE, _mark_unsupported, run_batch -from dokimi.repos.workspace import Task +from dokimi.repos.workspace import PrepError, Task, prepare_workspace from repo_eval_fixture import MANIFEST, FakeRunner, fake_harness, make_repo INTEL_CPUINFO = ("processor\t: 0\nvendor_id\t: GenuineIntel\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n" @@ -595,12 +595,65 @@ def test_a_host_miss_still_probes_isolation_and_skips_only_null(tmp_path, monkey assert "isolation" in race and "oracle" in race and "null" not in race -def test_warm_does_not_judge_the_host(tmp_path, monkeypatch, capsys): +def test_warm_judges_the_host_like_an_offline_check(tmp_path, monkeypatch, capsys): root = _repo(tmp_path) runner = HostRunner(_probe_out(affinity=2, cpuinfo=AMD_CPUINFO)) - _check(root, tmp_path / "a", runner, monkeypatch, "--task", "fix_race", "--warm") + with pytest.raises(SystemExit) as exc: + _check(root, tmp_path / "a", runner, monkeypatch, "--task", "fix_race", "--warm") + assert exc.value.code == 1 # unsupported, not a passing null reported as broken race = json.loads((tmp_path / "a/check/check.json").read_text())["tasks"][0] - assert race["verdict"] != "unsupported" and "null" in race and race["warm"] is True + assert race["verdict"] == "unsupported" and race["unsupported_by"] == "host" and race["warm"] is True + assert "null" not in race + _check(root, tmp_path / "b", runner, monkeypatch, "--task", "fix_race", "--warm", "--allow-unsupported") + assert "allowed by --allow-unsupported: fix_race" in capsys.readouterr().out + + +def test_a_prepare_error_on_the_null_step_is_a_problem_and_the_oracle_still_runs(tmp_path, monkeypatch): + root = _repo(tmp_path) + real = prepare_workspace + + def prepare(git_dir, manifest, task, arm, rd): + if rd.name.endswith("__null"): + raise PrepError("no such revision") + return real(git_dir, manifest, task, arm, rd) + + monkeypatch.setattr("dokimi.repos.selfcheck.prepare_workspace", prepare) + with pytest.raises(SystemExit) as exc: + _check(root, tmp_path / "a", HostRunner(_probe_out()), monkeypatch, "--task", "fix_add") + assert exc.value.code == 1 + add = json.loads((tmp_path / "a/check/check.json").read_text())["tasks"][0] + assert add["verdict"] == "broken" and add["problems"] == ["null: no such revision"] + assert "null" not in add and "oracle" in add + + +def test_an_oracle_that_exits_non_zero_is_a_problem_and_its_grade_is_not_run(tmp_path, monkeypatch): + class FailingOracle(HostRunner): + def run(self, spec, *, stdout, stderr, timeout_s): + if (spec.name or "").split("-")[-2] == "oracle": + stdout.write_text("") + stderr.write_text("") + return ContainerResult(returncode=3, timed_out=False, wall_s=0.0) + return super().run(spec, stdout=stdout, stderr=stderr, timeout_s=timeout_s) + + root = _repo(tmp_path) + with pytest.raises(SystemExit) as exc: + _check(root, tmp_path / "a", FailingOracle(_probe_out()), monkeypatch, "--task", "fix_add") + assert exc.value.code == 1 + add = json.loads((tmp_path / "a/check/check.json").read_text())["tasks"][0] + assert add["verdict"] == "broken" and len(add["problems"]) == 1 + assert add["problems"][0].startswith("oracle exited 3; see ") + assert add["oracle_run"]["rc"] == 3 and "oracle" not in add and "null" in add + + +def test_warm_runs_the_grade_with_the_bridge_network(tmp_path, monkeypatch): + root = _repo(tmp_path) + runner = HostRunner(_probe_out()) + _check(root, tmp_path / "a", runner, monkeypatch, "--task", "fix_add", "--warm") + grades = [s for s in runner.specs if (s.name or "").split("-")[-2] == "grade"] + assert len(grades) == 2 and {s.network for s in grades} == {"bridge"} + plain = HostRunner(_probe_out()) + _check(root, tmp_path / "b", plain, monkeypatch, "--task", "fix_add") + assert {s.network for s in plain.specs if (s.name or "").split("-")[-2] == "grade"} == {"none"} def test_a_probe_failure_leaves_no_marker_for_report_to_count(tmp_path, monkeypatch, capsys):