diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index b0c62f7c..a44766dc 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -79,6 +79,13 @@ on: required: false type: boolean default: false + allow-unsupported: + description: >- + Pass --allow-unsupported to `repo tasks check`: a task this runner misses the task.json + `host` requirements of passes its job (the reason is still in the log and summary) + required: false + type: boolean + default: false runs-on: description: Runner label for every job (a tasks-check job needs docker) required: false @@ -373,12 +380,14 @@ jobs: MANIFEST: ${{ inputs.manifest }} TASK: ${{ matrix.task }} WARM: ${{ inputs.warm }} + ALLOW_UNSUPPORTED: ${{ inputs.allow-unsupported }} # Each task's verdict in the log as it lands, not when the step ends. PYTHONUNBUFFERED: '1' run: | - warm=() - if [ "$WARM" = true ]; then warm=(--warm); fi - dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${warm[@]}" + flags=() + if [ "$WARM" = true ]; then flags+=(--warm); fi + if [ "$ALLOW_UNSUPPORTED" = true ]; then flags+=(--allow-unsupported); fi + dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${flags[@]}" # always(): a hung oracle ends in the job timeout, which cancels the job; a # cancelled job still runs these steps, so the run leaves a verdict. - name: Summary diff --git a/dokimi/changelog.d/20261021_120000_napetrov_repo-eval-allow-unsupported.md b/dokimi/changelog.d/20261021_120000_napetrov_repo-eval-allow-unsupported.md new file mode 100644 index 00000000..7a64a175 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_repo-eval-allow-unsupported.md @@ -0,0 +1,3 @@ +### Added + +- **repo-eval workflow: `allow-unsupported` input** — passes `--allow-unsupported` to `repo tasks check`, so a task whose task.json `host` requirements the runner misses (a race that needs more CPUs, another vendor) passes its job instead of failing it. Broken tasks and failed host probes still fail. Off by default. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5f31e13d..37dedf20 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -898,8 +898,10 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. same framework and dependencies. Moving the pin is a change to the calling workflow, so it runs the smoke set. - Other inputs: `manifest`, `max-parallel` (4), `warm` (`--warm`, for a cache that - starts empty on every runner), `runs-on` (`ubuntu-latest`), `timeout-minutes` - (180, per task) and `dokimi-repository` (`intel/dokimi`). + starts empty on every runner), `allow-unsupported` (`--allow-unsupported`: a + task whose `host` requirements the runner misses passes its job), `runs-on` + (`ubuntu-latest`), `timeout-minutes` (180, per task) and `dokimi-repository` + (`intel/dokimi`). ```yaml # .github/workflows/agent-evals.yml in the repository diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index eff57aa4..6d0790bc 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -361,3 +361,39 @@ def test_the_check_results_artifact_keeps_the_stderr_files(): upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] assert "${{ runner.temp }}/repo-check/check/*/*.err" in paths + + +def _tasks_check_argv(tmp_path: Path, inputs: dict[str, bool]) -> list[str]: + """Run the ``repo tasks check`` step with a stub ``dokimi`` and return its argv.""" + step = next(s for s in _steps() if s.get("name") == "repo tasks check") + values = {f"inputs.{k}": str(v).lower() for k, v in inputs.items()} + values |= {"inputs.manifest": "m.yml", "matrix.task": "t1"} + env = {**os.environ, "RUNNER_TEMP": str(tmp_path)} + for key, value in step["env"].items(): + ref = re.fullmatch(r"\$\{\{ ([\w.-]+) \}\}", str(value)) + env[key] = values[ref.group(1)] if ref else str(value) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + stub = bin_dir / "dokimi" + stub.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$RUNNER_TEMP/argv"\n', encoding="utf-8") + stub.chmod(0o755) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + subprocess.run(["bash", "-eo", "pipefail", "-c", step["run"]], env=env, check=True) + return (tmp_path / "argv").read_text(encoding="utf-8").splitlines() + + +def test_allow_unsupported_is_a_boolean_input_that_is_off_by_default(): + doc = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + declared = doc[True]["workflow_call"]["inputs"]["allow-unsupported"] # `on` reads as True + assert declared["type"] == "boolean" + assert declared["default"] is False + + +@pytest.mark.skipif(sys.platform == "win32", reason="the step is a bash script") +@pytest.mark.parametrize("warm", [False, True]) +@pytest.mark.parametrize("allow", [False, True]) +def test_the_tasks_check_step_forwards_warm_and_allow_unsupported(tmp_path, warm, allow): + argv = _tasks_check_argv(tmp_path, {"warm": warm, "allow-unsupported": allow}) + assert argv[:6] == ["repo", "tasks", "check", ".", "--manifest", "m.yml"] + assert ("--warm" in argv) is warm + assert ("--allow-unsupported" in argv) is allow