feat(ci): allow-unsupported input for the repo-eval workflow

.github/workflows/repo-eval.yml

@@ -79,6 +79,13 @@ on:
         required: false
         type: boolean
         default: false
+      allow-unsupported:
+        description: >-
+          Pass --allow-unsupported to `repo tasks check`: a task this runner misses the task.json
+          `host` requirements of passes its job (the reason is still in the log and summary)
+        required: false
+        type: boolean
+        default: false
       runs-on:
         description: Runner label for every job (a tasks-check job needs docker)
         required: false
@@ -373,12 +380,14 @@ jobs:
           MANIFEST: ${{ inputs.manifest }}
           TASK: ${{ matrix.task }}
           WARM: ${{ inputs.warm }}
+          ALLOW_UNSUPPORTED: ${{ inputs.allow-unsupported }}
           # Each task's verdict in the log as it lands, not when the step ends.
           PYTHONUNBUFFERED: '1'
         run: |
-          warm=()
-          if [ "$WARM" = true ]; then warm=(--warm); fi
-          dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${warm[@]}"
+          flags=()
+          if [ "$WARM" = true ]; then flags+=(--warm); fi
+          if [ "$ALLOW_UNSUPPORTED" = true ]; then flags+=(--allow-unsupported); fi
+          dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${flags[@]}"
       # always(): a hung oracle ends in the job timeout, which cancels the job; a
       # cancelled job still runs these steps, so the run leaves a verdict.
       - name: Summary

dokimi/changelog.d/20261021_120000_napetrov_repo-eval-allow-unsupported.md

@@ -0,0 +1,3 @@
+### Added
+
+- **repo-eval workflow: `allow-unsupported` input** — passes `--allow-unsupported` to `repo tasks check`, so a task whose task.json `host` requirements the runner misses (a race that needs more CPUs, another vendor) passes its job instead of failing it. Broken tasks and failed host probes still fail. Off by default.

dokimi/docs/repo-evaluation-contract.md

@@ -898,7 +898,8 @@ that runs the no-LLM checks on the caller's checkout and needs no secret.
   same framework and dependencies. Moving the pin is a change to the calling
   workflow, so it runs the smoke set.
 - Other inputs: `manifest`, `max-parallel` (4), `warm` (`--warm`, for a cache that
-  starts empty on every runner), `runs-on` (`ubuntu-latest`), `timeout-minutes`
+  starts empty on every runner), `allow-unsupported` (`--allow-unsupported`: a
+  task whose `host` requirements the runner misses passes its job), `runs-on` (`ubuntu-latest`), `timeout-minutes`
   (180, per task) and `dokimi-repository` (`intel/dokimi`).
 
 ```yaml

dokimi/tests/test_repo_eval_task_selection.py

@@ -361,3 +361,32 @@ def test_the_check_results_artifact_keeps_the_stderr_files():
     upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", "")))
     paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()]
     assert "${{ runner.temp }}/repo-check/check/*/*.err" in paths
+
+
+def _tasks_check_argv(tmp_path: Path, inputs: dict[str, bool]) -> list[str]:
+    """Run the ``repo tasks check`` step with a stub ``dokimi`` and return its argv."""
+    step = next(s for s in _steps() if s.get("name") == "repo tasks check")
+    values = {f"inputs.{k}": str(v).lower() for k, v in inputs.items()}
+    values |= {"inputs.manifest": "m.yml", "matrix.task": "t1"}
+    env = {**os.environ, "RUNNER_TEMP": str(tmp_path)}
+    for key, value in step["env"].items():
+        ref = re.fullmatch(r"\$\{\{ ([\w.-]+) \}\}", str(value))
+        env[key] = values[ref.group(1)] if ref else str(value)
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    stub = bin_dir / "dokimi"
+    stub.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$RUNNER_TEMP/argv"\n', encoding="utf-8")
+    stub.chmod(0o755)
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+    subprocess.run(["bash", "-eo", "pipefail", "-c", step["run"]], env=env, check=True)
+    return (tmp_path / "argv").read_text(encoding="utf-8").splitlines()
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="the step is a bash script")
+@pytest.mark.parametrize("warm", [False, True])
+@pytest.mark.parametrize("allow", [False, True])
+def test_the_tasks_check_step_forwards_warm_and_allow_unsupported(tmp_path, warm, allow):
+    argv = _tasks_check_argv(tmp_path, {"warm": warm, "allow-unsupported": allow})
+    assert argv[:6] == ["repo", "tasks", "check", ".", "--manifest", "m.yml"]
+    assert ("--warm" in argv) is warm
+    assert ("--allow-unsupported" in argv) is allow