diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 73e3b57403..acfd5b8a5e 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2163,7 +2163,12 @@ append_lm_eval_summary() { fi # Copy the complete allowlisted eval artifact set before removing its temp dir. - stage_eval_artifacts "$(pwd)" "$out_dir" || return $? + local artifact_dir + local artifact_sources=("$out_dir") + while IFS= read -r -d '' artifact_dir; do + [ "$artifact_dir" = "$out_dir" ] || artifact_sources+=("$artifact_dir") + done < <(find "$out_dir" -type d -print0 2>/dev/null) + stage_eval_artifacts "$(pwd)" "${artifact_sources[@]}" || return $? # Best-effort cleanup of the temp directory if [ -n "${out_dir}" ] && [ -d "${out_dir}" ]; then diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 469bd4aafd..6861291eef 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -171,6 +171,8 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench additionally uploads `agent_preds.json`, `predictions.jsonl`, `swebench_report_*.json`, and trajectory files in the single-node template. The aggregate is a navigation aid, not a substitute for raw samples and batch completeness. +Single-concurrency lm-eval can write `results*.json` and `sample*.jsonl` below a model-named subdirectory of `EVAL_RESULT_DIR`. `append_lm_eval_summary` recursively stages only the allowlisted eval artifacts into the workspace root before removing the temporary directory, where the workflow's flat upload patterns can find them. + ## 7. Run AgentX: fast feedback versus canonical evidence AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 501344bf25..e0d576fe0e 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -171,6 +171,8 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ 保留 `meta_env.json`、`results*.json` 和 `sample*.jsonl`。Agentic SWE-bench 在单节点模板中还会上传 `agent_preds.json`、`predictions.jsonl`、`swebench_report_*.json` 和 trajectory 文件。Aggregate 是导航工具,不能替代原始样本与 batch 完整性证据。 +单并发 lm-eval 可能会把 `results*.json` 和 `sample*.jsonl` 写入 `EVAL_RESULT_DIR` 下以模型命名的子目录。`append_lm_eval_summary` 会在删除临时目录前,递归地把允许列表中的 eval artifact 暂存到 workspace 根目录,使工作流的扁平上传模式能够找到这些文件。 + ## 7. 运行 AgentX:快速反馈与 canonical 证据 AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay,不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求,并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求,并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量;定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件([工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128))。 diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index f0e8007e53..e3b5bcb587 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1551,6 +1551,42 @@ def test_summary_stages_bfcl_upstream_archive_before_cleanup(tmp_path: Path) -> assert not results_dir.exists() +def test_summary_stages_nested_lm_eval_outputs_before_cleanup(tmp_path: Path) -> None: + work_dir = tmp_path / "work" + results_dir = tmp_path / "results" + nested_results_dir = results_dir / "local-chat-completions__test-model" + work_dir.mkdir() + nested_results_dir.mkdir(parents=True) + result = nested_results_dir / "results_2026-09-09T12-00-00.json" + sample = nested_results_dir / "samples_gsm8k_2026-09-09T12-00-00.jsonl" + result_content = '{"lm_eval_version": "test"}' + sample_content = '{"doc_id": 1}\n' + result.write_text(result_content) + sample.write_text(sample_content) + (nested_results_dir / "debug.log").write_text("do not stage") + script = r""" +source "$BENCHMARK_LIB" +cd "$WORK_DIR" +append_lm_eval_summary >/dev/null +""" + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "WORK_DIR": str(work_dir), + "EVAL_RESULT_DIR": str(results_dir), + "MODEL": "test-model", + "CONC": "7", + "KV_OFFLOADING": "none", + } + + subprocess.run(["bash", "-c", script], env=env, check=True) + + assert (work_dir / result.name).read_text() == result_content + assert (work_dir / sample.name).read_text() == sample_content + assert not (work_dir / "debug.log").exists() + assert not results_dir.exists() + + def test_stage_eval_artifacts_copies_eval_outputs_only(tmp_path: Path) -> None: source_one = tmp_path / "source-one" source_two = tmp_path / "source-two"