From a886801886b4b0271981e0466ccebc594bbc7269 Mon Sep 17 00:00:00 2001 From: Anuj Gupta Date: Fri, 10 Jul 2026 18:02:31 +0530 Subject: [PATCH 1/2] Add reported reproducer config test suite Signed-off-by: Anuj Gupta --- .gitignore | 3 + pyproject.toml | 5 +- tests/reproducer_configs/README.md | 80 + tests/reproducer_configs/conftest.py | 81 + .../test_reported_reproducer_configs.py | 1596 +++++++++++++++++ 5 files changed, 1763 insertions(+), 2 deletions(-) create mode 100644 tests/reproducer_configs/README.md create mode 100644 tests/reproducer_configs/conftest.py create mode 100644 tests/reproducer_configs/test_reported_reproducer_configs.py diff --git a/.gitignore b/.gitignore index 594b15010e..1feee9770b 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,6 @@ qeff_models # Agent tool-specific skill links (generated by `make codex` / `make claude`) .agents/ .claude/ + +# Generated reproducer config verdict report +tests/reproducer_configs/reproducer_config_results.md diff --git a/pyproject.toml b/pyproject.toml index ee4ca672b9..ffcb5264c9 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,5 +100,6 @@ markers = [ "qnn: marks QNN tests", "cli: marks CLI tests", "finetune: marks finetune tests", - "vllm: marks vLLM tests" -] \ No newline at end of file + "vllm: marks vLLM tests", + "reproducer: marks pre-merge reported reproducer config tests" +] diff --git a/tests/reproducer_configs/README.md b/tests/reproducer_configs/README.md new file mode 100644 index 0000000000..e817b057fa --- /dev/null +++ b/tests/reproducer_configs/README.md @@ -0,0 +1,80 @@ +# Reported Reproducer Config Tests + +This suite captures reported QEfficient regression reproducers as pre-merge tests. +Developers should add the reproducer config first, fix the bug, and rerun the +same scenario before review. + +## Run the suite + +Run the reduced/default matrix: + +```bash +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q -rs +``` + +Run one scenario by its `RegressionScenario.name`: + +```bash +pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ + --qeff-reproducer-scenario hf-transfer-import-order -q +``` + +Run multiple scenarios with repeated or comma-separated values: + +```bash +pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ + --qeff-reproducer-scenario hf-transfer-import-order \ + --qeff-reproducer-scenario ccl-default-enabled,embedding-model-compile -q +``` + +List available scenario names without running them: + +```bash +pytest tests/reproducer_configs/test_reported_reproducer_configs.py --collect-only -q +``` + +## Full-model reproducers + +Reduced/tiny models are used by default. Scenarios that need the official model +card are gated and must be enabled explicitly: + +```bash +QEFF_REPRODUCER_RUN_FULL_MODELS=1 \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ + --qeff-reproducer-scenario -q +``` + +Use official model cards instead of tiny reductions where both are present: + +```bash +QEFF_REPRODUCER_USE_OFFICIAL_MODELS=1 QEFF_REPRODUCER_RUN_FULL_MODELS=1 \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ + --qeff-reproducer-scenario -q +``` + +## Runtime report + +The suite writes a local Markdown verdict ledger by default at +`tests/reproducer_configs/reproducer_config_results.md`. The report header records +the Hugging Face cache path selected for the run, including fallback to `/tmp` +when the default cache parent is not writable. Override the report location when +needed: + +```bash +QEFF_REPRODUCER_REPORT_MD=/tmp/qeff_reproducer_results.md \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ + --qeff-reproducer-scenario -q +``` + +`--collect-only` does not create or update the verdict report. + +## Add a scenario + +1. Add one `RegressionScenario` entry in `test_reported_reproducer_configs.py`. +2. Keep reporter options exact in `config`, `load_kwargs`, `export_kwargs`, + `compile_kwargs`, or `inference_kwargs`. +3. Use a tiny or 2/4-layer model when it faithfully reproduces the issue. +4. Keep the official model card and gate it with `QEFF_REPRODUCER_RUN_FULL_MODELS=1` + when a reduction is not faithful. +5. Run only the new scenario with `--qeff-reproducer-scenario ` + before running the broader matrix. diff --git a/tests/reproducer_configs/conftest.py b/tests/reproducer_configs/conftest.py new file mode 100644 index 0000000000..46dc91fa11 --- /dev/null +++ b/tests/reproducer_configs/conftest.py @@ -0,0 +1,81 @@ +# ----------------------------------------------------------------------------- +# +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# ----------------------------------------------------------------------------- + +"""Pytest hooks for reported reproducer config tests.""" + +from __future__ import annotations + +import pytest + +SCENARIO_CLI_OPTION = "--qeff-reproducer-scenario" + + +# Parse comma-separated or repeated scenario names from pytest CLI values. +def _split_scenario_names(values: list[str]) -> tuple[str, ...]: + """Return unique scenario names from repeated or comma-separated options. + + The helper exists so developers can run one or more just-added reproducers + without collecting unrelated scenarios from the full reported-config matrix. + """ + names = [] + for value in values: + for name in value.split(","): + stripped_name = name.strip() + if stripped_name and stripped_name not in names: + names.append(stripped_name) + return tuple(names) + + +# Register a targeted reproducer selector for local developer loops. +def pytest_addoption(parser: pytest.Parser) -> None: + """Add the scenario-selection option used by reproducer config tests.""" + group = parser.getgroup("qeff reproducer configs") + group.addoption( + SCENARIO_CLI_OPTION, + action="append", + default=[], + metavar="NAME[,NAME...]", + help="Run only the named reproducer scenario(s). Can be repeated or comma-separated.", + ) + + +# Deselect non-targeted reproducer items after parametrization. +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Keep only requested reproducer scenarios when the CLI selector is set. + + The hook exists so ``--qeff-reproducer-scenario`` runs exactly the requested + scenario tests and drops catalog guards or empty-stage sentinel parameters. + """ + selected_names = set(_split_scenario_names(config.getoption(SCENARIO_CLI_OPTION) or [])) + if not selected_names: + return + + available_names = set() + for item in items: + scenario = getattr(getattr(item, "callspec", None), "params", {}).get("scenario") + scenario_name = getattr(scenario, "name", None) + if scenario_name: + available_names.add(scenario_name) + + missing_names = selected_names - available_names + if missing_names: + names = ", ".join(sorted(missing_names)) + raise pytest.UsageError(f"unknown reproducer scenario(s): {names}") + + kept_items = [] + deselected_items = [] + for item in items: + scenario = getattr(getattr(item, "callspec", None), "params", {}).get("scenario") + scenario_name = getattr(scenario, "name", None) + if scenario_name in selected_names: + kept_items.append(item) + else: + deselected_items.append(item) + + if deselected_items: + config.hook.pytest_deselected(items=deselected_items) + items[:] = kept_items diff --git a/tests/reproducer_configs/test_reported_reproducer_configs.py b/tests/reproducer_configs/test_reported_reproducer_configs.py new file mode 100644 index 0000000000..771e1bbb15 --- /dev/null +++ b/tests/reproducer_configs/test_reported_reproducer_configs.py @@ -0,0 +1,1596 @@ +# ----------------------------------------------------------------------------- +# +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# ----------------------------------------------------------------------------- + +"""Pre-merge reproducer configs for reported QEfficient regressions. + +Flow +---- +1. Each ``RegressionScenario`` is a self-contained bug-report reproducer entry. + The scenario entry, not any external Markdown format, is the source of truth. +2. Developers add the exact reproducer config before fixing a bug, then run + the same config after the fix before merge. +3. Active scenarios run the real QEfficient software stage named by ``stage``: + import/install, model download, model export, model compile, or inference. +4. Prefer tiny-random or 2/4-layer reductions of the same architecture. If a + reduced model cannot reproduce the bug, keep the official model card and run + it with ``QEFF_REPRODUCER_RUN_FULL_MODELS=1``. +5. Scenarios that require another machine's paths, private artifacts, vLLM + services, runtime generation, or full-memory measurement are explicit skips. + They remain visible in collection and cannot be mistaken for passing verdicts. +6. To add a future report, copy the commented template in ``SCENARIOS``, paste + exact reporter options into ``config``, choose the correct ``model_api``, and + implement the real stage assertion in the helper for that API. +7. To run one scenario during fix validation, pass + ``--qeff-reproducer-scenario `` to pytest. +""" + +from __future__ import annotations + +import contextlib +import inspect +import os +import re +import sys +import warnings +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +import pytest +import torch + +try: + import fcntl +except ImportError: # pragma: no cover - Linux CI has fcntl. + fcntl = None + +Stage = Literal["import_install", "download", "export", "compile", "inference"] +ModelAPI = Literal["causal_lm", "image_text_to_text", "wan_t2v", "wan_i2v", "replicate_kv_heads", "embedding"] + + +# Scenario entries are immutable so pytest parametrization cannot mutate reporter-derived options. +@dataclass(frozen=True) +class RegressionScenario: + """Store one reported regression with its exact stage and reduced test model. + + This class exists to keep model card, tiny checkpoint, failing options, + and explicit deferral reason together as a single auditable test record. + """ + + name: str + stage: Stage + source_model_card: str + tiny_model_id: str | None + summary: str + model_api: ModelAPI = "causal_lm" + official_model_id: str | None = None + config: dict[str, Any] = field(default_factory=dict) + load_kwargs: dict[str, Any] = field(default_factory=dict) + export_kwargs: dict[str, Any] = field(default_factory=dict) + compile_kwargs: dict[str, Any] = field(default_factory=dict) + inference_kwargs: dict[str, Any] = field(default_factory=dict) + skip_reason: str | None = None + follow_up: str | None = None + run_full_model_by_default: bool = False + + +TINY_GPT2 = "hf-internal-testing/tiny-random-GPT2LMHeadModel" +TINY_LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM" +TINY_MISTRAL = "hf-internal-testing/tiny-random-MistralForCausalLM" +TINY_FALCON = "hf-internal-testing/tiny-random-FalconForCausalLM" +TINY_MIXTRAL = "hf-internal-testing/tiny-random-MixtralForCausalLM" +TINY_BERT = "hf-internal-testing/tiny-random-BertModel" +TINY_GPT_OSS = "tiny-random/gpt-oss-bf16" +TINY_GEMMA3 = "tiny-random/gemma-3" +TINY_QWEN2_5_VL = "optimum-intel-internal-testing/tiny-random-qwen2.5-vl" +TINY_INTERNVL = "optimum-intel-internal-testing/tiny-random-internvl2" +TINY_QWEN3_MOE = "tiny-random/qwen3-moe" +TINY_QWEN3_5_MOE = "tiny-random/qwen3.5-moe" +TINY_QWEN3_VL_MOE = "tiny-random/qwen3-vl-moe" +FULL_MODEL_ENV = "QEFF_REPRODUCER_RUN_FULL_MODELS" +OFFICIAL_MODEL_ENV = "QEFF_REPRODUCER_USE_OFFICIAL_MODELS" +REPORT_PATH_ENV = "QEFF_REPRODUCER_REPORT_MD" +SCENARIO_CLI_OPTION = "--qeff-reproducer-scenario" +DEFAULT_REPORT_PATH = Path("tests/reproducer_configs/reproducer_config_results.md") +EXPECTED_SCENARIO_COUNT = 50 +EXTRA_QEFF_COMPILE_OPTIONS = frozenset( + { + "height", + "mdp_compiler_dump_path", + "mdp_dump_partition_config", + "mdp_load_partition_config", + "mdp_num_partitions", + "mdp_strategy", + "mdp_ts_num_devices", + "mm_processor_kwargs", + "node_precision_info", + "num_frames", + "parallel", + "vision_size", + "width", + } +) +COMPILER_ERROR_PATTERNS = tuple( + re.compile(pattern, re.IGNORECASE) + for pattern in ( + r"^Operator-", + r"QAIC_ERROR", + r"Error message:", + r"Unable to Compile", + r"Compilation failed", + r"Non-constant", + r"not supported", + ) +) + + +# Resolve the Hugging Face cache without mutating environment variables. +def _resolve_hf_cache_path() -> Path: + """Return the writable Hugging Face cache path used by reproducer tests. + + The helper keeps cache selection consistent between the runtime ledger and + the environment setup that precedes model downloads. + """ + requested_cache = Path( + os.environ.get("QEFF_REPRODUCER_HF_CACHE") + or os.environ.get("HF_HUB_CACHE") + or "/home/tmp/qeff_reproducer_hf_cache" + ) + if not os.access(requested_cache.parent, os.W_OK): + return Path("/tmp/qeff_reproducer_hf_cache") + return requested_cache + + +# Generated reports should not appear for pytest --collect-only runs. +def _is_pytest_collect_only() -> bool: + """Return whether pytest is collecting tests without executing them. + + The helper prevents collection-only invocations from creating a misleading + partial Markdown verdict report before any scenario has actually run. + """ + return "--collect-only" in sys.argv or "--co" in sys.argv + + +# Lock report writes so xdist workers cannot corrupt the runtime ledger. +@contextlib.contextmanager +def _locked_report(path: Path): + """Yield a locked report file handle for atomic read/append operations. + + The helper exists because pytest-xdist workers can record scenario verdicts + concurrently, and the generated Markdown report should remain readable. + """ + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8") as handle: + if fcntl is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield handle + finally: + if fcntl is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +# Tiny model IDs mirror the quickcheck suite and keep reports cheap to rerun. +# Official model IDs are retained for bugs that cannot be reproduced faithfully +# with a reduced checkpoint. They run only when FULL_MODEL_ENV is set. +SCENARIOS: tuple[RegressionScenario, ...] = ( + # Template for future bug reports; keep commented until a real report is added. + # RegressionScenario( + # name="short-neutral-slug", + # stage="compile", + # source_model_card="org/reported-model-card", + # tiny_model_id="hf-internal-testing/tiny-random-compatible-model", + # official_model_id="org/reported-model-card", + # summary="one-line failure mode", + # model_api="causal_lm", + # compile_kwargs={"copy_exact_reporter_flag": "copy_exact_reporter_value"}, + # skip_reason="only if exact repro requires external artifacts or runtime services", + # ), + RegressionScenario( + name="hf-transfer-import-order", + stage="import_install", + source_model_card="large gated Hugging Face checkpoints", + tiny_model_id=None, + summary="hf_transfer must be enabled before transformers import", + ), + RegressionScenario( + name="cloud-export-hf-token", + stage="download", + source_model_card="mistralai/Mistral-7B-v0.1", + tiny_model_id=TINY_MISTRAL, + summary="cloud export token path used an external cache directory in the report", + config={"hf_token": "", "full_batch_size": 2}, + skip_reason="reported config depends on an absolute cache path from another machine", + ), + RegressionScenario( + name="vlm-cli-config-dump", + stage="export", + source_model_card="VLM CLI/config dump", + tiny_model_id=TINY_QWEN2_5_VL, + summary="report did not include a concrete failing command or config payload", + skip_reason="exact reporter options are not available", + ), + RegressionScenario( + name="internvl25-decoder-language-model", + stage="export", + source_model_card="OpenGVLab/InternVL2_5-1B", + tiny_model_id=TINY_INTERNVL, + summary="InternVL language decoder wrapper export must expose language_model", + load_kwargs={"trust_remote_code": True}, + export_kwargs={"kv_offload": True}, + skip_reason="tiny InternVL export requires optional einops and timm dependencies", + ), + RegressionScenario( + name="internvl-dual-qpc-custom-num-patches", + stage="inference", + source_model_card="OpenGVLab/InternVL2_5-1B", + tiny_model_id=TINY_INTERNVL, + summary="dual-QPC custom num_patches=9 failed during AIC generation", + config={"kv_offload": True, "num_patches": 9, "generation_len": 128}, + skip_reason="verdict requires QAIC runtime generation with dual QPCs", + ), + RegressionScenario( + name="internvl-large-shape-mismatch", + stage="inference", + source_model_card="OpenGVLab/InternVL2_5-38B or OpenGVLab/InternVL2_5-78B", + tiny_model_id=TINY_INTERNVL, + summary="large InternVL shape mismatch report lacks exact compile/inference options", + skip_reason="exact reporter options are not available", + ), + RegressionScenario( + name="internvl25-dual-qpc-input-shape", + stage="inference", + source_model_card="OpenGVLab/InternVL2_5 family", + tiny_model_id=TINY_INTERNVL, + summary="dual-QPC input shape mismatch requires reported image/runtime setup", + skip_reason="verdict requires QAIC runtime generation with reported image input", + ), + RegressionScenario( + name="attention-mask-negative-infinity", + stage="export", + source_model_card="meta-llama/Llama-3.1-8B-Instruct", + tiny_model_id=TINY_LLAMA, + summary="attention masking export must use negative infinity semantics", + export_kwargs={}, + ), + RegressionScenario( + name="swiftkv-compile", + stage="compile", + source_model_card="Snowflake/Llama-3.1-SwiftKV-8B-Instruct", + tiny_model_id=None, + official_model_id="Snowflake/Llama-3.1-SwiftKV-8B-Instruct", + summary="SwiftKV cloud.infer compile options must produce a QPC", + compile_kwargs={ + "prefill_seq_len": 2048, + "ctx_len": 3072, + "batch_size": 1, + "num_devices": 4, + "num_cores": 16, + "full_batch_size": 1, + "mxfp6_matmul": True, + "mxint8_kv_cache": True, + "aic_enable_depth_first": True, + "allow_mxint8_mdp_io": True, + }, + ), + RegressionScenario( + name="kv-head-replication-num-heads", + stage="export", + source_model_card="meta-llama/Llama-3.1-8B-Instruct", + tiny_model_id=TINY_LLAMA, + summary="replicate_kv_heads script is absent from this checkout", + skip_reason="reported script entry point is not present in this repository", + ), + RegressionScenario( + name="mllama-dual-qpc-model-attr", + stage="compile", + source_model_card="meta-llama/Llama-3.2-11B-Vision-Instruct", + tiny_model_id=None, + official_model_id="meta-llama/Llama-3.2-11B-Vision-Instruct", + summary="Mllama dual-QPC VLM compile must expose language_model config", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True, "num_hidden_layers": 2}, + compile_kwargs={ + "prefill_seq_len": 32, + "ctx_len": 512, + "img_size": 560, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + }, + ), + RegressionScenario( + name="turbolora-export-hash", + stage="export", + source_model_card="TurboLoRA Llama path", + tiny_model_id=TINY_LLAMA, + summary="TurboLoRA needs PEFT/speculator fixtures", + skip_reason="speculator and adapter artifacts are not available", + ), + RegressionScenario( + name="external-mdp-load-config-duplicate", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=None, + official_model_id="openai/gpt-oss-20b", + summary="external mdp_load_partition_config appeared twice in compile command", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 2, + "num_cores": 8, + "mxfp6_matmul": True, + "mdp_num_partitions": 2, + "mdp_strategy": "onnx", + }, + ), + RegressionScenario( + name="llama4-dynamic-cache-export", + stage="compile", + source_model_card="meta-llama/Llama-4-Scout-17B-16E-Instruct", + tiny_model_id=None, + official_model_id="meta-llama/Llama-4-Scout-17B-16E-Instruct", + summary="Llama4 dynamic-cache VLM compile must preserve CtxGatherND path", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True, "num_hidden_layers": 4}, + compile_kwargs={ + "prefill_seq_len": 32, + "ctx_len": 3072, + "img_size": 336, + "batch_size": 1, + "num_devices": 4, + "num_cores": 8, + }, + ), + RegressionScenario( + name="swiftkv-replicate-kv-heads", + stage="export", + source_model_card="Snowflake/Llama-3.1-SwiftKV-8B-Instruct", + tiny_model_id=None, + official_model_id="Snowflake/Llama-3.1-SwiftKV-8B-Instruct", + summary="SwiftKV replicate_kv_heads export must accept attention_mask", + model_api="replicate_kv_heads", + export_kwargs={"prompt": "Hello, world!", "repeat": 2, "full_batch_size": 1, "num_hidden_layers": 2}, + ), + RegressionScenario( + name="falcon-input-layernorm-export", + stage="export", + source_model_card="tiiuae/falcon-40b", + tiny_model_id=TINY_FALCON, + summary="Falcon export failed because decoder layer lacked input_layernorm", + export_kwargs={}, + ), + RegressionScenario( + name="slow-downloads-hf-transfer", + stage="import_install", + source_model_card="large Hugging Face checkpoints", + tiny_model_id=None, + summary="hf_transfer enablement must happen before HF imports", + ), + RegressionScenario( + name="finetune-dataset-padding", + stage="inference", + source_model_card="fine-tuning datasets", + tiny_model_id=None, + summary="fine-tune DDP dataset padding belongs in fine-tune suite", + skip_reason="not a QEff export/compile/inference model-stage reproducer", + ), + RegressionScenario( + name="qwen-awq-replicate-kv-bias-scaling", + stage="export", + source_model_card="Qwen/Qwen2.5-72B-Instruct-AWQ", + tiny_model_id=None, + summary="report has no captured reproducer details", + skip_reason="exact reporter options are not available", + ), + RegressionScenario( + name="gpt-oss-ts2-cpl128-cl4096-compile", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=TINY_GPT_OSS, + summary="GPT-OSS requested TS2, CPL128, CL4096, cores8, MXFP6 compile must produce QPC", + load_kwargs={"trust_remote_code": True}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 2, + "num_cores": 8, + "mxfp6_matmul": True, + }, + ), + RegressionScenario( + name="gemma3-large-image-noisy-output", + stage="inference", + source_model_card="google/gemma-3-27b-it", + tiny_model_id=TINY_GEMMA3, + summary="Gemma3 noisy output report depends on attached OCR image and prompt", + skip_reason="attached image and expected text are not available in this repository", + ), + RegressionScenario( + name="wan22-n-layer-cache-hash", + stage="compile", + source_model_card="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + tiny_model_id=None, + official_model_id="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + summary="WAN2.2 n-layer cache hash must produce deterministic QPC paths", + model_api="wan_t2v", + compile_kwargs={"height": 192, "width": 320, "num_frames": 9, "parallel": False}, + ), + RegressionScenario( + name="disagg-prefill-continuous-batching", + stage="compile", + source_model_card="meta-llama/Llama-3.3-70B-Instruct", + tiny_model_id=None, + official_model_id="meta-llama/Llama-3.3-70B-Instruct", + summary="disagg prefill compile used full_batch_size with external MDP JSON", + compile_kwargs={ + "prefill_seq_len": 256, + "ctx_len": 11264, + "batch_size": 1, + "full_batch_size": 16, + "num_cores": 16, + "num_devices": 16, + "mxfp6_matmul": True, + "mxint8_kv_cache": True, + "prefill_only": True, + "aic_enable_depth_first": True, + "allow_mxint8_mdp_io": True, + "mdp_num_partitions": 4, + "mdp_strategy": "onnx", + }, + ), + RegressionScenario( + name="ccl-decode-fallback-context", + stage="compile", + source_model_card="meta-llama/Llama-3.2-1B", + tiny_model_id=TINY_LLAMA, + summary="decode CCL [4096] must include CL8192 fallback specialization", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 8192, + "num_cores": 16, + "num_devices": 1, + "mxint8_kv_cache": True, + "mxfp6_matmul": True, + "batch_size": 1, + "comp_ctx_lengths_prefill": [4000], + "comp_ctx_lengths_decode": [4096], + }, + ), + RegressionScenario( + name="ccl-none-prefill-with-decode", + stage="compile", + source_model_card="meta-llama/Llama-3.2-1B", + tiny_model_id=TINY_LLAMA, + summary="None prefill CCL and configured decode CCL must compile", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 8192, + "num_cores": 16, + "num_devices": 1, + "mxint8_kv_cache": True, + "mxfp6_matmul": True, + "batch_size": 1, + "comp_ctx_lengths_prefill": None, + "comp_ctx_lengths_decode": [4096], + }, + ), + RegressionScenario( + name="ccl-empty-prefill-with-decode", + stage="compile", + source_model_card="meta-llama/Llama-3.2-1B", + tiny_model_id=TINY_LLAMA, + summary="empty prefill CCL and configured decode CCL must compile", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 8192, + "num_cores": 16, + "num_devices": 1, + "mxint8_kv_cache": True, + "mxfp6_matmul": True, + "batch_size": 1, + "comp_ctx_lengths_prefill": [], + "comp_ctx_lengths_decode": [4096, 8192], + }, + ), + RegressionScenario( + name="mdp-dump-load-json-options", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=None, + official_model_id="openai/gpt-oss-20b", + summary="custom mdp dump/load config cases must work with local generated JSON artifacts", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 2, + "num_cores": 8, + "mxfp6_matmul": True, + "_mdp_intersection": True, + "mdp_num_partitions": 2, + }, + ), + RegressionScenario( + name="qwen25-vl-subfunction-split-compile", + stage="compile", + source_model_card="Qwen/Qwen2.5-VL-32B-Instruct", + tiny_model_id=TINY_QWEN2_5_VL, + official_model_id="Qwen/Qwen2.5-VL-32B-Instruct", + summary="Qwen2.5-VL subfunction compiler failure needs QAIC graph validation", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + "use_onnx_subfunctions": True, + }, + ), + RegressionScenario( + name="pipeline-prefill-mdp-dump-no-load", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=None, + official_model_id="openai/gpt-oss-20b", + summary="pipeline prefill with mdp dump must generate local compiler dump and use repo NPI", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 4, + "num_cores": 16, + "mxfp6_matmul": True, + "mxint8_kv_cache": True, + "prefill_only": True, + "use_onnx_subfunctions": True, + "node_precision_info": "examples/disagg_serving/subfunction_120b_npi.yaml", + "_mdp_intersection": True, + "mdp_num_partitions": 2, + }, + ), + RegressionScenario( + name="mdp-ts-files-in-qpc-dir", + stage="compile", + source_model_card="meta-llama/Llama-3.1-8B-Instruct", + tiny_model_id=TINY_LLAMA, + summary="generated mdp_ts artifacts should be owned by compile/QPC path", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 1024, + "num_cores": 16, + "num_devices": 4, + "mxint8_kv_cache": True, + "mxfp6_matmul": True, + "batch_size": 1, + "full_batch_size": 4, + "mos": 1, + "aic_enable_depth_first": True, + "comp_ctx_lengths_prefill": [4000], + "comp_ctx_lengths_decode": [4096], + }, + ), + RegressionScenario( + name="ccl-default-enabled", + stage="compile", + source_model_card="CCL-enabled causal LM", + tiny_model_id=TINY_LLAMA, + summary="default CCL-enabled causal LM compile must succeed", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 8192, + "num_cores": 16, + "num_devices": 1, + "mxint8_kv_cache": True, + "mxfp6_matmul": True, + "batch_size": 1, + }, + ), + RegressionScenario( + name="wan-skip-compile-existing-qpc", + stage="compile", + source_model_card="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + tiny_model_id=None, + official_model_id="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + summary="WAN compile must honor existing QPC path skip behavior", + model_api="wan_t2v", + compile_kwargs={"height": 192, "width": 320, "num_frames": 9, "parallel": False}, + ), + RegressionScenario( + name="qwen3-mixtral-subfunction-reducesum", + stage="compile", + source_model_card="Qwen/Qwen3-30B-A3B-Instruct and Mixtral-8x7B", + tiny_model_id=None, + official_model_id="Qwen/Qwen3-30B-A3B-Instruct", + summary="compiler ReduceSum failures require full ONNX graph validation", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 4, + "num_cores": 16, + "mxfp6_matmul": True, + "use_onnx_subfunctions": True, + }, + ), + RegressionScenario( + name="disagg-ccl-decode-gpt-oss", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=TINY_GPT_OSS, + summary="vLLM disagg CCL decode compile options must produce decode QPC", + compile_kwargs={ + "prefill_seq_len": 1, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 2, + "num_cores": 8, + "mxfp6_matmul": True, + "comp_ctx_lengths_decode": [4096], + }, + ), + RegressionScenario( + name="qwen25-vl-subfunction-internal-assert", + stage="compile", + source_model_card="Qwen/Qwen2.5-VL-32B-Instruct", + tiny_model_id=TINY_QWEN2_5_VL, + official_model_id="Qwen/Qwen2.5-VL-32B-Instruct", + summary="subfunction internal compiler assert requires QAIC graph validation", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + "use_onnx_subfunctions": True, + }, + ), + RegressionScenario( + name="wan22-excessive-prints", + stage="compile", + source_model_card="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + tiny_model_id=None, + official_model_id="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + summary="WAN2.2 export/compile must complete without excessive graph prints", + model_api="wan_t2v", + compile_kwargs={"height": 192, "width": 320, "num_frames": 9, "parallel": False}, + ), + RegressionScenario( + name="embedding-model-compile", + stage="compile", + source_model_card="BAAI/bge-reranker-v2-m3 and intfloat/multilingual-e5-large", + tiny_model_id=TINY_BERT, + summary="embedding compile failure requires sequence/reranker compile path", + model_api="embedding", + compile_kwargs={"seq_len": 128, "batch_size": 1, "num_devices": 1, "num_cores": 16}, + ), + RegressionScenario( + name="qwen3-vl-smart-resize-list-inputs", + stage="compile", + source_model_card="Qwen/Qwen3-VL-32B-Instruct", + tiny_model_id=None, + official_model_id="Qwen/Qwen3-VL-2B-Instruct", + summary="Qwen3-VL list height/width smart_resize failed before compile", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True, "num_hidden_layers": 1}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + }, + ), + RegressionScenario( + name="gemma4-dense-2qpc-cb2", + stage="compile", + source_model_card="google/gemma-4-E2B-it", + tiny_model_id="tiny-random/gemma-4-dense", + official_model_id="google/gemma-4-E2B-it", + summary="Gemma4 dense 2-QPC compile with decode batch size >1", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 64, + "ctx_len": 512, + "img_size": 384, + "batch_size": 1, + "full_batch_size": 2, + "num_devices": 1, + "num_cores": 16, + }, + ), + RegressionScenario( + name="qwen36-subfunction-custom-io", + stage="compile", + source_model_card="Qwen/Qwen3.6-35B-A3B", + tiny_model_id=None, + official_model_id="Qwen/Qwen3.6-35B-A3B", + summary="Qwen3.6 subfunction custom_io content mismatch", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + "use_onnx_subfunctions": True, + }, + ), + RegressionScenario( + name="qwen3vl-custom-io-subfunctions", + stage="compile", + source_model_card="Qwen/Qwen3-VL-30B-A3B-Instruct", + tiny_model_id=TINY_QWEN3_VL_MOE, + official_model_id="Qwen/Qwen3-VL-30B-A3B-Instruct", + summary="Qwen3-VL subfunction custom_io.yaml invalid entries", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + "use_onnx_subfunctions": True, + }, + ), + RegressionScenario( + name="qwen35-layerwise-no-decoder-artifacts", + stage="export", + source_model_card="Qwen3.5-397B-A17B", + tiny_model_id=TINY_QWEN3_5_MOE, + summary="layerwise 3-QPC export ran for many hours without decoder artifacts", + skip_reason="long-running layerwise export scenario requires full model artifacts", + ), + RegressionScenario( + name="gptoss-mdp-num-partitions-file", + stage="compile", + source_model_card="openai/gpt-oss-20b", + tiny_model_id=None, + official_model_id="openai/gpt-oss-20b", + summary="GPT-OSS mdp_num_partitions should generate partition file", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 2, + "num_cores": 8, + "mxfp6_matmul": True, + "mdp_num_partitions": 2, + "mdp_strategy": "onnx", + }, + ), + RegressionScenario( + name="qwen35-moe-multires-support", + stage="compile", + source_model_card="Qwen/Qwen3.5-35B-A3B", + tiny_model_id=None, + official_model_id="Qwen/Qwen3.5-35B-A3B", + summary="multi-resolution and vision_size support for Qwen3.5 MoE", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 64, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + }, + ), + RegressionScenario( + name="gptoss-gemma3-prefill-reducesum", + stage="compile", + source_model_card="openai/gpt-oss-20b and Gemma3", + tiny_model_id=None, + official_model_id="openai/gpt-oss-20b", + summary="prefill compiler ReduceSum failure requires reported full ONNX graph", + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "batch_size": 1, + "num_devices": 4, + "num_cores": 16, + "mxfp6_matmul": True, + "prefill_only": True, + }, + ), + RegressionScenario( + name="qwen3vl-moe-memory-after-prefill", + stage="export", + source_model_card="Qwen/Qwen3-VL-235B-A22B-Instruct", + tiny_model_id=TINY_QWEN3_VL_MOE, + summary="decode export RAM spike after prefill in same process", + skip_reason="full-model memory regression requires dedicated memory measurement job", + ), + RegressionScenario( + name="mdp-generation-example-script", + stage="compile", + source_model_card="Qwen/Qwen3-VL-30B-A3B-Instruct", + tiny_model_id=None, + official_model_id="Qwen/Qwen3-VL-30B-A3B-Instruct", + summary="Qwen3-VL MDP example must compile prefill without duplicate device options", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True, "layerwise": False}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "height": 354, + "width": 536, + "batch_size": 1, + "num_devices": 4, + "num_cores": 16, + "mos": 1, + "mxfp6_matmul": True, + "mxint8_kv_cache": True, + "retain_full_kv": True, + "prefill_only": True, + "enable_chunking": True, + "skip_vision": True, + "use_onnx_subfunctions": True, + "mdp_num_partitions": 2, + "mdp_strategy": "onnx", + }, + ), + RegressionScenario( + name="qwen-numpy-fix", + stage="export", + source_model_card="Qwen models", + tiny_model_id=None, + summary="report lacks concrete command/options", + skip_reason="exact reporter options are not available", + ), + RegressionScenario( + name="qwen3vl-moe-ctxgather-repeats", + stage="compile", + source_model_card="Qwen/Qwen3-VL-30B-A3B-Instruct", + tiny_model_id=None, + official_model_id="Qwen/Qwen3-VL-30B-A3B-Instruct", + summary="multi-resolution prefill CtxGatherCB non-constant repeats compiler failure", + model_api="image_text_to_text", + load_kwargs={"kv_offload": True}, + compile_kwargs={ + "prefill_seq_len": 128, + "ctx_len": 4096, + "img_size": 1540, + "batch_size": 1, + "num_devices": 1, + "num_cores": 16, + }, + ), + RegressionScenario( + name="mixtral-oom-export", + stage="export", + source_model_card="mistralai/Mixtral-8x7B-Instruct-v0.1", + tiny_model_id=TINY_MIXTRAL, + summary="full Mixtral export RAM spike requires memory measurement", + skip_reason="full-model memory regression requires dedicated memory measurement job", + ), +) + + +# Parse targeted scenario names from pytest CLI before parametrization writes reports. +def _selected_scenario_names_from_argv() -> tuple[str, ...]: + """Return scenario names requested by ``--qeff-reproducer-scenario``. + + The helper mirrors the registered pytest option so parametrization can avoid + creating skip-report rows for scenarios that the developer did not request. + """ + values = [] + args = sys.argv[1:] + index = 0 + while index < len(args): + arg = args[index] + if arg == SCENARIO_CLI_OPTION and index + 1 < len(args): + values.append(args[index + 1]) + index += 2 + continue + prefix = f"{SCENARIO_CLI_OPTION}=" + if arg.startswith(prefix): + values.append(arg[len(prefix) :]) + index += 1 + + names = [] + for value in values: + for name in value.split(","): + stripped_name = name.strip() + if stripped_name and stripped_name not in names: + names.append(stripped_name) + return tuple(names) + + +# Build pytest parameters once so skipped regressions remain visible in collection. +def _params_for(stage: Stage) -> tuple[Any, ...]: + """Return pytest parameters for one reproducer stage. + + The helper exists so every test function gets identical skip handling, + stable readable IDs, and QAIC markers for hardware-backed stages. + """ + selected_names = set(_selected_scenario_names_from_argv()) + params = [] + for scenario in SCENARIOS: + if scenario.stage != stage: + continue + if selected_names and scenario.name not in selected_names: + continue + marks = [] + skip_reason = _skip_reason_for(scenario) + if skip_reason: + _record_result(scenario, "SKIPPED", skip_reason) + marks.append(pytest.mark.skip(reason=skip_reason)) + if stage in {"compile", "inference"}: + marks.append(pytest.mark.on_qaic) + if scenario.model_api in {"wan_t2v", "wan_i2v"}: + marks.append(pytest.mark.wan) + if scenario.model_api == "image_text_to_text": + marks.append(pytest.mark.multimodal) + params.append(pytest.param(scenario, id=scenario.name, marks=marks)) + if not params: + params.append( + pytest.param(None, id=f"no-{stage}-configs", marks=pytest.mark.skip(reason=f"no selected {stage} configs")) + ) + return tuple(params) + + +# Full official models are gated so normal developer runs stay bounded. +def _skip_reason_for(scenario: RegressionScenario) -> str | None: + """Return the collection-time skip reason for a scenario, if any. + + This helper exists to keep full-model reproducers present in the catalog + while making expensive official-model execution an explicit developer choice. + """ + if scenario.skip_reason: + return scenario.skip_reason + if _requires_full_model(scenario) and not _run_full_models(): + return f"set {FULL_MODEL_ENV}=1 to run official model {scenario.official_model_id}" + return None + + +# Official model fallback is required when tiny reductions cannot reproduce a bug. +def _requires_full_model(scenario: RegressionScenario) -> bool: + """Return whether a scenario must use the official model card. + + The helper treats missing tiny checkpoints and explicit official-model runs + as full-model cases so they never run accidentally in quick local checks. + """ + if scenario.run_full_model_by_default: + return False + return bool(scenario.official_model_id and (not scenario.tiny_model_id or _use_official_models())) + + +# Env switches keep quick local validation separate from production repro runs. +def _run_full_models() -> bool: + """Return whether full official models should be loaded and compiled.""" + return os.environ.get(FULL_MODEL_ENV) == "1" + + +def _use_official_models() -> bool: + """Return whether official model cards override tiny reproducer models.""" + return os.environ.get(OFFICIAL_MODEL_ENV) == "1" + + +# Select the actual model card for the current run mode. +def _scenario_model_id(scenario: RegressionScenario) -> str: + """Resolve the tiny or official model card for a scenario. + + Tiny models are preferred for speed; official models are used when requested + or when no faithful reduced checkpoint exists. + """ + if scenario.official_model_id and (not scenario.tiny_model_id or _use_official_models()): + if not _run_full_models(): + pytest.skip(f"set {FULL_MODEL_ENV}=1 to run official model {scenario.official_model_id}") + return scenario.official_model_id + if scenario.tiny_model_id: + return scenario.tiny_model_id + pytest.skip("scenario has no runnable model") + + +# Report writer persists producer/consumer-readable results as tests execute. +def _record_result(scenario: RegressionScenario, status: str, detail: str) -> None: + """Append one scenario verdict to the Markdown reproducer report. + + The helper writes a real PASS/SKIP/FAIL ledger so developers can correlate + every reproducer config with its latest observed verdict while pytest runs. + """ + if _is_pytest_collect_only(): + return + path = Path(os.environ.get(REPORT_PATH_ENV, DEFAULT_REPORT_PATH)) + + if scenario.official_model_id and (not scenario.tiny_model_id or _use_official_models()): + model_id = scenario.official_model_id + else: + model_id = scenario.tiny_model_id or scenario.official_model_id or scenario.source_model_card + safe_detail = detail.replace("\n", "
").replace("|", "\\|") + follow_up_text = scenario.follow_up + if not follow_up_text and scenario.skip_reason: + follow_up_text = "add a portable runnable reproducer or confirm this remains non-runnable" + follow_up = (follow_up_text or "").replace("\n", "
").replace("|", "\\|") + row = ( + f"| {scenario.name} | {scenario.stage} | {scenario.model_api} | {model_id} | " + f"{status} | {follow_up} | {safe_detail} |\n" + ) + + with _locked_report(path) as handle: + handle.seek(0) + content = handle.read() + if not content.startswith("# QEfficient Reproducer Config Results"): + handle.seek(0) + handle.truncate() + handle.write( + "# QEfficient Reproducer Config Results\n\n" + "This file is generated by `tests/reproducer_configs/test_reported_reproducer_configs.py`.\n" + "Each row is a concrete verdict from collection or execution of one reproducer config.\n\n" + f"Hugging Face cache: `{_resolve_hf_cache_path()}`\n\n" + "| Scenario | Stage | API | Model Card | Verdict | Follow-up | Detail |\n" + "|---|---|---|---|---|---|---|\n" + ) + content = "" + if row not in content: + handle.write(row) + + +def _record_failure(scenario: RegressionScenario, exc: BaseException) -> None: + """Record a compact failure signature for a scenario. + + The helper keeps the Markdown report readable while preserving the exception + type, first message lines, and compiler operator/error lines that usually + carry the actionable failure signature. + """ + message_lines = [line.strip() for line in str(exc).splitlines() if line.strip()] + selected_lines = message_lines[:4] + for line in message_lines[4:]: + if any(pattern.search(line) for pattern in COMPILER_ERROR_PATTERNS) and line not in selected_lines: + selected_lines.append(line) + if len(selected_lines) >= 6: + break + signature = f"{type(exc).__name__}: {selected_lines[0] if selected_lines else ''}" + if len(selected_lines) > 1: + signature = "
".join([signature, *selected_lines[1:]]) + _record_result(scenario, "FAILED", signature) + + +def _run_with_verdict(scenario: RegressionScenario, runner, *args) -> None: + """Run one scenario helper and record FAIL before propagating exceptions.""" + try: + runner(scenario, *args) + except BaseException as exc: + if not isinstance(exc, pytest.skip.Exception): + _record_failure(scenario, exc) + raise + + +# Per-test environment restoration prevents cache and QEFF_HOME leakage. +@pytest.fixture(autouse=True) +def _restore_reproducer_environment(): + """Restore mutable reproducer environment variables after each test. + + The fixture exists because model loading and WAN compilation require process + environment variables, but those settings must not leak into later tests. + """ + keys = ( + "HF_HOME", + "HF_HUB_CACHE", + "HF_HUB_ENABLE_HF_TRANSFER", + "HF_MODULES_CACHE", + "QEFF_HOME", + ) + original_values = {key: os.environ.get(key) for key in keys} + yield + for key, value in original_values.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +# Normalize Hugging Face cache settings before any real model download or load. +def _ensure_hf_cache_env() -> None: + """Set the cache and transfer environment used by reduced reproducers. + + This helper keeps downloads in a writable local cache by default while + preserving explicit caller overrides for production reproducer runs. + """ + requested_cache = _resolve_hf_cache_path() + try: + requested_cache.mkdir(parents=True, exist_ok=True) + except OSError as exc: + pytest.skip(f"cannot create Hugging Face cache {requested_cache}: {exc}") + + os.environ["HF_HOME"] = str(requested_cache) + os.environ["HF_HUB_CACHE"] = str(requested_cache) + os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") + modules_cache = requested_cache / "modules" + try: + modules_cache.mkdir(parents=True, exist_ok=True) + except OSError as exc: + pytest.skip(f"cannot create Hugging Face modules cache {modules_cache}: {exc}") + os.environ["HF_MODULES_CACHE"] = str(modules_cache) + try: + import transformers.dynamic_module_utils as dynamic_module_utils + import transformers.utils.hub as transformers_hub + + dynamic_module_utils.HF_MODULES_CACHE = str(modules_cache) + transformers_hub.HF_MODULES_CACHE = str(modules_cache) + except (ImportError, AttributeError) as exc: + warnings.warn(f"could not update transformers module cache paths: {exc}", stacklevel=2) + + +# Hardware-backed scenarios must not silently pass on hosts without the compiler. +def _require_qaic_compile() -> None: + """Skip compile or inference scenarios when the QAIC compiler is absent. + + The check exists because active compile tests call QEfficient's real compile + API and need the production compiler binary to produce a QPC verdict. + """ + if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): + pytest.skip("qaic-compile is not available on this host") + + +# Runtime scenarios also require SDK Python bindings and a visible SoC device. +def _require_qaic_runtime() -> None: + """Skip inference scenarios when the QAIC runtime stack is unavailable. + + The helper is stricter than the compile gate because generation needs the + runtime libraries and an accessible ``/dev/qaic*`` device, not just QPC build. + """ + if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): + pytest.skip("QAIC runtime is unavailable because qaic-compile is not installed on this host") + if not any(Path("/dev").glob("qaic*")): + pytest.skip("no /dev/qaic* device is available on this host") + try: + from QEfficient.generation import cloud_infer + except Exception as exc: + pytest.skip(f"QAIC runtime import failed: {exc}") + if not (cloud_infer.is_qaicrt_imported and cloud_infer.is_aicapi_imported): + pytest.skip("qaicrt or QAicApi_pb2 runtime bindings are not available") + + +# Load a fresh model per stage to avoid re-exporting offloaded module state. +def _load_causal_model(scenario: RegressionScenario): + """Load a fresh causal QEfficient model for a runnable scenario. + + Each caller receives a new model instance so export, compile, and inference + never share mutated/offloaded weights across independent reproducer tests. + """ + from QEfficient import QEFFAutoModelForCausalLM + + model_id = _scenario_model_id(scenario) + _ensure_hf_cache_env() + load_kwargs = {"dtype": torch.float32, "trust_remote_code": True, **scenario.load_kwargs} + load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) + return QEFFAutoModelForCausalLM.from_pretrained(model_id, **load_kwargs) + + +# VLM official-model cases compile vision, prefill, and decode as separate QPCs. +def _load_vlm_model(scenario: RegressionScenario): + """Load a fresh image-text-to-text QEfficient model for VLM reproducers. + + The helper exists because VLM reports must exercise the dual-QPC API rather + than the causal-LM helper used by text-only models. + """ + from QEfficient import QEFFAutoModelForImageTextToText + + model_id = _scenario_model_id(scenario) + _ensure_hf_cache_env() + load_kwargs = {"trust_remote_code": True, **scenario.load_kwargs} + load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) + return QEFFAutoModelForImageTextToText.from_pretrained(model_id, **load_kwargs) + + +# Embedding reports use QEFFAutoModel rather than causal or VLM wrappers. +def _load_embedding_model(scenario: RegressionScenario): + """Load a fresh embedding QEfficient model for sequence/reranker reports. + + The helper exists so embedding compile regressions exercise the same API as + production embedding examples instead of the causal-LM wrapper. + """ + from QEfficient import QEFFAutoModel + + model_id = _scenario_model_id(scenario) + _ensure_hf_cache_env() + load_kwargs = {"trust_remote_code": True, **scenario.load_kwargs} + load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) + return QEFFAutoModel.from_pretrained(model_id, **load_kwargs) + + +# A compile verdict requires the QPC directory and its program binary. +def _qpc_succeeded(qpc_path: str | Path) -> None: + """Assert that QEfficient compile returned a usable QPC directory. + + This helper converts compile completion into an artifact-level verdict by + checking the directory and required ``programqpc.bin`` output. + """ + qpc = Path(qpc_path) + assert qpc.is_dir(), f"QPC directory was not created: {qpc}" + assert (qpc / "programqpc.bin").is_file(), f"programqpc.bin missing in QPC: {qpc}" + + +# Import/install reports are validated through real module imports and env checks. +def run_import_and_install_test(scenario: RegressionScenario) -> None: + """Validate import/install regressions without model artifacts. + + The helper exists for reports where package import order or environment + setup caused the bug before export, compile, or runtime was reached. + """ + _ensure_hf_cache_env() + + import transformers # noqa: F401 + + import QEfficient # noqa: F401 + + assert os.environ["HF_HUB_ENABLE_HF_TRANSFER"] == "1" + _record_result(scenario, "PASSED", "imported QEfficient and transformers with hf_transfer enabled") + + +# Download reports exercise QEfficient's real Hugging Face download wrapper. +def run_model_download_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Download the scenario's checkpoint through QEfficient utilities. + + The helper verifies model-resolution regressions at the download boundary + before any QEfficient model wrapping or export logic is involved. + """ + from QEfficient.utils import hf_download + + model_id = _scenario_model_id(scenario) + _ensure_hf_cache_env() + model_path = Path(hf_download(repo_id=model_id, cache_dir=str(tmp_path / "hf-cache"))) + assert model_path.exists() + _record_result(scenario, "PASSED", f"downloaded {model_path}") + + +# Export reports must call the same QEfficient export API users call. +def run_model_export_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Export the scenario's model and assert ONNX files are produced. + + The helper exists to turn reporter export options into a real PyTorch to + ONNX verdict instead of a placeholder option-plumbing assertion. + """ + if scenario.model_api == "replicate_kv_heads": + from scripts.replicate_kv_head.replicate_kv_heads import replicate_kv_heads + + model_id = _scenario_model_id(scenario) + cwd = Path.cwd() + os.chdir(tmp_path) + try: + replicate_kv_heads(model_name=model_id, **scenario.export_kwargs) + finally: + os.chdir(cwd) + _record_result(scenario, "PASSED", "replicate_kv_heads export completed") + return + + qeff_model = ( + _load_vlm_model(scenario) if scenario.model_api == "image_text_to_text" else _load_causal_model(scenario) + ) + onnx_path = qeff_model.export(export_dir=tmp_path / scenario.name, **scenario.export_kwargs) + paths = [Path(path) for path in onnx_path] if isinstance(onnx_path, list) else [Path(onnx_path)] + assert all(path.is_file() for path in paths), f"ONNX export did not create files: {paths}" + _record_result(scenario, "PASSED", "exported " + ", ".join(str(path) for path in paths)) + + +# Compiler helper signatures provide the public options that should stay in sync. +def _known_qeff_compile_options() -> set[str]: + """Return compile options accepted by QEfficient's helper APIs. + + The helper derives standard QAIC/QNN options from the public compile helper + signatures and keeps only MDP or model-specialization extras local here. + """ + from QEfficient.compile import compile_helper, qnn_compiler + + options = set(EXTRA_QEFF_COMPILE_OPTIONS) + options.update(inspect.signature(compile_helper.compile).parameters) + options.update(inspect.signature(qnn_compiler.compile).parameters) + return options + + +# Compile kwargs are checked before QEfficient receives reporter options. +def _assert_compile_kwargs_supported( + compile_owner, scenario: RegressionScenario, compile_kwargs: dict[str, Any] +) -> None: + """Assert that compile options match the public API or known compiler flags. + + The helper catches typos before long export/compile jobs start. ``node_precision_info`` + is allowed as a compiler option only when the referenced YAML exists in this checkout. + """ + parameters = inspect.signature(compile_owner.compile).parameters + has_var_kwargs = any(parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()) + private_kwargs = {key for key in compile_kwargs if key.startswith("_")} + assert not private_kwargs, f"{scenario.name}: private harness kwargs escaped to compile {private_kwargs}" + + accepted = set(parameters) + if has_var_kwargs: + accepted |= _known_qeff_compile_options() + unknown = set(compile_kwargs) - accepted + assert not unknown, f"{scenario.name}: unknown compile kwargs {unknown}" + + npi_path = compile_kwargs.get("node_precision_info") + if npi_path: + path = Path(npi_path) + if not path.is_absolute(): + path = Path.cwd() / path + assert path.is_file(), f"{scenario.name}: node_precision_info YAML does not exist: {path}" + + +# MDP intersection reproducers generate their compiler dump locally. +def _compile_with_optional_mdp(qeff_model, scenario: RegressionScenario, compile_dir: Path): + """Compile once or run the local two-pass MDP intersection flow. + + The helper replaces reporter-machine MDP JSON paths with artifacts generated + inside pytest's temp directory, following the Qwen3-VL MDP compile example. + """ + compile_kwargs = dict(scenario.compile_kwargs) + use_intersection = compile_kwargs.pop("_mdp_intersection", False) + if not use_intersection: + _assert_compile_kwargs_supported(qeff_model, scenario, compile_kwargs) + return qeff_model.compile(compile_dir=compile_dir, **compile_kwargs) + + dump_path = compile_dir / "mdp_compiler_dump.json" + (compile_dir / "mdp_dump").mkdir(parents=True, exist_ok=True) + (compile_dir / "mdp_intersection").mkdir(parents=True, exist_ok=True) + dump_kwargs = dict(compile_kwargs) + dump_kwargs.pop("mdp_num_partitions", None) + dump_kwargs.pop("mdp_strategy", None) + _assert_compile_kwargs_supported(qeff_model, scenario, {**dump_kwargs, "mdp_dump_partition_config": str(dump_path)}) + qeff_model.compile(compile_dir=compile_dir / "mdp_dump", mdp_dump_partition_config=str(dump_path), **dump_kwargs) + + final_kwargs = dict(compile_kwargs) + final_kwargs["mdp_strategy"] = "intersection" + final_kwargs["mdp_compiler_dump_path"] = str(dump_path) + _assert_compile_kwargs_supported(qeff_model, scenario, final_kwargs) + return qeff_model.compile(compile_dir=compile_dir / "mdp_intersection", **final_kwargs) + + +# Compile reports use QEfficient's compile API and validate the produced QPC. +def run_model_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Compile the scenario's model with reported compile options. + + The helper exists so active compile scenarios provide production-style + success/failure verdicts from the QEfficient export-plus-compile stack. + """ + _require_qaic_compile() + if scenario.model_api == "image_text_to_text": + run_vlm_compile_test(scenario, tmp_path) + return + if scenario.model_api in {"wan_t2v", "wan_i2v"}: + run_wan_compile_test(scenario, tmp_path) + return + + qeff_model = _load_embedding_model(scenario) if scenario.model_api == "embedding" else _load_causal_model(scenario) + compile_dir = tmp_path / scenario.name / "compile" + compile_dir.mkdir(parents=True, exist_ok=True) + qpc_path = _compile_with_optional_mdp(qeff_model, scenario, compile_dir) + _qpc_succeeded(qpc_path) + _record_result(scenario, "PASSED", f"QPC: {qpc_path}") + + +# Dual-QPC VLM compile is validated as vision + language prefill + language decode. +def run_vlm_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Compile all VLM deployment components for one reproducer config. + + The helper exists for full VLM reports where runtime generation is too heavy + but compilation of vision, language prefill, and language decode must be an + absolute pre-merge verdict. + """ + qeff_model = _load_vlm_model(scenario) + compile_dir = tmp_path / scenario.name / "compile" + compile_dir.mkdir(parents=True, exist_ok=True) + export_dir = tmp_path / scenario.name / "export" + export_kwargs = dict(scenario.export_kwargs) + export_kwargs.setdefault("offload_pt_weights", False) + onnx_paths = qeff_model.export(export_dir=export_dir, **export_kwargs) + vision_onnx_path, lang_onnx_path = onnx_paths + common_kwargs = dict(scenario.compile_kwargs) + use_intersection = common_kwargs.pop("_mdp_intersection", False) + common_kwargs.pop("skip_vision", None) + common_kwargs.pop("skip_lang", None) + vision_kwargs = dict(common_kwargs) + for language_only_key in ( + "prefill_only", + "enable_chunking", + "retain_full_kv", + "mdp_num_partitions", + "mdp_strategy", + "mdp_compiler_dump_path", + "mdp_dump_partition_config", + ): + vision_kwargs.pop(language_only_key, None) + _assert_compile_kwargs_supported(qeff_model, scenario, {**vision_kwargs, "skip_lang": True}) + vision_qpcs = qeff_model.compile( + compile_dir=compile_dir / "vision", + vision_onnx_path=vision_onnx_path, + lang_onnx_path=lang_onnx_path, + skip_lang=True, + **vision_kwargs, + ) + prefill_kwargs = dict(common_kwargs) + prefill_kwargs["prefill_only"] = True + if use_intersection: + dump_path = compile_dir / "lang_prefill_mdp_dump.json" + (compile_dir / "lang_prefill_mdp_dump").mkdir(parents=True, exist_ok=True) + (compile_dir / "lang_prefill").mkdir(parents=True, exist_ok=True) + dump_kwargs = dict(prefill_kwargs) + dump_kwargs.pop("mdp_num_partitions", None) + dump_kwargs.pop("mdp_strategy", None) + _assert_compile_kwargs_supported( + qeff_model, scenario, {**dump_kwargs, "skip_vision": True, "mdp_dump_partition_config": str(dump_path)} + ) + qeff_model.compile( + compile_dir=compile_dir / "lang_prefill_mdp_dump", + vision_onnx_path=vision_onnx_path, + lang_onnx_path=lang_onnx_path, + skip_vision=True, + mdp_dump_partition_config=str(dump_path), + **dump_kwargs, + ) + prefill_kwargs["mdp_strategy"] = "intersection" + prefill_kwargs["mdp_compiler_dump_path"] = str(dump_path) + _assert_compile_kwargs_supported(qeff_model, scenario, {**prefill_kwargs, "skip_vision": True}) + prefill_qpcs = qeff_model.compile( + compile_dir=compile_dir / "lang_prefill", + vision_onnx_path=vision_onnx_path, + lang_onnx_path=lang_onnx_path, + skip_vision=True, + **prefill_kwargs, + ) + decode_kwargs = {**common_kwargs, "prefill_seq_len": 1} + for prefill_only_key in ("prefill_only", "enable_chunking", "mdp_num_partitions", "mdp_strategy"): + decode_kwargs.pop(prefill_only_key, None) + _assert_compile_kwargs_supported(qeff_model, scenario, {**decode_kwargs, "skip_vision": True}) + decode_qpcs = qeff_model.compile( + compile_dir=compile_dir / "lang_decode", + vision_onnx_path=vision_onnx_path, + lang_onnx_path=lang_onnx_path, + skip_vision=True, + **decode_kwargs, + ) + for qpc_map in (vision_qpcs, prefill_qpcs, decode_qpcs): + for qpc_path in qpc_map.values(): + _qpc_succeeded(qpc_path) + _record_result(scenario, "PASSED", f"VLM QPCs: {vision_qpcs} {prefill_qpcs} {decode_qpcs}") + + +# WAN reports use the real diffusion pipeline compile API, not transformer helpers. +def run_wan_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Compile a WAN pipeline reproducer with the reported pipeline options. + + The helper exists so WAN regressions stay in the same reproducer catalog and + validate the real QEfficient WAN export/compile stack before merge. + """ + model_id = _scenario_model_id(scenario) + _ensure_hf_cache_env() + if scenario.model_api == "wan_i2v": + from QEfficient.diffusers.pipelines.wan.pipeline_wan_i2v import QEffWanImageToVideoPipeline as Pipeline + else: + from QEfficient.diffusers.pipelines.wan.pipeline_wan import QEffWanPipeline as Pipeline + + pipeline = Pipeline.from_pretrained(model_id, cache_dir=os.environ["HF_HUB_CACHE"]) + os.environ.setdefault("QEFF_HOME", str(tmp_path / scenario.name / "qeff_home")) + _assert_compile_kwargs_supported(pipeline, scenario, dict(scenario.compile_kwargs)) + qpc_path = pipeline.compile(**scenario.compile_kwargs) + if qpc_path: + _record_result(scenario, "PASSED", f"WAN QPC: {qpc_path}") + else: + _record_result(scenario, "PASSED", "WAN compile completed") + + +# Inference reports compile first, then run generation through QEfficient runtime. +def run_model_inference_test(scenario: RegressionScenario, tmp_path: Path) -> None: + """Compile and generate for runnable end-to-end inference scenarios. + + The helper exists for reports whose failure only appears after QPC creation + and validates that generation returns model output IDs. + """ + from QEfficient.utils._utils import load_hf_tokenizer + + _require_qaic_runtime() + qeff_model = _load_causal_model(scenario) + compile_dir = tmp_path / scenario.name / "compile" + compile_dir.mkdir(parents=True, exist_ok=True) + _assert_compile_kwargs_supported(qeff_model, scenario, dict(scenario.compile_kwargs)) + qpc_path = qeff_model.compile(compile_dir=compile_dir, **scenario.compile_kwargs) + _qpc_succeeded(qpc_path) + tokenizer = load_hf_tokenizer(_scenario_model_id(scenario)) + exec_info = qeff_model.generate(tokenizer=tokenizer, prompts=["My name is"], **scenario.inference_kwargs) + assert exec_info.generated_ids is not None + _record_result(scenario, "PASSED", "generation returned output IDs") + + +# Parametrized entrypoint for import/install-stage reported regressions. +@pytest.mark.reproducer +@pytest.mark.parametrize("scenario", _params_for("import_install")) +def test_import_and_install_reproducer(scenario: RegressionScenario) -> None: + """Run every import/install scenario through the real import helper.""" + _run_with_verdict(scenario, run_import_and_install_test) + + +# Parametrized entrypoint for model-download reported regressions. +@pytest.mark.reproducer +@pytest.mark.parametrize("scenario", _params_for("download")) +def test_model_download_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: + """Run every download scenario through the real QEfficient download helper.""" + _run_with_verdict(scenario, run_model_download_test, tmp_path) + + +# Parametrized entrypoint for model-export reported regressions. +@pytest.mark.reproducer +@pytest.mark.parametrize("scenario", _params_for("export")) +def test_model_export_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: + """Run every export scenario through the real QEfficient export helper.""" + _run_with_verdict(scenario, run_model_export_test, tmp_path) + + +# Parametrized entrypoint for compile-stage reported regressions. +@pytest.mark.reproducer +@pytest.mark.parametrize("scenario", _params_for("compile")) +def test_model_compile_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: + """Run every compile scenario through the real QEfficient compile helper.""" + _run_with_verdict(scenario, run_model_compile_test, tmp_path) + + +# Parametrized entrypoint for end-to-end inference reported regressions. +@pytest.mark.reproducer +@pytest.mark.parametrize("scenario", _params_for("inference")) +def test_inference_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: + """Run every inference scenario through real compile and generation.""" + _run_with_verdict(scenario, run_model_inference_test, tmp_path) + + +# Catalog guard prevents future edits from dropping reported configs silently. +def test_all_reported_configs_are_represented() -> None: + """Ensure the reported-regression catalog remains complete and unique. + + The test exists because skipped scenarios are still deliberate coverage + records and must not disappear while the suite evolves. + """ + assert len(SCENARIOS) == EXPECTED_SCENARIO_COUNT + assert len({scenario.name for scenario in SCENARIOS}) == len(SCENARIOS) + + +# Path guard recursively catches local reporter paths in nested option values. +def _contains_external_path(value: Any) -> bool: + """Return whether a config value contains a non-portable local path. + + The helper exists because reproducer configs may nest paths inside lists or + dictionaries, and active scenarios must remain runnable on production hosts. + """ + if isinstance(value, dict): + return any(_contains_external_path(key) or _contains_external_path(item) for key, item in value.items()) + if isinstance(value, (list, tuple, set)): + return any(_contains_external_path(item) for item in value) + text = str(value) + return "/home/" in text or "/local/" in text + + +# Active scenario guard keeps runnable tests independent of reporter machines. +def test_active_scenarios_are_real_stage_verdicts() -> None: + """Ensure runnable scenarios use models and portable config values. + + The test exists to prevent fake active tests and accidental dependencies on + absolute paths copied from a reporter's private machine. + """ + for scenario in SCENARIOS: + if scenario.skip_reason: + continue + if scenario.stage not in {"import_install"}: + assert scenario.tiny_model_id or scenario.official_model_id + combined = { + **scenario.config, + **scenario.load_kwargs, + **scenario.export_kwargs, + **scenario.compile_kwargs, + **scenario.inference_kwargs, + } + assert not _contains_external_path(combined) From a14f6145b42c4c6c144fe2f589b257a24b98ce84 Mon Sep 17 00:00:00 2001 From: Anuj Gupta Date: Mon, 13 Jul 2026 13:50:41 +0530 Subject: [PATCH 2/2] Simplify reproducer config test harness Signed-off-by: Anuj Gupta --- tests/reproducer_configs/README.md | 25 ++- tests/reproducer_configs/conftest.py | 81 -------- .../test_reported_reproducer_configs.py | 190 ++++++------------ 3 files changed, 78 insertions(+), 218 deletions(-) delete mode 100644 tests/reproducer_configs/conftest.py diff --git a/tests/reproducer_configs/README.md b/tests/reproducer_configs/README.md index e817b057fa..e8b77828a5 100644 --- a/tests/reproducer_configs/README.md +++ b/tests/reproducer_configs/README.md @@ -15,16 +15,15 @@ pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q -rs Run one scenario by its `RegressionScenario.name`: ```bash -pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ - --qeff-reproducer-scenario hf-transfer-import-order -q +QEFF_REPRODUCER_SCENARIO=hf-transfer-import-order \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q ``` -Run multiple scenarios with repeated or comma-separated values: +Run multiple scenarios with comma-separated values: ```bash -pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ - --qeff-reproducer-scenario hf-transfer-import-order \ - --qeff-reproducer-scenario ccl-default-enabled,embedding-model-compile -q +QEFF_REPRODUCER_SCENARIO=hf-transfer-import-order,ccl-default-enabled,embedding-model-compile \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q ``` List available scenario names without running them: @@ -40,16 +39,16 @@ card are gated and must be enabled explicitly: ```bash QEFF_REPRODUCER_RUN_FULL_MODELS=1 \ -pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ - --qeff-reproducer-scenario -q +QEFF_REPRODUCER_SCENARIO= \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q ``` Use official model cards instead of tiny reductions where both are present: ```bash QEFF_REPRODUCER_USE_OFFICIAL_MODELS=1 QEFF_REPRODUCER_RUN_FULL_MODELS=1 \ -pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ - --qeff-reproducer-scenario -q +QEFF_REPRODUCER_SCENARIO= \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q ``` ## Runtime report @@ -62,8 +61,8 @@ needed: ```bash QEFF_REPRODUCER_REPORT_MD=/tmp/qeff_reproducer_results.md \ -pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ - --qeff-reproducer-scenario -q +QEFF_REPRODUCER_SCENARIO= \ +pytest tests/reproducer_configs/test_reported_reproducer_configs.py -q ``` `--collect-only` does not create or update the verdict report. @@ -76,5 +75,5 @@ pytest tests/reproducer_configs/test_reported_reproducer_configs.py \ 3. Use a tiny or 2/4-layer model when it faithfully reproduces the issue. 4. Keep the official model card and gate it with `QEFF_REPRODUCER_RUN_FULL_MODELS=1` when a reduction is not faithful. -5. Run only the new scenario with `--qeff-reproducer-scenario ` +5. Run only the new scenario with `QEFF_REPRODUCER_SCENARIO=` before running the broader matrix. diff --git a/tests/reproducer_configs/conftest.py b/tests/reproducer_configs/conftest.py deleted file mode 100644 index 46dc91fa11..0000000000 --- a/tests/reproducer_configs/conftest.py +++ /dev/null @@ -1,81 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause -# -# ----------------------------------------------------------------------------- - -"""Pytest hooks for reported reproducer config tests.""" - -from __future__ import annotations - -import pytest - -SCENARIO_CLI_OPTION = "--qeff-reproducer-scenario" - - -# Parse comma-separated or repeated scenario names from pytest CLI values. -def _split_scenario_names(values: list[str]) -> tuple[str, ...]: - """Return unique scenario names from repeated or comma-separated options. - - The helper exists so developers can run one or more just-added reproducers - without collecting unrelated scenarios from the full reported-config matrix. - """ - names = [] - for value in values: - for name in value.split(","): - stripped_name = name.strip() - if stripped_name and stripped_name not in names: - names.append(stripped_name) - return tuple(names) - - -# Register a targeted reproducer selector for local developer loops. -def pytest_addoption(parser: pytest.Parser) -> None: - """Add the scenario-selection option used by reproducer config tests.""" - group = parser.getgroup("qeff reproducer configs") - group.addoption( - SCENARIO_CLI_OPTION, - action="append", - default=[], - metavar="NAME[,NAME...]", - help="Run only the named reproducer scenario(s). Can be repeated or comma-separated.", - ) - - -# Deselect non-targeted reproducer items after parametrization. -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - """Keep only requested reproducer scenarios when the CLI selector is set. - - The hook exists so ``--qeff-reproducer-scenario`` runs exactly the requested - scenario tests and drops catalog guards or empty-stage sentinel parameters. - """ - selected_names = set(_split_scenario_names(config.getoption(SCENARIO_CLI_OPTION) or [])) - if not selected_names: - return - - available_names = set() - for item in items: - scenario = getattr(getattr(item, "callspec", None), "params", {}).get("scenario") - scenario_name = getattr(scenario, "name", None) - if scenario_name: - available_names.add(scenario_name) - - missing_names = selected_names - available_names - if missing_names: - names = ", ".join(sorted(missing_names)) - raise pytest.UsageError(f"unknown reproducer scenario(s): {names}") - - kept_items = [] - deselected_items = [] - for item in items: - scenario = getattr(getattr(item, "callspec", None), "params", {}).get("scenario") - scenario_name = getattr(scenario, "name", None) - if scenario_name in selected_names: - kept_items.append(item) - else: - deselected_items.append(item) - - if deselected_items: - config.hook.pytest_deselected(items=deselected_items) - items[:] = kept_items diff --git a/tests/reproducer_configs/test_reported_reproducer_configs.py b/tests/reproducer_configs/test_reported_reproducer_configs.py index 771e1bbb15..800f0af92e 100644 --- a/tests/reproducer_configs/test_reported_reproducer_configs.py +++ b/tests/reproducer_configs/test_reported_reproducer_configs.py @@ -24,8 +24,8 @@ 6. To add a future report, copy the commented template in ``SCENARIOS``, paste exact reporter options into ``config``, choose the correct ``model_api``, and implement the real stage assertion in the helper for that API. -7. To run one scenario during fix validation, pass - ``--qeff-reproducer-scenario `` to pytest. +7. To run one scenario during fix validation, set + ``QEFF_REPRODUCER_SCENARIO=`` before pytest. """ from __future__ import annotations @@ -94,7 +94,7 @@ class RegressionScenario: FULL_MODEL_ENV = "QEFF_REPRODUCER_RUN_FULL_MODELS" OFFICIAL_MODEL_ENV = "QEFF_REPRODUCER_USE_OFFICIAL_MODELS" REPORT_PATH_ENV = "QEFF_REPRODUCER_REPORT_MD" -SCENARIO_CLI_OPTION = "--qeff-reproducer-scenario" +SCENARIO_FILTER_ENV = "QEFF_REPRODUCER_SCENARIO" DEFAULT_REPORT_PATH = Path("tests/reproducer_configs/reproducer_config_results.md") EXPECTED_SCENARIO_COUNT = 50 EXTRA_QEFF_COMPILE_OPTIONS = frozenset( @@ -128,23 +128,6 @@ class RegressionScenario: ) -# Resolve the Hugging Face cache without mutating environment variables. -def _resolve_hf_cache_path() -> Path: - """Return the writable Hugging Face cache path used by reproducer tests. - - The helper keeps cache selection consistent between the runtime ledger and - the environment setup that precedes model downloads. - """ - requested_cache = Path( - os.environ.get("QEFF_REPRODUCER_HF_CACHE") - or os.environ.get("HF_HUB_CACHE") - or "/home/tmp/qeff_reproducer_hf_cache" - ) - if not os.access(requested_cache.parent, os.W_OK): - return Path("/tmp/qeff_reproducer_hf_cache") - return requested_cache - - # Generated reports should not appear for pytest --collect-only runs. def _is_pytest_collect_only() -> bool: """Return whether pytest is collecting tests without executing them. @@ -883,33 +866,14 @@ def _locked_report(path: Path): ) -# Parse targeted scenario names from pytest CLI before parametrization writes reports. -def _selected_scenario_names_from_argv() -> tuple[str, ...]: - """Return scenario names requested by ``--qeff-reproducer-scenario``. - - The helper mirrors the registered pytest option so parametrization can avoid - creating skip-report rows for scenarios that the developer did not request. - """ - values = [] - args = sys.argv[1:] - index = 0 - while index < len(args): - arg = args[index] - if arg == SCENARIO_CLI_OPTION and index + 1 < len(args): - values.append(args[index + 1]) - index += 2 - continue - prefix = f"{SCENARIO_CLI_OPTION}=" - if arg.startswith(prefix): - values.append(arg[len(prefix) :]) - index += 1 - +# Parse targeted scenario names before parametrization writes reports. +def _selected_scenario_names() -> tuple[str, ...]: + """Return scenario names requested by ``QEFF_REPRODUCER_SCENARIO``.""" names = [] - for value in values: - for name in value.split(","): - stripped_name = name.strip() - if stripped_name and stripped_name not in names: - names.append(stripped_name) + for name in os.environ.get(SCENARIO_FILTER_ENV, "").split(","): + stripped_name = name.strip() + if stripped_name and stripped_name not in names: + names.append(stripped_name) return tuple(names) @@ -920,7 +884,11 @@ def _params_for(stage: Stage) -> tuple[Any, ...]: The helper exists so every test function gets identical skip handling, stable readable IDs, and QAIC markers for hardware-backed stages. """ - selected_names = set(_selected_scenario_names_from_argv()) + selected_names = set(_selected_scenario_names()) + unknown_names = selected_names - {scenario.name for scenario in SCENARIOS} + if unknown_names: + names = ", ".join(sorted(unknown_names)) + raise pytest.UsageError(f"unknown reproducer scenario(s): {names}") params = [] for scenario in SCENARIOS: if scenario.stage != stage: @@ -1023,6 +991,13 @@ def _record_result(scenario: RegressionScenario, status: str, detail: str) -> No f"| {scenario.name} | {scenario.stage} | {scenario.model_api} | {model_id} | " f"{status} | {follow_up} | {safe_detail} |\n" ) + hf_cache_path = Path( + os.environ.get("QEFF_REPRODUCER_HF_CACHE") + or os.environ.get("HF_HUB_CACHE") + or "/home/tmp/qeff_reproducer_hf_cache" + ) + if not os.access(hf_cache_path.parent, os.W_OK): + hf_cache_path = Path("/tmp/qeff_reproducer_hf_cache") with _locked_report(path) as handle: handle.seek(0) @@ -1034,7 +1009,7 @@ def _record_result(scenario: RegressionScenario, status: str, detail: str) -> No "# QEfficient Reproducer Config Results\n\n" "This file is generated by `tests/reproducer_configs/test_reported_reproducer_configs.py`.\n" "Each row is a concrete verdict from collection or execution of one reproducer config.\n\n" - f"Hugging Face cache: `{_resolve_hf_cache_path()}`\n\n" + f"Hugging Face cache: `{hf_cache_path}`\n\n" "| Scenario | Stage | API | Model Card | Verdict | Follow-up | Detail |\n" "|---|---|---|---|---|---|---|\n" ) @@ -1073,10 +1048,10 @@ def _run_with_verdict(scenario: RegressionScenario, runner, *args) -> None: raise -# Per-test environment restoration prevents cache and QEFF_HOME leakage. +# Per-test environment handling prevents cache and QEFF_HOME leakage. @pytest.fixture(autouse=True) def _restore_reproducer_environment(): - """Restore mutable reproducer environment variables after each test. + """Set and restore mutable reproducer environment variables for each test. The fixture exists because model loading and WAN compilation require process environment variables, but those settings must not leak into later tests. @@ -1089,35 +1064,23 @@ def _restore_reproducer_environment(): "QEFF_HOME", ) original_values = {key: os.environ.get(key) for key in keys} - yield - for key, value in original_values.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - - -# Normalize Hugging Face cache settings before any real model download or load. -def _ensure_hf_cache_env() -> None: - """Set the cache and transfer environment used by reduced reproducers. - - This helper keeps downloads in a writable local cache by default while - preserving explicit caller overrides for production reproducer runs. - """ - requested_cache = _resolve_hf_cache_path() + requested_cache = Path( + os.environ.get("QEFF_REPRODUCER_HF_CACHE") + or os.environ.get("HF_HUB_CACHE") + or "/home/tmp/qeff_reproducer_hf_cache" + ) + if not os.access(requested_cache.parent, os.W_OK): + requested_cache = Path("/tmp/qeff_reproducer_hf_cache") + modules_cache = requested_cache / "modules" try: requested_cache.mkdir(parents=True, exist_ok=True) + modules_cache.mkdir(parents=True, exist_ok=True) except OSError as exc: pytest.skip(f"cannot create Hugging Face cache {requested_cache}: {exc}") os.environ["HF_HOME"] = str(requested_cache) os.environ["HF_HUB_CACHE"] = str(requested_cache) os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") - modules_cache = requested_cache / "modules" - try: - modules_cache.mkdir(parents=True, exist_ok=True) - except OSError as exc: - pytest.skip(f"cannot create Hugging Face modules cache {modules_cache}: {exc}") os.environ["HF_MODULES_CACHE"] = str(modules_cache) try: import transformers.dynamic_module_utils as dynamic_module_utils @@ -1127,36 +1090,12 @@ def _ensure_hf_cache_env() -> None: transformers_hub.HF_MODULES_CACHE = str(modules_cache) except (ImportError, AttributeError) as exc: warnings.warn(f"could not update transformers module cache paths: {exc}", stacklevel=2) - - -# Hardware-backed scenarios must not silently pass on hosts without the compiler. -def _require_qaic_compile() -> None: - """Skip compile or inference scenarios when the QAIC compiler is absent. - - The check exists because active compile tests call QEfficient's real compile - API and need the production compiler binary to produce a QPC verdict. - """ - if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): - pytest.skip("qaic-compile is not available on this host") - - -# Runtime scenarios also require SDK Python bindings and a visible SoC device. -def _require_qaic_runtime() -> None: - """Skip inference scenarios when the QAIC runtime stack is unavailable. - - The helper is stricter than the compile gate because generation needs the - runtime libraries and an accessible ``/dev/qaic*`` device, not just QPC build. - """ - if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): - pytest.skip("QAIC runtime is unavailable because qaic-compile is not installed on this host") - if not any(Path("/dev").glob("qaic*")): - pytest.skip("no /dev/qaic* device is available on this host") - try: - from QEfficient.generation import cloud_infer - except Exception as exc: - pytest.skip(f"QAIC runtime import failed: {exc}") - if not (cloud_infer.is_qaicrt_imported and cloud_infer.is_aicapi_imported): - pytest.skip("qaicrt or QAicApi_pb2 runtime bindings are not available") + yield + for key, value in original_values.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value # Load a fresh model per stage to avoid re-exporting offloaded module state. @@ -1169,7 +1108,6 @@ def _load_causal_model(scenario: RegressionScenario): from QEfficient import QEFFAutoModelForCausalLM model_id = _scenario_model_id(scenario) - _ensure_hf_cache_env() load_kwargs = {"dtype": torch.float32, "trust_remote_code": True, **scenario.load_kwargs} load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) return QEFFAutoModelForCausalLM.from_pretrained(model_id, **load_kwargs) @@ -1185,7 +1123,6 @@ def _load_vlm_model(scenario: RegressionScenario): from QEfficient import QEFFAutoModelForImageTextToText model_id = _scenario_model_id(scenario) - _ensure_hf_cache_env() load_kwargs = {"trust_remote_code": True, **scenario.load_kwargs} load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) return QEFFAutoModelForImageTextToText.from_pretrained(model_id, **load_kwargs) @@ -1201,7 +1138,6 @@ def _load_embedding_model(scenario: RegressionScenario): from QEfficient import QEFFAutoModel model_id = _scenario_model_id(scenario) - _ensure_hf_cache_env() load_kwargs = {"trust_remote_code": True, **scenario.load_kwargs} load_kwargs.setdefault("cache_dir", os.environ["HF_HUB_CACHE"]) return QEFFAutoModel.from_pretrained(model_id, **load_kwargs) @@ -1220,14 +1156,12 @@ def _qpc_succeeded(qpc_path: str | Path) -> None: # Import/install reports are validated through real module imports and env checks. -def run_import_and_install_test(scenario: RegressionScenario) -> None: +def _run_import_and_install_test(scenario: RegressionScenario) -> None: """Validate import/install regressions without model artifacts. The helper exists for reports where package import order or environment setup caused the bug before export, compile, or runtime was reached. """ - _ensure_hf_cache_env() - import transformers # noqa: F401 import QEfficient # noqa: F401 @@ -1237,7 +1171,7 @@ def run_import_and_install_test(scenario: RegressionScenario) -> None: # Download reports exercise QEfficient's real Hugging Face download wrapper. -def run_model_download_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_model_download_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Download the scenario's checkpoint through QEfficient utilities. The helper verifies model-resolution regressions at the download boundary @@ -1246,14 +1180,13 @@ def run_model_download_test(scenario: RegressionScenario, tmp_path: Path) -> Non from QEfficient.utils import hf_download model_id = _scenario_model_id(scenario) - _ensure_hf_cache_env() model_path = Path(hf_download(repo_id=model_id, cache_dir=str(tmp_path / "hf-cache"))) assert model_path.exists() _record_result(scenario, "PASSED", f"downloaded {model_path}") # Export reports must call the same QEfficient export API users call. -def run_model_export_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_model_export_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Export the scenario's model and assert ONNX files are produced. The helper exists to turn reporter export options into a real PyTorch to @@ -1354,18 +1287,19 @@ def _compile_with_optional_mdp(qeff_model, scenario: RegressionScenario, compile # Compile reports use QEfficient's compile API and validate the produced QPC. -def run_model_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_model_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Compile the scenario's model with reported compile options. The helper exists so active compile scenarios provide production-style success/failure verdicts from the QEfficient export-plus-compile stack. """ - _require_qaic_compile() + if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): + pytest.skip("qaic-compile is not available on this host") if scenario.model_api == "image_text_to_text": - run_vlm_compile_test(scenario, tmp_path) + _run_vlm_compile_test(scenario, tmp_path) return if scenario.model_api in {"wan_t2v", "wan_i2v"}: - run_wan_compile_test(scenario, tmp_path) + _run_wan_compile_test(scenario, tmp_path) return qeff_model = _load_embedding_model(scenario) if scenario.model_api == "embedding" else _load_causal_model(scenario) @@ -1377,7 +1311,7 @@ def run_model_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None # Dual-QPC VLM compile is validated as vision + language prefill + language decode. -def run_vlm_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_vlm_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Compile all VLM deployment components for one reproducer config. The helper exists for full VLM reports where runtime generation is too heavy @@ -1463,14 +1397,13 @@ def run_vlm_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: # WAN reports use the real diffusion pipeline compile API, not transformer helpers. -def run_wan_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_wan_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Compile a WAN pipeline reproducer with the reported pipeline options. The helper exists so WAN regressions stay in the same reproducer catalog and validate the real QEfficient WAN export/compile stack before merge. """ model_id = _scenario_model_id(scenario) - _ensure_hf_cache_env() if scenario.model_api == "wan_i2v": from QEfficient.diffusers.pipelines.wan.pipeline_wan_i2v import QEffWanImageToVideoPipeline as Pipeline else: @@ -1487,7 +1420,7 @@ def run_wan_compile_test(scenario: RegressionScenario, tmp_path: Path) -> None: # Inference reports compile first, then run generation through QEfficient runtime. -def run_model_inference_test(scenario: RegressionScenario, tmp_path: Path) -> None: +def _run_model_inference_test(scenario: RegressionScenario, tmp_path: Path) -> None: """Compile and generate for runnable end-to-end inference scenarios. The helper exists for reports whose failure only appears after QPC creation @@ -1495,7 +1428,16 @@ def run_model_inference_test(scenario: RegressionScenario, tmp_path: Path) -> No """ from QEfficient.utils._utils import load_hf_tokenizer - _require_qaic_runtime() + if not Path("/opt/qti-aic/exec/qaic-compile").is_file(): + pytest.skip("QAIC runtime is unavailable because qaic-compile is not installed on this host") + if not any(Path("/dev").glob("qaic*")): + pytest.skip("no /dev/qaic* device is available on this host") + try: + from QEfficient.generation import cloud_infer + except Exception as exc: + pytest.skip(f"QAIC runtime import failed: {exc}") + if not (cloud_infer.is_qaicrt_imported and cloud_infer.is_aicapi_imported): + pytest.skip("qaicrt or QAicApi_pb2 runtime bindings are not available") qeff_model = _load_causal_model(scenario) compile_dir = tmp_path / scenario.name / "compile" compile_dir.mkdir(parents=True, exist_ok=True) @@ -1513,7 +1455,7 @@ def run_model_inference_test(scenario: RegressionScenario, tmp_path: Path) -> No @pytest.mark.parametrize("scenario", _params_for("import_install")) def test_import_and_install_reproducer(scenario: RegressionScenario) -> None: """Run every import/install scenario through the real import helper.""" - _run_with_verdict(scenario, run_import_and_install_test) + _run_with_verdict(scenario, _run_import_and_install_test) # Parametrized entrypoint for model-download reported regressions. @@ -1521,7 +1463,7 @@ def test_import_and_install_reproducer(scenario: RegressionScenario) -> None: @pytest.mark.parametrize("scenario", _params_for("download")) def test_model_download_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: """Run every download scenario through the real QEfficient download helper.""" - _run_with_verdict(scenario, run_model_download_test, tmp_path) + _run_with_verdict(scenario, _run_model_download_test, tmp_path) # Parametrized entrypoint for model-export reported regressions. @@ -1529,7 +1471,7 @@ def test_model_download_reproducer(scenario: RegressionScenario, tmp_path: Path) @pytest.mark.parametrize("scenario", _params_for("export")) def test_model_export_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: """Run every export scenario through the real QEfficient export helper.""" - _run_with_verdict(scenario, run_model_export_test, tmp_path) + _run_with_verdict(scenario, _run_model_export_test, tmp_path) # Parametrized entrypoint for compile-stage reported regressions. @@ -1537,7 +1479,7 @@ def test_model_export_reproducer(scenario: RegressionScenario, tmp_path: Path) - @pytest.mark.parametrize("scenario", _params_for("compile")) def test_model_compile_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: """Run every compile scenario through the real QEfficient compile helper.""" - _run_with_verdict(scenario, run_model_compile_test, tmp_path) + _run_with_verdict(scenario, _run_model_compile_test, tmp_path) # Parametrized entrypoint for end-to-end inference reported regressions. @@ -1545,7 +1487,7 @@ def test_model_compile_reproducer(scenario: RegressionScenario, tmp_path: Path) @pytest.mark.parametrize("scenario", _params_for("inference")) def test_inference_reproducer(scenario: RegressionScenario, tmp_path: Path) -> None: """Run every inference scenario through real compile and generation.""" - _run_with_verdict(scenario, run_model_inference_test, tmp_path) + _run_with_verdict(scenario, _run_model_inference_test, tmp_path) # Catalog guard prevents future edits from dropping reported configs silently.