From d7be28b15c5cc8f06e7f4c4127d99ebec85a0f04 Mon Sep 17 00:00:00 2001 From: ochougul Date: Tue, 23 Jun 2026 15:21:40 +0530 Subject: [PATCH] Reduce CI model test matrix runtime Signed-off-by: ochougul --- scripts/Jenkinsfile | 17 + tests/configs/causal_model_configs.json | 583 +++++++++++++----- tests/configs/embedding_model_configs.json | 26 +- tests/configs/feature_configs.json | 381 ++++++------ tests/configs/image_text_model_configs.json | 431 ++++++++----- tests/diffusers/flux_test_config.json | 228 ++++--- tests/diffusers/test_flux.py | 3 + tests/diffusers/test_wan.py | 5 + tests/diffusers/wan_i2v_test_config.json | 204 +++--- tests/diffusers/wan_test_config.json | 162 ++--- .../wan_test_non_unified_config.json | 196 +++--- tests/test_matrix.py | 115 ++++ .../caching/test_prefix_caching.py | 11 +- .../test_audio_embedding_models.py | 6 +- .../test_speech_seq2seq_models.py | 6 +- .../test_causal_lm_blocking_hqkv.py | 8 +- .../causal_lm_models/test_causal_lm_models.py | 10 +- .../causal_lm_models/test_causal_lm_pl1.py | 10 +- .../test_causal_tlm_models.py | 10 +- .../causal_lm_models/test_fp16_causal_lm.py | 10 +- .../embedding_models/test_embedding_models.py | 6 +- .../test_qwen3vl_embedding_mad.py | 10 +- .../test_continuous_batching.py | 10 +- .../image_text_to_text/test_custom_dtype.py | 10 +- .../test_image_text_to_text_models.py | 10 +- .../models/reranker/test_reranker_mad.py | 12 +- .../test_seq_classification.py | 6 +- .../sampler/test_greedy_sampler.py | 10 +- .../sampler/test_guided_sampler.py | 10 +- .../sampler/test_random_sampler.py | 10 +- .../sampler/test_sampler_transform.py | 10 +- tests/transformers/spd/test_pld_inference.py | 8 +- tests/transformers/spd/test_spd_inference.py | 8 +- .../test_causal_lm_blocking_subfunction.py | 10 +- .../subfunction/test_subfunction_vlm.py | 10 +- .../unit_test/models/test_model_quickcheck.py | 39 ++ 36 files changed, 1548 insertions(+), 1053 deletions(-) create mode 100644 tests/test_matrix.py diff --git a/scripts/Jenkinsfile b/scripts/Jenkinsfile index f437a1521a..598c6fc620 100644 --- a/scripts/Jenkinsfile +++ b/scripts/Jenkinsfile @@ -31,6 +31,14 @@ pipeline { ], description: 'Select test profile' ) + choice( + name: 'TEST_SCOPE', + choices: [ + 'pr', + 'exhaustive' + ], + description: 'Select model matrix scope. PR keeps architecture/feature coverage; exhaustive runs all configured models.' + ) string( name: 'SELECT_TEST_STAGES', defaultValue: 'ALL', @@ -46,6 +54,7 @@ pipeline { environment { TEST_FILTER = testFilter(params.TEST_PROFILE) + QEFF_TEST_SCOPE = "${params.TEST_SCOPE}" } stages { @@ -84,6 +93,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_cli_qaic && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_cli_qaic && pytest tests -m '(not on_qaic) and (not finetune) and ${TEST_FILTER}' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline -n 4 --junitxml=tests/tests_log1.xml --durations=10 && junitparser merge tests/tests_log1.xml tests/tests_log.xml && @@ -102,6 +112,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_qaic_llm && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_qaic_llm && pytest tests -m '(llm_model) and (not qnn) and ${TEST_FILTER}' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --junitxml=tests/tests_log2.xml --durations=10 && junitparser merge tests/tests_log2.xml tests/tests_log.xml && @@ -123,6 +134,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_qaic_feature && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_qaic_feature && pytest tests -m '(on_qaic) and (feature) and (not qnn) and ${TEST_FILTER}' --ignore tests/transformers/sampler --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --junitxml=tests/tests_log2_feature.xml --durations=10 && junitparser merge tests/tests_log2_feature.xml tests/tests_log.xml && @@ -141,6 +153,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_cli_qaic_multimodal && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_cli_qaic_multimodal && pytest tests -m '(multimodal) and (not qnn) and ${TEST_FILTER}' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --ignore tests/transformers/models/reranker/test_reranker_mad.py --junitxml=tests/tests_log6.xml --durations=10 && junitparser merge tests/tests_log6.xml tests/tests_log.xml && @@ -159,6 +172,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_cli_qaic_reranker && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_cli_qaic_reranker && export QEFF_RERANKER_DOC_LIMIT=1 && pytest -q tests/transformers/models/reranker/test_reranker_mad.py --maxfail=1 --junitxml=tests/tests_log_reranker.xml --durations=10 && @@ -178,6 +192,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/Non_cli_qaic_diffusion && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/Non_cli_qaic_diffusion && export HF_HUB_CACHE=/huggingface_hub && pytest tests -m 'diffusion_models' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --junitxml=tests/tests_log_diffusion.xml --durations=10 && @@ -200,6 +215,7 @@ pipeline { . preflight_qeff/bin/activate && mkdir -p $PWD/cli && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/cli && pytest tests -m '(cli and not qnn) and (not finetune)' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --junitxml=tests/tests_log3.xml --durations=10 && junitparser merge tests/tests_log3.xml tests/tests_log.xml && @@ -223,6 +239,7 @@ pipeline { pip install torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1 --index-url https://download.pytorch.org/whl/cpu && mkdir -p $PWD/cli_qaic_finetuning && export TOKENIZERS_PARALLELISM=false && + export QEFF_TEST_SCOPE=${QEFF_TEST_SCOPE} && export QEFF_HOME=$PWD/cli_qaic_finetuning && pytest tests -m '(finetune)' --ignore tests/vllm --ignore tests/unit_test --ignore tests/nightly_pipeline --junitxml=tests/tests_log_finetune.xml --durations=10 && junitparser merge tests/tests_log_finetune.xml tests/tests_log.xml && diff --git a/tests/configs/causal_model_configs.json b/tests/configs/causal_model_configs.json index 93f4e7ae2f..4937b6c37f 100644 --- a/tests/configs/causal_model_configs.json +++ b/tests/configs/causal_model_configs.json @@ -4,198 +4,268 @@ "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000, "num_key_value_heads": 1 - } + }, + "coverage_key": "llama", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "gpt2", "model_type": "gpt2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50257, "num_key_value_heads": 1 - } + }, + "coverage_key": "gpt2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "allenai/OLMo-2-0425-1B", "model_type": "olmo2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 100352, "num_key_value_heads": 1 - } + }, + "coverage_key": "olmo2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Salesforce/codegen-350M-mono", "model_type": "codegen", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 4, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 51200, "num_key_value_heads": 1, "rotary_dim": 16 - } + }, + "coverage_key": "codegen", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-3.1-1b-a400m-base", "model_type": "granitemoe", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granitemoe", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "microsoft/Phi-3-mini-4k-instruct", "model_type": "phi3", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32064, "num_key_value_heads": 1 - } + }, + "coverage_key": "phi3", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "tiiuae/falcon-7b", "model_type": "falcon", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 65024, "num_key_value_heads": 1 - } + }, + "coverage_key": "falcon", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", "model_type": "qwen3_moe", "additional_params": { - "hidden_size": 256, - "intermediate_size": 256, - "max_position_embeddings": 128, - "max_window_layers": 48, - "moe_intermediate_size": 768, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, + "moe_intermediate_size": 64, "num_attention_heads": 2, - "num_experts": 4, + "num_experts": 2, "num_experts_per_tok": 2, "num_hidden_layers": 1, "num_key_value_heads": 1, "vocab_size": 151936 - } + }, + "coverage_key": "qwen3_moe", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen2-0.5B", "model_type": "qwen2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 151936, "num_key_value_heads": 1 - } + }, + "coverage_key": "qwen2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "bigcode/starcoder2-3b", "model_type": "starcoder2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49152, "num_key_value_heads": 1 - } + }, + "coverage_key": "starcoder2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Felladrin/Minueza-32M-Base", "model_type": "mistral", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32002, "num_key_value_heads": 1 - } + }, + "coverage_key": "mistral", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "wtang06/mpt-125m-c4", "model_type": "mpt", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50368 - } + }, + "coverage_key": "mpt", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hakurei/gpt-j-random-tinier", "model_type": "gptj", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50400, "num_key_value_heads": 1, "rotary_dim": 16 - } + }, + "coverage_key": "gptj", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "mistralai/Mixtral-8x7B-Instruct-v0.1", "model_type": "mixtral", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000, "num_key_value_heads": 1 - } + }, + "coverage_key": "mixtral", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "meta-llama/Llama-3.2-1B", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 128256, "num_key_value_heads": 1, "rope_scaling": { @@ -206,134 +276,182 @@ "rope_type": "llama3", "rope_theta": 500000.0 } - } + }, + "coverage_key": "llama", + "tiers": [ + "exhaustive" + ] }, { "model_name": "unsloth/gemma-2b", "model_type": "gemma", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "unsloth/gemma-2-2b", "model_type": "gemma2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32003 - } + }, + "coverage_key": "llama:awq", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "TheBloke/Llama-2-7B-GPTQ", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000 - } + }, + "coverage_key": "llama:gptq", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-20b-code-base", "model_type": "gpt_bigcode", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49152, "num_key_value_heads": 1, "activation_function": "gelu", "architectures": [ "GPTBigCodeForCausalLM" ] - } + }, + "coverage_key": "gpt_bigcode", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "neuralmagic/Llama-3.2-3B-Instruct-FP8", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 128256 - } + }, + "coverage_key": "llama:fp8", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "neuralmagic/Qwen2-0.5B-Instruct-FP8", "model_type": "qwen2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 2, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 151936 - } + }, + "coverage_key": "qwen2:fp8", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-3.1-2b-instruct", "model_type": "granite", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granite", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-guardian-3.1-2b", "model_type": "granite", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granite", + "tiers": [ + "exhaustive" + ] }, { "model_name": "Snowflake/Llama-3.1-SwiftKV-8B-Instruct", "model_type": null, "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 2, "num_attention_heads": 2, - "hidden_size": 256, - "intermediate_size": 256, + "hidden_size": 64, + "intermediate_size": 128, "vocab_size": 128256, "num_key_value_layers": 1, "num_key_value_heads": 1, @@ -344,7 +462,12 @@ "original_max_position_embeddings": 8192, "rope_type": "llama3" } - } + }, + "coverage_key": "Snowflake/Llama-3.1-SwiftKV-8B-Instruct", + "tiers": [ + "pr", + "exhaustive" + ] } ], "causal_lm_fp16_test_models": [ @@ -352,117 +475,157 @@ "model_name": "gpt2", "model_type": "gpt2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50257, "num_key_value_heads": 1 - } + }, + "coverage_key": "gpt2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hf-internal-testing/tiny-random-Gemma2ForCausalLM", "model_type": "gemma2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hf-internal-testing/tiny-random-GPTBigCodeForCausalLM", "model_type": "gpt_bigcode", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49152, "num_key_value_heads": 1, "activation_function": "gelu", "architectures": [ "GPTBigCodeForCausalLM" ] - } + }, + "coverage_key": "gpt_bigcode", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen2-0.5B", "model_type": "qwen2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 151936, "num_key_value_heads": 1 - } + }, + "coverage_key": "qwen2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hf-internal-testing/tiny-random-MixtralForCausalLM", "model_type": "mixtral", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000, "num_key_value_heads": 1 - } + }, + "coverage_key": "mixtral", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hf-internal-testing/tiny-random-LlamaForCausalLM", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 128256, "num_key_value_heads": 1, "rope_scaling": { "factor": 32.0, "high_freq_factor": 4.0, - "low_freq_factor": 1.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3", - "rope_theta": 500000.0 - } - } - }, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3", + "rope_theta": 500000.0 + } + }, + "coverage_key": "llama", + "tiers": [ + "pr", + "exhaustive" + ] + }, { "model_name": "allenai/OLMo-2-0425-1B", "model_type": "olmo2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 100352, "num_key_value_heads": 1 - } + }, + "coverage_key": "olmo2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "hf-internal-testing/tiny-random-GraniteForCausalLM", "model_type": "granite", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granite", + "tiers": [ + "pr", + "exhaustive" + ] } ], "spd_causal_lm_models": [ @@ -470,27 +633,37 @@ "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000, "num_key_value_heads": 1 - } + }, + "coverage_key": "llama", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen2-0.5B", "model_type": "qwen2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 151936, "num_key_value_heads": 1 - } + }, + "coverage_key": "qwen2", + "tiers": [ + "pr", + "exhaustive" + ] } ], "qnn_causal_lm_models": [ @@ -498,24 +671,29 @@ "model_name": "mistralai/Mixtral-8x7B-Instruct-v0.1", "model_type": "mixtral", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 32000, "num_key_value_heads": 1 - } + }, + "coverage_key": "mixtral", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "meta-llama/Llama-3.2-1B", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 128256, "num_key_value_heads": 1, "rope_scaling": { @@ -526,33 +704,48 @@ "rope_type": "llama3", "rope_theta": 500000.0 } - } + }, + "coverage_key": "llama", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "unsloth/gemma-2b", "model_type": "gemma", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-guardian-3.1-2b", "model_type": "granite", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granite", + "tiers": [ + "pr", + "exhaustive" + ] } ], "prefix_caching_models": [ @@ -560,14 +753,19 @@ "model_name": "gpt2", "model_type": "gpt2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50257, "num_key_value_heads": 1 - } + }, + "coverage_key": "gpt2", + "tiers": [ + "pr", + "exhaustive" + ] } ], "blockedKV_causal_lm_models": [ @@ -575,50 +773,65 @@ "model_name": "unsloth/gemma-2b", "model_type": "gemma", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "unsloth/gemma-2-2b", "model_type": "gemma2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 256000, "num_key_value_heads": 1 - } + }, + "coverage_key": "gemma2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "ibm-granite/granite-3.1-1b-a400m-base", "model_type": "granitemoe", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49155, "num_key_value_heads": 1 - } + }, + "coverage_key": "granitemoe", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "meta-llama/Llama-3.2-1B", "model_type": "llama", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 128256, "num_key_value_heads": 1, "rope_scaling": { @@ -629,32 +842,47 @@ "rope_type": "llama3", "rope_theta": 500000.0 } - } + }, + "coverage_key": "llama", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "wtang06/mpt-125m-c4", "model_type": "mpt", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50368 - } + }, + "coverage_key": "mpt", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "bigcode/starcoder2-3b", "model_type": "starcoder2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 49152, "num_key_value_heads": 1 - } + }, + "coverage_key": "starcoder2", + "tiers": [ + "pr", + "exhaustive" + ] } ], "disaggregated_dummy_models": [ @@ -665,32 +893,42 @@ "additional_params": { "num_hidden_layers": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "num_attention_heads": 2, "num_key_value_heads": 1, - "num_local_experts": 4, + "num_local_experts": 2, "head_dim": 32, - "max_position_embeddings": 512, + "max_position_embeddings": 64, "vocab_size": 201088, - "sliding_window": 128 - } + "sliding_window": 64 + }, + "coverage_key": "gpt_oss", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", "model_type": "qwen3_moe", "additional_params": { - "hidden_size": 256, - "intermediate_size": 256, - "max_position_embeddings": 512, - "max_window_layers": 48, - "moe_intermediate_size": 768, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, + "moe_intermediate_size": 64, "num_attention_heads": 2, - "num_experts": 4, + "num_experts": 2, "num_experts_per_tok": 2, "num_hidden_layers": 2, "num_key_value_heads": 1, "vocab_size": 151936 - } + }, + "coverage_key": "qwen3_moe", + "tiers": [ + "pr", + "exhaustive" + ] } ], "causal_lm_models_pl1": [ @@ -698,14 +936,19 @@ "model_name": "gpt2", "model_type": "gpt2", "additional_params": { - "max_position_embeddings": 128, + "max_position_embeddings": 64, "num_hidden_layers": 1, "num_attention_heads": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "vocab_size": 50257, "num_key_value_heads": 1 - } + }, + "coverage_key": "gpt2", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "openai/gpt-oss-20b", @@ -713,11 +956,17 @@ "additional_params": { "num_hidden_layers": 2, "hidden_size": 64, - "intermediate_size": 256, + "intermediate_size": 128, "num_attention_heads": 2, "num_key_value_heads": 1, - "num_local_experts": 4 - } + "num_local_experts": 2, + "head_dim": 32 + }, + "coverage_key": "gpt_oss", + "tiers": [ + "pr", + "exhaustive" + ] } ] } diff --git a/tests/configs/embedding_model_configs.json b/tests/configs/embedding_model_configs.json index c10859886a..7265e87318 100644 --- a/tests/configs/embedding_model_configs.json +++ b/tests/configs/embedding_model_configs.json @@ -1,6 +1,22 @@ { - "embedding_models": [ - {"model_name": "jinaai/jina-embeddings-v2-base-code", "pooling": "mean"}, - {"model_name": "sentence-transformers/nli-bert-base-cls-pooling", "pooling": "cls"} - ] -} \ No newline at end of file + "embedding_models": [ + { + "model_name": "jinaai/jina-embeddings-v2-base-code", + "pooling": "mean", + "coverage_key": "embedding:jinaai/jina-embeddings-v2-base-code", + "tiers": [ + "pr", + "exhaustive" + ] + }, + { + "model_name": "sentence-transformers/nli-bert-base-cls-pooling", + "pooling": "cls", + "coverage_key": "embedding:sentence-transformers/nli-bert-base-cls-pooling", + "tiers": [ + "pr", + "exhaustive" + ] + } + ] +} diff --git a/tests/configs/feature_configs.json b/tests/configs/feature_configs.json index a8c41db64b..a8b8400dfc 100644 --- a/tests/configs/feature_configs.json +++ b/tests/configs/feature_configs.json @@ -1,182 +1,209 @@ { - "sampler_config": [ - { - "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "model_type": "llama", - "prompts": ["My name is","My name is"], - "prefill_seq_len": 32, - "ctx_len": 64, - "generation_len": 20, - "full_batch_size": 2, - "spec_length": 1, - "is_vlm": false, - "additional_params": { - "max_position_embeddings": 128, - "num_hidden_layers": 1, - "num_attention_heads": 2, - "hidden_size": 64, - "intermediate_size": 256, - "vocab_size": 32000, - "num_key_value_heads": 1 - }, - "full_layers_output":{ - "golden_texts": { - "w_sampler": null, - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - }, - "few_layers_output":{ - "golden_texts": { - "w_sampler": null, - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - }, - "dummy_layers_output":{ - "golden_texts": { - "w_sampler": "строиochastic bed particles pintfalseFrontounter RA official Linux thee Kat tienensimp Query garbagejsfiddle� deleting", - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - } + "sampler_config": [ + { + "model_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "model_type": "llama", + "prompts": [ + "My name is", + "My name is" + ], + "prefill_seq_len": 32, + "ctx_len": 64, + "generation_len": 20, + "full_batch_size": 2, + "spec_length": 1, + "is_vlm": false, + "additional_params": { + "max_position_embeddings": 64, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "hidden_size": 64, + "intermediate_size": 128, + "vocab_size": 32000, + "num_key_value_heads": 1 + }, + "full_layers_output": { + "golden_texts": { + "w_sampler": null, + "wo_sampler": null }, - { - "model_name": "OpenGVLab/InternVL2_5-1B", - "model_type": "llava", - "image_urls": [ - "https://picsum.photos/id/237/536/354", - "https://picsum.photos/id/237/536/354" - ], - "prompts": [ - "Can you describe the image in detail.", - "Can you describe the image in detail." - ], - "prefill_seq_len": 128, - "ctx_len": 4096, - "generation_len": 20, - "full_batch_size": 2, - "spec_length": null, - "is_vlm": true, - "additional_params": { - "force_image_size": 448, - "dtype": "float32", - "llm_config": { - "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct", - "architectures": [ - "Qwen2ForCausalLM" - ], - "layer_types": ["full_attention"], - "hidden_size": 896, - "intermediate_size": 4864, - "max_position_embeddings": 32768, - "max_window_layers": 21, - "num_attention_heads": 14, - "num_hidden_layers": 1, - "num_key_value_heads": 2, - "dtype": "float32", - "use_bfloat16": false, - "vocab_size": 151674 - }, - "vision_config": { - "architectures": [ - "InternVisionModel" - ], - "hidden_size": 1024, - "image_size": 448, - "intermediate_size": 4096, - "num_attention_heads": 16, - "num_channels": 3, - "num_hidden_layers": 1, - "norm_type": "layer_norm", - "qk_normalization": false, - "qkv_bias": true, - "dtype": "float32", - "use_bfloat16": false, - "patch_size": 14 - } - }, - "full_layers_output":{ - "golden_texts": { - "w_sampler": null, - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - }, - "few_layers_output":{ - "golden_texts": { - "w_sampler": null, - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - }, - "dummy_layers_output":{ - "golden_texts": { - "w_sampler": null, - "wo_sampler": null - }, - "golden_ids": { - "w_sampler": null, - "wo_sampler": null - } - } + "golden_ids": { + "w_sampler": null, + "wo_sampler": null } - ], - - "spd_config": [ - { - "id": "CB llama", - "draft_model_name": "JackFram/llama-160m", - "target_model_name": "JackFram/llama-160m", - "prompts": ["My name is"], - "num_speculative_tokens": 4, - "prefill_seq_len": 32, - "ctx_len": 128, - "prefill_bsz": 1, - "full_batch_size": 1, - "max_ngram_size": 3, - "additional_params": { - "max_position_embeddings": 128, - "num_hidden_layers": 1, - "num_attention_heads": 2, - "num_key_value_heads": 2, - "hidden_size": 64, - "intermediate_size": 256 - } + }, + "few_layers_output": { + "golden_texts": { + "w_sampler": null, + "wo_sampler": null }, - { - "id": "CB qwen", - "draft_model_name": "Qwen/Qwen2-0.5B", - "target_model_name": "Qwen/Qwen2-0.5B", - "prompts": ["My name is"], - "num_speculative_tokens": 4, - "prefill_seq_len": 32, - "ctx_len": 128, - "prefill_bsz": 1, - "full_batch_size": 1, - "additional_params": { - "max_position_embeddings": 128, - "num_hidden_layers": 1, - "num_attention_heads": 2, - "hidden_size": 64, - "intermediate_size": 256, - "num_key_value_heads": 1 - } + "golden_ids": { + "w_sampler": null, + "wo_sampler": null } - ] -} \ No newline at end of file + }, + "dummy_layers_output": { + "golden_texts": { + "w_sampler": "\u0441\u0442\u0440\u043e\u0438ochastic bed particles pintfalseFrontounter RA official Linux thee Kat tienensimp Query garbagejsfiddle\ufffd deleting", + "wo_sampler": null + }, + "golden_ids": { + "w_sampler": null, + "wo_sampler": null + } + }, + "coverage_key": "sampler:causal", + "tiers": [ + "pr", + "exhaustive" + ] + }, + { + "model_name": "OpenGVLab/InternVL2_5-1B", + "model_type": "llava", + "image_urls": [ + "https://picsum.photos/id/237/536/354", + "https://picsum.photos/id/237/536/354" + ], + "prompts": [ + "Can you describe the image in detail.", + "Can you describe the image in detail." + ], + "prefill_seq_len": 32, + "ctx_len": 64, + "generation_len": 20, + "full_batch_size": 2, + "spec_length": null, + "is_vlm": true, + "additional_params": { + "force_image_size": 224, + "dtype": "float32", + "llm_config": { + "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct", + "architectures": [ + "Qwen2ForCausalLM" + ], + "layer_types": [ + "full_attention" + ], + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, + "num_attention_heads": 2, + "num_hidden_layers": 1, + "num_key_value_heads": 2, + "dtype": "float32", + "use_bfloat16": false, + "vocab_size": 151674 + }, + "vision_config": { + "architectures": [ + "InternVisionModel" + ], + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 1, + "norm_type": "layer_norm", + "qk_normalization": false, + "qkv_bias": true, + "dtype": "float32", + "use_bfloat16": false, + "patch_size": 14 + } + }, + "full_layers_output": { + "golden_texts": { + "w_sampler": null, + "wo_sampler": null + }, + "golden_ids": { + "w_sampler": null, + "wo_sampler": null + } + }, + "few_layers_output": { + "golden_texts": { + "w_sampler": null, + "wo_sampler": null + }, + "golden_ids": { + "w_sampler": null, + "wo_sampler": null + } + }, + "dummy_layers_output": { + "golden_texts": { + "w_sampler": null, + "wo_sampler": null + }, + "golden_ids": { + "w_sampler": null, + "wo_sampler": null + } + }, + "coverage_key": "sampler:vlm", + "tiers": [ + "pr", + "exhaustive" + ] + } + ], + "spd_config": [ + { + "id": "CB llama", + "draft_model_name": "JackFram/llama-160m", + "target_model_name": "JackFram/llama-160m", + "prompts": [ + "My name is" + ], + "num_speculative_tokens": 2, + "prefill_seq_len": 16, + "ctx_len": 64, + "prefill_bsz": 1, + "full_batch_size": 1, + "max_ngram_size": 3, + "additional_params": { + "max_position_embeddings": 64, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "num_key_value_heads": 2, + "hidden_size": 64, + "intermediate_size": 128 + }, + "coverage_key": "spd", + "tiers": [ + "pr", + "exhaustive" + ] + }, + { + "id": "CB qwen", + "draft_model_name": "Qwen/Qwen2-0.5B", + "target_model_name": "Qwen/Qwen2-0.5B", + "prompts": [ + "My name is" + ], + "num_speculative_tokens": 2, + "prefill_seq_len": 16, + "ctx_len": 64, + "prefill_bsz": 1, + "full_batch_size": 1, + "additional_params": { + "max_position_embeddings": 64, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "hidden_size": 64, + "intermediate_size": 128, + "num_key_value_heads": 1 + }, + "coverage_key": "spd", + "tiers": [ + "exhaustive" + ] + } + ] +} diff --git a/tests/configs/image_text_model_configs.json b/tests/configs/image_text_model_configs.json index 85df559970..c70d534677 100644 --- a/tests/configs/image_text_model_configs.json +++ b/tests/configs/image_text_model_configs.json @@ -4,7 +4,7 @@ "model_name": "llava-hf/llava-1.5-7b-hf", "model_type": "llava", "batch_size": 1, - "prompt_len": 784, + "prompt_len": 384, "ctx_len": 1024, "img_size": 336, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg", @@ -23,10 +23,10 @@ "dtype": "float32", "text_config": { "dtype": "float32", - "head_dim": 128, - "hidden_size": 4096, - "intermediate_size": 11008, - "max_position_embeddings": 4096, + "head_dim": 2, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 1, @@ -35,27 +35,32 @@ }, "vision_config": { "dtype": "float32", - "hidden_size": 1024, - "image_size": 336, - "intermediate_size": 4096, + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, "model_type": "clip_vision_model", "num_attention_heads": 4, "num_hidden_layers": 1, "patch_size": 14, "vocab_size": 32000 } - } + }, + "coverage_key": "llava", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "model_type": "llama4", "batch_size": 1, "prompt_len": 32, - "ctx_len": 3072, + "ctx_len": 1024, "img_size": 336, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg", "text_prompt": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud", - "num_layers": 4, + "num_layers": 1, "img_url_list": [ "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg" @@ -65,15 +70,20 @@ "What are the objects in the image?" ], "full_batch_size": 2, - "additional_params": {} + "additional_params": {}, + "coverage_key": "llama4", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "google/gemma-3-4b-it", "model_type": "gemma3", "batch_size": 1, "prompt_len": 128, - "ctx_len": 3072, - "img_size": 896, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "text_prompt": "Can you describe the image in detail.", "num_layers": 2, @@ -90,9 +100,9 @@ "text_config": { "sliding_window_pattern": 2, "head_dim": 256, - "hidden_size": 2560, - "intermediate_size": 10240, - "num_hidden_layers": 2, + "hidden_size": 64, + "intermediate_size": 128, + "num_hidden_layers": 1, "layer_types": [ "sliding_attention", "full_attention" @@ -104,23 +114,28 @@ "sliding_window": 32 }, "vision_config": { - "hidden_size": 1152, - "image_size": 896, - "intermediate_size": 4304, + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, "num_attention_heads": 4, - "num_hidden_layers": 2, + "num_hidden_layers": 1, "patch_size": 14, "vision_use_head": false } - } + }, + "coverage_key": "gemma3", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", "model_type": "mistral3", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, @@ -135,9 +150,9 @@ "full_batch_size": 2, "additional_params": { "text_config": { - "head_dim": 128, - "hidden_size": 256, - "intermediate_size": 512, + "head_dim": 16, + "hidden_size": 64, + "intermediate_size": 128, "model_type": "mistral", "num_attention_heads": 4, "num_hidden_layers": 1, @@ -145,25 +160,30 @@ "vocab_size": 131072 }, "vision_config": { - "head_dim": 32, - "hidden_size": 128, - "image_size": 1540, - "intermediate_size": 256, + "head_dim": 16, + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, "model_type": "pixtral", "num_attention_heads": 4, "num_hidden_layers": 1, "patch_size": 14, "vocab_size": 32000 } - } + }, + "coverage_key": "mistral3", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen2.5-VL-3B-Instruct", "model_type": "qwen2_5_vl", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, @@ -177,21 +197,21 @@ ], "full_batch_size": 2, "additional_params": { - "hidden_size": 2048, + "hidden_size": 64, "text_config": { - "max_position_embeddings": 128000, - "hidden_size": 2048, - "intermediate_size": 11008, - "max_window_layers": 70, - "num_attention_heads": 16, + "max_position_embeddings": 64, + "hidden_size": 64, + "intermediate_size": 128, + "max_window_layers": 2, + "num_attention_heads": 2, "num_hidden_layers": 1, - "num_key_value_heads": 2, + "num_key_value_heads": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "mrope_section": [ - 16, - 24, - 24 + 2, + 3, + 3 ], "rope_type": "default", "type": "default" @@ -207,12 +227,12 @@ "vision_config": { "depth": 1, "num_hidden_layers": 1, - "hidden_size": 1280, + "hidden_size": 64, "in_chans": 3, - "intermediate_size": 3420, - "num_heads": 16, + "intermediate_size": 128, + "num_heads": 2, "spatial_patch_size": 14, - "out_hidden_size": 2048, + "out_hidden_size": 64, "tokens_per_second": 2, "torch_dtype": "float32" }, @@ -220,15 +240,20 @@ "vision_start_token_id": 151652, "vision_token_id": 151654, "vocab_size": 151936 - } + }, + "coverage_key": "qwen2_5_vl", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3-VL-2B-Instruct", "model_type": "qwen3_vl", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, @@ -244,9 +269,9 @@ "additional_params": { "text_config": { "dtype": "float32", - "head_dim": 128, - "hidden_size": 256, - "intermediate_size": 512, + "head_dim": 16, + "hidden_size": 64, + "intermediate_size": 128, "model_type": "qwen3_vl_text", "num_attention_heads": 4, "num_hidden_layers": 1, @@ -254,9 +279,9 @@ "rope_scaling": { "mrope_interleaved": true, "mrope_section": [ - 24, - 20, - 20 + 4, + 6, + 6 ], "rope_type": "default" }, @@ -268,24 +293,29 @@ 1 ], "depth": 2, - "hidden_size": 128, - "intermediate_size": 512, + "hidden_size": 64, + "intermediate_size": 128, "model_type": "qwen3_vl", "num_heads": 4, - "out_hidden_size": 256, + "out_hidden_size": 64, "patch_size": 16 }, "vision_end_token_id": 151653, "vision_start_token_id": 151652 - } + }, + "coverage_key": "qwen3_vl", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3-VL-30B-A3B-Instruct", "model_type": "qwen3_vl_moe", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, @@ -301,23 +331,23 @@ "additional_params": { "text_config": { "dtype": "float32", - "head_dim": 128, - "hidden_size": 256, - "intermediate_size": 512, + "head_dim": 16, + "hidden_size": 64, + "intermediate_size": 128, "mlp_only_layers": [], "model_type": "qwen3_vl_moe_text", - "moe_intermediate_size": 256, + "moe_intermediate_size": 64, "num_attention_heads": 4, - "num_experts": 8, - "num_experts_per_tok": 4, + "num_experts": 2, + "num_experts_per_tok": 2, "num_hidden_layers": 1, "num_key_value_heads": 2, "rope_scaling": { "mrope_interleaved": true, "mrope_section": [ - 24, - 20, - 20 + 4, + 6, + 6 ], "rope_type": "default" }, @@ -329,27 +359,32 @@ 1 ], "depth": 2, - "hidden_size": 128, + "hidden_size": 64, "in_channels": 3, - "intermediate_size": 512, + "intermediate_size": 128, "model_type": "qwen3_vl_moe", "num_heads": 4, - "out_hidden_size": 256 + "out_hidden_size": 64 }, "vision_end_token_id": 151653, "vision_start_token_id": 151652 - } + }, + "coverage_key": "qwen3_vl_moe", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3.5-0.8B", "model_type": "qwen3_5", "batch_size": 1, "prompt_len": 64, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", - "num_layers": 4, + "num_layers": 2, "img_url_list": [ "https://picsum.photos/id/237/536/354", "https://picsum.photos/id/238/536/354" @@ -363,9 +398,9 @@ "image_token_id": 248056, "text_config": { "dtype": "float32", - "head_dim": 128, - "hidden_size": 256, - "intermediate_size": 512, + "head_dim": 16, + "hidden_size": 64, + "intermediate_size": 128, "layer_types": [ "linear_attention", "linear_attention", @@ -375,14 +410,14 @@ "mlp_only_layers": [], "model_type": "qwen3_5_text", "num_attention_heads": 4, - "num_hidden_layers": 4, + "num_hidden_layers": 1, "num_key_value_heads": 2, "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ - 11, - 11, - 10 + 4, + 6, + 6 ], "partial_rotary_factor": 0.25, "rope_theta": 10000000, @@ -396,25 +431,30 @@ 1 ], "depth": 2, - "hidden_size": 128, + "hidden_size": 64, "in_channels": 3, - "intermediate_size": 512, + "intermediate_size": 128, "model_type": "qwen3_5", "num_heads": 4, - "out_hidden_size": 256, + "out_hidden_size": 64, "patch_size": 16 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 - } + }, + "coverage_key": "qwen3_5", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3.6-35B-A3B", "model_type": "qwen3_5_moe", "batch_size": 1, "prompt_len": 64, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, @@ -431,26 +471,26 @@ "image_token_id": 248056, "text_config": { "dtype": "float32", - "head_dim": 128, - "hidden_size": 256, + "head_dim": 16, + "hidden_size": 64, "layer_types": [ "linear_attention" ], "mlp_only_layers": [], "model_type": "qwen3_5_moe_text", - "moe_intermediate_size": 256, + "moe_intermediate_size": 64, "shared_expert_intermediate_size": 256, "num_attention_heads": 4, - "num_experts": 8, - "num_experts_per_tok": 4, + "num_experts": 2, + "num_experts_per_tok": 2, "num_hidden_layers": 1, "num_key_value_heads": 2, "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ - 11, - 11, - 10 + 4, + 6, + 6 ], "partial_rotary_factor": 0.25, "rope_theta": 10000000, @@ -464,28 +504,33 @@ 1 ], "depth": 2, - "hidden_size": 128, + "hidden_size": 64, "in_channels": 3, - "intermediate_size": 512, + "intermediate_size": 128, "model_type": "qwen3_5_moe", "num_heads": 4, - "out_hidden_size": 256, + "out_hidden_size": 64, "patch_size": 16 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 - } + }, + "coverage_key": "qwen3_5_moe", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "allenai/Molmo-7B-D-0924", "model_type": "molmo", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, + "ctx_len": 1024, "img_size": null, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", - "num_layers": 2, + "num_layers": 1, "img_url_list": [ "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg" @@ -495,7 +540,12 @@ "What are the objects in the image?" ], "full_batch_size": 2, - "additional_params": {} + "additional_params": {}, + "coverage_key": "molmo", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "OpenGVLab/InternVL2_5-1B", @@ -506,7 +556,7 @@ "img_size": null, "img_url": "https://image.slidesharecdn.com/azureintroduction-191206101932/75/Introduction-to-Microsoft-Azure-Cloud-1-2048.jpg", "text_prompt": "Please describe the image in detail.", - "num_layers": 2, + "num_layers": 1, "img_url_list": [ "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg" @@ -517,18 +567,18 @@ ], "full_batch_size": 2, "additional_params": { - "force_image_size": 448, + "force_image_size": 224, "llm_config": { "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct", "architectures": [ "Qwen2ForCausalLM" ], - "hidden_size": 896, - "intermediate_size": 4864, - "max_position_embeddings": 32768, - "max_window_layers": 21, - "num_attention_heads": 14, - "num_hidden_layers": 2, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, + "num_attention_heads": 2, + "num_hidden_layers": 1, "num_key_value_heads": 2, "torch_dtype": "bfloat16", "use_bfloat16": true, @@ -538,12 +588,12 @@ "architectures": [ "InternVisionModel" ], - "hidden_size": 1024, - "image_size": 448, - "intermediate_size": 4096, + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, "num_attention_heads": 16, "num_channels": 3, - "num_hidden_layers": 2, + "num_hidden_layers": 1, "norm_type": "layer_norm", "qk_normalization": false, "qkv_bias": true, @@ -551,7 +601,12 @@ "use_bfloat16": true, "patch_size": 14 } - } + }, + "coverage_key": "internvl_chat", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "OpenGVLab/InternVL3_5-1B", @@ -562,7 +617,7 @@ "img_size": null, "img_url": "https://image.slidesharecdn.com/azureintroduction-191206101932/75/Introduction-to-Microsoft-Azure-Cloud-1-2048.jpg", "text_prompt": "Please describe the image in detail.", - "num_layers": 2, + "num_layers": 1, "img_url_list": [ "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg" @@ -573,18 +628,18 @@ ], "full_batch_size": 2, "additional_params": { - "force_image_size": 448, + "force_image_size": 224, "llm_config": { "_name_or_path": "/root/codespace/checkpoints/Qwen3-0.6B", "architectures": [ "Qwen3ForCausalLM" ], - "hidden_size": 1024, - "intermediate_size": 3072, - "max_position_embeddings": 40960, - "max_window_layers": 28, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, "num_attention_heads": 16, - "num_hidden_layers": 2, + "num_hidden_layers": 1, "num_key_value_heads": 8, "torch_dtype": "bfloat16", "vocab_size": 151936 @@ -593,12 +648,12 @@ "architectures": [ "InternVisionModel" ], - "hidden_size": 1024, - "image_size": 448, - "intermediate_size": 4096, + "hidden_size": 64, + "image_size": 224, + "intermediate_size": 128, "num_attention_heads": 16, "num_channels": 3, - "num_hidden_layers": 2, + "num_hidden_layers": 1, "norm_type": "layer_norm", "qk_normalization": false, "qkv_bias": true, @@ -607,7 +662,11 @@ "use_flash_attn": true, "patch_size": 14 } - } + }, + "coverage_key": "internvl_chat", + "tiers": [ + "exhaustive" + ] } ], "image_text_subfunction_models": [ @@ -616,17 +675,17 @@ "model_type": "qwen2_5_vl", "batch_size": 1, "prompt_len": 128, - "ctx_len": 4096, - "img_size": 1540, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "text_prompt": "Can you describe the image in detail.", "num_layers": 1, "additional_params": { "dtype": "float32", - "hidden_size": 2048, - "intermediate_size": 11008, - "max_position_embeddings": 128000, - "max_window_layers": 70, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, "num_attention_heads": 16, "num_hidden_layers": 1, "num_key_value_heads": 2, @@ -638,20 +697,20 @@ "full_attention" ], "dtype": "float32", - "hidden_size": 2048, - "intermediate_size": 11008, - "max_position_embeddings": 128000, - "max_window_layers": 70, + "hidden_size": 64, + "intermediate_size": 128, + "max_position_embeddings": 64, + "max_window_layers": 2, "model_type": "qwen2_5_vl_text", - "num_attention_heads": 16, + "num_attention_heads": 2, "num_hidden_layers": 1, - "num_key_value_heads": 2, + "num_key_value_heads": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "mrope_section": [ - 16, - 24, - 24 + 2, + 3, + 3 ], "rope_type": "default", "type": "default" @@ -662,27 +721,29 @@ "depth": 1, "num_hidden_layers": 1, "hidden_act": "silu", - "hidden_size": 1280, - "intermediate_size": 3420, - "num_heads": 16, + "hidden_size": 64, + "intermediate_size": 128, + "num_heads": 2, "in_chans": 3, - "out_hidden_size": 2048, + "out_hidden_size": 64, "patch_size": 14, "spatial_merge_size": 2, "spatial_patch_size": 14, "window_size": 112, "fullatt_block_indexes": [ - 7, - 15, - 23, - 31 + 0 ], "tokens_per_second": 2, "temporal_patch_size": 2 }, "vision_start_token_id": 151652, "vocab_size": 151936 - } + }, + "coverage_key": "qwen2_5_vl", + "tiers": [ + "pr", + "exhaustive" + ] } ], "image_text_custom_dtype_models": [ @@ -696,32 +757,46 @@ "img_url": "https://image.slidesharecdn.com/azureintroduction-191206101932/75/Introduction-to-Microsoft-Azure-Cloud-1-2048.jpg", "text_prompt": "Please describe the image in detail.", "num_layers": 1, - "additional_params": {} + "additional_params": {}, + "coverage_key": "internvl_chat", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "google/gemma-3-4b-it", "model_type": "gemma3", "batch_size": 1, "prompt_len": 128, - "ctx_len": 3072, - "img_size": 896, + "ctx_len": 1024, + "img_size": 448, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png", "text_prompt": "Can you describe the image in detail.", "num_layers": 2, - "additional_params": {} + "additional_params": {}, + "coverage_key": "gemma3", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "llava-hf/llava-1.5-7b-hf", "model_type": "llava", "batch_size": 1, - "prompt_len": 784, + "prompt_len": 384, "ctx_len": 1024, "img_size": 336, "img_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg", "text_prompt": "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud", "num_layers": 1, - "additional_params": { - } + "additional_params": {}, + "coverage_key": "llava", + "tiers": [ + "pr", + "exhaustive" + ] } ], "image_text_reranker_models": [ @@ -730,28 +805,37 @@ "model_type": "qwen3_vl", "batch_size": 1, "prompt_len": 128, - "ctx_len": 1024, - "img_size": 1540, + "ctx_len": 512, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "instruction": "Retrieve candidates relevant to the query.", "query_text": "A woman playing with her dog on a beach at sunset.", "document_text": "A woman and her dog spend time together on a beach during sunset.", "num_layers": 1, - "additional_params": {} + "additional_params": {}, + "coverage_key": "qwen3_vl:reranker", + "tiers": [ + "pr", + "exhaustive" + ] }, { "model_name": "Qwen/Qwen3-VL-Reranker-8B", "model_type": "qwen3_vl", "batch_size": 1, "prompt_len": 128, - "ctx_len": 1024, - "img_size": 1540, + "ctx_len": 512, + "img_size": 448, "img_url": "https://picsum.photos/id/237/536/354", "instruction": "Retrieve candidates relevant to the query.", "query_text": "A woman playing with her dog on a beach at sunset.", "document_text": "A woman and her dog spend time together on a beach during sunset.", "num_layers": 1, - "additional_params": {} + "additional_params": {}, + "coverage_key": "qwen3_vl:reranker", + "tiers": [ + "exhaustive" + ] } ], "image_text_embedding_models": [ @@ -759,12 +843,17 @@ "model_name": "Qwen/Qwen3-VL-Embedding-8B", "model_type": "qwen3_vl", "batch_size": 1, - "ctx_len": 2048, + "ctx_len": 512, "num_layers": 1, "vision_depth": 9, "deepstack_index": 8, "compile_prefill_seq_len": null, - "mad_max_threshold": 0.002 + "mad_max_threshold": 0.002, + "coverage_key": "qwen3_vl:embedding", + "tiers": [ + "pr", + "exhaustive" + ] } ] } diff --git a/tests/diffusers/flux_test_config.json b/tests/diffusers/flux_test_config.json index d324a1aeb0..c2a6d1ad8d 100644 --- a/tests/diffusers/flux_test_config.json +++ b/tests/diffusers/flux_test_config.json @@ -1,127 +1,109 @@ { - "model_setup": { - "height": 64, - "width": 64, - "num_transformer_layers": 2, - "num_single_layers": 2 - }, - "mad_validation": { - "tolerances": { - "clip_text_encoder": 0.01, - "t5_text_encoder": 5, - "transformer": 0.1, - "vae_decoder": 0.01 - } + "model_setup": { + "height": 64, + "width": 64, + "num_transformer_layers": 1, + "num_single_layers": 1 + }, + "mad_validation": { + "tolerances": { + "clip_text_encoder": 0.01, + "t5_text_encoder": 5, + "transformer": 0.1, + "vae_decoder": 0.01 + } + }, + "pipeline_params": { + "test_prompt": "A cat holding a sign that says hello world", + "num_inference_steps": 1, + "guidance_scale": 0.0, + "max_sequence_length": 128, + "validate_gen_img": true, + "min_image_variance": 1.0, + "custom_config_path": null, + "use_onnx_subfunctions": true + }, + "validation_checks": { + "image_generation": true, + "onnx_export": true, + "compilation": true + }, + "modules": { + "text_encoder": { + "specializations": { + "batch_size": 1, + "seq_len": 77 + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } }, - "pipeline_params": { - "test_prompt": "A cat holding a sign that says hello world", - "num_inference_steps": 2, - "guidance_scale": 0.0, - "max_sequence_length": 256, - "validate_gen_img": true, - "min_image_variance": 1.0, - "custom_config_path": null, - "use_onnx_subfunctions": true - }, - "validation_checks": { - "image_generation": true, - "onnx_export": true, - "compilation": true - }, - "modules": - { - "text_encoder": - { - "specializations":{ - "batch_size": 1, - "seq_len": 77 - }, - "compilation": - { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16 - }, - "execute": - { - "device_ids": null, - "qpc_path" : null - } - - }, - "text_encoder_2": - { - "specializations": - { - "batch_size": 1, - "seq_len": 256 - }, - "compilation": - { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16 - }, - "execute": - { - "device_ids": null, - "qpc_path" : null - } - }, - "transformer": - { - "specializations": - { - "batch_size": 1, - "seq_len": 256, - "steps": 1 - }, - "compilation": - { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": true, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts-mos": 1, - "aic-enable-depth-first": true - }, - "execute": - { - "device_ids": null, - "qpc_path" : null - } - }, - "vae_decoder": - { - "specializations": - { - "batch_size": 1, - "channels": 16 - }, - "compilation": - { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16 - }, - "execute": - { - "device_ids": null, - "qpc_path" : null - } - } + "text_encoder_2": { + "specializations": { + "batch_size": 1, + "seq_len": 128 + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "transformer": { + "specializations": { + "batch_size": 1, + "seq_len": 128, + "steps": 1 + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": true, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts-mos": 1, + "aic-enable-depth-first": true + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "vae_decoder": { + "specializations": { + "batch_size": 1, + "channels": 16 + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + } } - } diff --git a/tests/diffusers/test_flux.py b/tests/diffusers/test_flux.py index fc7cae8c9b..47effe6add 100644 --- a/tests/diffusers/test_flux.py +++ b/tests/diffusers/test_flux.py @@ -24,6 +24,7 @@ from QEfficient.generation.cloud_infer import QAICInferenceSession from QEfficient.utils._utils import load_json from tests.diffusers.diffusers_utils import DiffusersTestUtils, MADValidator +from tests.test_matrix import EXHAUSTIVE_SCOPE, get_test_scope # Test Configuration for 256x256 resolution with 2 layers # update mad tolerance CONFIG_PATH = "tests/diffusers/flux_test_config.json" @@ -517,6 +518,8 @@ def test_flux_pipeline(flux_pipeline): @pytest.mark.diffusion_models @pytest.mark.on_qaic def test_flux_pipeline_first_block_cache(flux_pipeline_first_block_cache): + if get_test_scope() != EXHAUSTIVE_SCOPE: + pytest.skip("First-block-cache diffusion path runs in exhaustive scope.") _run_flux_pipeline_test_case( flux_pipeline_first_block_cache, INITIAL_TEST_CONFIG, diff --git a/tests/diffusers/test_wan.py b/tests/diffusers/test_wan.py index 9f920ae35f..c233076010 100644 --- a/tests/diffusers/test_wan.py +++ b/tests/diffusers/test_wan.py @@ -30,6 +30,7 @@ from QEfficient.utils import constants from QEfficient.utils._utils import load_json from tests.diffusers.diffusers_utils import DiffusersTestUtils, MADValidator +from tests.test_matrix import EXHAUSTIVE_SCOPE, get_test_scope # Test Configuration for 48 x 64 resolution with 1 layer CONFIG_PATH = "tests/diffusers/wan_test_config.json" @@ -519,6 +520,8 @@ def test_wan_pipeline(wan_pipeline): @pytest.mark.on_qaic @pytest.mark.wan def test_wan_pipeline_non_unified(wan_pipeline_non_unified): + if get_test_scope() != EXHAUSTIVE_SCOPE: + pytest.skip("WAN non-unified path runs in exhaustive scope.") _run_wan_pipeline_test_case( wan_pipeline_non_unified, NON_UNIFIED_TEST_CONFIG, @@ -531,6 +534,8 @@ def test_wan_pipeline_non_unified(wan_pipeline_non_unified): @pytest.mark.on_qaic @pytest.mark.wan def test_wan_pipeline_non_unified_first_block_cache(wan_pipeline_non_unified_first_block_cache): + if get_test_scope() != EXHAUSTIVE_SCOPE: + pytest.skip("WAN non-unified first-block-cache path runs in exhaustive scope.") _run_wan_pipeline_test_case( wan_pipeline_non_unified_first_block_cache, NON_UNIFIED_TEST_CONFIG, diff --git a/tests/diffusers/wan_i2v_test_config.json b/tests/diffusers/wan_i2v_test_config.json index a9cf17421e..71e611180b 100644 --- a/tests/diffusers/wan_i2v_test_config.json +++ b/tests/diffusers/wan_i2v_test_config.json @@ -1,108 +1,108 @@ { - "model_setup": { - "max_area": 3072, - "num_frames": 81, - "num_transformer_layers_high": 1, - "num_transformer_layers_low": 1 - }, - "mad_validation": { - "tolerances": { - "vae_encoder": 7, - "transformer_high": 0.7, - "transformer_low": 0.75, - "vae_decoder": 2.5 - } - }, - "pipeline_params": { - "test_prompt": "A cat walking in a garden with flowing motion", - "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg", - "num_inference_steps": 2, - "use_onnx_subfunctions": true, - "guidance_scale": 1.0, - "guidance_scale_2": 1.0, - "max_sequence_length": 512, - "validate_gen_video": true, - "min_video_variance": 1.0 - }, - "validation_checks": { - "video_generation": true, - "onnx_export": true, - "compilation": true, - "image_conditioning": true, - "dynamic_sizing": true + "model_setup": { + "max_area": 3072, + "num_frames": 5, + "num_transformer_layers_high": 1, + "num_transformer_layers_low": 1 + }, + "mad_validation": { + "tolerances": { + "vae_encoder": 7, + "transformer_high": 0.7, + "transformer_low": 0.75, + "vae_decoder": 2.5 + } + }, + "pipeline_params": { + "test_prompt": "A cat walking in a garden with flowing motion", + "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg", + "num_inference_steps": 1, + "use_onnx_subfunctions": true, + "guidance_scale": 1.0, + "guidance_scale_2": 1.0, + "max_sequence_length": 128, + "validate_gen_video": true, + "min_video_variance": 1.0 + }, + "validation_checks": { + "video_generation": true, + "onnx_export": true, + "compilation": true, + "image_conditioning": true, + "dynamic_sizing": true + }, + "modules": { + "vae_encoder": { + "specializations": { + "batch_size": "1", + "num_channels": "16" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } }, - "modules": { - "vae_encoder": { - "specializations": { - "batch_size": "1", - "num_channels": "16" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path": null - } - }, - "transformer": { - "specializations": [ - { - "batch_size": "1", - "num_channels": "36", - "steps": "1", - "sequence_length": "512", - "model_type": 1 - }, - { - "batch_size": "1", - "num_channels": "36", - "steps": "1", - "sequence_length": "512", - "model_type": 2 - } - ], - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": true, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path": null - } + "transformer": { + "specializations": [ + { + "batch_size": "1", + "num_channels": "36", + "steps": "1", + "sequence_length": "128", + "model_type": 1 }, - "vae_decoder": { - "specializations": { - "batch_size": "1", - "num_channels": "16" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path": null - } + { + "batch_size": "1", + "num_channels": "36", + "steps": "1", + "sequence_length": "128", + "model_type": 2 } + ], + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": true, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "vae_decoder": { + "specializations": { + "batch_size": "1", + "num_channels": "16" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } } + } } diff --git a/tests/diffusers/wan_test_config.json b/tests/diffusers/wan_test_config.json index 54bdf2a42d..35b5124540 100644 --- a/tests/diffusers/wan_test_config.json +++ b/tests/diffusers/wan_test_config.json @@ -1,86 +1,86 @@ { - "model_setup": { - "height": 48, - "width": 64, - "num_frames": 81, - "num_transformer_layers_high": 1, - "num_transformer_layers_low": 1 - }, - "mad_validation": { - "tolerances": { - "transformer": 0.1, - "transformer_high": 0.1, - "transformer_low": 0.1, - "vae_decoder" : 1.3 - } - }, - "pipeline_params": { - "test_prompt": "A cat walking in a garden", - "num_inference_steps": 2, - "guidance_scale": 1.0, - "guidance_scale_2": 1.0, - "max_sequence_length": 512, - "validate_gen_video": true, - "min_video_variance": 1.0, - "use_onnx_subfunctions": true - }, - "validation_checks": { - "video_generation": true, - "onnx_export": true, - "compilation": true - }, - "modules": { - "transformer": { - "specializations": [ - { - "batch_size": "1", - "num_channels": "16", - "steps": "1", - "sequence_length": "512", - "model_type": 1 - }, - { - "batch_size": "1", - "num_channels": "16", - "steps": "1", - "sequence_length": "512", - "model_type": 2 - } - ], - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": true, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path" : null - } + "model_setup": { + "height": 48, + "width": 64, + "num_frames": 5, + "num_transformer_layers_high": 1, + "num_transformer_layers_low": 1 + }, + "mad_validation": { + "tolerances": { + "transformer": 0.1, + "transformer_high": 0.1, + "transformer_low": 0.1, + "vae_decoder": 1.3 + } + }, + "pipeline_params": { + "test_prompt": "A cat walking in a garden", + "num_inference_steps": 1, + "guidance_scale": 1.0, + "guidance_scale_2": 1.0, + "max_sequence_length": 128, + "validate_gen_video": true, + "min_video_variance": 1.0, + "use_onnx_subfunctions": true + }, + "validation_checks": { + "video_generation": true, + "onnx_export": true, + "compilation": true + }, + "modules": { + "transformer": { + "specializations": [ + { + "batch_size": "1", + "num_channels": "16", + "steps": "1", + "sequence_length": "128", + "model_type": 1 }, - "vae_decoder": { - "specializations": { - "batch_size": "1", - "num_channels": "16" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path": null - } + { + "batch_size": "1", + "num_channels": "16", + "steps": "1", + "sequence_length": "128", + "model_type": 2 } + ], + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": true, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "vae_decoder": { + "specializations": { + "batch_size": "1", + "num_channels": "16" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } } + } } diff --git a/tests/diffusers/wan_test_non_unified_config.json b/tests/diffusers/wan_test_non_unified_config.json index c90f225ed6..f4726f46f3 100644 --- a/tests/diffusers/wan_test_non_unified_config.json +++ b/tests/diffusers/wan_test_non_unified_config.json @@ -1,101 +1,101 @@ { - "model_setup": { - "height": 48, - "width": 64, - "num_frames": 81, - "num_transformer_layers_high": 1, - "num_transformer_layers_low": 1 - }, - "mad_validation": { - "tolerances": { - "transformer": 0.1, - "transformer_high": 0.1, - "transformer_low": 0.1, - "vae_decoder" : 1.3 - } - }, - "pipeline_params": { - "test_prompt": "A cat walking in a garden", - "num_inference_steps": 2, - "guidance_scale": 1.0, - "guidance_scale_2": 1.0, - "max_sequence_length": 512, - "validate_gen_video": true, - "min_video_variance": 1.0, - "use_onnx_subfunctions": true - }, - "validation_checks": { - "video_generation": true, - "onnx_export": true, - "compilation": true - }, - "modules": { - "transformer_high": { - "specializations": { - "batch_size": "1", - "num_channels": "16", - "steps": "1", - "sequence_length": "512" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": true, - "convert_to_fp16": true, - "compile_only":true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path" : null - } - }, - "transformer_low": { - "specializations": { - "batch_size": "1", - "num_channels": "16", - "steps": "1", - "sequence_length": "512" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": true, - "convert_to_fp16": true, - "compile_only":true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path" : null - } - }, - "vae_decoder": { - "specializations": { - "batch_size": "1", - "num_channels": "16" - }, - "compilation": { - "onnx_path": null, - "compile_dir": null, - "mdp_ts_num_devices": 1, - "mxfp6_matmul": false, - "convert_to_fp16": true, - "compile_only": true, - "aic_num_cores": 16, - "mos": 1, - "mdts_mos": 1 - }, - "execute": { - "device_ids": null, - "qpc_path": null - } - } + "model_setup": { + "height": 48, + "width": 64, + "num_frames": 5, + "num_transformer_layers_high": 1, + "num_transformer_layers_low": 1 + }, + "mad_validation": { + "tolerances": { + "transformer": 0.1, + "transformer_high": 0.1, + "transformer_low": 0.1, + "vae_decoder": 1.3 } + }, + "pipeline_params": { + "test_prompt": "A cat walking in a garden", + "num_inference_steps": 1, + "guidance_scale": 1.0, + "guidance_scale_2": 1.0, + "max_sequence_length": 128, + "validate_gen_video": true, + "min_video_variance": 1.0, + "use_onnx_subfunctions": true + }, + "validation_checks": { + "video_generation": true, + "onnx_export": true, + "compilation": true + }, + "modules": { + "transformer_high": { + "specializations": { + "batch_size": "1", + "num_channels": "16", + "steps": "1", + "sequence_length": "128" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": true, + "convert_to_fp16": true, + "compile_only": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "transformer_low": { + "specializations": { + "batch_size": "1", + "num_channels": "16", + "steps": "1", + "sequence_length": "128" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": true, + "convert_to_fp16": true, + "compile_only": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + }, + "vae_decoder": { + "specializations": { + "batch_size": "1", + "num_channels": "16" + }, + "compilation": { + "onnx_path": null, + "compile_dir": null, + "mdp_ts_num_devices": 1, + "mxfp6_matmul": false, + "convert_to_fp16": true, + "compile_only": true, + "aic_num_cores": 16, + "mos": 1, + "mdts_mos": 1 + }, + "execute": { + "device_ids": null, + "qpc_path": null + } + } + } } diff --git a/tests/test_matrix.py b/tests/test_matrix.py new file mode 100644 index 0000000000..f439523d8a --- /dev/null +++ b/tests/test_matrix.py @@ -0,0 +1,115 @@ +# ----------------------------------------------------------------------------- +# +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# ----------------------------------------------------------------------------- + +import json +import os +from pathlib import Path +from typing import Any, Iterable + + +TEST_SCOPE_ENV = "QEFF_TEST_SCOPE" +PR_SCOPE = "pr" +EXHAUSTIVE_SCOPE = "exhaustive" +VALID_SCOPES = {PR_SCOPE, EXHAUSTIVE_SCOPE} + + +def get_test_scope() -> str: + scope = os.environ.get(TEST_SCOPE_ENV, EXHAUSTIVE_SCOPE).strip().lower() + if scope not in VALID_SCOPES: + raise ValueError(f"Unsupported {TEST_SCOPE_ENV}={scope!r}. Expected one of {sorted(VALID_SCOPES)}.") + return scope + + +def load_test_config(config_path: str | os.PathLike) -> dict[str, Any]: + with open(Path(config_path), "r", encoding="utf-8") as config_file: + return json.load(config_file) + + +def select_test_entries(config_path: str | os.PathLike, suite_name: str, scope: str | None = None) -> list[Any]: + entries = load_test_config(config_path)[suite_name] + scope = get_test_scope() if scope is None else scope + if scope == EXHAUSTIVE_SCOPE: + return entries + + selected_entries = [] + seen_coverage_keys = set() + for entry in entries: + if not _runs_in_scope(entry, scope): + continue + coverage_key = get_coverage_key(entry) + if coverage_key in seen_coverage_keys: + continue + seen_coverage_keys.add(coverage_key) + selected_entries.append(entry) + return selected_entries + + +def model_names(entries: Iterable[Any]) -> list[str]: + return [_entry_id(entry) for entry in entries] + + +def entries_by_model_name(entries: Iterable[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {entry["model_name"]: entry for entry in entries} + + +def entries_by_id(entries: Iterable[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {entry["id"]: entry for entry in entries} + + +def get_coverage_key(entry: Any) -> str: + if not isinstance(entry, dict): + return str(entry) + + if entry.get("coverage_key"): + return str(entry["coverage_key"]) + + model_name = str(entry.get("model_name") or entry.get("target_model_name") or entry.get("id") or entry) + model_type = entry.get("model_type") + if model_type: + return _typed_coverage_key(model_name, str(model_type)) + return model_name + + +def assert_pr_coverage(config_path: str | os.PathLike, suite_name: str, required_keys: set[str]) -> None: + selected_keys = {get_coverage_key(entry) for entry in select_test_entries(config_path, suite_name, PR_SCOPE)} + missing_keys = required_keys - selected_keys + assert not missing_keys, f"Missing PR coverage keys for {suite_name}: {sorted(missing_keys)}" + + +def _runs_in_scope(entry: Any, scope: str) -> bool: + if not isinstance(entry, dict): + return True + tiers = entry.get("tiers") + if tiers is None: + return True + return scope in tiers + + +def _entry_id(entry: Any) -> str: + if isinstance(entry, dict): + if "model_name" in entry: + return entry["model_name"] + if "id" in entry: + return entry["id"] + return str(entry) + + +def _typed_coverage_key(model_name: str, model_type: str) -> str: + lower_name = model_name.lower() + if "awq" in lower_name: + return f"{model_type}:awq" + if "gptq" in lower_name: + return f"{model_type}:gptq" + if "fp8" in lower_name: + return f"{model_type}:fp8" + if "swiftkv" in lower_name: + return f"{model_type}:swiftkv" + if "reranker" in lower_name: + return f"{model_type}:reranker" + if "embedding" in lower_name: + return f"{model_type}:embedding" + return model_type diff --git a/tests/transformers/caching/test_prefix_caching.py b/tests/transformers/caching/test_prefix_caching.py index 4413808c56..fad1a18d8e 100644 --- a/tests/transformers/caching/test_prefix_caching.py +++ b/tests/transformers/caching/test_prefix_caching.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os import numpy as np @@ -17,14 +16,12 @@ from QEfficient.utils._utils import create_json from QEfficient.utils.constants import QnnConstants from QEfficient.utils.test_utils import load_qeff_causal_lm_model +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - prefix_caching_models = config_data["prefix_caching_models"] - -test_models = [model["model_name"] for model in prefix_caching_models] -model_config_dict = {model["model_name"]: model for model in prefix_caching_models} +prefix_caching_models = select_test_entries(CONFIG_PATH, "prefix_caching_models") +test_models = model_names(prefix_caching_models) +model_config_dict = entries_by_model_name(prefix_caching_models) def prefix_caching_inference(model_name, qpc_path): diff --git a/tests/transformers/models/audio_models/test_audio_embedding_models.py b/tests/transformers/models/audio_models/test_audio_embedding_models.py index 82c613e557..78dce1a3c8 100644 --- a/tests/transformers/models/audio_models/test_audio_embedding_models.py +++ b/tests/transformers/models/audio_models/test_audio_embedding_models.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import List, Optional @@ -22,13 +21,12 @@ from QEfficient.utils import hf_download from QEfficient.utils._utils import create_json, load_hf_processor from QEfficient.utils.constants import WAV2VEC2_MAX_SEQ_LEN, QnnConstants +from tests.test_matrix import select_test_entries from ..check_model_results import dump_and_compare_results CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/audio_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - test_models = config_data["audio_embedding_models"] +test_models = select_test_entries(CONFIG_PATH, "audio_embedding_models") def load_ctc_model(model_config, torch_dtype: Optional[torch.dtype] = torch.float32): diff --git a/tests/transformers/models/audio_models/test_speech_seq2seq_models.py b/tests/transformers/models/audio_models/test_speech_seq2seq_models.py index 0c6fb29087..06d30fbb87 100644 --- a/tests/transformers/models/audio_models/test_speech_seq2seq_models.py +++ b/tests/transformers/models/audio_models/test_speech_seq2seq_models.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import List, Optional @@ -22,13 +21,12 @@ from QEfficient.utils import get_padding_shape_from_config, hf_download from QEfficient.utils._utils import create_json, load_hf_processor from QEfficient.utils.constants import Constants, QnnConstants +from tests.test_matrix import select_test_entries from ..check_model_results import dump_and_compare_results CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/audio_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - test_models = config_data["speech_seq2seq_models"] +test_models = select_test_entries(CONFIG_PATH, "speech_seq2seq_models") def load_seq2seq_model(model_config): diff --git a/tests/transformers/models/causal_lm_models/test_causal_lm_blocking_hqkv.py b/tests/transformers/models/causal_lm_models/test_causal_lm_blocking_hqkv.py index 0568939cd2..8423d35481 100644 --- a/tests/transformers/models/causal_lm_models/test_causal_lm_blocking_hqkv.py +++ b/tests/transformers/models/causal_lm_models/test_causal_lm_blocking_hqkv.py @@ -5,13 +5,13 @@ # # ----------------------------------------------------------------------------- -import json import os import pytest from transformers import AutoConfig from QEfficient.utils.test_utils import ModelConfig +from tests.test_matrix import model_names, select_test_entries from .check_causal_models import ( check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100, @@ -19,10 +19,8 @@ ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - blockedKV_models = config_data["blockedKV_causal_lm_models"] -test_models_blockedKV = [model["model_name"] for model in blockedKV_models] +blockedKV_models = select_test_entries(CONFIG_PATH, "blockedKV_causal_lm_models") +test_models_blockedKV = model_names(blockedKV_models) model_config_dict = {model["model_name"]: model for model in blockedKV_models} diff --git a/tests/transformers/models/causal_lm_models/test_causal_lm_models.py b/tests/transformers/models/causal_lm_models/test_causal_lm_models.py index 5011a670a6..bf1453c5f3 100644 --- a/tests/transformers/models/causal_lm_models/test_causal_lm_models.py +++ b/tests/transformers/models/causal_lm_models/test_causal_lm_models.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os import pytest @@ -14,6 +13,7 @@ from QEfficient.utils._utils import create_json from QEfficient.utils.constants import QnnConstants from QEfficient.utils.test_utils import ModelConfig +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from .check_causal_models import ( check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100, @@ -21,11 +21,9 @@ ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - causal_lm_models = config_data["causal_lm_models"] -test_models_causal = [model["model_name"] for model in causal_lm_models] -model_config_dict = {model["model_name"]: model for model in causal_lm_models} +causal_lm_models = select_test_entries(CONFIG_PATH, "causal_lm_models") +test_models_causal = model_names(causal_lm_models) +model_config_dict = entries_by_model_name(causal_lm_models) @pytest.mark.full_layers diff --git a/tests/transformers/models/causal_lm_models/test_causal_lm_pl1.py b/tests/transformers/models/causal_lm_models/test_causal_lm_pl1.py index f5f2384e67..cc0ac7242f 100644 --- a/tests/transformers/models/causal_lm_models/test_causal_lm_pl1.py +++ b/tests/transformers/models/causal_lm_models/test_causal_lm_pl1.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os import pytest @@ -13,17 +12,16 @@ from transformers import AutoConfig from QEfficient.utils.test_utils import ModelConfig +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from .check_causal_models import ( check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100, ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - causal_pl1_models = config_data["causal_lm_models_pl1"] -test_models_pl1 = [model["model_name"] for model in causal_pl1_models] -model_config_dict = {model["model_name"]: model for model in causal_pl1_models} +causal_pl1_models = select_test_entries(CONFIG_PATH, "causal_lm_models_pl1") +test_models_pl1 = model_names(causal_pl1_models) +model_config_dict = entries_by_model_name(causal_pl1_models) @pytest.mark.full_layers diff --git a/tests/transformers/models/causal_lm_models/test_causal_tlm_models.py b/tests/transformers/models/causal_lm_models/test_causal_tlm_models.py index 9d02acbd29..91962f8831 100644 --- a/tests/transformers/models/causal_lm_models/test_causal_tlm_models.py +++ b/tests/transformers/models/causal_lm_models/test_causal_tlm_models.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os import pytest @@ -13,6 +12,7 @@ from QEfficient.utils.constants import Constants from QEfficient.utils.test_utils import ModelConfig +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from .check_causal_models import ( check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100, @@ -20,11 +20,9 @@ ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - spd_models = config_data["spd_causal_lm_models"] -test_models_spd = [model["model_name"] for model in spd_models] -model_config_dict = {model["model_name"]: model for model in spd_models} +spd_models = select_test_entries(CONFIG_PATH, "spd_causal_lm_models") +test_models_spd = model_names(spd_models) +model_config_dict = entries_by_model_name(spd_models) @pytest.mark.full_layers diff --git a/tests/transformers/models/causal_lm_models/test_fp16_causal_lm.py b/tests/transformers/models/causal_lm_models/test_fp16_causal_lm.py index af8c3b70f0..b03762ae81 100644 --- a/tests/transformers/models/causal_lm_models/test_fp16_causal_lm.py +++ b/tests/transformers/models/causal_lm_models/test_fp16_causal_lm.py @@ -6,7 +6,6 @@ # ----------------------------------------------------------------------------- import copy -import json import os from typing import Optional @@ -20,17 +19,16 @@ from QEfficient.utils.constants import Constants from QEfficient.utils.run_utils import ApiRunner from QEfficient.utils.test_utils import ModelConfig, load_hf_causal_lm_model +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from .check_causal_models import ( get_custom_n_layers, ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - causal_lm_fp16_test_models = config_data["causal_lm_fp16_test_models"] -test_models = [model["model_name"] for model in causal_lm_fp16_test_models] -model_config_dict = {model["model_name"]: model for model in causal_lm_fp16_test_models} +causal_lm_fp16_test_models = select_test_entries(CONFIG_PATH, "causal_lm_fp16_test_models") +test_models = model_names(causal_lm_fp16_test_models) +model_config_dict = entries_by_model_name(causal_lm_fp16_test_models) def check_causal_lm_pytorch_vs_kv_vs_ai100( diff --git a/tests/transformers/models/embedding_models/test_embedding_models.py b/tests/transformers/models/embedding_models/test_embedding_models.py index bac1cb2193..b32c7fcdc7 100644 --- a/tests/transformers/models/embedding_models/test_embedding_models.py +++ b/tests/transformers/models/embedding_models/test_embedding_models.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause # # ----------------------------------------------------------------------------- -import json import os from typing import Optional @@ -19,13 +18,12 @@ from QEfficient.utils._utils import create_json from QEfficient.utils.constants import Constants, QnnConstants from QEfficient.utils.test_utils import ModelConfig +from tests.test_matrix import select_test_entries from ..check_model_results import dump_and_compare_results CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/embedding_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - embed_test_models = config_data["embedding_models"] +embed_test_models = select_test_entries(CONFIG_PATH, "embedding_models") def load_embedding_model(model_name: str, n_layer: int = -1): diff --git a/tests/transformers/models/embedding_models/test_qwen3vl_embedding_mad.py b/tests/transformers/models/embedding_models/test_qwen3vl_embedding_mad.py index d540593b86..f35eece063 100644 --- a/tests/transformers/models/embedding_models/test_qwen3vl_embedding_mad.py +++ b/tests/transformers/models/embedding_models/test_qwen3vl_embedding_mad.py @@ -5,7 +5,6 @@ # # ---------------------------------------------------------------------------- -import json from typing import Any, Dict, List import pytest @@ -23,15 +22,14 @@ resolve_model_source, ) from QEfficient.utils.test_utils import load_vlm_model +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = "tests/configs/image_text_model_configs.json" -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - embedding_models = config_data["image_text_embedding_models"] +embedding_models = select_test_entries(CONFIG_PATH, "image_text_embedding_models") -test_embedding_models = [model_config["model_name"] for model_config in embedding_models] -embedding_model_config_dict = {model["model_name"]: model for model in embedding_models} +test_embedding_models = model_names(embedding_models) +embedding_model_config_dict = entries_by_model_name(embedding_models) def _compute_cpu_embeddings(model_hf, embedder, model_inputs: List[Dict[str, Any]]) -> torch.Tensor: diff --git a/tests/transformers/models/image_text_to_text/test_continuous_batching.py b/tests/transformers/models/image_text_to_text/test_continuous_batching.py index b466b74eab..ce9436016d 100644 --- a/tests/transformers/models/image_text_to_text/test_continuous_batching.py +++ b/tests/transformers/models/image_text_to_text/test_continuous_batching.py @@ -6,7 +6,6 @@ # ---------------------------------------------------------------------------- import copy -import json import os from io import BytesIO from typing import Optional @@ -31,13 +30,12 @@ load_vlm_model_from_config, set_num_layers_vlm, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/image_text_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - multimodal_models = config_data["image_text_models"] -test_mm_models = [model_config["model_name"] for model_config in multimodal_models] -model_config_dict = {model["model_name"]: model for model in multimodal_models} +multimodal_models = select_test_entries(CONFIG_PATH, "image_text_models") +test_mm_models = model_names(multimodal_models) +model_config_dict = entries_by_model_name(multimodal_models) NEW_GENERATION_TOKENS = 10 diff --git a/tests/transformers/models/image_text_to_text/test_custom_dtype.py b/tests/transformers/models/image_text_to_text/test_custom_dtype.py index f291c5d12c..92ebc6943f 100644 --- a/tests/transformers/models/image_text_to_text/test_custom_dtype.py +++ b/tests/transformers/models/image_text_to_text/test_custom_dtype.py @@ -5,7 +5,6 @@ # # ---------------------------------------------------------------------------- -import json import os import pytest @@ -14,17 +13,16 @@ from QEfficient.utils.test_utils import ( ModelConfig, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from .test_image_text_to_text_models import ( check_image_text_to_text_pytorch_vs_kv_vs_ort_vs_ai100, ) CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/image_text_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - multimodal_models = config_data["image_text_custom_dtype_models"] -test_custom_dtype_support_models = [model_config["model_name"] for model_config in multimodal_models] -model_config_dict = {model["model_name"]: model for model in multimodal_models} +multimodal_models = select_test_entries(CONFIG_PATH, "image_text_custom_dtype_models") +test_custom_dtype_support_models = model_names(multimodal_models) +model_config_dict = entries_by_model_name(multimodal_models) NEW_GENERATION_TOKENS = 10 diff --git a/tests/transformers/models/image_text_to_text/test_image_text_to_text_models.py b/tests/transformers/models/image_text_to_text/test_image_text_to_text_models.py index 9b9e662e52..6ebe1a8028 100644 --- a/tests/transformers/models/image_text_to_text/test_image_text_to_text_models.py +++ b/tests/transformers/models/image_text_to_text/test_image_text_to_text_models.py @@ -6,7 +6,6 @@ # ---------------------------------------------------------------------------- import copy -import json import os from io import BytesIO from typing import List, Optional @@ -34,15 +33,14 @@ load_vlm_model_from_config, set_num_layers_vlm, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries from ..check_model_results import dump_and_compare_results CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/image_text_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - multimodal_models = config_data["image_text_models"] -test_mm_models = [model_config["model_name"] for model_config in multimodal_models] -model_config_dict = {model["model_name"]: model for model in multimodal_models} +multimodal_models = select_test_entries(CONFIG_PATH, "image_text_models") +test_mm_models = model_names(multimodal_models) +model_config_dict = entries_by_model_name(multimodal_models) NEW_GENERATION_TOKENS = 10 diff --git a/tests/transformers/models/reranker/test_reranker_mad.py b/tests/transformers/models/reranker/test_reranker_mad.py index 148935c5a7..626649cc53 100644 --- a/tests/transformers/models/reranker/test_reranker_mad.py +++ b/tests/transformers/models/reranker/test_reranker_mad.py @@ -5,7 +5,6 @@ # # ---------------------------------------------------------------------------- -import json import os from typing import Dict, List, Tuple @@ -38,6 +37,7 @@ truncate_tokens_optimized as _shared_truncate_tokens_optimized, ) from QEfficient.utils.test_utils import load_vlm_model, set_num_layers_vlm +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/image_text_model_configs.json") @@ -48,7 +48,7 @@ IMAGE_BASE_FACTOR = 16 IMAGE_FACTOR = IMAGE_BASE_FACTOR * 2 MIN_PIXELS = 4 * IMAGE_FACTOR * IMAGE_FACTOR -MAX_PIXELS = 1280 * IMAGE_FACTOR * IMAGE_FACTOR +MAX_PIXELS = 64 * IMAGE_FACTOR * IMAGE_FACTOR EXAMPLE_INPUTS = { "instruction": "Retrieve relevant content.", @@ -59,12 +59,10 @@ ], } -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - reranker_models = config_data["image_text_reranker_models"] +reranker_models = select_test_entries(CONFIG_PATH, "image_text_reranker_models") -test_reranker_models = [model_config["model_name"] for model_config in reranker_models] -reranker_model_config_dict = {model["model_name"]: model for model in reranker_models} +test_reranker_models = model_names(reranker_models) +reranker_model_config_dict = entries_by_model_name(reranker_models) def _resolve_model_source(model_name_or_path: str) -> str: diff --git a/tests/transformers/models/sequence_models/test_seq_classification.py b/tests/transformers/models/sequence_models/test_seq_classification.py index 0d76067c52..19ad678d70 100644 --- a/tests/transformers/models/sequence_models/test_seq_classification.py +++ b/tests/transformers/models/sequence_models/test_seq_classification.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import List, Optional, Union @@ -15,13 +14,12 @@ from transformers import AutoModelForSequenceClassification, AutoTokenizer from QEfficient.transformers.models.modeling_auto import QEFFAutoModelForSequenceClassification +from tests.test_matrix import select_test_entries from ..check_model_results import dump_and_compare_results CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../../configs/sequence_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - test_models = config_data["seq_classification_models"] +test_models = select_test_entries(CONFIG_PATH, "seq_classification_models") def check_seq_classification_pytorch_vs_ai100( diff --git a/tests/transformers/sampler/test_greedy_sampler.py b/tests/transformers/sampler/test_greedy_sampler.py index 8e0ffe1cca..35c5ed84da 100644 --- a/tests/transformers/sampler/test_greedy_sampler.py +++ b/tests/transformers/sampler/test_greedy_sampler.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import Optional @@ -20,13 +19,12 @@ load_hf_vlm_model, load_qeff_model_with_sampler, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - sampler_models = config_data["sampler_config"] -test_models = [model["model_name"] for model in sampler_models] -model_config_dict = {model["model_name"]: model for model in sampler_models} +sampler_models = select_test_entries(CONFIG_PATH, "sampler_config") +test_models = model_names(sampler_models) +model_config_dict = entries_by_model_name(sampler_models) def check_greedy_sampler( diff --git a/tests/transformers/sampler/test_guided_sampler.py b/tests/transformers/sampler/test_guided_sampler.py index 0b1c30f0cb..46a4b07425 100644 --- a/tests/transformers/sampler/test_guided_sampler.py +++ b/tests/transformers/sampler/test_guided_sampler.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import Optional @@ -20,13 +19,12 @@ load_hf_vlm_model, load_qeff_model_with_sampler, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - sampler_models = config_data["sampler_config"] -test_models = [model["model_name"] for model in sampler_models] -model_config_dict = {model["model_name"]: model for model in sampler_models} +sampler_models = select_test_entries(CONFIG_PATH, "sampler_config") +test_models = model_names(sampler_models) +model_config_dict = entries_by_model_name(sampler_models) def check_guided_decoding_sampler( diff --git a/tests/transformers/sampler/test_random_sampler.py b/tests/transformers/sampler/test_random_sampler.py index 336ef67577..682d3b0f90 100644 --- a/tests/transformers/sampler/test_random_sampler.py +++ b/tests/transformers/sampler/test_random_sampler.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import Optional @@ -20,13 +19,12 @@ load_hf_vlm_model, load_qeff_model_with_sampler, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - sampler_models = config_data["sampler_config"] -test_models = [model["model_name"] for model in sampler_models] -model_config_dict = {model["model_name"]: model for model in sampler_models} +sampler_models = select_test_entries(CONFIG_PATH, "sampler_config") +test_models = model_names(sampler_models) +model_config_dict = entries_by_model_name(sampler_models) def check_random_sampler( diff --git a/tests/transformers/sampler/test_sampler_transform.py b/tests/transformers/sampler/test_sampler_transform.py index 2010c1cba1..56740c3395 100644 --- a/tests/transformers/sampler/test_sampler_transform.py +++ b/tests/transformers/sampler/test_sampler_transform.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from typing import Optional @@ -18,13 +17,12 @@ from QEfficient.utils.test_utils import ( load_qeff_model_with_sampler, ) +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - sampler_models = config_data["sampler_config"] -test_models = [model["model_name"] for model in sampler_models] -model_config_dict = {model["model_name"]: model for model in sampler_models} +sampler_models = select_test_entries(CONFIG_PATH, "sampler_config") +test_models = model_names(sampler_models) +model_config_dict = entries_by_model_name(sampler_models) def check_sampler_transform( diff --git a/tests/transformers/spd/test_pld_inference.py b/tests/transformers/spd/test_pld_inference.py index ace7170d00..1f8626d0de 100644 --- a/tests/transformers/spd/test_pld_inference.py +++ b/tests/transformers/spd/test_pld_inference.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from dataclasses import dataclass from time import perf_counter @@ -19,14 +18,13 @@ from QEfficient.generation.cloud_infer import QAICInferenceSession from QEfficient.utils.constants import Constants from QEfficient.utils.test_utils import load_qeff_causal_lm_model +from tests.test_matrix import entries_by_id, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - spd_models = config_data["spd_config"] +spd_models = select_test_entries(CONFIG_PATH, "spd_config") test_models_id = [model["id"] for model in spd_models[:1]] -model_config_dict = {model["id"]: model for model in spd_models} +model_config_dict = entries_by_id(spd_models) @dataclass diff --git a/tests/transformers/spd/test_spd_inference.py b/tests/transformers/spd/test_spd_inference.py index feb0153e3c..38390ef0b2 100644 --- a/tests/transformers/spd/test_spd_inference.py +++ b/tests/transformers/spd/test_spd_inference.py @@ -5,7 +5,6 @@ # # ----------------------------------------------------------------------------- -import json import os from time import perf_counter from typing import List, Optional @@ -18,14 +17,13 @@ from QEfficient.generation.cloud_infer import QAICInferenceSession from QEfficient.utils.constants import Constants from QEfficient.utils.test_utils import load_qeff_causal_lm_model +from tests.test_matrix import entries_by_id, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/feature_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - spd_models = config_data["spd_config"] +spd_models = select_test_entries(CONFIG_PATH, "spd_config") test_models_id = [model["id"] for model in spd_models] -model_config_dict = {model["id"]: model for model in spd_models} +model_config_dict = entries_by_id(spd_models) def run_prefill_on_draft_and_target( diff --git a/tests/transformers/subfunction/test_causal_lm_blocking_subfunction.py b/tests/transformers/subfunction/test_causal_lm_blocking_subfunction.py index cef02bc4d0..ff7b1b60c5 100644 --- a/tests/transformers/subfunction/test_causal_lm_blocking_subfunction.py +++ b/tests/transformers/subfunction/test_causal_lm_blocking_subfunction.py @@ -6,7 +6,6 @@ # ---------------------------------------------------------------------------- -import json import os from typing import Optional @@ -17,13 +16,12 @@ from QEfficient.transformers.models.modeling_auto import QEFFAutoModelForCausalLM from QEfficient.utils.test_utils import ModelConfig, get_custom_n_layers, load_hf_causal_lm_model +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/causal_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - blockedKV_models = config_data["blockedKV_causal_lm_models"] -test_models_blockedKV = [model["model_name"] for model in blockedKV_models] -model_config_dict = {model["model_name"]: model for model in blockedKV_models} +blockedKV_models = select_test_entries(CONFIG_PATH, "blockedKV_causal_lm_models") +test_models_blockedKV = model_names(blockedKV_models) +model_config_dict = entries_by_model_name(blockedKV_models) torch.manual_seed(42) diff --git a/tests/transformers/subfunction/test_subfunction_vlm.py b/tests/transformers/subfunction/test_subfunction_vlm.py index 39e2c6d0ac..f0c9272150 100644 --- a/tests/transformers/subfunction/test_subfunction_vlm.py +++ b/tests/transformers/subfunction/test_subfunction_vlm.py @@ -5,7 +5,6 @@ # # ---------------------------------------------------------------------------- -import json import os from typing import Optional @@ -20,18 +19,17 @@ ) from QEfficient.utils.test_utils import load_qeff_vlm_model +from tests.test_matrix import entries_by_model_name, model_names, select_test_entries NEW_GENERATION_TOKENS = 10 CONFIG_PATH = os.path.join(os.path.dirname(__file__), "../../configs/image_text_model_configs.json") -with open(CONFIG_PATH, "r") as f: - config_data = json.load(f) - multimodal_models = config_data["image_text_subfunction_models"] +multimodal_models = select_test_entries(CONFIG_PATH, "image_text_subfunction_models") -test_mm_models = [model_config["model_name"] for model_config in multimodal_models] -model_config_dict = {model["model_name"]: model for model in multimodal_models} +test_mm_models = model_names(multimodal_models) +model_config_dict = entries_by_model_name(multimodal_models) def has_QwenLayer_function(onnx_path): diff --git a/tests/unit_test/models/test_model_quickcheck.py b/tests/unit_test/models/test_model_quickcheck.py index 5e61fbf79e..5c07d5ae70 100644 --- a/tests/unit_test/models/test_model_quickcheck.py +++ b/tests/unit_test/models/test_model_quickcheck.py @@ -54,6 +54,7 @@ from QEfficient.transformers.quantizers.auto import replace_transformers_quantizers from QEfficient.utils._utils import _infer_specialization_name, to_named_specializations from QEfficient.utils.run_utils import ApiRunner +from tests.test_matrix import EXHAUSTIVE_SCOPE, PR_SCOPE, get_coverage_key, load_test_config, select_test_entries ort.set_default_logger_severity(3) logging.getLogger("QEfficient").setLevel(logging.ERROR) @@ -152,6 +153,26 @@ MODEL_KWARGS = {"attn_implementation": "eager"} PREFIX_CACHING_MODEL_ID = "hf-internal-testing/tiny-random-GPT2LMHeadModel" +MATRIX_CONFIG_SUITES = { + "tests/configs/causal_model_configs.json": ( + "causal_lm_models", + "causal_lm_fp16_test_models", + "spd_causal_lm_models", + "prefix_caching_models", + "blockedKV_causal_lm_models", + "disaggregated_dummy_models", + "causal_lm_models_pl1", + ), + "tests/configs/image_text_model_configs.json": ( + "image_text_models", + "image_text_subfunction_models", + "image_text_custom_dtype_models", + "image_text_reranker_models", + "image_text_embedding_models", + ), + "tests/configs/feature_configs.json": ("sampler_config", "spd_config"), + "tests/configs/embedding_model_configs.json": ("embedding_models",), +} def _per_test_thread_budget() -> int: @@ -171,6 +192,24 @@ def _configure_torch_threads() -> None: torch.set_num_interop_threads(max(1, min(4, threads))) +def test_model_matrix_pr_scope_preserves_coverage_keys(): + for config_path, suite_names in MATRIX_CONFIG_SUITES.items(): + raw_config = load_test_config(config_path) + for suite_name in suite_names: + exhaustive_entries = raw_config[suite_name] + expected_keys = {get_coverage_key(entry) for entry in exhaustive_entries} + pr_keys = {get_coverage_key(entry) for entry in select_test_entries(config_path, suite_name, PR_SCOPE)} + assert pr_keys == expected_keys, f"{suite_name} PR matrix does not preserve coverage keys" + + +def test_model_matrix_exhaustive_scope_preserves_full_config_lists(): + for config_path, suite_names in MATRIX_CONFIG_SUITES.items(): + raw_config = load_test_config(config_path) + for suite_name in suite_names: + exhaustive_entries = select_test_entries(config_path, suite_name, EXHAUSTIVE_SCOPE) + assert exhaustive_entries == raw_config[suite_name] + + def _ort_session(onnx_path: Path) -> ort.InferenceSession: options = ort.SessionOptions() threads = _per_test_thread_budget()