Manedge qwen3 8 b qnn recipe

.aitk/configs/checks.json

@@ -1,19 +1,19 @@
 {
-    "configCheck": 174,
-    "copyCheck": 182,
+    "configCheck": 175,
+    "copyCheck": 184,
     "executePatchPyCheck": 0,
-    "executeRuntimeCheck": 106,
+    "executeRuntimeCheck": 107,
     "extensionCheck": 2,
-    "gitignoreCheck": 44,
+    "gitignoreCheck": 45,
     "inferenceModelCheck": 25,
-    "ipynbCheck": 52,
-    "licenseCheck": 41,
-    "modelProjectCheck": 46,
-    "oliveCheck": 84,
-    "oliveJsonCheck": 174,
-    "pathCheck": 1467,
-    "requirementsCheck": 44,
+    "ipynbCheck": 53,
+    "licenseCheck": 42,
+    "modelProjectCheck": 47,
+    "oliveCheck": 85,
+    "oliveJsonCheck": 175,
+    "pathCheck": 1480,
+    "requirementsCheck": 45,
     "templateCheck": 3,
-    "venvRequirementsCheck": 23,
-    "winmlCopyCheck": 38
+    "venvRequirementsCheck": 25,
+    "winmlCopyCheck": 39
 }

.aitk/configs/model_list.json

@@ -485,7 +485,7 @@
             "relativePath": "sam-vit-base/aitk",
             "version": 2,
             "pipeline_tags": [
-                "fill-mask"
+                "mask-generation"
             ]
         },
         {
@@ -504,7 +504,7 @@
             "relativePath": "sam2.1-hiera-small/aitk",
             "version": 3,
             "pipeline_tags": [
-                "fill-mask"
+                "mask-generation"
             ]
         },
         {
@@ -840,6 +840,22 @@
                 "text-generation"
             ]
         },
+        {
+            "displayName": "Qwen/Qwen3-0.6B",
+            "icon": "qwen",
+            "modelLink": "https://huggingface.co/Qwen/Qwen3-0.6B",
+            "id": "huggingface/Qwen/Qwen3-0.6B",
+            "runtimes": [
+                "QNN"
+            ],
+            "architecture": "Transformer",
+            "status": "Hide",
+            "relativePath": "Qwen-Qwen3-0.6B/aitk",
+            "version": 1,
+            "pipeline_tags": [
+                "text-generation"
+            ]
+        },
         {
             "displayName": "sd2-community/stable-diffusion-2-1",
             "icon": "HuggingFace",

.aitk/requirements/requirements-NvidiaGPU-Qwen3.txt

@@ -0,0 +1,3 @@
+olive-ai==0.13.0
+onnxruntime-gpu==1.27.0
+transformers==4.57.3

.aitk/requirements/requirements-QNN-Qwen3.txt

@@ -0,0 +1,4 @@
+# uvpip:uninstall onnxruntime-qnn;pre
+# uvpip:install onnxruntime-qnn==2.1.0 --no-deps;post
+olive-ai==0.13.0
+onnxruntime==1.24.4

.aitk/requirements/requirements-QNN.txt

@@ -38,6 +38,8 @@ numpy==2.2.4
 git+https://github.com/microsoft/Olive.git@b9c975f5651a4446283d9183348add9dcd4ed45f#egg=olive_ai
 onnx==1.17.0
 onnx-ir==0.1.10
+# Uninstall onnxruntime from Qwen3 feature
+# uvpip:uninstall onnxruntime onnxruntime-qnn;pre
 # uvpip:install onnxruntime-qnn==1.23.2 --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple --no-deps;post
 onnxscript==0.5.3
 optuna==4.2.1

.aitk/scripts/project_processor.py

@@ -223,7 +223,12 @@ def project_processor():
         # model info
         modelInfo = convert_yaml_to_model_info(root_dir, yml_file, yaml_object)
         if GlobalVars.fillPipelineTags:
-            modelInfo.pipeline_tags = fetch_pipeline_tags(modelInfo.modelLink)
+            fetched_tags = fetch_pipeline_tags(modelInfo.modelLink)
+            if fetched_tags is None:
+                modelInfo.pipeline_tags = existing_pipeline_tags.get(modelInfo.id)
+                print(f"Warning: Could not fetch pipeline tags for {modelInfo.id}, using existing")
+            else:
+                modelInfo.pipeline_tags = fetched_tags
         else:
             modelInfo.pipeline_tags = existing_pipeline_tags.get(modelInfo.id)
         if modelInfo.id.lower() in all_ids:

.aitk/scripts/sanitize/generator_amd.py

@@ -3,7 +3,7 @@
 from typing import Optional
 
 from .constants import EPNames, OlivePassNames, OlivePropertyNames, PhaseTypeEnum
-from .generator_common import create_model_parameter, set_optimization_path
+from .generator_common import apply_runtime_feature_overrides, create_model_parameter, set_optimization_path
 from .model_info import ModelList
 from .model_parameter import ModelParameter, OptimizationPath, Section
 from .parameters import Parameter
@@ -250,5 +250,7 @@ def generator_amd(id: str, recipe, folder: Path, modelList: ModelList):
     if quantize:
         parameter.sections.append(quantize)
 
+    apply_runtime_feature_overrides(aitk, parameter)
+
     parameter.writeIfChanged()
     print(f"\tGenerated AMD configuration for {file}")

.aitk/scripts/sanitize/generator_common.py

@@ -40,6 +40,17 @@ def create_model_parameter(aitk, name: str, configFile: Path):
     return parameter
 
 
+def apply_runtime_feature_overrides(aitk: dict, parameter: ModelParameter):
+    """Apply explicit runtime feature overrides from info.yml aitk block.
+
+    When set in info.yml, these replace any auto-generated values.
+    """
+    for field in ("executeRuntimeFeatures", "evaluationRuntimeFeatures", "pyEnvRuntimeFeatures"):
+        value = aitk.get(field)
+        if value is not None:
+            setattr(parameter, field, value)
+
+
 def add_optimization_wa(optimizationPaths: list[OptimizationPath], k: str, v: dict) -> bool:
     if OlivePropertyNames.Precision in v:
         optimizationPaths.append(

.aitk/scripts/sanitize/generator_dml.py

@@ -3,7 +3,7 @@
 from typing import Optional
 
 from .constants import OlivePassNames, OlivePropertyNames, ParameterTypeEnum, PhaseTypeEnum
-from .generator_common import create_model_parameter, set_optimization_path
+from .generator_common import apply_runtime_feature_overrides, create_model_parameter, set_optimization_path
 from .model_info import ModelList
 from .model_parameter import ModelParameter, OptimizationPath, Section
 from .parameters import Parameter
@@ -78,5 +78,7 @@ def generator_dml(id: str, recipe, folder: Path, modelList: ModelList):
     if quantize:
         parameter.sections.append(quantize)
 
+    apply_runtime_feature_overrides(aitk, parameter)
+
     parameter.writeIfChanged()
     print(f"\tGenerated DML configuration for {file}")

.aitk/scripts/sanitize/generator_intel.py

@@ -3,7 +3,7 @@
 from typing import Optional
 
 from .constants import OliveDeviceTypes, OlivePassNames, OlivePropertyNames, PhaseTypeEnum
-from .generator_common import create_model_parameter, set_optimization_path
+from .generator_common import apply_runtime_feature_overrides, create_model_parameter, set_optimization_path
 from .model_parameter import ModelParameter, OptimizationPath, Section
 from .parameters import Parameter
 from .utils import isLLM_by_id, open_ex
@@ -89,5 +89,7 @@ def generator_intel(id: str, recipe, folder: Path):
     if quantize:
         parameter.sections.append(quantize)
 
+    apply_runtime_feature_overrides(aitk, parameter)
+
     parameter.writeIfChanged()
     print(f"\tGenerated Intel configuration for {file}")

.aitk/scripts/sanitize/generator_qnn.py

@@ -3,7 +3,7 @@
 
 from .constants import OlivePassNames, OlivePropertyNames
 from .generator_amd import generate_quantization_config
-from .generator_common import create_model_parameter, set_optimization_path
+from .generator_common import apply_runtime_feature_overrides, create_model_parameter, set_optimization_path
 from .model_info import ModelList
 from .model_parameter import ModelParameter
 from .utils import isLLM_by_id, open_ex
@@ -53,6 +53,7 @@ def generator_qnn(id: str, recipe, folder: Path, modelList: ModelList):
         parameter.sections.append(quantize)
 
     setup_features(content, parameter)
+    apply_runtime_feature_overrides(aitk, parameter)
 
     parameter.writeIfChanged()
     print(f"\tGenerated QNN configuration for {file}")

.aitk/scripts/sanitize/generator_trtrtx.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 from .constants import OlivePassNames, OlivePropertyNames
-from .generator_common import create_model_parameter, set_optimization_path
+from .generator_common import apply_runtime_feature_overrides, create_model_parameter, set_optimization_path
 from .generator_dml import generate_quantization_config
 from .model_info import ModelList
 from .model_parameter import ModelParameter
@@ -41,6 +41,7 @@ def generator_trtrtx(id: str, recipe, folder: Path, modelList: ModelList):
     parameter.isLLM = isLLM
 
     generate_additional_config(configFile, parameter)
+    apply_runtime_feature_overrides(aitk, parameter)
 
     quantize = generate_quantization_config(configFile, parameter)
     if quantize:

Qwen-Qwen3-0.6B/aitk/.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+/cache
+/history/*/*
+!/history/*/history.config
+!/history/*/olive_config.json

Qwen-Qwen3-0.6B/aitk/README.md

@@ -0,0 +1,17 @@
+# Qwen3-0.6B Model Optimization
+
+This repository demonstrates the optimization of the [Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) model. The optimization process is divided into these workflows:
+
+- PTQ + AOT for QNN NPU
+
+## PTQ + AOT for QNN NPU
+
+This workflow contains two steps:
+
+### Post-training Quantization (PTQ)
+
+This step uses GPTQModel library, MatMul-NBits-QDQ and Static Quantization etc. They are resource-intensive and require GPU acceleration.
+
+### Ahead of Time (AOT) Compilation
+
+This step compiles model using QNN Execution Provider in a separate Python environment with onnxruntime-qnn installed. Note that after compilation, the model must run on an EP with same or higher version of QAIRT SDK as the package (https://github.com/onnxruntime/onnxruntime-qnn/releases#release-v2.1.0).

Qwen-Qwen3-0.6B/aitk/_copy.json.config

@@ -0,0 +1,18 @@
+{
+    "copies": [
+        {
+            "src": "../../deepseek-ai-DeepSeek-R1-Distill-Qwen-1.5B/aitk/inference_sample.ipynb",
+            "dst": "inference_sample.ipynb",
+            "replacements": [
+                {
+                    "find": "<｜User｜>{input}<｜Assistant｜><think>",
+                    "replace": "<|im_start|>user\\\\n{input}<|im_end|>\\\\n<|im_start|>assistant\\\\n"
+                }
+            ]
+        },
+        {
+            "src": "../../deepseek-ai-DeepSeek-R1-Distill-Qwen-1.5B/aitk/winml.py",
+            "dst": "winml.py"
+        }
+    ]
+}

Qwen-Qwen3-0.6B/aitk/inference_sample.ipynb

@@ -0,0 +1,115 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = 'Who is Isaac Newton?'\n",
+    "ExecutionProvider=\"QNNExecutionProvider\"\n",
+    "model_folder = \"./model\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from winml import register_execution_providers\n",
+    "register_execution_providers()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import onnxruntime_genai as og\n",
+    "import time\n",
+    "\n",
+    "# Load the base model and tokenizer\n",
+    "model = og.Model(model_folder)\n",
+    "tokenizer = og.Tokenizer(model)\n",
+    "tokenizer_stream = tokenizer.create_stream()\n",
+    "\n",
+    "# Set the max length to something sensible by default,\n",
+    "# since otherwise it will be set to the entire context length\n",
+    "search_options = {}\n",
+    "search_options[\"max_length\"] = 200\n",
+    "\n",
+    "chat_template = \"<|im_start|>user\\n{input}<|im_end|>\\n<|im_start|>assistant\\n\"\n",
+    "\n",
+    "# Generate prompt (prompt template + input)\n",
+    "prompt = f\"{chat_template.format(input=text)}\"\n",
+    "\n",
+    "# Encode the prompt using the tokenizer\n",
+    "input_tokens = tokenizer.encode(prompt)\n",
+    "\n",
+    "# Create params and generator\n",
+    "params = og.GeneratorParams(model)\n",
+    "params.set_search_options(**search_options)\n",
+    "generator = og.Generator(model, params)\n",
+    "\n",
+    "# Append input tokens to the generator\n",
+    "generator.append_tokens(input_tokens)\n",
+    "\n",
+    "print(\"\")\n",
+    "print(\"Output: \", end=\"\", flush=True)\n",
+    "\n",
+    "token_times = []\n",
+    "\n",
+    "# Stream the output\n",
+    "while True:\n",
+    "    start_time = time.time()\n",
+    "    if generator.is_done():\n",
+    "        break\n",
+    "    generator.generate_next_token()\n",
+    "    new_token = generator.get_next_tokens()[0]\n",
+    "    end_time = time.time()\n",
+    "    \n",
+    "    # Record the time for this token generation\n",
+    "    token_time = end_time - start_time\n",
+    "    token_times.append(token_time)\n",
+    "\n",
+    "    print(tokenizer_stream.decode(new_token), end=\"\", flush=True)\n",
+    "\n",
+    "print()\n",
+    "\n",
+    "# Calculate and display timing statistics\n",
+    "if token_times:\n",
+    "    total_tokens = len(token_times)\n",
+    "    avg_time = sum(token_times) / total_tokens\n",
+    "    \n",
+    "    print(f\"Total tokens generated: {total_tokens}\")\n",
+    "    print(f\"Average time per token: {avg_time:.4f} seconds\")\n",
+    "    print(f\"Tokens per second: {total_tokens / sum(token_times):.2f}\")\n",
+    "\n",
+    "del generator\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Qwen-Qwen3-0.6B/aitk/info.yml

@@ -0,0 +1,20 @@
+keywords:
+    aitk
+arch: qwen3
+recipes:
+    - file: "qwen3_qnn_config.json"
+      device: npu
+      ep: QNNExecutionProvider
+      aitk:
+        oliveFile: "QNN/config.json"
+        isGPURequired: true
+        executeRuntimeFeatures:
+          - GptqModel
+          - Qwen3
+        pyEnvRuntimeFeatures:
+          - Qwen3
+aitk:
+    modelInfo:
+        id: "huggingface/Qwen/Qwen3-0.6B"
+        version: 1
+        status: Hide