Add reproducible COCO accuracy gates for Vision v8 checkpoints

.github/workflows/vision-v8-coco-accuracy.yml

@@ -0,0 +1,181 @@
+name: Vision v8 COCO accuracy
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/vision-v8-coco-accuracy.yml"
+      - "configs/vision_v8_coco_accuracy_gate.json"
+      - "docs/vision-v8-coco-accuracy-gates.md"
+      - "docs/index.md"
+      - "scripts/check_vision_v8_coco_report.py"
+      - "scripts/evaluate_tr_hash_coco.py"
+      - "scripts/evaluate_onnx_coco.py"
+      - "scripts/merge_vision_v8_coco_reports.py"
+      - "complexity/deploy/onnx_detector/**"
+      - "tests/test_coco_release_evaluation.py"
+      - "tests/test_onnx_detector_core.py"
+      - "tests/test_vision_v8_coco_accuracy_gate.py"
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/vision-v8-coco-accuracy.yml"
+      - "configs/vision_v8_coco_accuracy_gate.json"
+      - "docs/vision-v8-coco-accuracy-gates.md"
+      - "docs/index.md"
+      - "scripts/check_vision_v8_coco_report.py"
+      - "scripts/evaluate_tr_hash_coco.py"
+      - "scripts/evaluate_onnx_coco.py"
+      - "scripts/merge_vision_v8_coco_reports.py"
+      - "complexity/deploy/onnx_detector/**"
+      - "tests/test_coco_release_evaluation.py"
+      - "tests/test_onnx_detector_core.py"
+      - "tests/test_vision_v8_coco_accuracy_gate.py"
+  workflow_dispatch:
+    inputs:
+      checkpoint:
+        description: "Local path to the Vision v8 checkpoint on the runner"
+        required: false
+      onnx_o2m_model:
+        description: "Local path to the O2M Vision v8 ONNX model on the runner"
+        required: false
+      onnx_o2m_metadata:
+        description: "Local path to the O2M Vision v8 ONNX metadata sidecar on the runner"
+        required: false
+      onnx_nms_free_model:
+        description: "Local path to the NMS-free Vision v8 ONNX model on the runner"
+        required: false
+      onnx_nms_free_metadata:
+        description: "Local path to the NMS-free Vision v8 ONNX metadata sidecar on the runner"
+        required: false
+      annotations:
+        description: "Local path to instances_val2017.json on the runner"
+        required: true
+      images:
+        description: "Local path to COCO val2017 images on the runner"
+        required: true
+      runner:
+        description: "Runner label with dataset/checkpoint access"
+        required: true
+        default: "self-hosted"
+      backend:
+        description: "Evaluator backend"
+        required: true
+        default: "pytorch"
+        type: choice
+        options:
+          - pytorch
+          - onnx
+      provider:
+        description: "ONNX Runtime provider alias for backend=onnx"
+        required: true
+        default: "cpu"
+      release_tag:
+        description: "Optional existing GitHub Release tag to receive gated report assets"
+        required: false
+      device:
+        description: "PyTorch device"
+        required: true
+        default: "cuda"
+      precision:
+        description: "Evaluation precision"
+        required: true
+        default: "bf16"
+        type: choice
+        options:
+          - bf16
+          - fp32
+
+permissions:
+  contents: read
+
+jobs:
+  fixture-gate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install CPU test environment
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+          python -m pip install -e ".[dev,detection]"
+      - name: Lint COCO accuracy gate
+        run: >-
+          ruff check
+          scripts/check_vision_v8_coco_report.py
+          scripts/evaluate_onnx_coco.py
+          scripts/evaluate_tr_hash_coco.py
+          scripts/merge_vision_v8_coco_reports.py
+          tests/test_coco_release_evaluation.py
+          tests/test_onnx_detector_core.py
+          tests/test_vision_v8_coco_accuracy_gate.py
+      - name: Test report validation and release metadata helpers
+        run: >-
+          pytest -q
+          tests/test_coco_release_evaluation.py
+          tests/test_onnx_detector_core.py
+          tests/test_vision_v8_coco_accuracy_gate.py
+
+  full-coco-eval:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 360
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install evaluation environment
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[detection,export]"
+      - name: Run deterministic COCO evaluation twice
+        run: |
+          if [ "${{ inputs.backend }}" = "pytorch" ]; then
+            test -n "${{ inputs.checkpoint }}"
+            python scripts/evaluate_tr_hash_coco.py "${{ inputs.checkpoint }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_a --branch both --device "${{ inputs.device }}" --precision "${{ inputs.precision }}" --seed 0
+            python scripts/evaluate_tr_hash_coco.py "${{ inputs.checkpoint }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_b --branch both --device "${{ inputs.device }}" --precision "${{ inputs.precision }}" --seed 0
+          else
+            test -n "${{ inputs.onnx_o2m_model }}"
+            test -n "${{ inputs.onnx_o2m_metadata }}"
+            test -n "${{ inputs.onnx_nms_free_model }}"
+            test -n "${{ inputs.onnx_nms_free_metadata }}"
+            python scripts/evaluate_onnx_coco.py --model "${{ inputs.onnx_o2m_model }}" --metadata "${{ inputs.onnx_o2m_metadata }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_a_o2m --branch o2m-nms --provider "${{ inputs.provider }}" --seed 0
+            python scripts/evaluate_onnx_coco.py --model "${{ inputs.onnx_nms_free_model }}" --metadata "${{ inputs.onnx_nms_free_metadata }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_a_nms_free --branch nms-free --provider "${{ inputs.provider }}" --seed 0
+            python scripts/merge_vision_v8_coco_reports.py artifacts/vision_v8_coco_eval/run_a_o2m/evaluation.json artifacts/vision_v8_coco_eval/run_a_nms_free/evaluation.json --output artifacts/vision_v8_coco_eval/run_a
+            python scripts/evaluate_onnx_coco.py --model "${{ inputs.onnx_o2m_model }}" --metadata "${{ inputs.onnx_o2m_metadata }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_b_o2m --branch o2m-nms --provider "${{ inputs.provider }}" --seed 0
+            python scripts/evaluate_onnx_coco.py --model "${{ inputs.onnx_nms_free_model }}" --metadata "${{ inputs.onnx_nms_free_metadata }}" --annotations "${{ inputs.annotations }}" --images "${{ inputs.images }}" --output artifacts/vision_v8_coco_eval/run_b_nms_free --branch nms-free --provider "${{ inputs.provider }}" --seed 0
+            python scripts/merge_vision_v8_coco_reports.py artifacts/vision_v8_coco_eval/run_b_o2m/evaluation.json artifacts/vision_v8_coco_eval/run_b_nms_free/evaluation.json --output artifacts/vision_v8_coco_eval/run_b
+          fi
+      - name: Gate full COCO report
+        run: >-
+          python scripts/check_vision_v8_coco_report.py
+          artifacts/vision_v8_coco_eval/run_a/evaluation.json
+          --repeat-report artifacts/vision_v8_coco_eval/run_b/evaluation.json
+          --config configs/vision_v8_coco_accuracy_gate.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: vision-v8-coco-accuracy-report
+          path: |
+            artifacts/vision_v8_coco_eval/run_a/evaluation.json
+            artifacts/vision_v8_coco_eval/run_a/evaluation.md
+            artifacts/vision_v8_coco_eval/run_b/evaluation.json
+            artifacts/vision_v8_coco_eval/run_b/evaluation.md
+      - name: Upload gated reports to GitHub Release
+        if: inputs.release_tag != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: >-
+          gh release upload "${{ inputs.release_tag }}"
+          artifacts/vision_v8_coco_eval/run_a/evaluation.json#vision-v8-coco-evaluation.json
+          artifacts/vision_v8_coco_eval/run_a/evaluation.md#vision-v8-coco-evaluation.md
+          artifacts/vision_v8_coco_eval/run_b/evaluation.json#vision-v8-coco-evaluation-repeat.json
+          artifacts/vision_v8_coco_eval/run_b/evaluation.md#vision-v8-coco-evaluation-repeat.md
+          --clobber

complexity/deploy/onnx_detector/pipeline.py

@@ -100,9 +100,19 @@ def postprocess_single_image(
     def create_session(
         model_path: str | Path,
         providers: Sequence[str] = ("CPUExecutionProvider",),
+        *,
+        warmup_runs: int = 1,
+        intra_op_num_threads: int | None = None,
+        inter_op_num_threads: int | None = None,
     ) -> OnnxDetectorSession:
         return OnnxDetectorSession(
-            OrtSessionConfig(Path(model_path), providers=tuple(providers))
+            OrtSessionConfig(
+                Path(model_path),
+                providers=tuple(providers),
+                warmup_runs=warmup_runs,
+                intra_op_num_threads=intra_op_num_threads,
+                inter_op_num_threads=inter_op_num_threads,
+            )
         )
 
     def _decode_and_postprocess(

complexity/deploy/onnx_detector/session.py

@@ -18,6 +18,8 @@ class OrtSessionConfig:
     model_path: Path | str
     providers: tuple[str, ...] = ("CPUExecutionProvider",)
     warmup_runs: int = 1
+    intra_op_num_threads: int | None = None
+    inter_op_num_threads: int | None = None
 
 
 class OnnxDetectorSession:
@@ -36,8 +38,14 @@ def open(self) -> "OnnxDetectorSession":
                 # Prefer NVIDIA site-package DLLs over an unrelated PyTorch CUDA build.
                 ort.preload_dlls(directory="")
             self._add_tensorrt_dll_directories()
+            session_options = ort.SessionOptions()
+            if self.config.intra_op_num_threads is not None:
+                session_options.intra_op_num_threads = self.config.intra_op_num_threads
+            if self.config.inter_op_num_threads is not None:
+                session_options.inter_op_num_threads = self.config.inter_op_num_threads
             self._session = ort.InferenceSession(
                 str(self.config.model_path),
+                sess_options=session_options,
                 providers=list(self.config.providers),
             )
         return self

configs/vision_v8_coco_accuracy_gate.json

@@ -0,0 +1,53 @@
+{
+  "schema_version": 1,
+  "dataset": {
+    "name": "coco-2017",
+    "split": "val2017",
+    "required_image_count": 5000,
+    "annotations_sha256": "e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f",
+    "image_list_sha256": "0e508226935f3b7b96afd98661e82d6b53a02a6d4a4a7ed1dec777768b1d3315"
+  },
+  "determinism": {
+    "seed": 0,
+    "metric_tolerance": 1e-12,
+    "cpu_metric_tolerance": 1e-12,
+    "cuda_metric_tolerance": 1e-6,
+    "tensorrt_metric_tolerance": 1e-5
+  },
+  "required_branches": ["o2m-nms", "nms-free"],
+  "branches": {
+    "o2m-nms": {
+      "baseline_source": "docs/tr-hash-object-detection.md independent reproduction",
+      "baseline_metrics": {
+        "map50_95": 0.200,
+        "map50": 0.325,
+        "ar_100": 0.379
+      },
+      "absolute_floors": {
+        "map50_95": 0.190,
+        "map50": 0.310,
+        "ar_100": 0.360
+      },
+      "max_regressions": {
+        "map50_95": 0.005,
+        "map50": 0.010,
+        "ar_100": 0.010
+      }
+    },
+    "nms-free": {
+      "baseline_source": "docs/tr-hash-object-detection.md independent reproduction",
+      "baseline_metrics": {
+        "map50_95": 0.096,
+        "map50": 0.140
+      },
+      "absolute_floors": {
+        "map50_95": 0.090,
+        "map50": 0.130
+      },
+      "max_regressions": {
+        "map50_95": 0.005,
+        "map50": 0.010
+      }
+    }
+  }
+}

docs/index.md

@@ -91,6 +91,7 @@ GQA + TR-MoE architecture.
 - [TR-Hash text-to-image](tr-hash-text-to-image.md)
 - [TR-Hash object detection and serving](tr-hash-object-detection.md)
 - [TR-Hash Vision ONNX deployment](onnx_deploy.md)
+- [Vision v8 COCO accuracy gates](vision-v8-coco-accuracy-gates.md)
 - [Detector specialization and ablations](TR_HASH_DETECTOR_SPECIALIZATION.md)
 - [TR-Hash sensor fusion](tr_hash_sensor_fusion.md)
 - [Vision dependency stack](vision-dependency-stack.md)