chore: keep nox session weights complete

.github/workflows/update-session-weights.yaml

@@ -12,7 +12,7 @@ permissions:
 
 jobs:
   measure:
-    runs-on: windows-2025
+    runs-on: ubuntu-24.04
     timeout-minutes: 45
 
     strategy:
@@ -25,11 +25,11 @@ jobs:
       - name: Setup Python environment
         uses: ./.github/actions/setup-python-env
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Run nox tests (shard ${{ matrix.shard }}/4)
         shell: bash
         run: |
-          mise exec python@3.10 -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 4 \
+          mise exec python@3.12 -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 4 \
             --exclude-static-checks \
             --exclude-session test_api_codegen \
             --output-durations measured-durations-${{ matrix.shard }}.json
@@ -46,6 +46,10 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-python-env
+        with:
+          python-version: "3.12"
       - name: Download measured durations
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -76,7 +80,7 @@ jobs:
           body: |
             Automated weekly update of `py/scripts/session-weights.json`.
 
-            Session durations were re-measured on `windows-2025` with Python 3.10
+            Session durations were re-measured on `ubuntu-24.04` with Python 3.12
             and at least one session drifted beyond the 50% threshold.
 
             This keeps nox shard balancing accurate.

py/scripts/check-session-weights.py

@@ -6,9 +6,8 @@
 whose actual duration drifted significantly from the recorded weight.
 
 Exit codes:
-    0 — all weights are within tolerance (new/missing sessions are reported
-        but do not cause a non-zero exit; they receive the default weight)
-    1 — at least one weight drifted beyond the threshold
+    0 — all weights are within tolerance
+    1 — weights drifted beyond the threshold or an update is missing measurements
 
 Usage:
     python check-session-weights.py measured-shard-0.json measured-shard-1.json ...
@@ -19,6 +18,8 @@
 
 import argparse
 import json
+import re
+import subprocess
 import sys
 from pathlib import Path
 
@@ -31,20 +32,37 @@
 # have high relative variance and aren't worth chasing.
 MIN_DURATION_FOR_DRIFT = 8
 
+# These sessions are measured in separate CI jobs or excluded from the shard
+# matrix, matching the arguments in checks.yaml and update-session-weights.yaml.
+EXCLUDED_SESSIONS = {"pylint", "test_types", "test_api_codegen"}
 
-def update_weights(weights_path: Path, weights_data: dict, measured: dict[str, int]) -> None:
+
+def get_nox_sessions() -> set[str]:
+    """Return sessions in nox -l, using the SDK's locked project environment."""
+    root = Path(__file__).resolve().parents[2]
+    result = subprocess.run(
+        ["uv", "run", "--project", str(root / "py"), "nox", "-l", "-f", str(root / "py/noxfile.py")],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    return {match.group(1) for line in result.stdout.splitlines() if (match := re.match(r"[*-] (\S+)", line))}
+
+
+def update_weights(
+    weights_path: Path, weights_data: dict, measured: dict[str, int], current_sessions: set[str]
+) -> None:
     """Overwrite session-weights.json with measured durations."""
     meta_keys = {k for k in weights_data if k.startswith("_")}
     updated = {k: weights_data[k] for k in sorted(meta_keys)}
     # Merge: keep measured values, drop sessions that no longer exist
-    all_sessions = sorted(set(weights_data.keys() - meta_keys) | set(measured.keys()))
+    all_sessions = sorted(current_sessions | set(measured.keys()))
     for session in all_sessions:
         if session in measured:
             updated[session] = measured[session]
         else:
-            # Session wasn't measured — keep the old weight (may be platform-specific
-            # or skipped; a full run across all shards would cover everything)
-            updated[session] = weights_data[session]
+            # Unmeasured static-check sessions still get an explicit weight.
+            updated[session] = weights_data.get(session, weights_data.get("_default", 15))
     with open(weights_path, "w") as f:
         json.dump(updated, f, indent=2, sort_keys=True)
         f.write("\n")
@@ -78,14 +96,22 @@ def main() -> None:
             measured.update(json.load(f))
 
     if not measured:
-        print("⚠️  No measured durations found — nothing to check.")
-        sys.exit(0)
+        print("❌ No measured durations found — unable to verify session weights.")
+        sys.exit(1)
 
     with open(args.weights) as f:
         weights_data: dict[str, int] = json.load(f)
 
+    current_sessions = get_nox_sessions()
     if args.update:
-        update_weights(args.weights, weights_data, measured)
+        expected_sessions = current_sessions - EXCLUDED_SESSIONS
+        missing_measurements = sorted(expected_sessions - set(measured))
+        if missing_measurements:
+            print("❌ No measured duration for current session(s):")
+            for session in missing_measurements:
+                print(f"   {session}")
+            sys.exit(1)
+        update_weights(args.weights, weights_data, measured, current_sessions)
         return
 
     meta_keys = {k for k in weights_data if k.startswith("_")}
@@ -118,9 +144,10 @@ def main() -> None:
 
         print(f"{session:<50} {expected:>7}s {actual:>7}s {drift_pct:>+7.0%}{flag}")
 
-    # Check for sessions in weights but not measured (may have been removed)
-    known_sessions = {k for k in weights_data if k not in meta_keys}
+    # Check for current sharded sessions missing from measurements.
+    known_sessions = current_sessions - EXCLUDED_SESSIONS
     missing = sorted(known_sessions - set(measured))
+    stale = sorted({k for k in weights_data if k not in meta_keys} - current_sessions)
 
     print()
 
@@ -137,7 +164,16 @@ def main() -> None:
             print(f"   {s}")
         print()
 
-    if drifted:
+    if stale:
+        print(f"🗑️  {len(stale)} stale session weight(s) no longer in nox -l:")
+        for s in stale:
+            print(f"   {s}")
+        print()
+
+    if new_sessions or missing or stale:
+        print("❌ Session weights need updating or measurement is incomplete.")
+        sys.exit(1)
+    elif drifted:
         print(f"⚠️  {len(drifted)} session(s) drifted beyond {DRIFT_THRESHOLD:.0%} threshold.")
         print("   Consider updating py/scripts/session-weights.json")
         sys.exit(1)

py/scripts/session-weights.json

@@ -1,15 +1,17 @@
 {
-  "_comment": "Approximate session durations in seconds from CI (windows, Python 3.10). Used by nox-matrix.py for balanced shard assignment. Re-measure periodically and update.",
+  "_comment": "Approximate session durations in seconds from CI (ubuntu-24.04, Python 3.12). Used by nox-matrix.py for balanced shard assignment. Re-measure periodically and update.",
   "_default": 15,
+  "pylint": 15,
   "test_agentscope(1.0.0)": 31,
   "test_agentscope(latest)": 31,
   "test_agno(2.1.0)": 15,
   "test_agno(2.4.0)": 18,
   "test_agno(latest)": 15,
+  "test_ai_sdk(0.4.0)": 15,
+  "test_ai_sdk(latest)": 15,
   "test_anthropic(0.48.0)": 8,
-  "test_anthropic(0.49.0)": 13,
-  "test_anthropic(0.50.0)": 13,
   "test_anthropic(latest)": 12,
+  "test_api_codegen": 15,
   "test_autoevals(0.0.129)": 54,
   "test_autoevals(latest)": 51,
   "test_autogen(0.7.0)": 11,
@@ -31,6 +33,8 @@
   "test_crewai(latest)": 42,
   "test_cursor_sdk(1.0.25)": 21,
   "test_cursor_sdk(latest)": 38,
+  "test_deepagents(0.6.0)": 15,
+  "test_deepagents(latest)": 15,
   "test_dspy(2.6.0)": 48,
   "test_dspy(latest)": 49,
   "test_google_adk(1.14.1)": 29,
@@ -40,31 +44,31 @@
   "test_google_genai(1.30.0)": 10,
   "test_google_genai(1.75.0)": 12,
   "test_google_genai(latest)": 10,
+  "test_harbor(0.16.0)": 15,
+  "test_harbor(latest)": 15,
   "test_huggingface_hub(0.32.0)": 11,
   "test_huggingface_hub(latest)": 10,
   "test_instructor(1.11.0)": 19,
   "test_instructor(latest)": 12,
   "test_langchain(0.3.28)": 20,
   "test_langchain(latest)": 24,
   "test_litellm(1.74.0)": 17,
-  "test_litellm(1.83.0)": 34,
   "test_litellm(latest)": 34,
+  "test_livekit_agents(1.3.1)": 15,
+  "test_livekit_agents(latest)": 15,
   "test_llamaindex(0.13.0)": 19,
   "test_llamaindex(latest)": 18,
   "test_mistral(1.12.4)": 9,
+  "test_mistral(2.10.1)": 15,
   "test_mistral(latest)": 12,
-  "test_openai(1.71)": 24,
   "test_openai(1.71.0)": 14,
   "test_openai(1.77.0)": 14,
-  "test_openai(1.91)": 15,
-  "test_openai(1.92)": 15,
   "test_openai(1.92.0)": 16,
   "test_openai(latest)": 20,
   "test_openai_agents(0.0.19)": 24,
   "test_openai_agents(0.13.6)": 10,
   "test_openai_agents(latest)": 18,
   "test_openai_ddtrace": 25,
-  "test_openai_http2_streaming": 10,
   "test_openai_http2_streaming(1.71.0)": 7,
   "test_openai_http2_streaming(1.77.0)": 7,
   "test_openai_http2_streaming(1.92.0)": 6,
@@ -75,9 +79,10 @@
   "test_otel(1.44.0)": 5,
   "test_otel(latest)": 6,
   "test_otel_not_installed": 5,
+  "test_pipecat(1.3.0)": 15,
+  "test_pipecat(latest)": 15,
   "test_pydantic_ai_integration(1.10.0)": 31,
   "test_pydantic_ai_integration(latest)": 44,
-  "test_pydantic_ai_logfire": 25,
   "test_pydantic_ai_logfire(1.10.0)": 43,
   "test_pydantic_ai_logfire(latest)": 21,
   "test_pydantic_ai_wrap_openai(0.1.9)": 21,
@@ -92,5 +97,6 @@
   "test_temporal(latest)": 12,
   "test_transformers(4.42.0)": 52,
   "test_transformers(latest)": 58,
+  "test_types": 15,
   "test_typesafe(latest)": 6
 }