From f91a54cde3cc36ec838b372e76ac6b84e3d1d66c Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 9 Oct 2026 11:50:09 -0400 Subject: [PATCH 1/2] fix(ci): keep nox session weights complete --- .github/workflows/update-session-weights.yaml | 12 ++-- py/scripts/check-session-weights.py | 55 ++++++++++++++----- py/scripts/session-weights.json | 24 +++++--- 3 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.github/workflows/update-session-weights.yaml b/.github/workflows/update-session-weights.yaml index 634bef674..39f9fa7e9 100644 --- a/.github/workflows/update-session-weights.yaml +++ b/.github/workflows/update-session-weights.yaml @@ -12,7 +12,7 @@ permissions: jobs: measure: - runs-on: windows-2025 + runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: @@ -25,11 +25,11 @@ jobs: - name: Setup Python environment uses: ./.github/actions/setup-python-env with: - python-version: "3.10" + python-version: "3.12" - name: Run nox tests (shard ${{ matrix.shard }}/4) shell: bash run: | - mise exec python@3.10 -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 4 \ + mise exec python@3.12 -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 4 \ --exclude-static-checks \ --exclude-session test_api_codegen \ --output-durations measured-durations-${{ matrix.shard }}.json @@ -46,6 +46,10 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Python environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" - name: Download measured durations uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -76,7 +80,7 @@ jobs: body: | Automated weekly update of `py/scripts/session-weights.json`. - Session durations were re-measured on `windows-2025` with Python 3.10 + Session durations were re-measured on `ubuntu-24.04` with Python 3.12 and at least one session drifted beyond the 50% threshold. This keeps nox shard balancing accurate. diff --git a/py/scripts/check-session-weights.py b/py/scripts/check-session-weights.py index e7fea1284..85ddcd7a4 100644 --- a/py/scripts/check-session-weights.py +++ b/py/scripts/check-session-weights.py @@ -6,9 +6,8 @@ whose actual duration drifted significantly from the recorded weight. Exit codes: - 0 — all weights are within tolerance (new/missing sessions are reported - but do not cause a non-zero exit; they receive the default weight) - 1 — at least one weight drifted beyond the threshold + 0 — all weights are within tolerance + 1 — weights drifted beyond the threshold or an update is missing measurements Usage: python check-session-weights.py measured-shard-0.json measured-shard-1.json ... @@ -19,6 +18,8 @@ import argparse import json +import re +import subprocess import sys from pathlib import Path @@ -31,20 +32,37 @@ # have high relative variance and aren't worth chasing. MIN_DURATION_FOR_DRIFT = 8 +# These sessions are measured in separate CI jobs or excluded from the shard +# matrix, matching the arguments in checks.yaml and update-session-weights.yaml. +EXCLUDED_SESSIONS = {"pylint", "test_types", "test_api_codegen"} -def update_weights(weights_path: Path, weights_data: dict, measured: dict[str, int]) -> None: + +def get_nox_sessions() -> set[str]: + """Return sessions in nox -l, using the SDK's locked project environment.""" + root = Path(__file__).resolve().parents[2] + result = subprocess.run( + ["uv", "run", "--project", str(root / "py"), "nox", "-l", "-f", str(root / "py/noxfile.py")], + capture_output=True, + check=True, + text=True, + ) + return {match.group(1) for line in result.stdout.splitlines() if (match := re.match(r"[*-] (\S+)", line))} + + +def update_weights( + weights_path: Path, weights_data: dict, measured: dict[str, int], current_sessions: set[str] +) -> None: """Overwrite session-weights.json with measured durations.""" meta_keys = {k for k in weights_data if k.startswith("_")} updated = {k: weights_data[k] for k in sorted(meta_keys)} # Merge: keep measured values, drop sessions that no longer exist - all_sessions = sorted(set(weights_data.keys() - meta_keys) | set(measured.keys())) + all_sessions = sorted(current_sessions | set(measured.keys())) for session in all_sessions: if session in measured: updated[session] = measured[session] else: - # Session wasn't measured — keep the old weight (may be platform-specific - # or skipped; a full run across all shards would cover everything) - updated[session] = weights_data[session] + # Unmeasured static-check sessions still get an explicit weight. + updated[session] = weights_data.get(session, weights_data.get("_default", 15)) with open(weights_path, "w") as f: json.dump(updated, f, indent=2, sort_keys=True) f.write("\n") @@ -78,14 +96,22 @@ def main() -> None: measured.update(json.load(f)) if not measured: - print("⚠️ No measured durations found — nothing to check.") - sys.exit(0) + print("❌ No measured durations found — unable to verify session weights.") + sys.exit(1) with open(args.weights) as f: weights_data: dict[str, int] = json.load(f) + current_sessions = get_nox_sessions() if args.update: - update_weights(args.weights, weights_data, measured) + expected_sessions = current_sessions - EXCLUDED_SESSIONS + missing_measurements = sorted(expected_sessions - set(measured)) + if missing_measurements: + print("❌ No measured duration for current session(s):") + for session in missing_measurements: + print(f" {session}") + sys.exit(1) + update_weights(args.weights, weights_data, measured, current_sessions) return meta_keys = {k for k in weights_data if k.startswith("_")} @@ -119,7 +145,7 @@ def main() -> None: print(f"{session:<50} {expected:>7}s {actual:>7}s {drift_pct:>+7.0%}{flag}") # Check for sessions in weights but not measured (may have been removed) - known_sessions = {k for k in weights_data if k not in meta_keys} + known_sessions = current_sessions - EXCLUDED_SESSIONS missing = sorted(known_sessions - set(measured)) print() @@ -137,7 +163,10 @@ def main() -> None: print(f" {s}") print() - if drifted: + if new_sessions or missing: + print("❌ Session weights need updating or measurement is incomplete.") + sys.exit(1) + elif drifted: print(f"⚠️ {len(drifted)} session(s) drifted beyond {DRIFT_THRESHOLD:.0%} threshold.") print(" Consider updating py/scripts/session-weights.json") sys.exit(1) diff --git a/py/scripts/session-weights.json b/py/scripts/session-weights.json index b689d48b1..25c5fd0b4 100644 --- a/py/scripts/session-weights.json +++ b/py/scripts/session-weights.json @@ -1,15 +1,17 @@ { - "_comment": "Approximate session durations in seconds from CI (windows, Python 3.10). Used by nox-matrix.py for balanced shard assignment. Re-measure periodically and update.", + "_comment": "Approximate session durations in seconds from CI (ubuntu-24.04, Python 3.12). Used by nox-matrix.py for balanced shard assignment. Re-measure periodically and update.", "_default": 15, + "pylint": 15, "test_agentscope(1.0.0)": 31, "test_agentscope(latest)": 31, "test_agno(2.1.0)": 15, "test_agno(2.4.0)": 18, "test_agno(latest)": 15, + "test_ai_sdk(0.4.0)": 15, + "test_ai_sdk(latest)": 15, "test_anthropic(0.48.0)": 8, - "test_anthropic(0.49.0)": 13, - "test_anthropic(0.50.0)": 13, "test_anthropic(latest)": 12, + "test_api_codegen": 15, "test_autoevals(0.0.129)": 54, "test_autoevals(latest)": 51, "test_autogen(0.7.0)": 11, @@ -31,6 +33,8 @@ "test_crewai(latest)": 42, "test_cursor_sdk(1.0.25)": 21, "test_cursor_sdk(latest)": 38, + "test_deepagents(0.6.0)": 15, + "test_deepagents(latest)": 15, "test_dspy(2.6.0)": 48, "test_dspy(latest)": 49, "test_google_adk(1.14.1)": 29, @@ -40,6 +44,8 @@ "test_google_genai(1.30.0)": 10, "test_google_genai(1.75.0)": 12, "test_google_genai(latest)": 10, + "test_harbor(0.16.0)": 15, + "test_harbor(latest)": 15, "test_huggingface_hub(0.32.0)": 11, "test_huggingface_hub(latest)": 10, "test_instructor(1.11.0)": 19, @@ -47,24 +53,22 @@ "test_langchain(0.3.28)": 20, "test_langchain(latest)": 24, "test_litellm(1.74.0)": 17, - "test_litellm(1.83.0)": 34, "test_litellm(latest)": 34, + "test_livekit_agents(1.3.1)": 15, + "test_livekit_agents(latest)": 15, "test_llamaindex(0.13.0)": 19, "test_llamaindex(latest)": 18, "test_mistral(1.12.4)": 9, + "test_mistral(2.10.1)": 15, "test_mistral(latest)": 12, - "test_openai(1.71)": 24, "test_openai(1.71.0)": 14, "test_openai(1.77.0)": 14, - "test_openai(1.91)": 15, - "test_openai(1.92)": 15, "test_openai(1.92.0)": 16, "test_openai(latest)": 20, "test_openai_agents(0.0.19)": 24, "test_openai_agents(0.13.6)": 10, "test_openai_agents(latest)": 18, "test_openai_ddtrace": 25, - "test_openai_http2_streaming": 10, "test_openai_http2_streaming(1.71.0)": 7, "test_openai_http2_streaming(1.77.0)": 7, "test_openai_http2_streaming(1.92.0)": 6, @@ -75,9 +79,10 @@ "test_otel(1.44.0)": 5, "test_otel(latest)": 6, "test_otel_not_installed": 5, + "test_pipecat(1.3.0)": 15, + "test_pipecat(latest)": 15, "test_pydantic_ai_integration(1.10.0)": 31, "test_pydantic_ai_integration(latest)": 44, - "test_pydantic_ai_logfire": 25, "test_pydantic_ai_logfire(1.10.0)": 43, "test_pydantic_ai_logfire(latest)": 21, "test_pydantic_ai_wrap_openai(0.1.9)": 21, @@ -92,5 +97,6 @@ "test_temporal(latest)": 12, "test_transformers(4.42.0)": 52, "test_transformers(latest)": 58, + "test_types": 15, "test_typesafe(latest)": 6 } From 7e95b8b702b216a541534484748d04728744fe18 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 9 Oct 2026 11:57:18 -0400 Subject: [PATCH 2/2] fix(ci): detect stale nox session weights --- py/scripts/check-session-weights.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/py/scripts/check-session-weights.py b/py/scripts/check-session-weights.py index 85ddcd7a4..272aa01c3 100644 --- a/py/scripts/check-session-weights.py +++ b/py/scripts/check-session-weights.py @@ -144,9 +144,10 @@ def main() -> None: print(f"{session:<50} {expected:>7}s {actual:>7}s {drift_pct:>+7.0%}{flag}") - # Check for sessions in weights but not measured (may have been removed) + # Check for current sharded sessions missing from measurements. known_sessions = current_sessions - EXCLUDED_SESSIONS missing = sorted(known_sessions - set(measured)) + stale = sorted({k for k in weights_data if k not in meta_keys} - current_sessions) print() @@ -163,7 +164,13 @@ def main() -> None: print(f" {s}") print() - if new_sessions or missing: + if stale: + print(f"🗑️ {len(stale)} stale session weight(s) no longer in nox -l:") + for s in stale: + print(f" {s}") + print() + + if new_sessions or missing or stale: print("❌ Session weights need updating or measurement is incomplete.") sys.exit(1) elif drifted: