diff --git a/.github/workflows/reusable-linux.yml b/.github/workflows/reusable-linux.yml index e5d1022d..e5d60190 100644 --- a/.github/workflows/reusable-linux.yml +++ b/.github/workflows/reusable-linux.yml @@ -75,6 +75,15 @@ jobs: - name: Verify version policy working-directory: loop run: python3 scripts/ci/check_version_policy.py + - name: Verify resource-envelope contracts + working-directory: loop + run: | + python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python3 scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json + python3 scripts/qualification/validate_resource_envelope_evidence.py + python3 scripts/resource_envelope/pathological_workload.py \ + --output "${RUNNER_TEMP}/loop-pathological-vector.pdf" \ + --page-count 256 --operations 256 --family pathological-vector - name: Verify processing-budget exhaustion corpus working-directory: loop run: python3 scripts/budget_exhaustion/generate_corpus.py --check diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index d45552f4..2fd91ea3 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -82,8 +82,9 @@ jobs: working-directory: loop shell: pwsh run: | - python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload -v + python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v python scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json + python scripts/qualification/validate_resource_envelope_evidence.py python scripts/resource_envelope/pathological_workload.py ` --output "$env:RUNNER_TEMP\loop-pathological-vector.pdf" ` --page-count 256 --operations 256 --family pathological-vector diff --git a/changes/cursor-session-11-resource-envelope.md b/changes/cursor-session-11-resource-envelope.md new file mode 100644 index 00000000..2f163863 --- /dev/null +++ b/changes/cursor-session-11-resource-envelope.md @@ -0,0 +1,4 @@ +Category: internal +Audience: developers +Breaking-Change: no +Summary: Freeze Session 11 resource-envelope fixture manifest, fail-closed matrix evidence, budget-exhaustion corpus verification, and R-01 closeout updates; hosted Linux and full external fixtures remain incomplete. diff --git a/docs/SESSION_11_HANDOFF.md b/docs/SESSION_11_HANDOFF.md new file mode 100644 index 00000000..20cbdc1f --- /dev/null +++ b/docs/SESSION_11_HANDOFF.md @@ -0,0 +1,30 @@ +# Session 11 — Prove production resource envelope + +## Scope + +Session 11 closes issue #242 measurement gaps for gate **R-01** on the merged +`dev` candidate SHA recorded at PR merge time, anchored to `origin/dev` @ +`6a55130c…`. + +Fixtures remain outside the repository per #242. + +## Issue 34 — Fixture manifest + +Partial manifest with digests for office-2mb, pathological-vector, transparency-spots. +Requirement catalog documents all issue #242 IDs. External bytes under +`C:\.dev\qualification\session-11\fixtures\`. + +## Issue 35 — Measure envelopes + +Local Windows `run_matrix.py --strict`: exit 1, `measured: 0`, fail-closed on stale +PdfTool commit and missing fixtures. `-1` fields never promoted to pass. + +## Issue 36 — Archive + +Frozen `docs/evidence/session-11-resource-envelope/evidence.json` with +`disposition: incomplete`. CI validates via +`scripts/qualification/validate_resource_envelope_evidence.py`. + +## Exit gate + +R-01 remains **Partial** until hosted Linux + full fixture bundle on candidate-SHA PdfTool. diff --git a/docs/evidence/session-11-resource-envelope/evidence.json b/docs/evidence/session-11-resource-envelope/evidence.json new file mode 100644 index 00000000..c73cab65 --- /dev/null +++ b/docs/evidence/session-11-resource-envelope/evidence.json @@ -0,0 +1,40 @@ +{ + "schema_kind": "loop-resource-envelope-qualification-evidence", + "schema_version": 1, + "session": 11, + "branch": "cursor/session-11-resource-envelope", + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "session_09_baseline_sha": "1c3f9d6d2312045264134f07b4d00e26de2058bf", + "disposition": "incomplete", + "disposition_reason": "Strict matrix fail-closed: stale PdfTool commit, missing image-heavy-500mb and ten-thousand-page fixtures, -1 preflight/cancel/recovery. Linux hosted matrix not run.", + "fixture_status": { + "office-2mb": {"status": "failed", "preflight_high_water_bytes": -1, "cancellation_latency_ms": -1, "recovery_ms": -1}, + "image-heavy-500mb": {"status": "unavailable", "reason": "fixture-not-supplied"}, + "ten-thousand-page": {"status": "unavailable", "reason": "DIV2K corpus absent locally"}, + "pathological-vector": {"status": "failed", "preflight_high_water_bytes": -1, "cancellation_latency_ms": -1, "recovery_ms": -1}, + "transparency-spots": {"status": "failed", "preflight_high_water_bytes": -1, "cancellation_latency_ms": -1, "recovery_ms": -1}, + "multi-gb": {"status": "unavailable", "reason": "optional"} + }, + "matrix_runs": [ + { + "platform": "windows-local", + "path": "docs/evidence/session-11-resource-envelope/matrix-windows-local.json", + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "pdf_tool_commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "strict_exit_code": 1, + "summary": {"total": 6, "measured": 0, "flagged": 2, "failed": 3, "candidate_sha_verified": true} + } + ], + "hosted_gaps": [ + "Rebuild PdfTool from candidate_sha on Windows and Linux", + "Supply image-heavy-500mb external fixture", + "Build ten-thousand-page PDF from DIV2K with --hash-all manifest", + "Run Linux hosted run_matrix.py --strict --repetitions 3", + "Cancellation probe on pathological-vector" + ], + "verifiers": [ + {"command": "resource_envelope unit tests", "result": "pass"}, + {"command": "validate_resource_envelope_evidence.py", "result": "pass"}, + {"command": "run_matrix.py --strict windows-local", "result": "fail-closed"} + ] +} diff --git a/docs/evidence/session-11-resource-envelope/fixture-manifest.json b/docs/evidence/session-11-resource-envelope/fixture-manifest.json new file mode 100644 index 00000000..a631b1b1 --- /dev/null +++ b/docs/evidence/session-11-resource-envelope/fixture-manifest.json @@ -0,0 +1,29 @@ +{ + "schema_kind": "loop-resource-envelope-fixtures", + "schema_version": 1, + "fixtures": [ + { + "fixture_id": "office-2mb", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\office-2mb.pdf", + "sha256": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "size_bytes": 2169858, + "provenance": "Session 11 synthetic office workload PDF" + }, + { + "fixture_id": "pathological-vector", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\pathological-vector.pdf", + "sha256": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "size_bytes": 6369566, + "provenance": "pathological_workload.py --family pathological-vector", + "page_count": 256 + }, + { + "fixture_id": "transparency-spots", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\transparency-spots.pdf", + "sha256": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "size_bytes": 3266105, + "provenance": "pathological_workload.py --family transparency-spots", + "page_count": 256 + } + ] +} diff --git a/docs/evidence/session-11-resource-envelope/fixture-requirements.json b/docs/evidence/session-11-resource-envelope/fixture-requirements.json new file mode 100644 index 00000000..33937674 --- /dev/null +++ b/docs/evidence/session-11-resource-envelope/fixture-requirements.json @@ -0,0 +1,31 @@ +{ + "schema_kind": "loop-resource-envelope-fixture-requirements", + "schema_version": 1, + "external_root": "C:\\.dev\\qualification\\session-11\\fixtures", + "required_fixtures": [ + { + "fixture_id": "office-2mb", + "availability": "local", + "sha256": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9" + }, + { + "fixture_id": "image-heavy-500mb", + "availability": "hosted-required" + }, + { + "fixture_id": "ten-thousand-page", + "availability": "hosted-required", + "provenance": "div2k_workload.py from external DIV2K corpus" + }, + { + "fixture_id": "pathological-vector", + "availability": "local", + "sha256": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f" + }, + { + "fixture_id": "transparency-spots", + "availability": "local", + "sha256": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3" + } + ] +} diff --git a/docs/evidence/session-11-resource-envelope/matrix-windows-local.json b/docs/evidence/session-11-resource-envelope/matrix-windows-local.json new file mode 100644 index 00000000..34e24a90 --- /dev/null +++ b/docs/evidence/session-11-resource-envelope/matrix-windows-local.json @@ -0,0 +1,853 @@ +{ + "schema_kind": "loop-resource-envelope-matrix", + "schema_version": 2, + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "candidate_identity": { + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "source": "git-head", + "environment_sha": "", + "verified": true + }, + "generated_at_utc": "2026-09-06T23:50:51.501345+00:00", + "fixtures": [ + { + "fixture_id": "office-2mb", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\office-2mb.pdf", + "expected_page_count": null, + "workload": null, + "profile": { + "render_hw_accel": false, + "render_rasterizers": 8 + }, + "command": [ + "C:\\.dev\\build\\loop-native-qml-deploy-msvc\\install\\usr\\bin\\PdfTool.exe", + "benchmark", + "C:\\.dev\\qualification\\session-11\\fixtures\\office-2mb.pdf", + "--render-hw-accel", + "0", + "--render-rasterizers", + "8", + "--console-format", + "json" + ], + "fixture_sha256": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "input_bytes": 2169858, + "provenance": "Session 11 synthetic office workload PDF", + "manifest_sha256": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "result": { + "cache_high_water_bytes": 103957205, + "cancellation_latency_ms": -1, + "elapsed_ms": 355, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 677, + "pages_materialized": 23, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 654, + "process_commit_high_water_bytes": 160018432, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 2973929, + "evictions": 0, + "high_water_bytes": 2973929, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 3276, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 654 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 2973929, + "resident_high_water_bytes": 103957205 + }, + "rss_high_water_bytes": 148635648, + "status": "budget-exceeded" + }, + "statistics": { + "repetitions": 1, + "elapsed_ms": { + "median": 355, + "min": 355, + "max": 355 + }, + "rss_high_water_bytes": { + "median": 148635648, + "min": 148635648, + "max": 148635648 + }, + "unstable": false + }, + "runs": [ + { + "run": 1, + "status": "recorded", + "process_exit_code": 5, + "result": { + "cache_high_water_bytes": 103957205, + "cancellation_latency_ms": -1, + "elapsed_ms": 355, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "0f0a45ba88d63f6218f81467ea5a8713ca457c8cb7bcd8f619742d266bbe67c9", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 677, + "pages_materialized": 23, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 654, + "process_commit_high_water_bytes": 160018432, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 2973929, + "evictions": 0, + "high_water_bytes": 2973929, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 3276, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 654 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 2973929, + "resident_high_water_bytes": 103957205 + }, + "rss_high_water_bytes": 148635648, + "status": "budget-exceeded" + } + } + ], + "validation_errors": [ + "PdfTool identity commit does not match checkout HEAD", + "run 1: identity.commit '2d95b4a6019504d4e1ffbf658a0e078ab38e34df' does not match candidate '6e65be48e261d14c64653694320c0185b84fc560'" + ], + "regressions": [], + "status": "failed", + "required": true + }, + { + "fixture_id": "image-heavy-500mb", + "status": "unavailable", + "reason": "fixture-not-supplied", + "result": null, + "runs": [], + "validation_errors": [], + "regressions": [], + "required": true + }, + { + "fixture_id": "multi-gb", + "status": "unavailable", + "reason": "fixture-not-supplied-optional", + "result": null, + "runs": [], + "validation_errors": [], + "regressions": [], + "required": false + }, + { + "fixture_id": "ten-thousand-page", + "status": "unavailable", + "reason": "fixture-not-supplied", + "result": null, + "runs": [], + "validation_errors": [], + "regressions": [], + "required": true + }, + { + "fixture_id": "pathological-vector", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\pathological-vector.pdf", + "expected_page_count": 256, + "workload": "pathological-vector", + "profile": { + "render_hw_accel": false, + "render_rasterizers": 8 + }, + "command": [ + "C:\\.dev\\build\\loop-native-qml-deploy-msvc\\install\\usr\\bin\\PdfTool.exe", + "benchmark", + "C:\\.dev\\qualification\\session-11\\fixtures\\pathological-vector.pdf", + "--render-hw-accel", + "0", + "--render-rasterizers", + "8", + "--console-format", + "json" + ], + "fixture_sha256": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "input_bytes": 6369566, + "provenance": "pathological_workload.py --family pathological-vector", + "manifest_sha256": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "result": { + "cache_high_water_bytes": 108754618, + "cancellation_latency_ms": -1, + "elapsed_ms": 303, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 256, + "pages_materialized": 9, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 247, + "process_commit_high_water_bytes": 131481600, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 6664198, + "evictions": 0, + "high_water_bytes": 6664198, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 1110420, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 247 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 6664198, + "resident_high_water_bytes": 108754618 + }, + "rss_high_water_bytes": 147341312, + "status": "budget-exceeded" + }, + "statistics": { + "repetitions": 1, + "elapsed_ms": { + "median": 303, + "min": 303, + "max": 303 + }, + "rss_high_water_bytes": { + "median": 147341312, + "min": 147341312, + "max": 147341312 + }, + "unstable": false + }, + "runs": [ + { + "run": 1, + "status": "recorded", + "process_exit_code": 5, + "result": { + "cache_high_water_bytes": 108754618, + "cancellation_latency_ms": -1, + "elapsed_ms": 303, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "5517185518d353635366270ff41e3e2a56cf1d6d21187e84aba8fc6a44aa6c0f", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 256, + "pages_materialized": 9, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 247, + "process_commit_high_water_bytes": 131481600, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 6664198, + "evictions": 0, + "high_water_bytes": 6664198, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 1110420, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 247 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 6664198, + "resident_high_water_bytes": 108754618 + }, + "rss_high_water_bytes": 147341312, + "status": "budget-exceeded" + } + } + ], + "validation_errors": [ + "PdfTool identity commit does not match checkout HEAD", + "run 1: identity.commit '2d95b4a6019504d4e1ffbf658a0e078ab38e34df' does not match candidate '6e65be48e261d14c64653694320c0185b84fc560'" + ], + "regressions": [], + "status": "failed", + "required": true + }, + { + "fixture_id": "transparency-spots", + "path": "C:\\.dev\\qualification\\session-11\\fixtures\\transparency-spots.pdf", + "expected_page_count": 256, + "workload": null, + "profile": { + "render_hw_accel": false, + "render_rasterizers": 8 + }, + "command": [ + "C:\\.dev\\build\\loop-native-qml-deploy-msvc\\install\\usr\\bin\\PdfTool.exe", + "benchmark", + "C:\\.dev\\qualification\\session-11\\fixtures\\transparency-spots.pdf", + "--render-hw-accel", + "0", + "--render-rasterizers", + "8", + "--console-format", + "json" + ], + "fixture_sha256": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "input_bytes": 3266105, + "provenance": "pathological_workload.py --family transparency-spots", + "manifest_sha256": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "result": { + "cache_high_water_bytes": 105247528, + "cancellation_latency_ms": -1, + "elapsed_ms": 226, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 256, + "pages_materialized": 3, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 253, + "process_commit_high_water_bytes": 124989440, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 3707748, + "evictions": 0, + "high_water_bytes": 3707748, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 559780, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 253 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 3707748, + "resident_high_water_bytes": 105247528 + }, + "rss_high_water_bytes": 140754944, + "status": "budget-exceeded" + }, + "statistics": { + "repetitions": 1, + "elapsed_ms": { + "median": 226, + "min": 226, + "max": 226 + }, + "rss_high_water_bytes": { + "median": 140754944, + "min": 140754944, + "max": 140754944 + }, + "unstable": false + }, + "runs": [ + { + "run": 1, + "status": "recorded", + "process_exit_code": 5, + "result": { + "cache_high_water_bytes": 105247528, + "cancellation_latency_ms": -1, + "elapsed_ms": 226, + "family": "benchmark-render", + "identity": { + "build": "Release", + "commit": "2d95b4a6019504d4e1ffbf658a0e078ab38e34df", + "compiler": "msvc 1944", + "cpu": "x86_64", + "fixture_digest": "d4316776aaacb9f056da41925cea51fd68bf1b00ae7b53e1326306e1d85810b3", + "gpu": "unspecified", + "operation_version": "resource-envelope-v2", + "os": "Windows 11 Version 25H2 10.0.26200", + "product_version": "0.2.0-alpha", + "profile_version": "", + "qt": "6.11.1", + "renderer": "loop" + }, + "incomplete_reason": "resource-budget-exceeded", + "interaction_slot_held": false, + "open_to_first_view_ms": -1, + "page_count": 256, + "pages_materialized": 3, + "prefetch_shed": false, + "preflight_high_water_bytes": -1, + "pressure_shed_count": 253, + "process_commit_high_water_bytes": 124989440, + "recovery_ms": -1, + "resources": { + "config": { + "pool_limits_bytes": { + "active-document-model": 268435456, + "compiled-evidence-cache": 134217728, + "decoded-stream-image-cache": 268435456, + "gpu-texture-cache": 134217728, + "raster-tile-cache": 134217728, + "rollback-storage": 2147483648, + "undo-history": 268435456 + }, + "resident_limit_bytes": 805306368 + }, + "pools": { + "active-document-model": { + "current_bytes": 3707748, + "evictions": 0, + "high_water_bytes": 3707748, + "limit_bytes": 268435456, + "shed": 0 + }, + "compiled-evidence-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 559780, + "limit_bytes": 134217728, + "shed": 0 + }, + "decoded-stream-image-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + }, + "gpu-texture-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 134217728, + "shed": 0 + }, + "raster-tile-cache": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 100980000, + "limit_bytes": 134217728, + "shed": 253 + }, + "rollback-storage": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 2147483648, + "shed": 0 + }, + "undo-history": { + "current_bytes": 0, + "evictions": 0, + "high_water_bytes": 0, + "limit_bytes": 268435456, + "shed": 0 + } + }, + "pressure": "normal", + "resident_bytes": 3707748, + "resident_high_water_bytes": 105247528 + }, + "rss_high_water_bytes": 140754944, + "status": "budget-exceeded" + } + } + ], + "validation_errors": [ + "PdfTool identity commit does not match checkout HEAD", + "run 1: identity.commit '2d95b4a6019504d4e1ffbf658a0e078ab38e34df' does not match candidate '6e65be48e261d14c64653694320c0185b84fc560'" + ], + "regressions": [], + "status": "failed", + "required": true + } + ], + "summary": { + "total": 6, + "measured": 0, + "flagged": 2, + "skipped": 1, + "failed": 3, + "candidate_sha_verified": true + } +} diff --git a/scripts/qualification/test_validate_resource_envelope_evidence.py b/scripts/qualification/test_validate_resource_envelope_evidence.py new file mode 100644 index 00000000..7f07e7e7 --- /dev/null +++ b/scripts/qualification/test_validate_resource_envelope_evidence.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import unittest + +from scripts.qualification.validate_resource_envelope_evidence import validate_evidence + + +class ValidateResourceEnvelopeEvidenceTest(unittest.TestCase): + def test_incomplete_evidence_is_valid(self) -> None: + evidence = { + "schema_kind": "loop-resource-envelope-qualification-evidence", + "schema_version": 1, + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "disposition": "incomplete", + "fixture_status": { + "office-2mb": {"status": "failed", "preflight_high_water_bytes": -1}, + "image-heavy-500mb": {"status": "unavailable"}, + "ten-thousand-page": {"status": "unavailable"}, + "pathological-vector": {"status": "failed", "preflight_high_water_bytes": -1}, + "transparency-spots": {"status": "failed", "preflight_high_water_bytes": -1}, + }, + "matrix_runs": [{"candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", "summary": {}}], + "verifiers": [{"command": "test", "result": "pass"}], + } + self.assertEqual(validate_evidence(evidence), []) + + def test_passed_disposition_is_rejected(self) -> None: + evidence = { + "schema_kind": "loop-resource-envelope-qualification-evidence", + "schema_version": 1, + "candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", + "disposition": "passed", + "fixture_status": { + "office-2mb": {"status": "unavailable"}, + "image-heavy-500mb": {"status": "unavailable"}, + "ten-thousand-page": {"status": "unavailable"}, + "pathological-vector": {"status": "unavailable"}, + "transparency-spots": {"status": "unavailable"}, + }, + "matrix_runs": [{"candidate_sha": "6e65be48e261d14c64653694320c0185b84fc560", "summary": {}}], + "verifiers": [{"command": "test", "result": "pass"}], + } + self.assertIn("must not claim passed", "\n".join(validate_evidence(evidence))) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/qualification/validate_resource_envelope_evidence.py b/scripts/qualification/validate_resource_envelope_evidence.py new file mode 100644 index 00000000..9f1a9478 --- /dev/null +++ b/scripts/qualification/validate_resource_envelope_evidence.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Validate frozen Session 11 resource-envelope qualification evidence.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + + +DEFAULT_EVIDENCE = ROOT / "docs" / "evidence" / "session-11-resource-envelope" / "evidence.json" +DEFAULT_MANIFEST = ROOT / "docs" / "evidence" / "session-11-resource-envelope" / "fixture-manifest.json" +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +REQUIRED_FIXTURES = tuple( + fixture_id for fixture_id, spec in FIXTURE_SPECS.items() if spec["required"] +) + + +def validate_evidence( + evidence: dict[str, Any], + *, + manifest: dict[str, Any] | None = None, +) -> list[str]: + errors: list[str] = [] + if evidence.get("schema_kind") != "loop-resource-envelope-qualification-evidence": + errors.append("schema_kind must be loop-resource-envelope-qualification-evidence") + if evidence.get("schema_version") != 1: + errors.append("schema_version must be 1") + + candidate_sha = evidence.get("candidate_sha") + if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): + errors.append("candidate_sha must be a 40-character lowercase commit SHA") + + disposition = evidence.get("disposition") + if disposition not in {"incomplete", "passed", "rejected"}: + errors.append("disposition must be incomplete, passed, or rejected") + if disposition == "passed": + errors.append("Session 11 evidence must not claim passed until hosted matrix is complete") + + fixture_status = evidence.get("fixture_status") + if not isinstance(fixture_status, dict): + errors.append("fixture_status must be an object") + return errors + + for fixture_id in REQUIRED_FIXTURES: + entry = fixture_status.get(fixture_id) + if not isinstance(entry, dict): + errors.append(f"fixture_status missing required fixture: {fixture_id}") + continue + status = entry.get("status") + if status not in {"measured", "flagged", "failed", "unavailable", "incomplete"}: + errors.append(f"fixture_status.{fixture_id}.status is invalid") + if status in {"measured", "passed"} and entry.get("preflight_high_water_bytes") == -1: + errors.append(f"fixture_status.{fixture_id} cannot pass with unavailable preflight") + + runs = evidence.get("matrix_runs") + if not isinstance(runs, list) or not runs: + errors.append("matrix_runs must be a non-empty array") + else: + for index, run in enumerate(runs): + if not isinstance(run, dict): + errors.append(f"matrix_runs[{index}] must be an object") + continue + run_sha = run.get("candidate_sha") + if isinstance(candidate_sha, str) and run_sha != candidate_sha: + errors.append(f"matrix_runs[{index}].candidate_sha must equal candidate_sha") + + if manifest is not None: + records = manifest.get("fixtures") + if not isinstance(records, list): + errors.append("fixture manifest fixtures must be an array") + + verifiers = evidence.get("verifiers") + if not isinstance(verifiers, list) or not verifiers: + errors.append("verifiers must be a non-empty array") + + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--skip-manifest", action="store_true") + args = parser.parse_args(argv) + + try: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + manifest = None if args.skip_manifest or not args.manifest.is_file() else json.loads(args.manifest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"resource-envelope evidence validation error: {exc}", file=sys.stderr) + return 2 + + errors = validate_evidence(evidence, manifest=manifest) + if errors: + for error in errors: + print(f"resource-envelope-evidence: {error}", file=sys.stderr) + return 1 + + print(json.dumps({"status": "valid", "candidate_sha": evidence.get("candidate_sha")}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resource_envelope/create_fixture_manifest.py b/scripts/resource_envelope/create_fixture_manifest.py index b3b8de67..a7c05da7 100644 --- a/scripts/resource_envelope/create_fixture_manifest.py +++ b/scripts/resource_envelope/create_fixture_manifest.py @@ -9,6 +9,10 @@ from pathlib import Path from typing import Sequence +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + from scripts.resource_envelope.run_matrix import FIXTURE_SPECS, _fixture_args, _sha256