diff --git a/.claude/policy-brief.md b/.claude/policy-brief.md index 9e7fc7410..d34ed5510 100644 --- a/.claude/policy-brief.md +++ b/.claude/policy-brief.md @@ -62,7 +62,7 @@ Every agent-authored or materially agent-modified diff, as a final pass before t - Remove generated boilerplate, needless wrappers and abstractions, and other local-style drift. Preserve required validation, security, cancellation, provenance, and failure handling. -Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence anti-slop summary in the handoff. +Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence quality summary in the handoff. The pass is review judgment on the diff; do not add brittle automated style metrics. Canonical policy: Berry Studio BSP-002 §4.2 and §4.3 (https://app.notion.com/p/3b39cb079ddb811d8527ed129c3ac511). diff --git a/.cursor/agent-policy.md b/.cursor/agent-policy.md index 5bc5d2b83..2aba0ef90 100644 --- a/.cursor/agent-policy.md +++ b/.cursor/agent-policy.md @@ -62,7 +62,7 @@ Every agent-authored or materially agent-modified diff, as a final pass before t - Remove generated boilerplate, needless wrappers and abstractions, and other local-style drift. Preserve required validation, security, cancellation, provenance, and failure handling. -Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence anti-slop summary in the handoff. +Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence quality summary in the handoff. The pass is review judgment on the diff; do not add brittle automated style metrics. Canonical policy: Berry Studio BSP-002 §4.2 and §4.3 (https://app.notion.com/p/3b39cb079ddb811d8527ed129c3ac511). diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 186840154..5d7ca7f5b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -22,7 +22,7 @@ - [ ] Invalid state stops before partial mutation or publication and returns a descriptive error or result - [ ] Names carry the domain intent, and comments explain rationale rather than restating the code -## Anti-slop pass +## Quality pass - [ ] Redundant or explanatory comments that do not match the file's style removed - [ ] Abnormal defensive checks and broad try/catch blocks removed where a trusted upstream boundary already guarantees the invariant, with real boundary and safety checks kept @@ -31,7 +31,7 @@ - [ ] Generated boilerplate, needless wrappers, and local-style drift removed - [ ] Validation, security, cancellation, provenance, and failure handling preserved -Anti-slop summary (1-3 sentences): +Quality summary (1-3 sentences): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c40305b..35430facd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,8 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Test release assets against paired package evidence run: python -m unittest scripts.ci.test_verify_release_assets -v + - name: Test Core qualification provenance + run: python -m unittest scripts.qualification.test_check_core_qualification scripts.ci.test_compare_package_boundary_evidence -v - name: Test package lifecycle scripts with fake packages run: python -m unittest scripts.ci.test_run_qt_relink_test -v - name: Test Linux AppImage Qt relink script with fake AppImage diff --git a/.github/workflows/issue-promotion.yml b/.github/workflows/issue-promotion.yml index ce2059661..78b785ee3 100644 --- a/.github/workflows/issue-promotion.yml +++ b/.github/workflows/issue-promotion.yml @@ -1,13 +1,15 @@ -name: Track issue promotion +name: Track issue promotion stage on: push: branches: - dev + - unstable - stable concurrency: - group: issue-promotion-${{ github.ref }} + group: loop-issue-promotion + queue: max cancel-in-progress: false permissions: diff --git a/.github/workflows/resource-envelope-qualification.yml b/.github/workflows/resource-envelope-qualification.yml new file mode 100644 index 000000000..793fc5fd2 --- /dev/null +++ b/.github/workflows/resource-envelope-qualification.yml @@ -0,0 +1,302 @@ +name: Resource envelope qualification + +# Hosted qualification for issue #19: builds PdfTool from the candidate SHA, +# generates the deterministic synthetic fixture bundle, and runs the strict +# resource-envelope matrix (measured fixtures, cancellation/recovery probe, +# hostile corpus) on Linux and Windows. The evidence job turns both matrices +# into one record carrying this run's id. + +on: + workflow_dispatch: + pull_request: + paths: + - 'scripts/resource_envelope/**' + - 'scripts/qualification/*resource_envelope*' + - 'PdfTool/main.cpp' + - 'PdfTool/pdftoolrender.*' + - 'LoopLibCore/sources/pdfworkloadenvelope.*' + - 'LoopLibCore/sources/pdfresourcebudget.*' + - 'LoopLibCore/sources/pdfrenderer.*' + - 'docs/RESOURCE_ENVELOPE_BUDGETS.json' + - '.github/workflows/resource-envelope-qualification.yml' + +permissions: + contents: read + +concurrency: + group: resource-envelope-${{ github.ref }} + cancel-in-progress: true + +env: + REPETITIONS: 3 + CANCEL_FIXTURE: ten-thousand-page + CANCEL_AFTER_SECONDS: 3 + TIMEOUT_SECONDS: 1800 + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}/loop/vcpkg/overlays/linux:${{ github.workspace }}/loop/vcpkg/overlays/general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Install Dependencies + run: | + mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE" + sudo apt update + sudo apt install -y autoconf autoconf-archive automake libtool libcups2 libcups2-dev libfontconfig1-dev + + - name: 'VCPKG: Set up VCPKG' + run: | + VCPKG_COMMIT="$(python3 -c 'import json; print(json.load(open("loop/vcpkg-configuration.json"))["default-registry"]["baseline"])')" + if ! [[ "$VCPKG_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::Invalid vcpkg baseline: $VCPKG_COMMIT" + exit 1 + fi + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach "$VCPKG_COMMIT" + test "$(git -C vcpkg rev-parse HEAD)" = "$VCPKG_COMMIT" + ./vcpkg/bootstrap-vcpkg.sh + ./vcpkg/vcpkg integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg_installed + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + run: | + ./vcpkg install --x-manifest-root=$GITHUB_WORKSPACE/loop --x-install-root=$VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'linux' + target: 'desktop' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + + - name: Build PdfTool + working-directory: loop + run: | + cmake -B build -S . -DLOOP_LOOP_DISTRIBUTION=ON -DLOOP_INSTALL_QT_DEPENDENCIES=0 -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release + cmake --build build --target PdfTool --config Release -j6 + PDF_TOOL="$(find "$PWD/build" -type f -name PdfTool -perm -u+x | head -n 1)" + test -n "$PDF_TOOL" + echo "PDF_TOOL=$PDF_TOOL" >> "$GITHUB_ENV" + "$PDF_TOOL" help --console-format json > /dev/null + + - name: Generate synthetic fixture bundle + working-directory: loop + run: python3 scripts/resource_envelope/synthetic_workload.py --output-dir "$RUNNER_TEMP/fixtures" --manifest "$RUNNER_TEMP/fixtures/fixtures.json" + + - name: Run strict resource-envelope matrix + working-directory: loop + run: | + mkdir -p "$RUNNER_TEMP/qualification" + cp "$RUNNER_TEMP/fixtures/fixtures.json" "$RUNNER_TEMP/qualification/fixtures.json" + python3 -m scripts.resource_envelope.run_matrix \ + --pdf-tool "$PDF_TOOL" \ + --manifest "$RUNNER_TEMP/fixtures/fixtures.json" \ + --repetitions "$REPETITIONS" --timeout-seconds "$TIMEOUT_SECONDS" \ + --cancel-fixture "$CANCEL_FIXTURE" --cancel-after-seconds "$CANCEL_AFTER_SECONDS" \ + --strict \ + --output "$RUNNER_TEMP/qualification/matrix-linux.json" + + - name: Upload Linux matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-linux + path: ${{ runner.temp }}/qualification/ + if-no-files-found: warn + + windows: + runs-on: windows-2022 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}\loop\vcpkg\overlays\general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}\vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}\vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Exclude workspace from Windows Defender real-time scanning + shell: pwsh + run: | + Add-MpPreference -ExclusionPath "${env:GITHUB_WORKSPACE}" -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $env:VCPKG_DEFAULT_BINARY_CACHE | Out-Null + + - name: 'VCPKG: Set up VCPKG' + shell: pwsh + run: | + $config = Get-Content (Join-Path $env:GITHUB_WORKSPACE "loop\vcpkg-configuration.json") -Raw | ConvertFrom-Json + $vcpkgCommit = $config.'default-registry'.baseline + if ($vcpkgCommit -notmatch '^[0-9a-f]{40}$') { throw "Invalid vcpkg baseline: $vcpkgCommit" } + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach $vcpkgCommit + if ((git -C vcpkg rev-parse HEAD).Trim() -ne $vcpkgCommit) { throw "vcpkg checkout does not match manifest baseline" } + .\vcpkg\bootstrap-vcpkg.bat -disableMetrics + .\vcpkg\vcpkg.exe integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg/installed + ./vcpkg/archives + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + shell: pwsh + run: | + $manifestRoot = Join-Path $env:GITHUB_WORKSPACE 'loop' + .\vcpkg.exe install --triplet x64-windows --x-manifest-root=$manifestRoot --x-install-root=$env:VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + if ($LASTEXITCODE -ne 0) { throw "vcpkg install failed with exit code $LASTEXITCODE." } + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'windows' + target: 'desktop' + arch: 'win64_msvc2022_64' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + # Same aqtinstall pin as reusable-windows.yml (miurahr/aqtinstall#1007). + aqtsource: git+https://github.com/miurahr/aqtinstall.git@8c3695d4a4e1ceabf6a74dc6c79681656dc6b74b + + - name: Build PdfTool + working-directory: loop + shell: pwsh + run: | + cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release -DLOOP_LOOP_DISTRIBUTION=OFF -DLOOP_INSTALL_QT_DEPENDENCIES=OFF -DCMAKE_TOOLCHAIN_FILE="${env:GITHUB_WORKSPACE}\vcpkg\scripts\buildsystems\vcpkg.cmake" -DLOOP_QT_ROOT="${env:QT_ROOT_DIR}" + if ($LASTEXITCODE -ne 0) { throw "cmake configure failed with exit code $LASTEXITCODE." } + cmake --build build --target PdfTool --config Release -j6 + if ($LASTEXITCODE -ne 0) { throw "cmake --build PdfTool failed with exit code $LASTEXITCODE." } + $pdfTool = Get-ChildItem build -Recurse -Filter PdfTool.exe | Select-Object -First 1 + if (-not $pdfTool) { throw "PdfTool.exe was not produced." } + "PDF_TOOL=$($pdfTool.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "QT_PLUGIN_PATH=$(Join-Path $env:QT_ROOT_DIR 'plugins')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "$(Join-Path $env:QT_ROOT_DIR 'bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + "$(Join-Path $env:VCPKG_INSTALLED_DIR 'x64-windows\bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + + - name: Smoke PdfTool runtime + working-directory: loop + shell: pwsh + run: | + & $env:PDF_TOOL help --console-format json | Out-Null + if ($LASTEXITCODE -ne 0) { throw "PdfTool help failed with exit code $LASTEXITCODE." } + + - name: Generate synthetic fixture bundle + working-directory: loop + shell: pwsh + run: | + python scripts/resource_envelope/synthetic_workload.py --output-dir "$env:RUNNER_TEMP\fixtures" --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" + if ($LASTEXITCODE -ne 0) { throw "fixture generation failed with exit code $LASTEXITCODE." } + + - name: Run strict resource-envelope matrix + working-directory: loop + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:RUNNER_TEMP\qualification" | Out-Null + python -m scripts.resource_envelope.run_matrix ` + --pdf-tool "$env:PDF_TOOL" ` + --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" ` + --repetitions $env:REPETITIONS --timeout-seconds $env:TIMEOUT_SECONDS ` + --cancel-fixture $env:CANCEL_FIXTURE --cancel-after-seconds $env:CANCEL_AFTER_SECONDS ` + --strict ` + --output "$env:RUNNER_TEMP\qualification\matrix-windows.json" + if ($LASTEXITCODE -ne 0) { throw "strict resource-envelope matrix failed with exit code $LASTEXITCODE." } + + - name: Upload Windows matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-windows + path: ${{ runner.temp }}\qualification\ + if-no-files-found: warn + + evidence: + needs: [linux, windows] + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + + - name: Download matrices + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: resource-envelope-matrix-* + path: ${{ runner.temp }}/matrices + + - name: Build and validate qualification evidence + working-directory: loop + run: | + set -euo pipefail + MATRICES=() + for platform in linux windows; do + matrix="$RUNNER_TEMP/matrices/resource-envelope-matrix-$platform/matrix-$platform.json" + if [ -f "$matrix" ]; then MATRICES+=(--matrix "$platform=$matrix"); fi + done + manifest="$(find "$RUNNER_TEMP/matrices" -name fixtures.json | head -n 1)" + test -n "$manifest" || { echo "::error::no fixture manifest was uploaded"; exit 1; } + python3 scripts/qualification/build_resource_envelope_evidence.py \ + "${MATRICES[@]}" \ + --fixture-manifest "$manifest" \ + --run-id "$GITHUB_RUN_ID" \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + --output "$RUNNER_TEMP/evidence/evidence.json" + python3 scripts/qualification/validate_resource_envelope_evidence.py --evidence "$RUNNER_TEMP/evidence/evidence.json" --skip-manifest + cp "$manifest" "$RUNNER_TEMP/evidence/fixture-manifest.json" + python3 -c 'import json,sys; e=json.load(open(sys.argv[1])); print("disposition:", e["disposition"]); [print(" -", r) for r in e["disposition_reasons"]]; sys.exit(0 if e["disposition"] == "passed" else 1)' "$RUNNER_TEMP/evidence/evidence.json" + + - name: Upload qualification evidence + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-evidence + path: ${{ runner.temp }}/evidence/ + if-no-files-found: warn diff --git a/.github/workflows/reusable-linux.yml b/.github/workflows/reusable-linux.yml index 2fb039748..58a30314a 100644 --- a/.github/workflows/reusable-linux.yml +++ b/.github/workflows/reusable-linux.yml @@ -78,7 +78,7 @@ jobs: - name: Verify resource-envelope contracts working-directory: loop run: | - python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python3 scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python3 scripts/qualification/validate_resource_envelope_evidence.py python3 scripts/resource_envelope/pathological_workload.py \ @@ -284,7 +284,13 @@ jobs: env: FAST_MODE: ${{ inputs.fast }} run: | - cmake -B build -S . -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLOOP_LOOP_DISTRIBUTION=ON -DLOOP_BUILD_QUICK_SHELL_SMOKE=ON -DLOOP_BUILD_CANVAS_BENCHMARK=ON -DLOOP_BUILD_QUICK_CANVAS=ON -DLOOP_BUILD_PRODUCT_QUICK_A11Y_SMOKE=ON -DLOOP_INSTALL_QT_DEPENDENCIES=0 -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release + # The agent-fast proof builds and lints every changed target, so it needs the + # Widgets developer tools configured; the full release-profile build keeps them off. + TOOL_TARGETS="" + if [ "${FAST_MODE}" = "true" ]; then + TOOL_TARGETS="-DLOOP_BUILD_CODE_GENERATOR=ON -DLOOP_BUILD_JBIG2_VIEWER=ON -DLOOP_BUILD_EXAMPLE_GENERATOR=ON" + fi + cmake -B build -S . ${TOOL_TARGETS} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DLOOP_LOOP_DISTRIBUTION=ON -DLOOP_BUILD_QUICK_SHELL_SMOKE=ON -DLOOP_BUILD_CANVAS_BENCHMARK=ON -DLOOP_BUILD_QUICK_CANVAS=ON -DLOOP_BUILD_PRODUCT_QUICK_A11Y_SMOKE=ON -DLOOP_INSTALL_QT_DEPENDENCIES=0 -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release if [ "${FAST_MODE}" != "true" ]; then cmake --build build --target all release_translations --config Release -j6 cmake --install build diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index b0d9c1414..938f7256c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -82,7 +82,7 @@ jobs: working-directory: loop shell: pwsh run: | - python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python scripts/qualification/validate_resource_envelope_evidence.py python scripts/resource_envelope/pathological_workload.py ` diff --git a/AGENTS.md b/AGENTS.md index cfb0d0d95..f2109f846 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ Every agent-authored or materially agent-modified diff, as a final pass before t - Remove generated boilerplate, needless wrappers and abstractions, and other local-style drift. Preserve required validation, security, cancellation, provenance, and failure handling. -Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence anti-slop summary in the handoff. +Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence quality summary in the handoff. The pass is review judgment on the diff; do not add brittle automated style metrics. Canonical policy: Berry Studio BSP-002 §4.2 and §4.3 (https://app.notion.com/p/3b39cb079ddb811d8527ed129c3ac511). diff --git a/CMakeLists.txt b/CMakeLists.txt index faa3068fe..ef2a53204 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,27 +319,27 @@ if(NOT LOOP_BUILD_ONLY_CORE_LIBRARY) endif() if(LOOP_BUILD_CODE_GENERATOR) - add_subdirectory(CodeGenerator) + add_subdirectory(tools/CodeGenerator) endif() if(LOOP_BUILD_JBIG2_VIEWER) - add_subdirectory(JBIG2_Viewer) + add_subdirectory(tools/JBIG2_Viewer) endif() if(LOOP_BUILD_EXAMPLE_GENERATOR) - add_subdirectory(PdfExampleGenerator) + add_subdirectory(tools/PdfExampleGenerator) endif() add_subdirectory(PdfTool) add_subdirectory(loop-preflight/tools) add_subdirectory(loop-ocr/tools) if(LOOP_BUILD_QUICK_SHELL_SMOKE) - add_subdirectory(QuickShellSmoke) + add_subdirectory(tools/QuickShellSmoke) endif() if(LOOP_BUILD_CANVAS_BENCHMARK) - add_subdirectory(CanvasBenchmark) + add_subdirectory(tools/CanvasBenchmark) endif() if(LOOP_BUILD_QUICK_CANVAS) add_subdirectory(LoopEditor) if(LOOP_BUILD_PRODUCT_QUICK_A11Y_SMOKE) - add_subdirectory(ProductQuickAccessibilitySmoke) + add_subdirectory(tools/ProductQuickAccessibilitySmoke) endif() endif() add_subdirectory(WixInstaller) diff --git a/LoopEditor/editorhost.cpp b/LoopEditor/editorhost.cpp index 4bf452777..c557836ca 100644 --- a/LoopEditor/editorhost.cpp +++ b/LoopEditor/editorhost.cpp @@ -292,6 +292,10 @@ QVariantMap descriptorToVariant(const pdfinteraction::CommandDescriptor& descrip struct EditorHost::PreflightWorkerOutcome { pdf::PreflightResult result; + QString effectiveProfileDigest; + QString documentPath; + QByteArray auditBytes; + QJsonObject auditSummary; }; EditorHost::EditorHost(QObject* parent) : @@ -1449,6 +1453,7 @@ bool EditorHost::runPreflight() profile.effectiveDigest = pdf::computeProfileDigest(bound.profile); profile.profileIdentity = imported.identity.toJson(); profile.profileIdentity.insert(QStringLiteral("effective_digest"), profile.effectiveDigest); + outcome->effectiveProfileDigest = profile.effectiveDigest; context.reportProgress(5); std::unique_ptr session( @@ -1481,26 +1486,9 @@ bool EditorHost::runPreflight() context.reportProgress(15); outcome->result = engine.run(profile); pdf::finalizePreflightResult(outcome->result, revisionHash, resolved); - - pdf::PDFOperationHistoryStatus auditStatus = pdf::PDFOperationHistoryStatus::Accepted; - if (context.isCancellationRequested()) - auditStatus = pdf::PDFOperationHistoryStatus::Cancelled; - else if (pdf::reducePreflightVerdict(outcome->result).state == pdf::PreflightVerdictState::Error) - auditStatus = pdf::PDFOperationHistoryStatus::Failed; - - const QJsonObject auditSummary = - pdf::preflightAuditReportSummary(outcome->result, documentPath); - if (const pdf::PDFOperationResult auditResult = - pdf::appendPreflightAuditRun(documentPath, - auditBytes, - outcome->result, - auditStatus, - QStringLiteral("LoopEditor"), - auditSummary); - !auditResult) - { - throw std::runtime_error(auditResult.getErrorMessage().toStdString()); - } + outcome->documentPath = documentPath; + outcome->auditBytes = std::move(auditBytes); + outcome->auditSummary = pdf::preflightAuditReportSummary(outcome->result, documentPath); if (context.isCancellationRequested()) return; @@ -2476,19 +2464,23 @@ bool EditorHost::requestFixRollback(const QString& rollbackId) return false; } + // The restored revision and its rolled-back event are already published, so a retention + // failure is reported alongside them rather than hiding the new revision. const pdf::PDFHistoryRetentionResult retention = history.enforceRetention({}, artifacts); - if (!retention.success) + const QString revision = point->documentRevisionDigest.left(12); + const QString destinationName = QFileInfo(destination).fileName(); + if (retention.success) { - announceDocumentState(tr("The revision was restored, but history retention could not be enforced: %1") - .arg(retention.errorMessage)); - return false; + announceDocumentState(tr("Returned to revision %1 as %2.").arg(revision, destinationName)); + } + else + { + announceDocumentState(tr("Returned to revision %1 as %2, but history retention could not be enforced: %3") + .arg(revision, destinationName, retention.errorMessage)); } - - announceDocumentState(tr("Returned to revision %1 as %2.") - .arg(point->documentRevisionDigest.left(12), QFileInfo(destination).fileName())); openFileUrl(QUrl::fromLocalFile(destination)); refreshFixRollbackPoints(); - return true; + return retention.success; } void EditorHost::discardActionListPlan() @@ -3209,11 +3201,47 @@ void EditorHost::finishPreflightJob(const pdf::PDFJobSnapshot& snapshot) return; } + const auto profile = std::find_if(m_preflightProfiles.cbegin(), m_preflightProfiles.cend(), + [this](const PreflightProfileChoice& choice) + { return choice.id == m_selectedPreflightProfileId; }); + if (!hasDocument() || !m_session->revisionSource() || snapshot.kind != pdf::PDFJobKind::Preflight || + snapshot.documentKey != m_preflight.documentKey() || + snapshot.documentKey != m_session->revisionSource()->documentKey() || + snapshot.documentRevision != m_preflight.documentRevision() || + snapshot.documentRevision != m_session->facade().currentRevision().toString() || + profile == m_preflightProfiles.cend() || !profile->valid || + profile->digest != m_preflight.profileDigest() || + snapshot.operationId != QStringLiteral("preflight.%1").arg(profile->id)) + { + m_preflight.markProfileStale(); + return; + } + switch (snapshot.status) { case pdf::PDFJobStatus::Succeeded: if (outcome) { + if (outcome->effectiveProfileDigest.isEmpty() || + outcome->result.effectiveProfileDigest != outcome->effectiveProfileDigest || + outcome->documentPath != m_session->facade().source().path) + { + m_preflight.failRun(snapshot.jobId, snapshot.documentRevision, + tr("Preflight result identity did not match the request.")); + break; + } + const pdf::PDFOperationHistoryStatus auditStatus = + pdf::reducePreflightVerdict(outcome->result).state == pdf::PreflightVerdictState::Error + ? pdf::PDFOperationHistoryStatus::Failed + : pdf::PDFOperationHistoryStatus::Accepted; + const pdf::PDFOperationResult auditResult = pdf::appendPreflightAuditRun( + outcome->documentPath, outcome->auditBytes, outcome->result, auditStatus, + QStringLiteral("LoopEditor"), outcome->auditSummary); + if (!auditResult) + { + m_preflight.failRun(snapshot.jobId, snapshot.documentRevision, auditResult.getErrorMessage()); + break; + } acceptPreflightResult(snapshot.jobId, snapshot.documentRevision, outcome->result); } else @@ -3254,6 +3282,21 @@ void EditorHost::finishActionListJob(const pdf::PDFJobSnapshot& snapshot) return; } + const pdfinteraction::ActionListRecipeEntry* recipe = m_actionListCatalog.recipe(m_selectedActionListRecipeId); + if (!hasDocument() || !m_session->revisionSource() || snapshot.kind != pdf::PDFJobKind::Other || + snapshot.documentKey != m_actionListController.documentKey() || + snapshot.documentKey != m_session->revisionSource()->documentKey() || + snapshot.documentRevision != m_actionListController.documentRevision() || + snapshot.documentRevision != m_session->facade().currentRevision().toString() || + m_selectedActionListRecipeId != m_actionListController.recipeId() || + !recipe || !recipe->valid || + snapshot.operationId != QStringLiteral("action-list.%1").arg(recipe->actionList.id) || + snapshot.checkId != recipe->actionList.name) + { + m_actionListController.markRecipeStale(); + return; + } + switch (snapshot.status) { case pdf::PDFJobStatus::Succeeded: @@ -3263,6 +3306,13 @@ void EditorHost::finishActionListJob(const pdf::PDFJobSnapshot& snapshot) tr("Action List result was unavailable.")); break; } + if (state != pdfinteraction::ActionListController::State::Validating && + outcome->executionResult.recipeHash != recipe->recipeHash) + { + m_actionListController.failRun(snapshot.jobId, snapshot.documentRevision, + tr("Action List result did not match the recipe.")); + break; + } if (state == pdfinteraction::ActionListController::State::Validating) { if (m_acceptActionListResults) @@ -3286,9 +3336,14 @@ void EditorHost::finishActionListJob(const pdf::PDFJobSnapshot& snapshot) } else if (state == pdfinteraction::ActionListController::State::Running) { + if (!outcome->candidate) + { + m_actionListController.failRun(snapshot.jobId, snapshot.documentRevision, + tr("Action List produced no document.")); + break; + } if (m_acceptActionListResults && - m_actionListController.acceptExecution(snapshot.jobId, snapshot.documentRevision, outcome->executionResult) && - outcome->candidate) + m_actionListController.acceptExecution(snapshot.jobId, snapshot.documentRevision, outcome->executionResult)) { m_session->context().setDocument(outcome->candidate); m_preflight.markProfileStale(); diff --git a/LoopLibCore/CMakeLists.txt b/LoopLibCore/CMakeLists.txt index 496f46a46..6a5d44998 100644 --- a/LoopLibCore/CMakeLists.txt +++ b/LoopLibCore/CMakeLists.txt @@ -118,6 +118,7 @@ add_library(LoopLibCore SHARED sources/preflightengine.h sources/preflightclirun.cpp sources/preflightclirun.h + sources/pdfpreflightreceipt.cpp sources/pdfpreflightverdict.cpp sources/pdfpreflightverdict.h sources/pdfpreflightaudit.cpp diff --git a/LoopLibCore/sources/pdfcolorinventory.cpp b/LoopLibCore/sources/pdfcolorinventory.cpp index 55c1c4857..9d8ec01f1 100644 --- a/LoopLibCore/sources/pdfcolorinventory.cpp +++ b/LoopLibCore/sources/pdfcolorinventory.cpp @@ -125,6 +125,17 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin for (PDFInteger pageIndex = 0; pageIndex < pageCount; ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + result.cancelled = true; + break; + } + + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { @@ -139,7 +150,12 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin continue; } - const QSize imageSize(qMax(1, int(widthPxReal)), qMax(1, int(heightPxReal))); + // The probe holds several float bitmaps per pixel, so an oversized page is probed + // at a coarser resolution instead of allocating memory proportional to its size. + const double probeScale = settings.maxProbePixels > 0 && widthPxReal * heightPxReal > double(settings.maxProbePixels) + ? std::sqrt(double(settings.maxProbePixels) / (widthPxReal * heightPxReal)) + : 1.0; + const QSize imageSize(qMax(1, int(widthPxReal * probeScale)), qMax(1, int(heightPxReal * probeScale))); const QTransform pagePointToDevice = PDFRenderer::createPagePointToDevicePointMatrix( page, QRect(QPoint(0, 0), imageSize)); PDFTransparencyRenderer renderer(page, diff --git a/LoopLibCore/sources/pdfcolorinventory.h b/LoopLibCore/sources/pdfcolorinventory.h index a9f08c786..d18902e27 100644 --- a/LoopLibCore/sources/pdfcolorinventory.h +++ b/LoopLibCore/sources/pdfcolorinventory.h @@ -24,12 +24,16 @@ #define PDFCOLORINVENTORY_H #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include "pdftransparencyrenderer.h" #include #include +#include #include +#include + namespace pdf { @@ -59,18 +63,27 @@ struct LOOPLIBCORESHARED_EXPORT PDFColorInventoryResult QList spotColors; QList richBlackPages; PDFRenderDiagnostics diagnostics; + /// True when the inspection stopped early because the operation was cancelled; + /// the lists above then cover only the pages probed before the stop. + bool cancelled = false; }; struct LOOPLIBCORESHARED_EXPORT PDFColorInventorySettings { int probeDpi = 150; qreal richBlackKThreshold = 0.10; + /// Largest probe raster in pixels; larger pages are probed at a proportionally lower DPI. + /// A letter page at the default 150 DPI is about 1.9 million pixels. + qint64 maxProbePixels = 2'500'000; + const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages to probe; unset probes every page. + std::optional> pageIndices; }; /// Shared rich-black predicate used by preflight and Output Preview. LOOPLIBCORESHARED_EXPORT bool isRichBlackPixel(PDFConstColorBuffer buffer, - const PDFPixelFormat& format, - PDFColorComponent kThreshold); + const PDFPixelFormat& format, + PDFColorComponent kThreshold); class LOOPLIBCORESHARED_EXPORT PDFColorInventory { diff --git a/LoopLibCore/sources/pdfdiff.cpp b/LoopLibCore/sources/pdfdiff.cpp index 8d9c0b9f3..cc8d4ce0d 100644 --- a/LoopLibCore/sources/pdfdiff.cpp +++ b/LoopLibCore/sources/pdfdiff.cpp @@ -170,7 +170,15 @@ void PDFDiff::start() { return; } - onComparationPerformed(snapshot.status == pdf::PDFJobStatus::Cancelled); + if (snapshot.status != pdf::PDFJobStatus::Succeeded) + { + m_result = PDFDiffResult(); + m_result.setResult(pdf::PDFOperationResult( + snapshot.errorMessage.isEmpty() + ? QStringLiteral("Comparison job did not complete.") + : snapshot.errorMessage)); + } + onComparationPerformed(snapshot.status != pdf::PDFJobStatus::Succeeded); }); } else @@ -192,6 +200,8 @@ void PDFDiff::stop() m_cancelled = true; pdf::PDFJobScheduler::global().cancel(jobId); pdf::PDFJobScheduler::global().waitForFinished(jobId); + m_result = PDFDiffResult(); + m_result.setResult(pdf::PDFOperationResult(QStringLiteral("Comparison cancelled."))); m_activeJobId.clear(); if (m_jobFinishedConnection) { diff --git a/LoopLibCore/sources/pdfdocumentsession.cpp b/LoopLibCore/sources/pdfdocumentsession.cpp index 14ec58cbe..3fcf27c1d 100644 --- a/LoopLibCore/sources/pdfdocumentsession.cpp +++ b/LoopLibCore/sources/pdfdocumentsession.cpp @@ -258,9 +258,14 @@ PDFDocumentSession* PDFDocumentSession::create(PDFDocument* document, return new PDFDocumentSession(document, context, std::move(pageCacheBudget)); } -PDFDocumentSession* PDFDocumentSession::createForInspection(PDFDocument* document) +PDFDocumentSession* PDFDocumentSession::createForInspection(PDFDocument* document, const QString& documentId) { - return new PDFDocumentSession(document, nullptr, nullptr, PDFDocumentSessionAdmission::Inspection); + auto* session = new PDFDocumentSession(document, nullptr, nullptr, PDFDocumentSessionAdmission::Inspection); + if (!documentId.isEmpty()) + { + session->m_localDocumentIdentity.documentId = documentId; + } + return session; } void PDFDocumentSession::destroy(PDFDocumentSession* session) noexcept diff --git a/LoopLibCore/sources/pdfdocumentsession.h b/LoopLibCore/sources/pdfdocumentsession.h index d896c2c13..aee154d6a 100644 --- a/LoopLibCore/sources/pdfdocumentsession.h +++ b/LoopLibCore/sources/pdfdocumentsession.h @@ -88,7 +88,7 @@ class LOOPLIBCORESHARED_EXPORT PDFDocumentSession static PDFDocumentSession* create(PDFDocument* document, PDFDocumentContext* context = nullptr, std::shared_ptr pageCacheBudget = nullptr); - static PDFDocumentSession* createForInspection(PDFDocument* document); + static PDFDocumentSession* createForInspection(PDFDocument* document, const QString& documentId = QString()); static void destroy(PDFDocumentSession* session) noexcept; /// Estimates the resident model owned by a parsed document, including raw diff --git a/LoopLibCore/sources/pdfevidencegraph.cpp b/LoopLibCore/sources/pdfevidencegraph.cpp index 5e2879bb2..a1cfe6772 100644 --- a/LoopLibCore/sources/pdfevidencegraph.cpp +++ b/LoopLibCore/sources/pdfevidencegraph.cpp @@ -159,6 +159,7 @@ namespace { constexpr int EVIDENCE_MAX_FORM_DEPTH = 32; +constexpr const char* EVIDENCE_CANCELLED_REASON = "cancelled"; PDFArtifactIdentity artifactIdentityFromDocument(const PDFDocument* document) { @@ -1288,8 +1289,16 @@ void collectColorants(PDFDocumentSession* session, PDFEvidenceGraph* graph, cons PDFColorInventorySettings inventorySettings; inventorySettings.probeDpi = settings.colorProbeDpi; inventorySettings.richBlackKThreshold = settings.richBlackKThreshold; + inventorySettings.operationControl = settings.operationControl; + inventorySettings.pageIndices = settings.pageIndices; PDFColorInventory inventory(session); const PDFColorInventoryResult result = inventory.inspect(inventorySettings); + if (result.cancelled) + { + graph->complete = false; + graph->incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return; + } if (!result.diagnostics.isExact()) { const QString diagnostic = result.diagnostics.reasons.join(QStringLiteral("; ")).isEmpty() @@ -1398,6 +1407,17 @@ PDFEvidenceGraph PDFEvidenceCollector::collect(PDFDocumentSession* session, const PDFCatalog* catalog = document->getCatalog(); for (PDFInteger pageIndex = 0; pageIndex < catalog->getPageCount(); ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + graph.complete = false; + graph.incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return graph; + } + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { diff --git a/LoopLibCore/sources/pdfevidencegraph.h b/LoopLibCore/sources/pdfevidencegraph.h index 98a0685f2..efc7df0b1 100644 --- a/LoopLibCore/sources/pdfevidencegraph.h +++ b/LoopLibCore/sources/pdfevidencegraph.h @@ -26,13 +26,17 @@ #include "pdfartifactidentity.h" #include "pdfdocumentcontext.h" #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include #include #include #include +#include #include +#include + namespace pdf { @@ -100,6 +104,12 @@ struct LOOPLIBCORESHARED_EXPORT PDFEvidenceCollectSettings qreal richBlackKThreshold = 0.10; qreal minEffectiveStrokeWidthPt = 0.0; qreal zeroWidthEpsilonPt = 1.0e-6; + /// Polled between pages; a cancelled collection returns an incomplete graph + /// with incompleteReason "cancelled". + const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages whose content is walked and probed; unset + /// covers every page. Document-level evidence is collected either way. + std::optional> pageIndices; }; class LOOPLIBCORESHARED_EXPORT PDFEvidenceCollector diff --git a/LoopLibCore/sources/pdfjobscheduler.cpp b/LoopLibCore/sources/pdfjobscheduler.cpp index f5b05ffc3..0db6cbd2f 100644 --- a/LoopLibCore/sources/pdfjobscheduler.cpp +++ b/LoopLibCore/sources/pdfjobscheduler.cpp @@ -163,6 +163,7 @@ void PDFJobContext::setOutputArtifact(PDFArtifactIdentity artifact) struct PDFJobScheduler::JobEntry { PDFJobSpec spec; + quint64 revisionEpoch = 0; PDFJobWork work; PDFJobCancellationTokenPtr cancellationToken; quint64 sequence = 0; @@ -275,6 +276,13 @@ QString PDFJobScheduler::submit(PDFJobSpec spec, } job->sequence = ++m_sequence; + const QString documentKey = resolvedDocumentKey(job->spec); + const auto revision = m_currentRevisions.find(documentKey); + if (!documentKey.isEmpty() && revision != m_currentRevisions.end() && + revision->second.revision == job->spec.documentRevision) + { + job->revisionEpoch = revision->second.epoch; + } job->queueDepth = static_cast(m_queue.size()); m_jobs.emplace(job->spec.jobId, job); m_queue.push(job); @@ -408,7 +416,16 @@ void PDFJobScheduler::setCurrentRevision(QString documentKey, QString documentRe return; } std::lock_guard lock(m_mutex); - m_currentRevisions[std::move(documentKey)] = std::move(documentRevision); + if (documentRevision.isEmpty()) + { + m_currentRevisions.erase(documentKey); + return; + } + auto& current = m_currentRevisions[std::move(documentKey)]; + if (current.revision != documentRevision) + { + current = CurrentRevision{ std::move(documentRevision), ++m_sequence }; + } } void PDFJobScheduler::clearCurrentRevision(const QString& documentKey) @@ -488,7 +505,12 @@ void PDFJobScheduler::workerLoop() Q_EMIT jobStarted(startedSnapshot); - if (isStale(job->spec) && job->spec.staleResultPolicy == PDFJobStaleResultPolicy::Discard) + bool staleBeforeWork = false; + { + std::lock_guard lock(m_mutex); + staleBeforeWork = isStaleLocked(*job); + } + if (staleBeforeWork && job->spec.staleResultPolicy == PDFJobStaleResultPolicy::Discard) { finishJob(job, PDFJobStatus::Stale, QStringLiteral("Document revision is no longer current.")); continue; @@ -542,10 +564,6 @@ void PDFJobScheduler::workerLoop() { finishJob(job, PDFJobStatus::Failed, std::move(errorMessage)); } - else if (isStale(job->spec) && job->spec.staleResultPolicy == PDFJobStaleResultPolicy::Discard) - { - finishJob(job, PDFJobStatus::Stale, QStringLiteral("Document revision changed while the job was running.")); - } else { finishJob(job, PDFJobStatus::Succeeded); @@ -565,6 +583,18 @@ void PDFJobScheduler::finishJob(const std::shared_ptr& job, return; } + if (status == PDFJobStatus::Succeeded && job->cancellationToken->isCancellationRequested()) + { + status = PDFJobStatus::Cancelled; + errorMessage = QStringLiteral("Cancellation requested during execution."); + } + else if (status == PDFJobStatus::Succeeded && + job->spec.staleResultPolicy == PDFJobStaleResultPolicy::Discard && isStaleLocked(*job)) + { + status = PDFJobStatus::Stale; + errorMessage = QStringLiteral("Document revision changed while the job was running."); + } + if (job->slotAcquired) { job->slotAcquired = false; @@ -575,6 +605,11 @@ void PDFJobScheduler::finishJob(const std::shared_ptr& job, } job->status = status; job->errorMessage = std::move(errorMessage); + if (status != PDFJobStatus::Succeeded) + { + job->resultSummary.clear(); + job->outputArtifact = {}; + } job->finishedAtUtc = QDateTime::currentDateTimeUtc(); if (job->startedAtUtc.isValid()) { @@ -660,16 +695,16 @@ void PDFJobScheduler::appendTrace(const std::shared_ptr& job, } } -bool PDFJobScheduler::isStale(const PDFJobSpec& spec) const +bool PDFJobScheduler::isStaleLocked(const JobEntry& job) const { - const QString key = resolvedDocumentKey(spec); + const QString key = resolvedDocumentKey(job.spec); if (key.isEmpty()) { return false; } - std::lock_guard lock(m_mutex); const auto it = m_currentRevisions.find(key); - return it != m_currentRevisions.end() && it->second != spec.documentRevision; + return it == m_currentRevisions.end() || job.revisionEpoch == 0 || + it->second.epoch != job.revisionEpoch || it->second.revision != job.spec.documentRevision; } PDFJobSnapshot PDFJobScheduler::snapshotLocked(const JobEntry& job) const diff --git a/LoopLibCore/sources/pdfjobscheduler.h b/LoopLibCore/sources/pdfjobscheduler.h index 12a6d9f5e..90aef18e7 100644 --- a/LoopLibCore/sources/pdfjobscheduler.h +++ b/LoopLibCore/sources/pdfjobscheduler.h @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -241,6 +240,11 @@ class LOOPLIBCORESHARED_EXPORT PDFJobScheduler final : public QObject private: struct JobEntry; + struct CurrentRevision + { + QString revision; + quint64 epoch = 0; + }; struct JobCompare { bool operator()(const std::shared_ptr& left, @@ -251,7 +255,7 @@ class LOOPLIBCORESHARED_EXPORT PDFJobScheduler final : public QObject void ensureWorkersStarted(); void finishJob(const std::shared_ptr& job, PDFJobStatus status, QString errorMessage = {}); void appendTrace(const std::shared_ptr& job, PDFJobStatus status, qint64 elapsedMs = 0); - bool isStale(const PDFJobSpec& spec) const; + bool isStaleLocked(const JobEntry& job) const; PDFJobSnapshot snapshotLocked(const JobEntry& job) const; static QString resolvedDocumentKey(const PDFJobSpec& spec); @@ -264,7 +268,7 @@ class LOOPLIBCORESHARED_EXPORT PDFJobScheduler final : public QObject int m_activeBackgroundJobs = 0; std::priority_queue, std::vector>, JobCompare> m_queue; std::unordered_map, PDFJobStringHash> m_jobs; - std::unordered_map m_currentRevisions; + std::unordered_map m_currentRevisions; std::unordered_map, PDFJobStringHash> m_traces; std::vector m_workers; std::once_flag m_workersOnce; diff --git a/LoopLibCore/sources/pdfpreflightreceipt.cpp b/LoopLibCore/sources/pdfpreflightreceipt.cpp new file mode 100644 index 000000000..bb118d5a2 --- /dev/null +++ b/LoopLibCore/sources/pdfpreflightreceipt.cpp @@ -0,0 +1,307 @@ +// MIT License +// +// Copyright (c) 2018-2025 Jakub Melka and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "pdfpreflightverdict.h" +#include "preflightprofileresolver.h" + +#include +#include +#include +#include + +namespace pdf +{ +namespace +{ +QString receiptIdentity(const PreflightInspectionReceipt& receipt) +{ + const QJsonObject identity{ + { QStringLiteral("kind"), QStringLiteral("loop.inspection-receipt-identity.v1") }, + { QStringLiteral("input_digest"), receipt.inputDigest }, + { QStringLiteral("effective_profile_digest"), receipt.effectiveProfileDigest }, + { QStringLiteral("coverage_scope"), receipt.coverageScope } + }; + return QString::fromLatin1(QCryptographicHash::hash(canonicalJson(identity), QCryptographicHash::Sha256).toHex()); +} + +bool strings(const QJsonValue& value, QStringList& output) +{ + if (!value.isArray()) + { + return false; + } + for (const QJsonValue item : value.toArray()) + { + if (!item.isString() || item.toString().isEmpty()) + { + return false; + } + output.append(item.toString()); + } + return true; +} + +bool counter(const QJsonValue& value, quint64& output) +{ + if (!value.isString()) + { + return false; + } + const QString text = value.toString(); + bool ok = false; + output = text.toULongLong(&ok); + return ok && text == QString::number(output); +} +} + +QJsonObject PreflightInspectionReceipt::toJson() const +{ + QJsonArray checkArray; + for (const PreflightReceiptCheck& check : checks) + { + checkArray.append(QJsonObject{ + { QStringLiteral("id"), check.id }, { QStringLiteral("required"), check.required }, { QStringLiteral("complete"), check.complete }, { QStringLiteral("status"), check.status }, { QStringLiteral("reason"), check.reason } }); + } + return QJsonObject{ + { QStringLiteral("schema"), QStringLiteral("loop.inspection-receipt.v1") }, + { QStringLiteral("identity"), identity }, + { QStringLiteral("input_digest"), inputDigest }, + { QStringLiteral("revision"), QJsonObject{ + { QStringLiteral("source_sha256"), QString::fromLatin1(revision.document.sourceDataHash.toHex()) }, + { QStringLiteral("document_id"), revision.document.documentId }, + { QStringLiteral("document_revision"), QString::number(revision.documentRevision) }, + { QStringLiteral("cache_generation"), QString::number(revision.cacheGeneration) }, + { QStringLiteral("effective_profile_identity"), revision.effectiveProfileIdentity } } }, + { QStringLiteral("effective_profile_digest"), effectiveProfileDigest }, + { QStringLiteral("profile_identity"), profileIdentity }, + { QStringLiteral("coverage_scope"), coverageScope }, + { QStringLiteral("checks"), checkArray }, + { QStringLiteral("evidence_refs"), QJsonArray::fromStringList(evidenceRefs) }, + { QStringLiteral("fidelity"), fidelity }, + { QStringLiteral("limitations"), QJsonArray::fromStringList(limitations) }, + { QStringLiteral("verdict"), verdict.toJson() } + }; +} + +bool preflightInspectionReceiptFromJson(const QJsonObject& object, PreflightInspectionReceipt& receipt, QString& errorMessage) +{ + receipt = {}; + errorMessage = QStringLiteral("Invalid inspection receipt."); + if (object.value(QStringLiteral("schema")) != QJsonValue(QStringLiteral("loop.inspection-receipt.v1"))) + { + return false; + } + for (const QString& key : { QStringLiteral("identity"), QStringLiteral("input_digest"), + QStringLiteral("effective_profile_digest"), QStringLiteral("fidelity") }) + { + if (!object.value(key).isString()) + { + return false; + } + } + for (const QString& key : { QStringLiteral("revision"), QStringLiteral("profile_identity"), + QStringLiteral("coverage_scope"), QStringLiteral("verdict") }) + { + if (!object.value(key).isObject()) + { + return false; + } + } + PreflightInspectionReceipt candidate; + candidate.identity = object.value(QStringLiteral("identity")).toString(); + candidate.inputDigest = object.value(QStringLiteral("input_digest")).toString(); + candidate.effectiveProfileDigest = object.value(QStringLiteral("effective_profile_digest")).toString(); + candidate.fidelity = object.value(QStringLiteral("fidelity")).toString(); + candidate.profileIdentity = object.value(QStringLiteral("profile_identity")).toObject(); + candidate.coverageScope = object.value(QStringLiteral("coverage_scope")).toObject(); + const QJsonObject revision = object.value(QStringLiteral("revision")).toObject(); + const QString source = revision.value(QStringLiteral("source_sha256")).toString(); + if (!revision.value(QStringLiteral("source_sha256")).isString() || (!source.isEmpty() && (!isPDFSha256(source) || source != candidate.inputDigest)) || + !revision.value(QStringLiteral("document_id")).isString() || + !revision.value(QStringLiteral("effective_profile_identity")).isString() || + !counter(revision.value(QStringLiteral("document_revision")), candidate.revision.documentRevision) || + !counter(revision.value(QStringLiteral("cache_generation")), candidate.revision.cacheGeneration)) + { + return false; + } + candidate.revision.document.sourceDataHash = QByteArray::fromHex(source.toLatin1()); + candidate.revision.document.documentId = revision.value(QStringLiteral("document_id")).toString(); + candidate.revision.effectiveProfileIdentity = revision.value(QStringLiteral("effective_profile_identity")).toString(); + if (!candidate.revision.isValid()) + return false; + if ((!candidate.inputDigest.isEmpty() && !isPDFSha256(candidate.inputDigest)) || + (!candidate.effectiveProfileDigest.isEmpty() && !isPDFSha256(candidate.effectiveProfileDigest)) || + candidate.inputDigest != candidate.inputDigest.toLower() || + candidate.effectiveProfileDigest != candidate.effectiveProfileDigest.toLower() || + candidate.identity != receiptIdentity(candidate) || + !strings(object.value(QStringLiteral("evidence_refs")), candidate.evidenceRefs) || + !strings(object.value(QStringLiteral("limitations")), candidate.limitations)) + { + return false; + } + const QJsonObject verdict = object.value(QStringLiteral("verdict")).toObject(); + const QString state = verdict.value(QStringLiteral("state")).toString(); + if (!QStringList{ QStringLiteral("pass"), QStringLiteral("fail"), QStringLiteral("incomplete"), QStringLiteral("error") }.contains(state) || + !verdict.value(QStringLiteral("reason_code")).isString() || !verdict.value(QStringLiteral("reason")).isString()) + { + return false; + } + candidate.verdict = preflightVerdictFromJson(verdict); + candidate.verdict.blockingFindingIds.clear(); + candidate.verdict.waivedFindingIds.clear(); + if (!strings(verdict.value(QStringLiteral("blocking_finding_ids")), candidate.verdict.blockingFindingIds) || + !strings(verdict.value(QStringLiteral("waived_finding_ids")), candidate.verdict.waivedFindingIds) || + !object.value(QStringLiteral("checks")).isArray()) + { + return false; + } + QSet ids; + bool incomplete = candidate.coverageScope.isEmpty(); + const QStringList statuses{ QString(), QStringLiteral("ok"), QStringLiteral("warning"), QStringLiteral("failed"), + QStringLiteral("incomplete"), QStringLiteral("unsupported"), QStringLiteral("skipped"), + QStringLiteral("not_inspected"), QStringLiteral("not_applicable") }; + for (const QJsonValue value : object.value(QStringLiteral("checks")).toArray()) + { + if (!value.isObject()) + { + return false; + } + const QJsonObject check = value.toObject(); + PreflightReceiptCheck parsed; + if (!check.value(QStringLiteral("id")).isString() || !check.value(QStringLiteral("required")).isBool() || + !check.value(QStringLiteral("complete")).isBool() || !check.value(QStringLiteral("status")).isString() || + !check.value(QStringLiteral("reason")).isString()) + { + return false; + } + parsed.id = check.value(QStringLiteral("id")).toString(); + parsed.required = check.value(QStringLiteral("required")).toBool(); + parsed.complete = check.value(QStringLiteral("complete")).toBool(); + parsed.status = check.value(QStringLiteral("status")).toString(); + parsed.reason = check.value(QStringLiteral("reason")).toString(); + const bool completedStatus = parsed.status == QLatin1String("ok") || parsed.status == QLatin1String("warning") || + parsed.status == QLatin1String("failed"); + if (parsed.id.isEmpty() || ids.contains(parsed.id) || !statuses.contains(parsed.status) || (parsed.complete && !completedStatus)) + { + return false; + } + ids.insert(parsed.id); + incomplete |= !parsed.complete; + candidate.checks.append(parsed); + } + incomplete |= candidate.checks.isEmpty() || candidate.evidenceRefs.isEmpty() || candidate.fidelity == QLatin1String("unsupported") || + candidate.fidelity == QLatin1String("not-recorded"); + if (!QStringList{ QStringLiteral("exact"), QStringLiteral("sampled"), QStringLiteral("catalog"), + QStringLiteral("unsupported"), QStringLiteral("not-recorded") } + .contains(candidate.fidelity) || + (candidate.verdict.isPass() && (incomplete || !candidate.verdict.blockingFindingIds.isEmpty())) || + (candidate.verdict.state == PreflightVerdictState::Fail && candidate.verdict.blockingFindingIds.isEmpty())) + { + return false; + } + const bool unknownIdentity = candidate.inputDigest.isEmpty() || candidate.effectiveProfileDigest.isEmpty(); + if (unknownIdentity && + (candidate.verdict.state != PreflightVerdictState::Incomplete || !candidate.coverageScope.isEmpty() || + !candidate.checks.isEmpty() || !candidate.evidenceRefs.isEmpty() || !candidate.profileIdentity.isEmpty() || + candidate.fidelity != QLatin1String("not-recorded") || candidate.limitations.isEmpty())) + { + return false; + } + receipt = std::move(candidate); + errorMessage.clear(); + return true; +} + +bool validatePreflightInspectionReceipt(const PreflightInspectionReceipt& receipt, const QString& inputDigest, + const PDFRevisionIdentity& revision, const PreflightProfileData& profile, + QString& errorMessage) +{ + PreflightInspectionReceipt parsed; + if (!preflightInspectionReceiptFromJson(receipt.toJson(), parsed, errorMessage)) + { + return false; + } + errorMessage = QStringLiteral("Inspection receipt does not match the requested input, revision or profile coverage."); + QJsonObject expectedCoverage = profile.coverageScope; + if (!profile.restrictions.isUnrestricted()) + { + expectedCoverage.insert(QStringLiteral("scope_restrictions"), profile.restrictions.toJson()); + } + if (expectedCoverage.isEmpty() || parsed.inputDigest != inputDigest || parsed.revision != revision || + parsed.effectiveProfileDigest != profile.effectiveDigest || parsed.profileIdentity != profile.profileIdentity || + parsed.coverageScope != expectedCoverage) + { + return false; + } + QSet expected; + for (const PreflightCheckConfig& check : profile.checks) + { + if (!check.enabled) + { + continue; + } + if (expected.contains(check.id)) + { + return false; + } + expected.insert(check.id); + const auto found = std::find_if(parsed.checks.cbegin(), parsed.checks.cend(), + [&](const PreflightReceiptCheck& value) + { return value.id == check.id; }); + if (found == parsed.checks.cend() || found->required != check.required) + { + return false; + } + } + if (profile.pdfx.has_value()) + { + expected.insert(QStringLiteral("pdfx")); + const auto found = std::find_if(parsed.checks.cbegin(), parsed.checks.cend(), + [](const PreflightReceiptCheck& check) + { return check.id == QLatin1String("pdfx"); }); + if (found == parsed.checks.cend() || !found->required) + return false; + } + if (expected.size() != parsed.checks.size()) + { + return false; + } + errorMessage.clear(); + return true; +} + +PreflightInspectionReceipt buildTerminalPreflightReceipt(const QString& inputDigest, const PDFRevisionIdentity& revision, + const QString& profileDigest, const QString& reasonCode) +{ + PreflightInspectionReceipt receipt; + receipt.inputDigest = inputDigest; + receipt.revision = revision; + receipt.effectiveProfileDigest = profileDigest; + receipt.fidelity = QStringLiteral("not-recorded"); + receipt.limitations = { QStringLiteral("Inspection did not complete; no coverage is admitted.") }; + receipt.verdict.state = PreflightVerdictState::Incomplete; + receipt.verdict.reasonCode = reasonCode; + receipt.verdict.reason = QStringLiteral("Isolated inspection did not complete."); + receipt.identity = receiptIdentity(receipt); + return receipt; +} +} // namespace pdf diff --git a/LoopLibCore/sources/pdfpreflightverdict.cpp b/LoopLibCore/sources/pdfpreflightverdict.cpp index 2703882b5..489996b50 100644 --- a/LoopLibCore/sources/pdfpreflightverdict.cpp +++ b/LoopLibCore/sources/pdfpreflightverdict.cpp @@ -28,10 +28,13 @@ #include "preflightprofileresolver.h" #include +#include #include +#include #include #include +#include namespace pdf { @@ -483,6 +486,203 @@ PreflightVerdict reducePreflightVerdict(const PreflightResult& result, return verdict; } +bool buildPreflightInspectionReceipt(const PreflightResult& result, + const PreflightProfileData& profile, + const PDFRevisionIdentity& revision, + const PDFEvidenceGraph& evidence, + PreflightInspectionReceipt& receipt, + QString& errorMessage) +{ + receipt = {}; + if (!isPDFSha256(result.documentRevisionDigest) || !isPDFSha256(profile.effectiveDigest) || + result.effectiveProfileDigest.compare(profile.effectiveDigest, Qt::CaseInsensitive) != 0 || + !revision.isValid()) + { + errorMessage = QStringLiteral("Inspection receipt requires a valid input digest, matching effective profile digest and document revision."); + return false; + } + if ((!evidence.artifact.sha256.isEmpty() && + evidence.artifact.sha256.compare(result.documentRevisionDigest, Qt::CaseInsensitive) != 0) || + (evidence.revision.isValid() && evidence.revision != revision)) + { + errorMessage = QStringLiteral("Inspection evidence belongs to a different input or revision."); + return false; + } + + PreflightInspectionReceipt candidate; + candidate.inputDigest = result.documentRevisionDigest.toLower(); + candidate.revision = revision; + candidate.effectiveProfileDigest = profile.effectiveDigest.toLower(); + candidate.profileIdentity = result.profileIdentity.isEmpty() ? profile.profileIdentity : result.profileIdentity; + candidate.coverageScope = result.coverageScope; + candidate.verdict = reducePreflightVerdict(result, &profile); + + bool incompleteCoverage = false; + QSet declaredChecks; + const auto appendCheck = [&](const QString& id, bool required) + { + PreflightReceiptCheck check; + check.id = id; + check.required = required; + const auto status = std::find_if(result.checkStatuses.cbegin(), result.checkStatuses.cend(), + [&id](const PreflightCheckStatus& value) + { return value.id == id; }); + if (status != result.checkStatuses.cend()) + { + check.status = status->status; + check.reason = status->reason; + const bool unique = std::find_if(std::next(status), result.checkStatuses.cend(), + [&id](const PreflightCheckStatus& value) + { return value.id == id; }) == result.checkStatuses.cend(); + check.complete = unique && status->budgetKind.isEmpty() && + (status->status == QLatin1String("ok") || + status->status == QLatin1String("warning") || + status->status == QLatin1String("failed")); + } + if (!check.complete) + { + incompleteCoverage = true; + QString reason = check.reason; + if (reason.isEmpty()) + { + reason = check.status.isEmpty() ? QStringLiteral("no status") : check.status; + } + candidate.limitations.append(QStringLiteral("Check '%1' did not complete: %2") + .arg(id, reason)); + } + candidate.checks.append(std::move(check)); + }; + + for (const PreflightCheckConfig& check : profile.checks) + { + if (!check.enabled) + { + continue; + } + if (check.id.isEmpty() || declaredChecks.contains(check.id)) + { + errorMessage = QStringLiteral("Inspection profile has an empty or duplicate enabled check ID."); + return false; + } + declaredChecks.insert(check.id); + appendCheck(check.id, check.required); + } + if (profile.pdfx.has_value()) + { + appendCheck(QStringLiteral("pdfx"), true); + } + if (candidate.checks.isEmpty() || candidate.coverageScope.isEmpty()) + { + incompleteCoverage = true; + candidate.limitations.append(QStringLiteral("Inspection check coverage or scope was not recorded.")); + } + for (const PreflightCheckStatus& status : result.checkStatuses) + { + if (!declaredChecks.contains(status.id) && status.id != QLatin1String("pdfx") && + (!status.budgetKind.isEmpty() || status.status == QLatin1String("incomplete") || + status.status == QLatin1String("unsupported") || status.status == QLatin1String("skipped") || + status.status == QLatin1String("not_inspected"))) + { + incompleteCoverage = true; + candidate.limitations.append(QStringLiteral("Inspection status '%1' did not complete.").arg(status.id)); + } + } + + QSet evidenceIds; + int fidelityRank = 3; + for (const PDFEvidenceRecord& record : evidence.records) + { + if ((!record.artifact.sha256.isEmpty() && + record.artifact.sha256.compare(result.documentRevisionDigest, Qt::CaseInsensitive) != 0) || + (record.revision.isValid() && record.revision != revision)) + { + errorMessage = QStringLiteral("Inspection evidence record belongs to a different input or revision."); + return false; + } + if (!record.id.isEmpty()) + { + evidenceIds.insert(record.id); + } + if (!record.incompleteReason.isEmpty()) + { + incompleteCoverage = true; + candidate.limitations.append(record.incompleteReason); + } + if (record.fidelity == QLatin1String("catalog")) + { + fidelityRank = std::min(fidelityRank, 1); + } + else if (record.fidelity == QLatin1String("sampled")) + { + fidelityRank = std::min(fidelityRank, 2); + } + else if (record.fidelity != QLatin1String("exact")) + { + fidelityRank = 0; + incompleteCoverage = true; + candidate.limitations.append(QStringLiteral("Evidence fidelity is unsupported or unknown.")); + } + } + const auto appendFindingEvidence = [&evidenceIds](const QList& findings) + { + for (const PreflightFinding& finding : findings) + { + for (const QString& id : finding.evidenceIds) + { + if (!id.isEmpty()) + { + evidenceIds.insert(id); + } + } + } + }; + appendFindingEvidence(result.errors); + appendFindingEvidence(result.warnings); + candidate.evidenceRefs = evidenceIds.values(); + candidate.evidenceRefs.sort(); + candidate.fidelity = evidence.records.isEmpty() ? QStringLiteral("not-recorded") + : fidelityRank == 3 ? QStringLiteral("exact") + : fidelityRank == 2 ? QStringLiteral("sampled") + : fidelityRank == 1 ? QStringLiteral("catalog") + : QStringLiteral("unsupported"); + if (candidate.evidenceRefs.isEmpty()) + { + incompleteCoverage = true; + candidate.limitations.append(QStringLiteral("No evidence was recorded for this inspection.")); + } + if (!evidence.isComplete()) + { + incompleteCoverage = true; + candidate.limitations.append(evidence.incompleteReason.isEmpty() + ? QStringLiteral("Evidence collection did not complete.") + : evidence.incompleteReason); + } + const QString coverageClaim = candidate.coverageScope.value(QStringLiteral("claim")).toString(); + if (!coverageClaim.isEmpty()) + { + candidate.limitations.append(coverageClaim); + } + candidate.limitations.removeDuplicates(); + if (candidate.verdict.isPass() && incompleteCoverage) + { + candidate.verdict.state = PreflightVerdictState::Incomplete; + candidate.verdict.reasonCode = QStringLiteral("receipt-evidence-incomplete"); + candidate.verdict.reason = QStringLiteral("Required inspection coverage or evidence did not complete."); + } + + const QJsonObject identityData{ + { QStringLiteral("kind"), QStringLiteral("loop.inspection-receipt-identity.v1") }, + { QStringLiteral("input_digest"), candidate.inputDigest }, + { QStringLiteral("effective_profile_digest"), candidate.effectiveProfileDigest }, + { QStringLiteral("coverage_scope"), candidate.coverageScope } + }; + candidate.identity = QString::fromLatin1( + QCryptographicHash::hash(canonicalJson(identityData), QCryptographicHash::Sha256).toHex()); + receipt = std::move(candidate); + errorMessage.clear(); + return true; +} + PDFOperationResult runMandatoryPostflight(PDFDocument* document, const QString& profilePath, PreflightVerdict* verdictOut, diff --git a/LoopLibCore/sources/pdfpreflightverdict.h b/LoopLibCore/sources/pdfpreflightverdict.h index 5199c27ef..fa90c1bf1 100644 --- a/LoopLibCore/sources/pdfpreflightverdict.h +++ b/LoopLibCore/sources/pdfpreflightverdict.h @@ -77,6 +77,54 @@ LOOPLIBCORESHARED_EXPORT QString preflightGateFailureMessage(const QString& file LOOPLIBCORESHARED_EXPORT PreflightVerdict reducePreflightVerdict(const PreflightResult& result, const PreflightProfileData* effectiveProfile = nullptr); +struct LOOPLIBCORESHARED_EXPORT PreflightReceiptCheck +{ + QString id; + bool required = false; + bool complete = false; + QString status; + QString reason; +}; + +struct LOOPLIBCORESHARED_EXPORT PreflightInspectionReceipt +{ + QString identity; + QString inputDigest; + PDFRevisionIdentity revision; + QString effectiveProfileDigest; + QJsonObject profileIdentity; + QJsonObject coverageScope; + QList checks; + QStringList evidenceRefs; + QString fidelity; + QStringList limitations; + PreflightVerdict verdict; + + QJsonObject toJson() const; +}; + +LOOPLIBCORESHARED_EXPORT bool preflightInspectionReceiptFromJson( + const QJsonObject& object, PreflightInspectionReceipt& receipt, QString& errorMessage); + +LOOPLIBCORESHARED_EXPORT bool validatePreflightInspectionReceipt( + const PreflightInspectionReceipt& receipt, const QString& inputDigest, + const PDFRevisionIdentity& revision, const PreflightProfileData& profile, QString& errorMessage); + +LOOPLIBCORESHARED_EXPORT PreflightInspectionReceipt buildTerminalPreflightReceipt( + const QString& inputDigest, const PDFRevisionIdentity& revision, + const QString& profileDigest, const QString& reasonCode); + + +/// Binds one Core result to its input revision and evidence. The identity is +/// stable for the same input, effective profile and coverage policy; a missing +/// required check or unsupported evidence cannot produce PASS. +LOOPLIBCORESHARED_EXPORT bool buildPreflightInspectionReceipt(const PreflightResult& result, + const PreflightProfileData& profile, + const PDFRevisionIdentity& revision, + const PDFEvidenceGraph& evidence, + PreflightInspectionReceipt& receipt, + QString& errorMessage); + /// The single Core planner used by step postflight, check selection and impact /// qualification. An operation-wide/uncertain declaration cannot be narrowed /// by page-local repair targets. diff --git a/LoopLibCore/sources/pdfrenderer.cpp b/LoopLibCore/sources/pdfrenderer.cpp index 45c20b493..e17b83da4 100644 --- a/LoopLibCore/sources/pdfrenderer.cpp +++ b/LoopLibCore/sources/pdfrenderer.cpp @@ -250,12 +250,10 @@ PDFRasterizer::PDFRasterizer(QObject* parent) : BaseClass(parent), m_rendererEngine(RendererEngine::Blend2D_SingleThread) { - } PDFRasterizer::~PDFRasterizer() { - } void PDFRasterizer::reset(RendererEngine rendererEngine) @@ -446,16 +444,16 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, const QSize imageSize = imageSizeGetter(page); const bool validImageSize = imageSize.width() > 0 && imageSize.height() > 0; const qint64 imageBytes = !validImageSize - ? 0 - : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 - ? static_cast(imageSize.width()) * imageSize.height() * 4 - : std::numeric_limits::max(); + ? 0 + : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 + ? static_cast(imageSize.width()) * imageSize.height() * 4 + : std::numeric_limits::max(); std::optional imageReservation; if (m_resourceBudget && imageBytes > 0 && !m_resourceBudget->tryReserve(PDFResourcePool::RasterTileCache, - imageBytes, - PDFResourcePriority::Visible, - QStringLiteral("benchmark raster image"))) + imageBytes, + PDFResourcePriority::Visible, + QStringLiteral("benchmark raster image"))) { m_resourceBudget->recordShed(PDFResourcePool::RasterTileCache); m_resourceBudgetExhausted.store(true, std::memory_order_release); @@ -476,17 +474,30 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, QImage image = rasterizer->render(pageIndex, page, &precompiledPage, imageSize, m_features, &annotationManager, cms.data(), PageRotation::None); qint64 pageRenderTime = pageTimer.elapsed(); - release(rasterizer); + // A budgeted run keeps the rasterizer until the image is consumed, so the images + // alive at once never exceed the rasterizer count the raster tile pool was sized for. + if (!imageReservation) + { + release(rasterizer); + } // Now, process the image - PDFRenderedPageImage renderedPageImage; - renderedPageImage.pageIndex = pageIndex; - renderedPageImage.pageImage = qMove(image); - renderedPageImage.pageCompileTime = pageCompileTime; - renderedPageImage.pageWaitTime = pageWaitTime; - renderedPageImage.pageRenderTime = pageRenderTime; - renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); - processImage(renderedPageImage); + { + PDFRenderedPageImage renderedPageImage; + renderedPageImage.pageIndex = pageIndex; + renderedPageImage.pageImage = qMove(image); + renderedPageImage.pageCompileTime = pageCompileTime; + renderedPageImage.pageWaitTime = pageWaitTime; + renderedPageImage.pageRenderTime = pageRenderTime; + renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); + processImage(renderedPageImage); + } + + if (imageReservation) + { + imageReservation.reset(); + release(rasterizer); + } if (progress) { diff --git a/LoopLibCore/sources/pdfresourcebudget.h b/LoopLibCore/sources/pdfresourcebudget.h index 93ac07334..0c351bc9c 100644 --- a/LoopLibCore/sources/pdfresourcebudget.h +++ b/LoopLibCore/sources/pdfresourcebudget.h @@ -83,7 +83,7 @@ struct LOOPLIBCORESHARED_EXPORT PDFResourceBudgetConfig /// resident ceiling is reached; active model and a visible request remain /// hard admission boundaries. std::array poolLimits = { - 256 * MiB, // active document model + 640 * MiB, // active document model 128 * MiB, // compiled/evidence cache 128 * MiB, // raster/tile cache 128 * MiB, // GPU/texture accounted proxy diff --git a/LoopLibCore/sources/pdfworkloadenvelope.cpp b/LoopLibCore/sources/pdfworkloadenvelope.cpp index c4a0d0e88..8b7148bee 100644 --- a/LoopLibCore/sources/pdfworkloadenvelope.cpp +++ b/LoopLibCore/sources/pdfworkloadenvelope.cpp @@ -172,15 +172,17 @@ qint64 PDFWorkloadEnvelope::currentRssHighWaterBytes() return -1; } - while (!status.atEnd()) + // procfs reports size 0, so QFile::atEnd() is true before the first read. + const QList lines = status.readAll().split('\n'); + for (const QByteArray& rawLine : lines) { - const QByteArray line = status.readLine().trimmed(); + const QByteArray line = rawLine.trimmed(); if (!line.startsWith("VmHWM:")) { continue; } - const QList parts = line.split(' '); + const QList parts = line.mid(6).simplified().split(' '); for (const QByteArray& part : parts) { bool ok = false; diff --git a/LoopLibCore/sources/preflightclirun.cpp b/LoopLibCore/sources/preflightclirun.cpp index 51e64fc1d..3a1a34b6e 100644 --- a/LoopLibCore/sources/preflightclirun.cpp +++ b/LoopLibCore/sources/preflightclirun.cpp @@ -145,7 +145,7 @@ PreflightFileInspectionOutcome inspectPreflightFile(const PreflightFileInspectio outcome.sourceData = reader.getSource(); std::unique_ptr session( - PDFDocumentSession::createForInspection(document.get()), + PDFDocumentSession::createForInspection(document.get(), request.receiptDocumentId), &PDFDocumentSession::destroy); PreflightEngine engine(session.get()); @@ -166,7 +166,19 @@ PreflightFileInspectionOutcome inspectPreflightFile(const PreflightFileInspectio } else { - outcome.report = engine.run(request.profile, request.jobSpec, request.cliBindings, request.plan, selectedPages); + PreflightProfileData effectiveProfile; + outcome.report = engine.run(request.profile, request.jobSpec, request.cliBindings, request.plan, + selectedPages, request.createReceipt ? &effectiveProfile : nullptr); + if (request.createReceipt) + { + outcome.report.documentRevisionDigest = QString::fromLatin1(session->getRevision().document.sourceDataHash.toHex()); + PreflightInspectionReceipt receipt; + if (buildPreflightInspectionReceipt(outcome.report, effectiveProfile, session->getRevision(), + engine.lastEvidenceGraph(), receipt, outcome.receiptError)) + { + outcome.receipt = std::move(receipt); + } + } } outcome.inspectionRan = true; } diff --git a/LoopLibCore/sources/preflightclirun.h b/LoopLibCore/sources/preflightclirun.h index 182eaa86e..c67c25fbd 100644 --- a/LoopLibCore/sources/preflightclirun.h +++ b/LoopLibCore/sources/preflightclirun.h @@ -26,6 +26,8 @@ #include "pdfglobal.h" #include "pdfdocumentreader.h" #include "preflightengine.h" +#include "pdfpreflightverdict.h" +#include #include #include @@ -41,6 +43,8 @@ struct PreflightResolvedProfile; struct LOOPLIBCORESHARED_EXPORT PreflightFileInspectionRequest { QString documentPath; + QString receiptDocumentId; + bool createReceipt = false; QString password; bool permissiveReading = false; QJsonObject profile; @@ -62,6 +66,8 @@ struct LOOPLIBCORESHARED_EXPORT PreflightFileInspectionOutcome QString readErrorMessage; QStringList readWarnings; bool inspectionRan = false; + std::optional receipt; + QString receiptError; }; /// Runs a file-backed preflight inspection entirely inside LoopLibCore so host diff --git a/LoopLibCore/sources/preflightengine.cpp b/LoopLibCore/sources/preflightengine.cpp index 8b8889e43..14c686b18 100644 --- a/LoopLibCore/sources/preflightengine.cpp +++ b/LoopLibCore/sources/preflightengine.cpp @@ -917,9 +917,10 @@ QJsonObject preflightCoverageScopeFor(const PreflightProfileData& profile) } } return QJsonObject{ - { QStringLiteral("claim"), QStringLiteral("This run does not evaluate formal GWG 2022/2024 conformance for " - "sheetfed-offset or packaging. A clean result covers only its " - "enabled checks, not either family certificate.") }, + { QStringLiteral("claim"), QStringLiteral("Loop does not claim formal GWG conformance. This run does not " + "evaluate GWG 2022/2024 certificate requirements for " + "sheetfed-offset or packaging; a clean result covers only its " + "enabled checks.") }, { QStringLiteral("matrix_id"), QStringLiteral("loop-gwg-pdfx-v1") }, { QStringLiteral("enabled_checks"), checkIds } }; @@ -6033,7 +6034,8 @@ PreflightResult PreflightEngine::run(const QJsonObject& profile, const QJsonObject& jobSpecBindings, const QJsonObject& cliBindings, const PDFRevalidationPlan& plan, - const std::optional>& cliPages) + const std::optional>& cliPages, + PreflightProfileData* effectiveProfile) { const PreflightProfileImportResult imported = importPreflightProfile(profile); if (!imported.ok) @@ -6124,6 +6126,10 @@ PreflightResult PreflightEngine::run(const QJsonObject& profile, data.profileIdentity = imported.identity.toJson(); data.profileIdentity.insert(QStringLiteral("digest"), data.fileDigest); data.profileIdentity.insert(QStringLiteral("effective_digest"), data.effectiveDigest); + if (effectiveProfile) + { + *effectiveProfile = data; + } return run(data, plan); } @@ -6193,7 +6199,12 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const const PDFEvidenceDomains graphDomains = effectivePlan.full ? evidenceDomainsForProfile(profile) : evidenceDomainsForCheckIds(effectivePlan.checkIds); if (graphDomains != PDFEvidenceDomains()) { - m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettingsForProfile(profile)); + PDFEvidenceCollectSettings evidenceSettings = evidenceSettingsForProfile(profile); + evidenceSettings.operationControl = m_operationControl; + // Records outside the profile's page scope are dropped below, so do not + // spend the render and content walk on those pages. + evidenceSettings.pageIndices = profile.restrictions.pages; + m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettings); if (profile.restrictions.pages.has_value() || (!plan.full && !plan.pages.isEmpty())) { QList kept; @@ -6213,7 +6224,12 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const if (!m_activeGraph.isComplete()) { result.inspectionComplete = false; - if (!m_activeGraph.budgetKind.isEmpty()) + if (PDFOperationControl::isOperationCancelled(m_operationControl)) + { + result.errorCode = QStringLiteral("cancelled"); + result.errorMessage = PDFTranslationContext::tr("Preflight was cancelled."); + } + else if (!m_activeGraph.budgetKind.isEmpty()) { result.errorCode = QStringLiteral("budget-exceeded"); result.errorMessage = PDFTranslationContext::tr("Evidence collection exceeded the %1 processing budget.") diff --git a/LoopLibCore/sources/preflightengine.h b/LoopLibCore/sources/preflightengine.h index c1bb35322..7ae0eef1c 100644 --- a/LoopLibCore/sources/preflightengine.h +++ b/LoopLibCore/sources/preflightengine.h @@ -421,7 +421,8 @@ class LOOPLIBCORESHARED_EXPORT PreflightEngine const QJsonObject& jobSpecBindings, const QJsonObject& cliBindings, const PDFRevalidationPlan& plan, - const std::optional>& cliPages); + const std::optional>& cliPages, + PreflightProfileData* effectiveProfile = nullptr); PreflightResult run(const PreflightProfileData& profile); PreflightResult run(const PreflightProfileData& profile, const PDFRevalidationPlan& plan); PreflightResult revalidate(const PreflightProfileData& profile, diff --git a/LoopLibInteraction/sources/documentfacade.cpp b/LoopLibInteraction/sources/documentfacade.cpp index 594368dd1..792924bfd 100644 --- a/LoopLibInteraction/sources/documentfacade.cpp +++ b/LoopLibInteraction/sources/documentfacade.cpp @@ -24,6 +24,7 @@ #include "pdfresourcebudget.h" +#include #include #include @@ -434,6 +435,24 @@ void DocumentFacade::admitLoadResult(CommandInvocationId invocation, return; } + const pdf::PDFJobSnapshot job = m_submitter->snapshot(m_pendingJobId); + if (job.jobId == m_pendingJobId && + (job.status == pdf::PDFJobStatus::Queued || job.status == pdf::PDFJobStatus::Running)) + { + QTimer::singleShot(1, this, [this, invocation, generation, result = std::move(result)]() mutable + { admitLoadResult(invocation, generation, std::move(result)); }); + return; + } + if (job.jobId != m_pendingJobId || job.status != pdf::PDFJobStatus::Succeeded || + job.kind != pdf::PDFJobKind::Other) + { + result = {}; + result.outcome = job.status == pdf::PDFJobStatus::Cancelled + ? DocumentLoadOutcome::Cancelled + : DocumentLoadOutcome::Failed; + result.typedError = QStringLiteral("document/job-not-admitted"); + } + m_pendingJobId.clear(); pdf::PDFDocumentContext* documentContext = context(); @@ -548,6 +567,24 @@ void DocumentFacade::admitWriteResult(CommandInvocationId invocation, return; } + const pdf::PDFJobSnapshot job = m_submitter->snapshot(m_pendingJobId); + if (job.jobId == m_pendingJobId && + (job.status == pdf::PDFJobStatus::Queued || job.status == pdf::PDFJobStatus::Running)) + { + QTimer::singleShot(1, this, [this, invocation, generation, target = std::move(target), result = std::move(result)]() mutable + { admitWriteResult(invocation, generation, std::move(target), std::move(result)); }); + return; + } + if (job.jobId != m_pendingJobId || job.status != pdf::PDFJobStatus::Succeeded || + job.kind != pdf::PDFJobKind::Export || m_publishedKey.isEmpty() || job.documentKey != m_publishedKey || + job.documentRevision != m_revisionSource.currentRevision().toString()) + { + result.outcome = job.status == pdf::PDFJobStatus::Cancelled + ? DocumentWriteOutcome::Cancelled + : DocumentWriteOutcome::Failed; + result.typedError = QStringLiteral("document/job-not-admitted"); + } + m_pendingJobId.clear(); switch (result.outcome) @@ -583,8 +620,7 @@ void DocumentFacade::detachDocument() { if (!m_publishedKey.isEmpty()) { - // A key with no entry is never stale, so this belongs at close and at - // replacement, not between submissions. + // Closing the fence also rejects any completion from this session. m_submitter->clearCurrentRevision(m_publishedKey); m_publishedKey.clear(); } diff --git a/LoopLibInteraction/sources/jobsubmitter.h b/LoopLibInteraction/sources/jobsubmitter.h index e191ad80b..b92e0ae6e 100644 --- a/LoopLibInteraction/sources/jobsubmitter.h +++ b/LoopLibInteraction/sources/jobsubmitter.h @@ -64,8 +64,8 @@ class IJobSubmitter virtual void publishCurrentRevision(const QString& documentKey, const pdf::PDFRevisionIdentity& revision) = 0; - /// Drops the fence entry for a document key. A key with no entry is never - /// stale, so this belongs at document close, not between submissions. + /// Drops the fence entry for a document key. Bound jobs then become stale, + /// including if the same revision is published after a reopen. virtual void clearCurrentRevision(const QString& documentKey) = 0; }; diff --git a/LoopLibInteraction/sources/pagesurfacecoordinator.cpp b/LoopLibInteraction/sources/pagesurfacecoordinator.cpp index 2e450991a..b99b1f9da 100644 --- a/LoopLibInteraction/sources/pagesurfacecoordinator.cpp +++ b/LoopLibInteraction/sources/pagesurfacecoordinator.cpp @@ -24,6 +24,7 @@ #include "pdfpagecachebudget.h" +#include #include #include @@ -93,7 +94,18 @@ PageSurfaceCoordinator::~PageSurfaceCoordinator() void PageSurfaceCoordinator::setDocumentKey(QString documentKey) { + if (m_documentKey == documentKey) + { + return; + } + const bool hadDocumentKey = !m_documentKey.isEmpty(); + cancelInFlight(); + clearCache(); m_documentKey = std::move(documentKey); + if (hadDocumentKey || m_initialSnapshotPrimed) + { + rebuildSnapshot(); + } } void PageSurfaceCoordinator::setResourceBudget(std::shared_ptr budget) @@ -627,6 +639,33 @@ void PageSurfaceCoordinator::admit(quint64 requestId, std::shared_ptr resourceReservation) { const auto inFlight = m_inFlight.find(requestId); + if (inFlight != m_inFlight.end()) + { + const pdf::PDFJobSnapshot job = m_submitter->snapshot(inFlight->jobId); + if (job.jobId == inFlight->jobId && + (job.status == pdf::PDFJobStatus::Queued || job.status == pdf::PDFJobStatus::Running)) + { + QTimer::singleShot(1, this, [this, requestId, result = std::move(result), resourceReservation = std::move(resourceReservation)]() mutable + { admit(requestId, std::move(result), std::move(resourceReservation)); }); + return; + } + if (job.jobId != inFlight->jobId || job.status != pdf::PDFJobStatus::Succeeded) + { + const SurfaceTerminalState terminal = job.status == pdf::PDFJobStatus::Cancelled + ? SurfaceTerminalState::Cancelled + : job.status == pdf::PDFJobStatus::Stale + ? SurfaceTerminalState::Stale + : SurfaceTerminalState::Failed; + finishInFlight(requestId, terminal); + return; + } + if (!(result.key == inFlight->key) || !(result.token == inFlight->token)) + { + ++m_counters.rejectedSuperseded; + finishInFlight(requestId, SurfaceTerminalState::Stale); + return; + } + } if (!resourceReservation && inFlight != m_inFlight.end()) { resourceReservation = inFlight->resourceReservation; diff --git a/PdfTool/CMakeLists.txt b/PdfTool/CMakeLists.txt index 0c1d215a4..673bd85c8 100644 --- a/PdfTool/CMakeLists.txt +++ b/PdfTool/CMakeLists.txt @@ -98,6 +98,9 @@ set(_loop_pdftool_sources pdftoolworker.h pdfworkerclient.cpp pdfworkerclient.h + pdfworkerprocess.cpp + pdfworkerprocess_win.cpp + pdfworkerprocess.h pdfworkerprotocol.h ) @@ -171,8 +174,9 @@ set(_loop_pdf_worker_sources ) add_executable(loop-pdf-worker ${_loop_pdf_worker_sources}) target_link_libraries(loop-pdf-worker PRIVATE LoopLibCore Qt6::Core Qt6::Gui) -if(UNIX AND NOT APPLE) - target_compile_definitions(loop-pdf-worker PRIVATE LOOP_PDF_WORKER_REQUIRE_SANDBOX=1) +if(WIN32) + target_compile_definitions(PdfTool PRIVATE _WIN32_WINNT=0x0A00) + target_link_libraries(PdfTool PRIVATE userenv advapi32 ole32) endif() set_target_properties(loop-pdf-worker PROPERTIES WIN32_EXECUTABLE OFF @@ -182,3 +186,10 @@ set_target_properties(loop-pdf-worker PROPERTIES ) install(TARGETS loop-pdf-worker RUNTIME DESTINATION ${LOOP_INSTALL_BIN_DIR} LIBRARY DESTINATION ${LOOP_INSTALL_LIB_DIR}) add_dependencies(PdfTool loop-pdf-worker) + +include(${CMAKE_CURRENT_SOURCE_DIR}/worker-runtime.cmake) +loop_stage_worker_runtime(loop-pdf-worker) +if(WIN32) + install(FILES "$.runtime" DESTINATION ${LOOP_INSTALL_BIN_DIR}) + install(DIRECTORY "$/worker-runtime/" DESTINATION ${LOOP_INSTALL_BIN_DIR}/worker-runtime) +endif() diff --git a/PdfTool/loop-pdf-worker-main.cpp b/PdfTool/loop-pdf-worker-main.cpp index be4e78e76..7d24f4038 100644 --- a/PdfTool/loop-pdf-worker-main.cpp +++ b/PdfTool/loop-pdf-worker-main.cpp @@ -33,13 +33,25 @@ #include #include +#if defined(Q_OS_WIN) +#include +#endif namespace { int writeLine(const QJsonObject& object) { - const QByteArray line = QJsonDocument(object).toJson(QJsonDocument::Compact) + '\n'; + QByteArray line = QJsonDocument(object).toJson(QJsonDocument::Compact) + '\n'; + if (line.size() > pdftool::worker::MAX_RESPONSE_BYTES) + { + line = QJsonDocument(pdftool::worker::makeErrorResponse( + object.value(QStringLiteral("id")).toString(), object.value(QStringLiteral("op")).toString(), + QStringLiteral("incomplete"), QStringLiteral("worker.response-limit"), + QStringLiteral("Inspection response exceeded its limit."))) + .toJson(QJsonDocument::Compact) + + '\n'; + } if (fwrite(line.constData(), 1, static_cast(line.size()), stdout) != static_cast(line.size())) { return 1; @@ -52,9 +64,9 @@ int writeLine(const QJsonObject& object) int main(int argc, char* argv[]) { - // loop-pdf-worker never initializes Sentry or an out-of-process crash - // reporter. A hostile PDF that crashes this process must not produce a - // customer-content minidump (R-008). +#if defined(Q_OS_WIN) + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +#endif QCoreApplication application(argc, argv); pdf::initializeApplicationIdentity(pdf::PDFApplicationSurface::LoopPdfWorker); @@ -96,29 +108,35 @@ int main(int argc, char* argv[]) pdftool::worker::WorkerSandboxLimits limits; if (parser.isSet(rssOption)) { - limits.rssBytes = parser.value(rssOption).toLongLong(); + bool valid = false; + limits.rssBytes = parser.value(rssOption).toLongLong(&valid); + if (!valid || limits.rssBytes <= 0 || limits.rssBytes > pdftool::worker::DEFAULT_RSS_LIMIT_BYTES) + return 2; } if (parser.isSet(cpuOption)) { - limits.cpuSeconds = parser.value(cpuOption).toLongLong(); + bool valid = false; + limits.cpuSeconds = parser.value(cpuOption).toLongLong(&valid); + if (!valid || limits.cpuSeconds <= 0 || limits.cpuSeconds > pdftool::worker::DEFAULT_CPU_SECONDS) + return 2; } QString sandboxError; const bool sandboxApplied = pdftool::worker::applyWorkerSandbox(paths, limits, &sandboxError); -#if defined(LOOP_PDF_WORKER_REQUIRE_SANDBOX) && LOOP_PDF_WORKER_REQUIRE_SANDBOX if (!sandboxApplied) { - QTextStream(stderr) << "loop-pdf-worker: sandbox required but failed: " << sandboxError << '\n'; - return 3; + const QStringList codes{ QStringLiteral("worker.sandbox.token"), QStringLiteral("worker.sandbox.capabilities"), + QStringLiteral("worker.sandbox.job-query"), QStringLiteral("worker.sandbox.job-limits"), + QStringLiteral("worker.sandbox.children") }; + const int index = codes.indexOf(sandboxError); + return index >= 0 ? 10 + index : 3; } +#if defined(Q_OS_WIN) + const QString sandboxDetail = QStringLiteral("windows-appcontainer-job"); #else - if (!sandboxApplied) - { - QTextStream(stderr) << "loop-pdf-worker: sandbox unavailable: " << sandboxError << '\n'; - } + const QString sandboxDetail = QStringLiteral("linux-landlock-seccomp-rlimit"); #endif - - pdftool::worker::WorkerRuntime runtime(paths); + pdftool::worker::WorkerRuntime runtime(paths, pdftool::worker::sandboxStatusJson(sandboxApplied, sandboxDetail, limits)); QFile input; if (!input.open(stdin, QIODevice::ReadOnly)) { @@ -128,11 +146,15 @@ int main(int argc, char* argv[]) while (true) { - const QByteArray line = input.readLine(); + const QByteArray line = input.readLine(pdftool::worker::MAX_REQUEST_BYTES + 1); if (line.isNull()) { break; } + if (line.size() > pdftool::worker::MAX_REQUEST_BYTES || !line.endsWith('\n')) + { + return 6; + } const QByteArray trimmed = line.trimmed(); if (trimmed.isEmpty()) { @@ -146,7 +168,7 @@ int main(int argc, char* argv[]) writeLine(pdftool::worker::makeErrorResponse(QString(), QStringLiteral("unknown"), QStringLiteral("invalid-invocation"), QStringLiteral("worker.bad-json"), - parseError.errorString())); + QStringLiteral("Invalid JSON request."))); continue; } diff --git a/PdfTool/main.cpp b/PdfTool/main.cpp index 7850d096e..91743d061 100644 --- a/PdfTool/main.cpp +++ b/PdfTool/main.cpp @@ -20,6 +20,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include + #include "pdftoolabstractapplication.h" #include "pdftoolcancel.h" #include "pdftoolresult.h" @@ -234,11 +236,29 @@ int main(int argc, char* argv[]) pdf::initializeApplicationIdentity(pdf::PDFApplicationSurface::PdfTool); pdf::PDFSettings::migrateLegacySettings(); - const pdf::PDFLogSession logSession(QStringLiteral("pdftool")); - const pdf::PDFSentrySession sentrySession(QStringLiteral("pdftool")); - const QStringList arguments = QCoreApplication::arguments(); const QString command = requestedCommand(arguments); + const bool isolatedOperation = command == QStringLiteral("worker-ping") || + command == QStringLiteral("worker-open") || + command == QStringLiteral("worker-preflight"); +#if defined(Q_OS_WIN) + if (isolatedOperation) + { + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); + } +#endif + if (isolatedOperation) + { + qInstallMessageHandler([](QtMsgType, const QMessageLogContext&, const QString&) {}); + } + std::unique_ptr logSession; + std::unique_ptr sentrySession; + if (!isolatedOperation) + { + logSession = std::make_unique(QStringLiteral("pdftool")); + sentrySession = std::make_unique(QStringLiteral("pdftool")); + } + const bool wantsJson = commandLineRequestsJson(arguments) || ((command == QStringLiteral("preflight") || command == QStringLiteral("verify-certificate") || command == QStringLiteral("ocr") || command == QStringLiteral("capabilities") || command == QStringLiteral("schema") || @@ -348,7 +368,11 @@ int main(int argc, char* argv[]) pdftool::resetCancelRequested(); std::signal(SIGINT, handleTerminationSignal); -#ifndef Q_OS_WIN +#ifdef Q_OS_WIN + // CTRL_BREAK_EVENT is the only console interrupt deliverable to a child + // started in its own process group; the CRT raises it as SIGBREAK. + std::signal(SIGBREAK, handleTerminationSignal); +#else std::signal(SIGTERM, handleTerminationSignal); #endif diff --git a/PdfTool/pdftoolabstractapplication.cpp b/PdfTool/pdftoolabstractapplication.cpp index d5c81b2bc..b2a419ce7 100644 --- a/PdfTool/pdftoolabstractapplication.cpp +++ b/PdfTool/pdftoolabstractapplication.cpp @@ -388,6 +388,11 @@ QList PDFToolAbstractApplication::describeOptions(Optio { add(QStringLiteral("report-file"), { QStringLiteral("--report-file") }, QStringLiteral("file"), PDFToolValueType::Path); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + add(QStringLiteral("profile"), { QStringLiteral("--profile") }, QStringLiteral("profile"), PDFToolValueType::Path); + add(QStringLiteral("preflight-page-last"), { QStringLiteral("--preflight-page-last") }, QStringLiteral("page"), PDFToolValueType::Integer, {}, QStringLiteral("0")); + } if (optionFlags.testFlag(CapabilityDiscovery)) { add(QStringLiteral("command"), { QStringLiteral("--command") }, QStringLiteral("id"), PDFToolValueType::String); @@ -900,6 +905,12 @@ void PDFToolAbstractApplication::initializeCommandLineParser(QCommandLineParser* addDescribedOption(parser, optionDescriptors, QStringLiteral("report-file"), QStringLiteral("Write the preflight report JSON this run is certified over to this file.")); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + addDescribedOption(parser, optionDescriptors, QStringLiteral("profile"), QStringLiteral("Run a preflight phase with this profile before rendering and record its memory high-water.")); + addDescribedOption(parser, optionDescriptors, QStringLiteral("preflight-page-last"), QStringLiteral("Limit the preflight phase to pages 1 through this page (0 covers the whole document); rendering still covers every selected page.")); + } + if (optionFlags.testFlag(CapabilityDiscovery)) { addDescribedOption(parser, optionDescriptors, QStringLiteral("command"), QStringLiteral("Limit discovery to one stable command ID.")); @@ -1483,6 +1494,17 @@ PDFToolOptions PDFToolAbstractApplication::getOptions(QCommandLineParser* parser options.preflightReportPath = parser->value("report-file"); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + options.preflightProfilePath = parser->value("profile"); + bool preflightPageLastOk = false; + const int preflightPageLast = parser->value("preflight-page-last").toInt(&preflightPageLastOk); + if (preflightPageLastOk && preflightPageLast > 0) + { + options.preflightPageLast = preflightPageLast; + } + } + if (optionFlags.testFlag(VerifyPreflightCertificate)) { options.preflightCertificatePath = positionalArguments.value(0); diff --git a/PdfTool/pdftoolabstractapplication.h b/PdfTool/pdftoolabstractapplication.h index fabe7644b..0926748b4 100644 --- a/PdfTool/pdftoolabstractapplication.h +++ b/PdfTool/pdftoolabstractapplication.h @@ -256,6 +256,9 @@ struct PDFToolOptions // figures" is a legitimate answer - so the fail-closed reading is opt-in. bool failIfEmpty = false; + // For option 'BenchmarkPreflightProfile': last page of the preflight phase, 0 for all pages. + int preflightPageLast = 0; + // For option 'PreflightProfile' QString preflightProfilePath; QString preflightJobContextPath; @@ -431,6 +434,7 @@ class PDFToolAbstractApplication EvidenceBundleExport = 0x80000000000ULL, ///< Export a portable proof-of-preflight bundle EvidenceBundleVerify = 0x100000000000ULL, ///< Verify a portable proof-of-preflight bundle PreflightReportFile = 0x200000000000ULL, ///< Preflight --report-file output path + BenchmarkPreflightProfile = 0x400000000000ULL, ///< Benchmark --profile for a measured preflight phase }; Q_DECLARE_FLAGS(Options, Option) diff --git a/PdfTool/pdftoolrender.cpp b/PdfTool/pdftoolrender.cpp index a1910fc98..99a6b8a07 100644 --- a/PdfTool/pdftoolrender.cpp +++ b/PdfTool/pdftoolrender.cpp @@ -25,20 +25,40 @@ #include "pdfdocumentsession.h" #include "pdffont.h" #include "pdfconstants.h" +#include "pdfoperationcontrol.h" #include "pdfsafefilewriter.h" #include "pdfworkloadenvelope.h" +#include "preflightclirun.h" +#include "preflightengine.h" +#include "preflightprofileresolver.h" #include #include #include #include +#include + namespace pdftool { static PDFToolRender s_toolRenderApplication; static PDFToolBenchmark s_toolBenchmarkApplication; +namespace +{ + +class CliCancellationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override + { + return isCancelRequested(); + } +}; + +} // namespace + QString PDFToolRender::getStandardString(PDFToolAbstractApplication::StandardString standardString) const { switch (standardString) @@ -160,7 +180,54 @@ QString PDFToolBenchmark::getStandardString(PDFToolAbstractApplication::Standard PDFToolAbstractApplication::Options PDFToolBenchmark::getOptionsFlags() const { - return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags; + return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags | BenchmarkPreflightProfile; +} + +PDFToolExitCode PDFToolBenchmark::execute(const PDFToolOptions& options) +{ + m_preflightPhase = PreflightPhase(); + if (options.preflightProfilePath.isEmpty()) + { + return PDFToolRenderBase::execute(options); + } + + QJsonObject profileJson; + QString profileError; + if (!pdf::PreflightEngine::loadProfile(options.preflightProfilePath, profileJson, profileError)) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("cli.invalid-arguments"), profileError); + return PDFToolExitCode::InvalidInvocation; + } + const pdf::PreflightProfileImportResult imported = pdf::importPreflightProfile(profileJson, options.preflightProfilePath); + if (!imported.ok) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, imported.errorCode, imported.errorMessage); + return PDFToolExitCode::InvalidInvocation; + } + + CliCancellationControl cancellationControl; + pdf::PreflightFileInspectionRequest request; + request.documentPath = options.document; + request.password = options.password; + request.permissiveReading = options.permissiveReading; + request.profile = imported.profile; + request.plan.full = true; + request.plan.reason = QStringLiteral("benchmark-preflight-phase"); + request.firstPage = options.pageSelectorFirstPage; + request.lastPage = options.preflightPageLast > 0 ? QString::number(options.preflightPageLast) : options.pageSelectorLastPage; + request.selectedPages = options.pageSelectorSelection; + request.cancellation = &cancellationControl; + + // The outcome owns the full source bytes; it is released here, before the + // render phase opens the document again. + { + const pdf::PreflightFileInspectionOutcome outcome = pdf::inspectPreflightFile(request); + m_preflightPhase.requested = true; + m_preflightPhase.inspected = outcome.documentReadOk && outcome.inspectionRan && !isCancelRequested(); + m_preflightPhase.highWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + } + + return PDFToolRenderBase::execute(options); } void PDFToolBenchmark::finish(const PDFToolOptions& options) @@ -207,26 +274,52 @@ void PDFToolBenchmark::finish(const PDFToolOptions& options) pdf::PDFWorkloadEnvelope envelope; envelope.identity = identity; envelope.family = QStringLiteral("benchmark-render"); - const bool cancelled = isCancelRequested(); - envelope.status = cancelled - ? QStringLiteral("cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("budget-exceeded") - : QStringLiteral("incomplete"); envelope.pageCount = static_cast(m_pageInfo.size()); - envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); - envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); - envelope.elapsedMs = m_wallTime; - envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; - envelope.incompleteReason = cancelled - ? QStringLiteral("operation-cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("resource-budget-exceeded") - : QStringLiteral("preflight-measurement-unavailable"); qint64 pagesMaterialized = 0; for (const PageInfo& page : m_pageInfo) { pagesMaterialized += page.isRendered ? 1 : 0; } envelope.pagesMaterialized = pagesMaterialized; + + // A record is complete only when every phase it claims was measured: + // an unmeasured preflight or an unrendered page keeps it incomplete. + const bool cancelled = isCancelRequested(); + if (cancelled) + { + envelope.status = QStringLiteral("cancelled"); + envelope.incompleteReason = QStringLiteral("operation-cancelled"); + } + else if (m_resourceBudgetExhausted) + { + envelope.status = QStringLiteral("budget-exceeded"); + envelope.incompleteReason = QStringLiteral("resource-budget-exceeded"); + } + else if (!m_preflightPhase.requested) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-measurement-unavailable"); + } + else if (!m_preflightPhase.inspected) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-phase-failed"); + } + else if (pagesMaterialized != envelope.pageCount) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("pages-not-materialized"); + } + else + { + envelope.status = QStringLiteral("complete"); + } + + envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); + envelope.preflightHighWaterBytes = m_preflightPhase.highWaterBytes; + envelope.elapsedMs = m_wallTime; + envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; envelope.recordResources(*m_resourceBudget); data.insert(QStringLiteral("workload_envelope"), envelope.toJson()); options.executionContext->setData(data); @@ -373,10 +466,22 @@ PDFToolExitCode PDFToolRenderBase::execute(const PDFToolOptions& options) QElapsedTimer timer; timer.start(); - rasterizerPool.render(pageIndices, imageSizeGetter, std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1), nullptr); + // Render in slices and check for cancellation between them, so an interrupt + // stops the run within one slice instead of after the whole page range. + const auto processImage = std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1); + const std::size_t sliceSize = std::size_t(pdf::PDFRasterizerPool::getCorrectedRasterizerCount(options.renderRasterizerCount)) * 4; + bool resourceBudgetExhausted = false; + for (std::size_t first = 0; first < pageIndices.size() && !isCancelRequested(); first += sliceSize) + { + const std::size_t last = std::min(pageIndices.size(), first + sliceSize); + const std::vector slice(pageIndices.cbegin() + first, pageIndices.cbegin() + last); + rasterizerPool.render(slice, imageSizeGetter, processImage, nullptr); + // render() resets the pool's flag per call, so exhaustion is accumulated here. + resourceBudgetExhausted = resourceBudgetExhausted || rasterizerPool.resourceBudgetExhausted(); + } m_wallTime = timer.elapsed(); - m_resourceBudgetExhausted = rasterizerPool.resourceBudgetExhausted(); + m_resourceBudgetExhausted = resourceBudgetExhausted; fontCache.setCacheShrinkEnabled(nullptr, true); diff --git a/PdfTool/pdftoolrender.h b/PdfTool/pdftoolrender.h index b57f76397..1068f5fcc 100644 --- a/PdfTool/pdftoolrender.h +++ b/PdfTool/pdftoolrender.h @@ -82,12 +82,25 @@ class PDFToolBenchmark : public PDFToolRenderBase public: virtual QString getStandardString(StandardString standardString) const override; virtual Options getOptionsFlags() const override; + virtual PDFToolExitCode execute(const PDFToolOptions& options) override; protected: virtual void finish(const PDFToolOptions& options) override; virtual void onPageRendered(const PDFToolOptions& options, pdf::PDFRenderedPageImage& renderedPageImage) override; + +private: + /// Preflight runs before rendering, so the process high-water recorded when + /// it ends is the peak of open plus preflight, independent of rendering. + struct PreflightPhase + { + bool requested = false; + bool inspected = false; + qint64 highWaterBytes = -1; + }; + + PreflightPhase m_preflightPhase; }; } // namespace pdftool -#endif // PDFTOOLRENDER_H +#endif // PDFTOOLRENDER_H diff --git a/PdfTool/pdftoolrepair.cpp b/PdfTool/pdftoolrepair.cpp index 38bdeb887..0410d47b5 100644 --- a/PdfTool/pdftoolrepair.cpp +++ b/PdfTool/pdftoolrepair.cpp @@ -806,15 +806,6 @@ PDFToolExitCode PDFToolRepair::execute(const PDFToolOptions& options) return PDFToolExitCode::ProcessingFailure; } - const pdf::PDFHistoryRetentionResult retention = operationHistory.enforceRetention({}, historyArtifacts); - if (!retention.success) - { - reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("history.retention-failed"), - QStringLiteral("The repair history was recorded, but retention could not be enforced: %1") - .arg(retention.errorMessage)); - return PDFToolExitCode::ProcessingFailure; - } - if (!options.repairReportFile.isEmpty()) { QString reportError; @@ -841,6 +832,17 @@ PDFToolExitCode PDFToolRepair::execute(const PDFToolOptions& options) { PDFConsole::writeText(QString::fromUtf8(QJsonDocument(reportJson).toJson(QJsonDocument::Indented)), options.outputCodec); } + + // Retention runs after the accepted output is fully reported so a retention failure + // cannot suppress the repair report or output registration. + const pdf::PDFHistoryRetentionResult retention = operationHistory.enforceRetention({}, historyArtifacts); + if (!retention.success) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("history.retention-failed"), + QStringLiteral("The repair history was recorded, but retention could not be enforced: %1") + .arg(retention.errorMessage)); + return PDFToolExitCode::ProcessingFailure; + } return PDFToolExitCode::Success; } diff --git a/PdfTool/pdftoolworker.cpp b/PdfTool/pdftoolworker.cpp index 7b270d778..37dc57991 100644 --- a/PdfTool/pdftoolworker.cpp +++ b/PdfTool/pdftoolworker.cpp @@ -66,34 +66,6 @@ QString resolveWorkerExecutable() return PdfWorkerClient::defaultWorkerExecutable(); } -bool stageProfile(const QString& profilePath, const QString& tempDir, QString* stagedPath, QString* error) -{ - const QFileInfo info(profilePath); - if (!info.exists() || !info.isFile()) - { - if (error) - { - *error = PDFToolTranslationContext::tr("Profile not found: %1").arg(profilePath); - } - return false; - } - const QString target = QDir(tempDir).filePath(info.fileName()); - if (QFile::exists(target)) - { - QFile::remove(target); - } - if (!QFile::copy(profilePath, target)) - { - if (error) - { - *error = PDFToolTranslationContext::tr("Failed to stage profile into worker temp."); - } - return false; - } - *stagedPath = target; - return true; -} - void publishWorkerResult(const PDFToolOptions& options, const WorkerClientResult& result) { if (options.executionContext) @@ -212,13 +184,8 @@ PDFToolExitCode PDFToolWorkerOpenApplication::execute(const PDFToolOptions& opti PdfWorkerClient client; QString error; - if (!client.start(resolveWorkerExecutable(), inputPath, tempDir.path(), outputDir.path(), &error)) - { - reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("worker.unavailable"), error); - return PDFToolExitCode::ProcessingFailure; - } + client.start(resolveWorkerExecutable(), inputPath, tempDir.path(), outputDir.path(), &error); - const qint64 firstPid = client.workerPid(); const WorkerClientResult result = client.openDocument(inputPath, options.password, options.permissiveReading); publishWorkerResult(options, result); @@ -227,21 +194,6 @@ PDFToolExitCode PDFToolWorkerOpenApplication::execute(const PDFToolOptions& opti reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, result.code.isEmpty() ? QStringLiteral("worker.unavailable") : result.code, result.reason.isEmpty() ? PDFToolTranslationContext::tr("Worker open failed.") : result.reason); - // Prove the supervisor can replace a dead worker without restarting. - if (!client.isRunning()) - { - QString replaceError; - if (client.replaceWorker(&replaceError) && options.executionContext) - { - const WorkerClientResult ping = client.ping(); - QJsonObject data = result.response; - data.insert(QStringLiteral("replaced_worker"), true); - data.insert(QStringLiteral("previous_pid"), firstPid); - data.insert(QStringLiteral("replacement_pid"), client.workerPid()); - data.insert(QStringLiteral("replacement_ping_ok"), ping.outcome == WorkerClientOutcome::Success); - options.executionContext->setData(data); - } - } return mapWorkerOutcome(result.outcome); } @@ -303,23 +255,11 @@ PDFToolExitCode PDFToolWorkerPreflightApplication::execute(const PDFToolOptions& return PDFToolExitCode::InternalError; } - QString stagedProfile; - QString stageError; - if (!stageProfile(options.preflightProfilePath, tempDir.path(), &stagedProfile, &stageError)) - { - reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("worker.profile"), stageError); - return PDFToolExitCode::InputError; - } - PdfWorkerClient client; QString error; - if (!client.start(resolveWorkerExecutable(), inputPath, tempDir.path(), outputDir.path(), &error)) - { - reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("worker.unavailable"), error); - return PDFToolExitCode::ProcessingFailure; - } + client.start(resolveWorkerExecutable(), inputPath, tempDir.path(), outputDir.path(), &error); - const WorkerClientResult result = client.preflight(inputPath, stagedProfile, outputDir.path(), + const WorkerClientResult result = client.preflight(inputPath, options.preflightProfilePath, outputDir.path(), options.password, options.permissiveReading); publishWorkerResult(options, result); @@ -331,8 +271,6 @@ PDFToolExitCode PDFToolWorkerPreflightApplication::execute(const PDFToolOptions& .arg(result.response.value(QStringLiteral("status")).toString()), options.outputCodec); } - // Findings are still a successful isolated run (not PASS-as-clean when - // status is findings); map findings to Findings exit when reported. if (result.response.value(QStringLiteral("status")).toString() == QLatin1String("findings")) { return PDFToolExitCode::Findings; diff --git a/PdfTool/pdfworkerclient.cpp b/PdfTool/pdfworkerclient.cpp index dea52ef7b..9572b0e8c 100644 --- a/PdfTool/pdfworkerclient.cpp +++ b/PdfTool/pdfworkerclient.cpp @@ -21,286 +21,408 @@ // SOFTWARE. #include "pdfworkerclient.h" - #include "pdfworkerprotocol.h" +#include "pdfartifactidentity.h" +#include "preflightprofileresolver.h" #include +#include #include #include +#include #include #include #include namespace pdftool { - namespace { - WorkerClientOutcome outcomeFromStatus(const QString& status) { if (status == QLatin1String("success") || status == QLatin1String("findings")) - { return WorkerClientOutcome::Success; - } - if (status == QLatin1String("incomplete") || status == QLatin1String("preflight-incomplete")) - { - return WorkerClientOutcome::Incomplete; - } if (status == QLatin1String("cancelled")) - { return WorkerClientOutcome::Cancelled; - } if (status == QLatin1String("invalid-invocation")) - { return WorkerClientOutcome::InvalidInvocation; - } if (status == QLatin1String("input-error")) - { return WorkerClientOutcome::InputError; - } - if (status == QLatin1String("unavailable") || status == QLatin1String("preflight-error")) - { - return status == QLatin1String("unavailable") ? WorkerClientOutcome::Unavailable - : WorkerClientOutcome::Incomplete; - } - return WorkerClientOutcome::Unavailable; + if (status == QLatin1String("unavailable")) + return WorkerClientOutcome::Unavailable; + return WorkerClientOutcome::Incomplete; } -} // namespace - -PdfWorkerClient::PdfWorkerClient() +pdf::PDFRevisionIdentity requestRevision(const QJsonObject& request) { - m_process.setProcessChannelMode(QProcess::SeparateChannels); + pdf::PDFRevisionIdentity revision; + revision.document.sourceDataHash = QByteArray::fromHex(request.value(QStringLiteral("input_sha256")).toString().toLatin1()); + revision.document.documentId = request.value(QStringLiteral("id")).toString(); + return revision; } -PdfWorkerClient::~PdfWorkerClient() +bool responseEnvelopeValid(const QJsonObject& response, const QJsonObject& request) { - killWorker(); + if (response.value(QStringLiteral("v")) != QJsonValue(worker::PROTOCOL_VERSION) || + response.value(QStringLiteral("id")) != request.value(QStringLiteral("id")) || + response.value(QStringLiteral("op")) != request.value(QStringLiteral("op")) || + !response.value(QStringLiteral("ok")).isBool() || !response.value(QStringLiteral("status")).isString()) + { + return false; + } + const QString status = response.value(QStringLiteral("status")).toString(); + const bool success = status == QLatin1String("success") || status == QLatin1String("findings"); + const QStringList failures{ QStringLiteral("incomplete"), QStringLiteral("cancelled"), QStringLiteral("invalid-invocation"), + QStringLiteral("input-error"), QStringLiteral("unavailable"), QStringLiteral("preflight-error") }; + if (response.value(QStringLiteral("ok")).toBool() != success || (!success && !failures.contains(status)) || + (status == QLatin1String("findings") && request.value(QStringLiteral("op")) != QJsonValue(QStringLiteral("preflight")))) + { + return false; + } + return success || (response.value(QStringLiteral("code")).isString() && response.value(QStringLiteral("reason")).isString()); +} } +PdfWorkerClient::PdfWorkerClient() = default; +PdfWorkerClient::~PdfWorkerClient() { killWorker(); } + QString PdfWorkerClient::defaultWorkerExecutable() { const QDir dir(QCoreApplication::applicationDirPath()); -#if defined(Q_OS_WIN) +#ifdef Q_OS_WIN return dir.filePath(QStringLiteral("loop-pdf-worker.exe")); #else return dir.filePath(QStringLiteral("loop-pdf-worker")); #endif } -bool PdfWorkerClient::start(const QString& workerExecutable, - const QString& sandboxInput, - const QString& sandboxTemp, - const QString& sandboxOutput, - QString* errorMessage) +bool PdfWorkerClient::start(const QString& workerExecutable, const QString& sandboxInput, const QString& sandboxTemp, + const QString& sandboxOutput, QString* errorMessage) { + Q_UNUSED(sandboxInput); killWorker(); m_workerExecutable = workerExecutable; - m_sandboxInput = sandboxInput; + m_sandboxInput = m_snapshots.path(); m_sandboxTemp = sandboxTemp; m_sandboxOutput = sandboxOutput; - - if (!QFileInfo::exists(workerExecutable)) + QString error; + const QStringList arguments{ + QStringLiteral("--sandbox-input"), m_sandboxInput, + QStringLiteral("--sandbox-temp"), sandboxTemp, QStringLiteral("--sandbox-output"), sandboxOutput + }; + if (!m_snapshots.isValid() || + !m_process.start(workerExecutable, arguments, m_sandboxInput, sandboxTemp, sandboxOutput, error)) { if (errorMessage) - { - *errorMessage = QStringLiteral("Worker executable not found: %1").arg(workerExecutable); - } + *errorMessage = error.isEmpty() ? QStringLiteral("worker.launch.snapshots") : error; return false; } - - m_process.setProgram(workerExecutable); - m_process.setArguments({ - QStringLiteral("--sandbox-input"), - sandboxInput, - QStringLiteral("--sandbox-temp"), - sandboxTemp, - QStringLiteral("--sandbox-output"), - sandboxOutput, - }); - m_process.start(); - if (!m_process.waitForStarted(30000)) + const WorkerClientResult handshake = ping(); + if (handshake.outcome != WorkerClientOutcome::Success) { + killWorker(); if (errorMessage) - { - *errorMessage = QStringLiteral("Failed to start worker: %1").arg(m_process.errorString()); - } + *errorMessage = QStringLiteral("Worker containment handshake failed (exit %1).").arg(m_process.exitCode()); return false; } return true; } -bool PdfWorkerClient::isRunning() const -{ - return m_process.state() != QProcess::NotRunning; -} - -qint64 PdfWorkerClient::workerPid() const +bool PdfWorkerClient::isRunning() const { return m_process.running(); } +qint64 PdfWorkerClient::workerPid() const { return m_process.pid(); } +void PdfWorkerClient::killWorker() { m_process.stop(); } +bool PdfWorkerClient::replaceWorker(QString* errorMessage) { - return m_process.processId(); + return start(m_workerExecutable, m_sandboxInput, m_sandboxTemp, m_sandboxOutput, errorMessage); } -void PdfWorkerClient::killWorker() +bool PdfWorkerClient::stageSnapshot(const QString& source, const QString& name, QString& target, QString& digest) { - if (m_process.state() == QProcess::NotRunning) + QFile input(source); + target = QDir(m_snapshots.path()).filePath(QUuid::createUuid().toString(QUuid::WithoutBraces) + name); + QFile output(target); + if (!input.open(QIODevice::ReadOnly) || !output.open(QIODevice::WriteOnly | QIODevice::NewOnly)) { - return; + return false; } - m_process.kill(); - m_process.waitForFinished(5000); -} - -bool PdfWorkerClient::replaceWorker(QString* errorMessage) -{ - return start(m_workerExecutable, m_sandboxInput, m_sandboxTemp, m_sandboxOutput, errorMessage); + QCryptographicHash hash(QCryptographicHash::Sha256); + while (!input.atEnd()) + { + const QByteArray chunk = input.read(65536); + if (chunk.isEmpty() || output.write(chunk) != chunk.size()) + { + output.close(); + QFile::remove(target); + return false; + } + hash.addData(chunk); + } + if (input.error() != QFileDevice::NoError || !output.flush()) + { + output.close(); + QFile::remove(target); + return false; + } + output.close(); + if (!QFile::setPermissions(target, QFileDevice::ReadOwner)) + { + QFile::remove(target); + return false; + } + digest = QString::fromLatin1(hash.result().toHex()); + return true; } -WorkerClientResult PdfWorkerClient::fromWorkerFailure(const QString& op, const QString& reason) +WorkerClientResult PdfWorkerClient::fromWorkerFailure(const QJsonObject& request, const QString& code) { WorkerClientResult result; - result.outcome = WorkerClientOutcome::Unavailable; - result.code = QStringLiteral("worker.unavailable"); - result.reason = reason; - result.response = worker::makeErrorResponse(QString(), op, QStringLiteral("unavailable"), - result.code, reason); + result.outcome = code == QLatin1String("worker.cancelled") ? WorkerClientOutcome::Cancelled : WorkerClientOutcome::Incomplete; + result.code = code; + result.reason = QStringLiteral("Isolated operation did not complete."); + result.response = worker::makeErrorResponse(request.value(QStringLiteral("id")).toString(), + request.value(QStringLiteral("op")).toString(), + result.outcome == WorkerClientOutcome::Cancelled ? QStringLiteral("cancelled") : QStringLiteral("incomplete"), code, result.reason); + if (request.value(QStringLiteral("op")) == QJsonValue(QStringLiteral("preflight"))) + { + const QString profileDigest = m_expectedProfile ? m_expectedProfile->effectiveDigest : request.value(QStringLiteral("profile_sha256")).toString(); + const auto receipt = pdf::buildTerminalPreflightReceipt(request.value(QStringLiteral("input_sha256")).toString(), + requestRevision(request), profileDigest, code); + result.response.insert(QStringLiteral("receipt"), receipt.toJson()); + } return result; } WorkerClientResult PdfWorkerClient::call(const QJsonObject& request, int timeoutMs) { - const QString op = request.value(QStringLiteral("op")).toString(); + m_cancelled.store(false); if (!isRunning()) - { - return fromWorkerFailure(op, QStringLiteral("Worker process is not running.")); - } - + return fromWorkerFailure(request, QStringLiteral("worker.unavailable")); const QByteArray line = QJsonDocument(request).toJson(QJsonDocument::Compact) + '\n'; - if (m_process.write(line) != line.size()) + if (line.size() > worker::MAX_REQUEST_BYTES || timeoutMs <= 0) { - return fromWorkerFailure(op, QStringLiteral("Failed to write request to worker.")); + return fromWorkerFailure(request, QStringLiteral("worker.request-limit")); } - if (!m_process.waitForBytesWritten(timeoutMs)) + QElapsedTimer timer; + timer.start(); + QByteArray frame; + if (!m_process.write(line, timer, timeoutMs, m_cancelled)) { killWorker(); - return fromWorkerFailure(op, QStringLiteral("Timed out writing request to worker.")); + return fromWorkerFailure(request, m_cancelled.load() ? QStringLiteral("worker.cancelled") : QStringLiteral("worker.write-failed")); } - - QElapsedTimer timer; - timer.start(); - while (timer.elapsed() < timeoutMs) + while (timer.elapsed() < timeoutMs && !m_cancelled.load()) { - if (m_process.state() == QProcess::NotRunning) + const QByteArray chunk = m_process.read(worker::MAX_RESPONSE_BYTES + 1 - frame.size()); + frame.append(chunk); + if (frame.size() > worker::MAX_RESPONSE_BYTES) { - return fromWorkerFailure(op, QStringLiteral("Worker exited unexpectedly (exit %1, status %2).") - .arg(m_process.exitCode()) - .arg(static_cast(m_process.exitStatus()))); + killWorker(); + return fromWorkerFailure(request, QStringLiteral("worker.response-limit")); } - if (!m_process.canReadLine()) + const qsizetype end = frame.indexOf('\n'); + if (end >= 0) { - if (!m_process.waitForReadyRead(qMax(1, timeoutMs - int(timer.elapsed())))) + QJsonParseError error; + const QJsonDocument document = QJsonDocument::fromJson(frame.first(end), &error); + if (end != frame.size() - 1 || error.error != QJsonParseError::NoError || !document.isObject() || + !responseEnvelopeValid(document.object(), request)) { - continue; + killWorker(); + return fromWorkerFailure(request, QStringLiteral("worker.invalid-response")); } + const QJsonObject response = document.object(); + const QString op = request.value(QStringLiteral("op")).toString(); + const QString status = response.value(QStringLiteral("status")).toString(); + bool valid = true; + QJsonObject admitted; + if (op == QLatin1String("ping")) + { + const QJsonObject sandbox = response.value(QStringLiteral("sandbox")).toObject(); + valid = status == QLatin1String("success") && + sandbox.value(QStringLiteral("applied")) == QJsonValue(true) && + sandbox.value(QStringLiteral("rss_limit_bytes")) == QJsonValue(worker::DEFAULT_RSS_LIMIT_BYTES) && + sandbox.value(QStringLiteral("cpu_limit_seconds")) == QJsonValue(worker::DEFAULT_CPU_SECONDS); + admitted.insert(QStringLiteral("sandbox"), QJsonObject{ + { QStringLiteral("applied"), true }, +#if defined(Q_OS_WIN) + { QStringLiteral("platform"), QStringLiteral("windows") }, +#else + { QStringLiteral("platform"), QStringLiteral("linux") }, +#endif + { QStringLiteral("detail"), QStringLiteral("enforced") }, + { QStringLiteral("rss_limit_bytes"), worker::DEFAULT_RSS_LIMIT_BYTES }, + { QStringLiteral("cpu_limit_seconds"), worker::DEFAULT_CPU_SECONDS } }); + } + else if (response.value(QStringLiteral("ok")).toBool() || response.contains(QStringLiteral("receipt"))) + { + valid = response.value(QStringLiteral("input_sha256")) == request.value(QStringLiteral("input_sha256")); + if (op == QLatin1String("open")) + { + const QJsonObject artifact = response.value(QStringLiteral("artifact")).toObject(); + const QJsonValue pages = response.value(QStringLiteral("page_count")); + pdf::PDFArtifactIdentity expectedArtifact; + expectedArtifact.sha256 = request.value(QStringLiteral("input_sha256")).toString(); + expectedArtifact.size = QFileInfo(request.value(QStringLiteral("input_path")).toString()).size(); + expectedArtifact.mediaType = QStringLiteral("application/pdf"); + expectedArtifact.logicalName = QStringLiteral("input.pdf"); + expectedArtifact.storageToken = QStringLiteral("worker-input"); + const QJsonObject expected = expectedArtifact.toJson(); + for (auto it = expected.begin(); it != expected.end(); ++it) + valid &= artifact.value(it.key()) == it.value(); + valid &= + pages.isDouble() && pages.toDouble() > 0 && pages.toDouble() == pages.toInt(-1); + admitted.insert(QStringLiteral("artifact"), expected); + admitted.insert(QStringLiteral("page_count"), pages.toInt()); + } + if (op == QLatin1String("preflight")) + { + pdf::PreflightInspectionReceipt receipt; + QString receiptError; + valid &= m_expectedProfile.has_value() && + response.value(QStringLiteral("profile_sha256")) == request.value(QStringLiteral("profile_sha256")) && + response.value(QStringLiteral("receipt")).isObject() && + pdf::preflightInspectionReceiptFromJson(response.value(QStringLiteral("receipt")).toObject(), receipt, receiptError) && + pdf::validatePreflightInspectionReceipt(receipt, request.value(QStringLiteral("input_sha256")).toString(), + requestRevision(request), *m_expectedProfile, receiptError); + const QString expectedStatus = receipt.verdict.isPass() ? QStringLiteral("success") : receipt.verdict.state == pdf::PreflightVerdictState::Fail ? QStringLiteral("findings") + : receipt.verdict.state == pdf::PreflightVerdictState::Error ? QStringLiteral("preflight-error") + : QStringLiteral("incomplete"); + valid &= status == expectedStatus; + admitted.insert(QStringLiteral("receipt"), receipt.toJson()); + admitted.insert(QStringLiteral("profile_sha256"), request.value(QStringLiteral("profile_sha256"))); + } + } + if (op != QLatin1String("ping")) + admitted.insert(QStringLiteral("input_sha256"), request.value(QStringLiteral("input_sha256"))); + if (!valid) + { + killWorker(); + return fromWorkerFailure(request, QStringLiteral("worker.identity-mismatch")); + } + if (!response.value(QStringLiteral("ok")).toBool() && !response.contains(QStringLiteral("receipt"))) + { + return fromWorkerFailure(request, QStringLiteral("worker.operation-failed")); + } + WorkerClientResult result; + result.outcome = outcomeFromStatus(status); + result.response = worker::makeOkResponse(request.value(QStringLiteral("id")).toString(), op, status, admitted); + result.response.insert(QStringLiteral("ok"), response.value(QStringLiteral("ok"))); + // Raw worker diagnostics never cross the host's diagnostic gateway. + if (result.outcome != WorkerClientOutcome::Success) + { + result.code = QStringLiteral("worker.incomplete"); + result.reason = QStringLiteral("Isolated inspection did not complete."); + result.response.insert(QStringLiteral("code"), result.code); + result.response.insert(QStringLiteral("reason"), result.reason); + } + return result; } - if (!m_process.canReadLine()) - { - continue; - } - - const QByteArray responseLine = m_process.readLine().trimmed(); - if (responseLine.isEmpty()) - { - continue; - } - - QJsonParseError parseError; - const QJsonDocument document = QJsonDocument::fromJson(responseLine, &parseError); - if (parseError.error != QJsonParseError::NoError || !document.isObject()) + if (!isRunning()) { killWorker(); - return fromWorkerFailure(op, QStringLiteral("Worker returned invalid JSON.")); + return fromWorkerFailure(request, QStringLiteral("worker.exited")); } - - WorkerClientResult result; - result.response = document.object(); - result.code = result.response.value(QStringLiteral("code")).toString(); - result.reason = result.response.value(QStringLiteral("reason")).toString(); - const QString status = result.response.value(QStringLiteral("status")).toString(); - if (result.response.value(QStringLiteral("ok")).toBool()) - { - result.outcome = outcomeFromStatus(status.isEmpty() ? QStringLiteral("success") : status); - } - else - { - result.outcome = outcomeFromStatus(status.isEmpty() ? QStringLiteral("unavailable") : status); - } - return result; } - killWorker(); - WorkerClientResult timedOut = fromWorkerFailure(op, QStringLiteral("Timed out waiting for worker response.")); - timedOut.outcome = WorkerClientOutcome::Incomplete; - timedOut.code = QStringLiteral("worker.incomplete"); - timedOut.response = worker::makeErrorResponse(request.value(QStringLiteral("id")).toString(), op, - QStringLiteral("incomplete"), timedOut.code, timedOut.reason); - return timedOut; + return fromWorkerFailure(request, m_cancelled.load() ? QStringLiteral("worker.cancelled") : QStringLiteral("worker.timeout")); } WorkerClientResult PdfWorkerClient::ping(int timeoutMs) { - return call(QJsonObject{ - { QStringLiteral("v"), worker::PROTOCOL_VERSION }, - { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, - { QStringLiteral("op"), QStringLiteral("ping") }, - }, + return call(QJsonObject{ { QStringLiteral("v"), worker::PROTOCOL_VERSION }, + { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, + { QStringLiteral("op"), QStringLiteral("ping") } }, timeoutMs); } -WorkerClientResult PdfWorkerClient::openDocument(const QString& inputPath, - const QString& password, - bool permissive, - int timeoutMs) +WorkerClientResult PdfWorkerClient::openDocument(const QString& inputPath, const QString& password, bool permissive, int timeoutMs) { - return call(QJsonObject{ - { QStringLiteral("v"), worker::PROTOCOL_VERSION }, - { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, - { QStringLiteral("op"), QStringLiteral("open") }, - { QStringLiteral("input_path"), inputPath }, - { QStringLiteral("password"), password }, - { QStringLiteral("permissive"), permissive }, - }, - timeoutMs); + QJsonObject request{ { QStringLiteral("v"), worker::PROTOCOL_VERSION }, + { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, + { QStringLiteral("op"), QStringLiteral("open") }, + { QStringLiteral("password"), password }, + { QStringLiteral("permissive"), permissive } }; + QString snapshot, digest; + if (!stageSnapshot(inputPath, QStringLiteral("-input.pdf"), snapshot, digest)) + return fromWorkerFailure(request, QStringLiteral("worker.snapshot-failed")); + request.insert(QStringLiteral("input_path"), snapshot); + request.insert(QStringLiteral("input_sha256"), digest); + auto result = call(request, timeoutMs); + if (!QFile::setPermissions(snapshot, QFileDevice::ReadOwner | QFileDevice::WriteOwner) || !QFile::remove(snapshot)) + { + killWorker(); + return fromWorkerFailure(request, QStringLiteral("worker.snapshot-cleanup-failed")); + } + return result; } -WorkerClientResult PdfWorkerClient::preflight(const QString& inputPath, - const QString& profilePath, - const QString& outputDir, - const QString& password, - bool permissive, - int timeoutMs) +WorkerClientResult PdfWorkerClient::preflight(const QString& inputPath, const QString& profilePath, const QString& outputDir, + const QString& password, bool permissive, int timeoutMs) { - return call(QJsonObject{ - { QStringLiteral("v"), worker::PROTOCOL_VERSION }, - { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, - { QStringLiteral("op"), QStringLiteral("preflight") }, - { QStringLiteral("input_path"), inputPath }, - { QStringLiteral("profile_path"), profilePath }, - { QStringLiteral("output_dir"), outputDir }, - { QStringLiteral("password"), password }, - { QStringLiteral("permissive"), permissive }, - }, - timeoutMs); + Q_UNUSED(outputDir); + m_expectedProfile.reset(); + QJsonObject request{ { QStringLiteral("v"), worker::PROTOCOL_VERSION }, + { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, + { QStringLiteral("op"), QStringLiteral("preflight") }, + { QStringLiteral("password"), password }, + { QStringLiteral("permissive"), permissive } }; + QString input, digest, profile, profileDigest; + if (!stageSnapshot(inputPath, QStringLiteral("-input.pdf"), input, digest)) + { + return fromWorkerFailure(request, QStringLiteral("worker.snapshot-failed")); + } + request.insert(QStringLiteral("input_path"), input); + request.insert(QStringLiteral("input_sha256"), digest); + if (!stageSnapshot(profilePath, QStringLiteral("-profile.json"), profile, profileDigest)) + { + const bool removed = QFile::setPermissions(input, QFileDevice::ReadOwner | QFileDevice::WriteOwner) && QFile::remove(input); + if (!removed) + killWorker(); + return fromWorkerFailure(request, removed ? QStringLiteral("worker.snapshot-failed") : QStringLiteral("worker.snapshot-cleanup-failed")); + } + request.insert(QStringLiteral("profile_path"), profile); + request.insert(QStringLiteral("profile_sha256"), profileDigest); + QFile profileFile(profile); + QString error; + pdf::PreflightProfileData data; + bool valid = profileFile.open(QIODevice::ReadOnly) && profileFile.size() <= worker::MAX_RESPONSE_BYTES; + QJsonParseError parseError; + const QJsonDocument document = valid ? QJsonDocument::fromJson(profileFile.readAll(), &parseError) : QJsonDocument(); + const auto imported = pdf::importPreflightProfile(document.object()); + const auto bound = pdf::bindPreflightProfileVariables(imported.profile, QJsonObject{}); + valid &= parseError.error == QJsonParseError::NoError && document.isObject() && imported.ok && bound.ok && + pdf::PreflightEngine::parseProfile(bound.profile, data, error); + if (valid) + { + data.variableBindings = bound.bindings; + data.fileDigest = imported.identity.digest; + data.effectiveDigest = pdf::computeProfileDigest(bound.profile); + data.provisional = imported.identity.provisional; + data.profileIdentity = imported.identity.toJson(); + data.profileIdentity.insert(QStringLiteral("digest"), data.fileDigest); + data.profileIdentity.insert(QStringLiteral("effective_digest"), data.effectiveDigest); + m_expectedProfile = data; + } + auto result = valid ? call(request, timeoutMs) : fromWorkerFailure(request, QStringLiteral("worker.profile-invalid")); + profileFile.close(); + const bool inputRemoved = QFile::setPermissions(input, QFileDevice::ReadOwner | QFileDevice::WriteOwner) && QFile::remove(input); + const bool profileRemoved = QFile::setPermissions(profile, QFileDevice::ReadOwner | QFileDevice::WriteOwner) && QFile::remove(profile); + if (!inputRemoved || !profileRemoved) + { + killWorker(); + return fromWorkerFailure(request, QStringLiteral("worker.snapshot-cleanup-failed")); + } + return result; } WorkerClientResult PdfWorkerClient::cancel(int timeoutMs) { - return call(QJsonObject{ - { QStringLiteral("v"), worker::PROTOCOL_VERSION }, - { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, - { QStringLiteral("op"), QStringLiteral("cancel") }, - }, - timeoutMs); + Q_UNUSED(timeoutMs); + m_cancelled.store(true); + WorkerClientResult result; + result.outcome = WorkerClientOutcome::Cancelled; + result.code = QStringLiteral("worker.cancelled"); + result.reason = QStringLiteral("Isolated operation cancelled."); + return result; +} } - -} // namespace pdftool diff --git a/PdfTool/pdfworkerclient.h b/PdfTool/pdfworkerclient.h index 19dcf0cfd..a4faba346 100644 --- a/PdfTool/pdfworkerclient.h +++ b/PdfTool/pdfworkerclient.h @@ -24,7 +24,11 @@ #define PDFWORKERCLIENT_H #include -#include +#include "pdfworkerprocess.h" +#include "pdfpreflightverdict.h" +#include +#include +#include #include namespace pdftool @@ -91,9 +95,14 @@ class PdfWorkerClient private: WorkerClientResult call(const QJsonObject& request, int timeoutMs); - WorkerClientResult fromWorkerFailure(const QString& op, const QString& reason); + WorkerClientResult fromWorkerFailure(const QJsonObject& request, const QString& code); + bool stageSnapshot(const QString& source, const QString& name, QString& target, QString& digest); + + WorkerProcess m_process; + QTemporaryDir m_snapshots; + std::atomic_bool m_cancelled{ false }; + std::optional m_expectedProfile; - QProcess m_process; QString m_workerExecutable; QString m_sandboxInput; QString m_sandboxTemp; diff --git a/PdfTool/pdfworkerprocess.cpp b/PdfTool/pdfworkerprocess.cpp new file mode 100644 index 000000000..f5895f319 --- /dev/null +++ b/PdfTool/pdfworkerprocess.cpp @@ -0,0 +1,113 @@ +// MIT License +// +// Copyright (c) 2018-2025 Jakub Melka and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "pdfworkerprocess.h" +#include "pdfworkerprotocol.h" + +#ifndef Q_OS_WIN +#include +#include + +namespace pdftool +{ +struct WorkerProcess::State +{ + QProcess process; +}; + +WorkerProcess::WorkerProcess() : + m_state(std::make_unique()) +{ + auto* process = &m_state->process; + QObject::connect(process, &QProcess::readyReadStandardOutput, process, [process] + { + if (process->bytesAvailable() > worker::MAX_RESPONSE_BYTES) + { + process->kill(); + } }); +} +WorkerProcess::~WorkerProcess() { stop(); } + +bool WorkerProcess::start(const QString& executable, const QStringList& arguments, const QString& inputDir, + const QString& tempDir, const QString& outputDir, QString& error) +{ + Q_UNUSED(inputDir); + Q_UNUSED(outputDir); + stop(); + auto& process = m_state->process; + process.setStandardErrorFile(QProcess::nullDevice()); + QProcessEnvironment environment; + environment.insert(QStringLiteral("LANG"), QStringLiteral("C.UTF-8")); + environment.insert(QStringLiteral("TMPDIR"), tempDir); + process.setProcessEnvironment(environment); + process.setWorkingDirectory(tempDir); +#if defined(Q_OS_UNIX) + process.setUnixProcessParameters({ QProcess::UnixProcessFlag::CloseFileDescriptors | + QProcess::UnixProcessFlag::DisableCoreDumps }); +#endif + process.start(executable, arguments); + if (!process.waitForStarted(10000)) + { + error = QStringLiteral("Isolated worker launch failed."); + return false; + } + return true; +} + +bool WorkerProcess::running() const { return m_state->process.state() != QProcess::NotRunning; } +qint64 WorkerProcess::pid() const { return m_state->process.processId(); } +qint64 WorkerProcess::exitCode() const { return m_state->process.exitCode(); } + +void WorkerProcess::stop() +{ + if (running()) + { + m_state->process.kill(); + m_state->process.waitForFinished(5000); + } + m_state->process.readAllStandardOutput(); +} + +bool WorkerProcess::write(const QByteArray& bytes, QElapsedTimer& timer, int timeoutMs, const std::atomic_bool& cancelled) +{ + auto& process = m_state->process; + if (process.write(bytes) != bytes.size()) + { + return false; + } + while (process.bytesToWrite() > 0 && timer.elapsed() < timeoutMs && !cancelled.load()) + { + if (process.bytesAvailable() > worker::MAX_RESPONSE_BYTES) + return false; + process.waitForBytesWritten(qMin(10, timeoutMs - int(timer.elapsed()))); + } + return process.bytesToWrite() == 0 && timer.elapsed() < timeoutMs && !cancelled.load(); +} + +QByteArray WorkerProcess::read(qint64 maximum) +{ + auto& process = m_state->process; + process.waitForReadyRead(10); + return process.read(qMin(maximum, qint64(65536))); +} +} +#endif diff --git a/PdfTool/pdfworkerprocess.h b/PdfTool/pdfworkerprocess.h new file mode 100644 index 000000000..744216887 --- /dev/null +++ b/PdfTool/pdfworkerprocess.h @@ -0,0 +1,53 @@ +// MIT License +// +// Copyright (c) 2018-2025 Jakub Melka and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef PDFWORKERPROCESS_H +#define PDFWORKERPROCESS_H + +#include +#include +#include +#include +#include + +namespace pdftool +{ +class WorkerProcess +{ +public: + WorkerProcess(); + ~WorkerProcess(); + bool start(const QString& executable, const QStringList& arguments, const QString& inputDir, + const QString& tempDir, const QString& outputDir, QString& error); + void stop(); + bool running() const; + qint64 pid() const; + qint64 exitCode() const; + bool write(const QByteArray& bytes, QElapsedTimer& timer, int timeoutMs, const std::atomic_bool& cancelled); + QByteArray read(qint64 maximum); + +private: + struct State; + std::unique_ptr m_state; +}; +} +#endif diff --git a/PdfTool/pdfworkerprocess_win.cpp b/PdfTool/pdfworkerprocess_win.cpp new file mode 100644 index 000000000..57446b8fc --- /dev/null +++ b/PdfTool/pdfworkerprocess_win.cpp @@ -0,0 +1,450 @@ +// MIT License +// +// Copyright (c) 2018-2025 Jakub Melka and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "pdfworkerprocess.h" +#include "pdfworkerprotocol.h" + +#ifdef Q_OS_WIN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace pdftool +{ +namespace +{ +class Handle +{ +public: + HANDLE value = nullptr; + ~Handle() { reset(); } + void reset(HANDLE handle = nullptr) + { + if (value && value != INVALID_HANDLE_VALUE) + CloseHandle(value); + value = handle; + } +}; + +bool grantDirectory(const QString& path, const QString& containerSid, const QString& userSid, bool writable) +{ + const QString sddl = QStringLiteral("D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;%1)(A;OICI;%2;;;%3)") + .arg(userSid, writable ? QStringLiteral("FA") : QStringLiteral("GRGX"), containerSid) + + (writable ? QStringLiteral("S:(ML;OICI;NW;;;LW)") : QString()); + PSECURITY_DESCRIPTOR descriptor = nullptr; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + reinterpret_cast(sddl.utf16()), SDDL_REVISION_1, &descriptor, nullptr)) + { + return false; + } + PACL dacl = nullptr, sacl = nullptr; + BOOL present = FALSE, defaulted = FALSE; + GetSecurityDescriptorDacl(descriptor, &present, &dacl, &defaulted); + GetSecurityDescriptorSacl(descriptor, &present, &sacl, &defaulted); + auto nativePath = QDir::toNativeSeparators(path).toStdWString(); + const DWORD result = SetNamedSecurityInfoW(nativePath.data(), SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION | + (writable ? LABEL_SECURITY_INFORMATION : 0), + nullptr, nullptr, dacl, sacl); + LocalFree(descriptor); + return result == ERROR_SUCCESS; +} + +QString sidText(PSID sid) +{ + LPWSTR text = nullptr; + if (!ConvertSidToStringSidW(sid, &text)) + return {}; + const QString result = QString::fromWCharArray(text); + LocalFree(text); + return result; +} + +QString currentUserSid() +{ + Handle token; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token.value)) + return {}; + DWORD size = 0; + GetTokenInformation(token.value, TokenUser, nullptr, 0, &size); + std::vector bytes(size); + if (!GetTokenInformation(token.value, TokenUser, bytes.data(), size, &size)) + return {}; + return sidText(reinterpret_cast(bytes.data())->User.Sid); +} + +QString quoteArgument(const QString& argument) +{ + QString result = QStringLiteral("\""); + qsizetype slashes = 0; + for (const QChar ch : argument) + { + if (ch == QLatin1Char('\\')) + { + ++slashes; + continue; + } + result += QString(slashes * (ch == QLatin1Char('"') ? 2 : 1), QLatin1Char('\\')); + slashes = 0; + if (ch == QLatin1Char('"')) + result += QLatin1Char('\\'); + result += ch; + } + return result + QString(slashes * 2, QLatin1Char('\\')) + QLatin1Char('"'); +} + +bool pipePair(Handle& parent, Handle& child, bool parentWrites) +{ + const QString name = QStringLiteral("\\\\.\\pipe\\loop-worker-") + QUuid::createUuid().toString(QUuid::WithoutBraces); + parent.value = CreateNamedPipeW(reinterpret_cast(name.utf16()), + (parentWrites ? PIPE_ACCESS_OUTBOUND : PIPE_ACCESS_INBOUND) | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, 1, 65536, 65536, 0, nullptr); + if (parent.value == INVALID_HANDLE_VALUE) + return false; + SECURITY_ATTRIBUTES security{ sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE }; + child.value = CreateFileW(reinterpret_cast(name.utf16()), parentWrites ? GENERIC_READ : GENERIC_WRITE, + 0, &security, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + // Opening the client end connects the local pipe before process creation. + return child.value != INVALID_HANDLE_VALUE; +} +} + +struct WorkerProcess::State +{ + Handle process, job, input, output; + DWORD processId = 0; + qint64 lastExitCode = -1; + QString profileName; + PSID containerSid = nullptr; + std::unique_ptr runtime; + ~State() + { + process.reset(); + job.reset(); + input.reset(); + output.reset(); + if (containerSid) + FreeSid(containerSid); + if (!profileName.isEmpty()) + DeleteAppContainerProfile(reinterpret_cast(profileName.utf16())); + } +}; + +WorkerProcess::WorkerProcess() : + m_state(std::make_unique()) +{ +} +WorkerProcess::~WorkerProcess() { stop(); } + +bool WorkerProcess::start(const QString& executable, const QStringList& arguments, const QString& inputDir, + const QString& tempDir, const QString& outputDir, QString& error) +{ + stop(); + error = QStringLiteral("Windows worker containment could not be established."); + auto state = std::make_unique(); + state->profileName = QStringLiteral("Loop.Worker.") + QUuid::createUuid().toString(QUuid::WithoutBraces); + error = QStringLiteral("worker.launch.profile"); + if (FAILED(CreateAppContainerProfile(reinterpret_cast(state->profileName.utf16()), + L"Loop PDF Worker", L"Isolated PDF inspection", nullptr, 0, &state->containerSid))) + { + return false; + } + const QString containerSid = sidText(state->containerSid); + const QString userSid = currentUserSid(); + error = QStringLiteral("worker.launch.profile-path"); + PWSTR profileFolder = nullptr; + if (containerSid.isEmpty() || userSid.isEmpty() || + FAILED(GetAppContainerFolderPath(reinterpret_cast(containerSid.utf16()), &profileFolder))) + { + return false; + } + const QString appDataPath = QString::fromWCharArray(profileFolder); + CoTaskMemFree(profileFolder); + const QString profilePath = QFileInfo(appDataPath).dir().canonicalPath(); + if (QFileInfo(appDataPath).fileName() != QLatin1String("AC") || + QFileInfo(profilePath).fileName().compare(state->profileName, Qt::CaseInsensitive) != 0) + { + return false; + } + error = QStringLiteral("worker.launch.profile-acl"); + QDirIterator entries(profilePath, QDir::AllEntries | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System, + QDirIterator::Subdirectories); + QStringList profileEntries; + while (entries.hasNext()) + profileEntries.append(entries.next()); + for (const QString& path : profileEntries) + { + if (!grantDirectory(path, containerSid, userSid, false)) + return false; + } + if (!grantDirectory(profilePath, containerSid, userSid, false)) + return false; + error = QStringLiteral("worker.launch.directories"); + state->runtime = std::make_unique(); + if (containerSid.isEmpty() || userSid.isEmpty() || !state->runtime->isValid() || + !grantDirectory(state->runtime->path(), containerSid, userSid, false) || + !grantDirectory(inputDir, containerSid, userSid, false) || + !grantDirectory(tempDir, containerSid, userSid, true) || + !grantDirectory(outputDir, containerSid, userSid, true)) + { + return false; + } + const QFileInfo workerInfo(executable); + const QString stagedExecutable = state->runtime->filePath(workerInfo.fileName()); + error = QStringLiteral("worker.launch.manifest"); + QFile manifest(executable + QStringLiteral(".runtime")); + if (!workerInfo.isFile() || !manifest.open(QIODevice::ReadOnly) || manifest.size() > 65536 || + !QFile::copy(executable, stagedExecutable) || + !grantDirectory(stagedExecutable, containerSid, userSid, false)) + { + return false; + } + error = QStringLiteral("worker.launch.runtime-copy"); + QSet names{ workerInfo.fileName().toLower() }; + while (!manifest.atEnd()) + { + const QString relative = QString::fromUtf8(manifest.readLine()).trimmed(); + const QFileInfo dependency(workerInfo.dir().filePath(relative)); + const QString canonical = dependency.canonicalFilePath(); + const QString root = workerInfo.dir().canonicalPath() + QLatin1Char('/'); + if (relative.isEmpty()) + continue; + if (!relative.startsWith(QStringLiteral("worker-runtime/")) || + !canonical.startsWith(root, Qt::CaseInsensitive) || !dependency.isFile() || + dependency.isSymLink() || names.contains(dependency.fileName().toLower()) || + !QFile::copy(canonical, state->runtime->filePath(dependency.fileName())) || + !grantDirectory(state->runtime->filePath(dependency.fileName()), containerSid, userSid, false)) + { + return false; + } + names.insert(dependency.fileName().toLower()); + } + error = QStringLiteral("worker.launch.pipes"); + Handle childInput, childOutput, childError; + if (!pipePair(state->input, childInput, true) || !pipePair(state->output, childOutput, false)) + { + return false; + } + SECURITY_ATTRIBUTES security{ sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE }; + childError.value = CreateFileW(L"NUL", GENERIC_WRITE, FILE_SHARE_WRITE | FILE_SHARE_READ, &security, OPEN_EXISTING, 0, nullptr); + error = QStringLiteral("worker.launch.job"); + state->job.value = CreateJobObjectW(nullptr, nullptr); + if (childError.value == INVALID_HANDLE_VALUE || !state->job.value) + return false; + error = QStringLiteral("worker.launch.job-limits"); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits{}; + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | + JOB_OBJECT_LIMIT_ACTIVE_PROCESS | JOB_OBJECT_LIMIT_PROCESS_MEMORY | JOB_OBJECT_LIMIT_PROCESS_TIME; + limits.BasicLimitInformation.ActiveProcessLimit = 1; + limits.BasicLimitInformation.PerProcessUserTimeLimit.QuadPart = worker::DEFAULT_CPU_SECONDS * 10000000; + limits.ProcessMemoryLimit = static_cast(worker::DEFAULT_RSS_LIMIT_BYTES); + if (!SetInformationJobObject(state->job.value, JobObjectExtendedLimitInformation, &limits, sizeof(limits))) + return false; + error = QStringLiteral("worker.launch.attributes"); + SIZE_T bytes = 0; + InitializeProcThreadAttributeList(nullptr, 4, 0, &bytes); + std::vector attributes(bytes); + auto* list = reinterpret_cast(attributes.data()); + if (!InitializeProcThreadAttributeList(list, 4, 0, &bytes)) + return false; + SECURITY_CAPABILITIES capabilities{}; + capabilities.AppContainerSid = state->containerSid; + HANDLE handles[]{ childInput.value, childOutput.value, childError.value }; + DWORD childPolicy = PROCESS_CREATION_CHILD_PROCESS_RESTRICTED; + const bool applied = + UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, &capabilities, sizeof(capabilities), nullptr, nullptr) && + UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles, sizeof(handles), nullptr, nullptr) && + UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_JOB_LIST, &state->job.value, sizeof(HANDLE), nullptr, nullptr) && + UpdateProcThreadAttribute(list, 0, PROC_THREAD_ATTRIBUTE_CHILD_PROCESS_POLICY, &childPolicy, sizeof(childPolicy), nullptr, nullptr); + if (!applied) + { + const DWORD attributeError = GetLastError(); + DeleteProcThreadAttributeList(list); + error += QStringLiteral(".%1").arg(attributeError); + return false; + } + STARTUPINFOEXW startup{}; + startup.StartupInfo.cb = sizeof(startup); + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.StartupInfo.hStdInput = childInput.value; + startup.StartupInfo.hStdOutput = childOutput.value; + startup.StartupInfo.hStdError = childError.value; + startup.lpAttributeList = list; + QString command = quoteArgument(QDir::toNativeSeparators(stagedExecutable)); + for (const QString& argument : arguments) + command += QLatin1Char(' ') + quoteArgument(argument); + auto commandBuffer = command.toStdWString(); + // No inherited PATH, Qt plugin overrides, credentials or reporting configuration. + wchar_t windowsDirectory[MAX_PATH]{}; + if (!GetWindowsDirectoryW(windowsDirectory, MAX_PATH)) + { + DeleteProcThreadAttributeList(list); + return false; + } + QMap systemEnvironment; + for (const QString& key : { QStringLiteral("APPDATA"), QStringLiteral("LOCALAPPDATA"), QStringLiteral("USERPROFILE"), + QStringLiteral("ProgramData"), QStringLiteral("SystemDrive") }) + { + const QString value = qEnvironmentVariable(qPrintable(key)); + if (!value.isEmpty()) + systemEnvironment.insert(key, value); + } + systemEnvironment.insert(QStringLiteral("SystemRoot"), QString::fromWCharArray(windowsDirectory)); + systemEnvironment.insert(QStringLiteral("windir"), QString::fromWCharArray(windowsDirectory)); + systemEnvironment.insert(QStringLiteral("ALLUSERSPROFILE"), qEnvironmentVariable("ProgramData")); + systemEnvironment.insert(QStringLiteral("TEMP"), QDir::toNativeSeparators(tempDir)); + systemEnvironment.insert(QStringLiteral("TMP"), QDir::toNativeSeparators(tempDir)); + QString environment; + for (auto it = systemEnvironment.cbegin(); it != systemEnvironment.cend(); ++it) + { + environment += it.key() + QLatin1Char('=') + it.value() + QChar(0); + } + environment += QChar(0); + const auto currentDirectory = QDir::toNativeSeparators(state->runtime->path()).toStdWString(); + PROCESS_INFORMATION process{}; + error = QStringLiteral("worker.launch.create-process"); + const BOOL created = CreateProcessW(nullptr, commandBuffer.data(), nullptr, nullptr, TRUE, + EXTENDED_STARTUPINFO_PRESENT | DETACHED_PROCESS | CREATE_UNICODE_ENVIRONMENT, + const_cast(environment.utf16()), currentDirectory.c_str(), &startup.StartupInfo, &process); + const DWORD creationError = created ? ERROR_SUCCESS : GetLastError(); + DeleteProcThreadAttributeList(list); + if (!created) + { + error += QStringLiteral(".%1").arg(creationError); + return false; + } + CloseHandle(process.hThread); + state->process.value = process.hProcess; + state->processId = process.dwProcessId; + m_state = std::move(state); + error.clear(); + return true; +} + +void WorkerProcess::stop() +{ + qint64 exitCode = m_state->lastExitCode; + if (m_state->job.value) + { + TerminateJobObject(m_state->job.value, 1); + m_state->job.reset(); + } + if (m_state->process.value) + { + WaitForSingleObject(m_state->process.value, 5000); + DWORD status = 0; + if (GetExitCodeProcess(m_state->process.value, &status)) + exitCode = status; + } + m_state = std::make_unique(); + m_state->lastExitCode = exitCode; +} + +bool WorkerProcess::running() const +{ + return m_state->process.value && WaitForSingleObject(m_state->process.value, 0) == WAIT_TIMEOUT; +} +qint64 WorkerProcess::pid() const { return m_state->processId; } +qint64 WorkerProcess::exitCode() const +{ + DWORD status = 0; + if (m_state->process.value && GetExitCodeProcess(m_state->process.value, &status) && status != STILL_ACTIVE) + return status; + return m_state->lastExitCode; +} + +bool WorkerProcess::write(const QByteArray& bytes, QElapsedTimer& timer, int timeoutMs, const std::atomic_bool& cancelled) +{ + OVERLAPPED operation{}; + Handle event; + event.value = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!event.value) + return false; + operation.hEvent = event.value; + DWORD written = 0; + if (WriteFile(m_state->input.value, bytes.constData(), static_cast(bytes.size()), &written, &operation)) + return written == bytes.size(); + if (GetLastError() != ERROR_IO_PENDING) + return false; + while (timer.elapsed() < timeoutMs && !cancelled.load()) + { + if (WaitForSingleObject(event.value, 10) == WAIT_OBJECT_0) + return GetOverlappedResult(m_state->input.value, &operation, &written, FALSE) && written == bytes.size(); + } + CancelIoEx(m_state->input.value, &operation); + GetOverlappedResult(m_state->input.value, &operation, &written, TRUE); + return false; +} + +QByteArray WorkerProcess::read(qint64 maximum) +{ + DWORD available = 0; + if (!PeekNamedPipe(m_state->output.value, nullptr, 0, nullptr, &available, nullptr)) + return {}; + if (!available) + { + QThread::msleep(5); + return {}; + } + QByteArray bytes(qMin(qMin(maximum, qint64(available)), qint64(65536)), Qt::Uninitialized); + OVERLAPPED operation{}; + Handle event; + event.value = CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!event.value) + return {}; + operation.hEvent = event.value; + DWORD count = 0; + const BOOL completed = ReadFile(m_state->output.value, bytes.data(), static_cast(bytes.size()), &count, &operation); + if (!completed) + { + if (GetLastError() != ERROR_IO_PENDING) + return {}; + if (WaitForSingleObject(event.value, 10) != WAIT_OBJECT_0) + { + CancelIoEx(m_state->output.value, &operation); + GetOverlappedResult(m_state->output.value, &operation, &count, TRUE); + return {}; + } + if (!GetOverlappedResult(m_state->output.value, &operation, &count, FALSE)) + return {}; + } + bytes.resize(count); + return bytes; +} +} +#endif diff --git a/PdfTool/pdfworkerprotocol.h b/PdfTool/pdfworkerprotocol.h index dcbe72050..1aa933e1e 100644 --- a/PdfTool/pdfworkerprotocol.h +++ b/PdfTool/pdfworkerprotocol.h @@ -33,7 +33,10 @@ namespace pdftool::worker /// Wire protocol version for PdfTool supervisor ↔ loop-pdf-worker IPC. /// Newline-delimited JSON; allowlist ops only (not a general RPC). -inline constexpr int PROTOCOL_VERSION = 1; +inline constexpr int PROTOCOL_VERSION = 2; + +inline constexpr qint64 MAX_REQUEST_BYTES = 64 * 1024; +inline constexpr qint64 MAX_RESPONSE_BYTES = 64 * 1024 * 1024; inline constexpr qint64 DEFAULT_RSS_LIMIT_BYTES = qint64(768) * 1024 * 1024; inline constexpr qint64 DEFAULT_CPU_SECONDS = 120; diff --git a/PdfTool/pdfworkerruntime.cpp b/PdfTool/pdfworkerruntime.cpp index c3b0b75b9..bb3ba9de5 100644 --- a/PdfTool/pdfworkerruntime.cpp +++ b/PdfTool/pdfworkerruntime.cpp @@ -21,16 +21,14 @@ // SOFTWARE. #include "pdfworkerruntime.h" - #include "pdfworkerprotocol.h" #include "pdfartifactidentity.h" #include "pdfdocumentreader.h" +#include "pdfoperationcontrol.h" #include "pdfpreflightverdict.h" -#include "pdfsafefilewriter.h" #include "preflightclirun.h" #include "preflightengine.h" -#include "preflightprofileresolver.h" #include #include @@ -41,115 +39,91 @@ namespace pdftool::worker { - namespace { - -class CancelFence final : public pdf::PDFOperationControl -{ -public: - explicit CancelFence(const std::atomic_bool* flag) : - m_flag(flag) - { - } - - bool isOperationCancelled() const override - { - return m_flag && m_flag->load(); - } - -private: - const std::atomic_bool* m_flag = nullptr; -}; - -pdf::PDFArtifactIdentity artifactFromBytes(const QByteArray& bytes, const QString& logicalName) +pdf::PDFArtifactIdentity artifactFromBytes(const QByteArray& bytes) { pdf::PDFArtifactIdentity identity; identity.sha256 = QString::fromLatin1(QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex()); identity.size = bytes.size(); identity.mediaType = QStringLiteral("application/pdf"); - identity.logicalName = pdf::sanitizeArtifactLogicalName(logicalName); + identity.logicalName = QStringLiteral("input.pdf"); identity.storageToken = QStringLiteral("worker-input"); return identity; } -QJsonObject revisionJson(const QByteArray& sourceData) +bool fileDigestMatches(const QString& path, const QString& digest) { - const QByteArray hash = QCryptographicHash::hash(sourceData, QCryptographicHash::Sha256); - return QJsonObject{ - { QStringLiteral("source_sha256"), QString::fromLatin1(hash.toHex()) }, - { QStringLiteral("document_revision"), 0 }, - { QStringLiteral("cache_generation"), 0 }, - { QStringLiteral("effective_profile_identity"), QString() }, - }; + if (!pdf::isPDFSha256(digest)) + return false; + QFile file(path); + QCryptographicHash hash(QCryptographicHash::Sha256); + return file.open(QIODevice::ReadOnly) && hash.addData(&file) && + QString::fromLatin1(hash.result().toHex()) == digest; } - -} // namespace - -WorkerRuntime::WorkerRuntime(WorkerSandboxPaths sandboxPaths) : - m_sandboxPaths(std::move(sandboxPaths)) -{ - m_sandboxStatus = sandboxStatusJson(true, QStringLiteral("linux-landlock-seccomp-rlimit")); } -void WorkerRuntime::requestCancel() +WorkerRuntime::WorkerRuntime(WorkerSandboxPaths sandboxPaths, QJsonObject sandboxStatus) : + m_sandboxPaths(std::move(sandboxPaths)), + m_sandboxStatus(std::move(sandboxStatus)) { - m_cancelRequested.store(true); } bool WorkerRuntime::pathIsInsideSandbox(const QString& candidate, const QString& root) const { - const QString absoluteCandidate = QFileInfo(candidate).absoluteFilePath(); - const QString absoluteRoot = QFileInfo(root).absoluteFilePath(); - if (absoluteCandidate == absoluteRoot) - { - return true; - } - const QString prefix = absoluteRoot.endsWith(QLatin1Char('/')) ? absoluteRoot : absoluteRoot + QLatin1Char('/'); - return absoluteCandidate.startsWith(prefix); + const QFileInfo file(candidate); + const QString canonical = file.canonicalFilePath(); + const QString canonicalRoot = QFileInfo(root).canonicalFilePath(); +#ifdef Q_OS_WIN + constexpr Qt::CaseSensitivity sensitivity = Qt::CaseInsensitive; +#else + constexpr Qt::CaseSensitivity sensitivity = Qt::CaseSensitive; +#endif + return !canonical.isEmpty() && !canonicalRoot.isEmpty() && !file.isSymLink() && + (canonical.compare(canonicalRoot, sensitivity) == 0 || + canonical.startsWith(canonicalRoot + QLatin1Char('/'), sensitivity)); } QJsonObject WorkerRuntime::handleRequest(const QJsonObject& request) { - const int version = request.value(QStringLiteral("v")).toInt(); const QString id = request.value(QStringLiteral("id")).toString(); const QString op = request.value(QStringLiteral("op")).toString(); - - if (version != PROTOCOL_VERSION) - { - return makeErrorResponse(id, op.isEmpty() ? QStringLiteral("unknown") : op, - QStringLiteral("invalid-invocation"), - QStringLiteral("worker.protocol-version"), - QStringLiteral("Unsupported protocol version.")); - } - if (id.isEmpty() || !isAllowedOperation(op)) + if (m_sandboxStatus.value(QStringLiteral("applied")) != QJsonValue(true) || + request.value(QStringLiteral("v")) != QJsonValue(PROTOCOL_VERSION) || + !request.value(QStringLiteral("id")).isString() || id.isEmpty() || id.size() > 128 || + !request.value(QStringLiteral("op")).isString() || !isAllowedOperation(op)) { - return makeErrorResponse(id, op.isEmpty() ? QStringLiteral("unknown") : op, - QStringLiteral("invalid-invocation"), - QStringLiteral("worker.op-not-allowed"), - QStringLiteral("Operation is outside the worker allowlist.")); + return makeErrorResponse(id, op, QStringLiteral("invalid-invocation"), + QStringLiteral("worker.invalid-request"), QStringLiteral("Invalid worker request.")); } - if (op == QLatin1String("ping")) - { return handlePing(id); - } if (op == QLatin1String("cancel")) - { return handleCancel(id); + if (!request.value(QStringLiteral("input_path")).isString() || + !request.value(QStringLiteral("input_sha256")).isString() || + !request.value(QStringLiteral("password")).isString() || !request.value(QStringLiteral("permissive")).isBool() || + !pathIsInsideSandbox(request.value(QStringLiteral("input_path")).toString(), m_sandboxPaths.inputPath)) + { + return makeErrorResponse(id, op, QStringLiteral("invalid-invocation"), + QStringLiteral("worker.invalid-request"), QStringLiteral("Invalid worker request.")); } - if (op == QLatin1String("open")) + if (op == QLatin1String("preflight") && + (!request.value(QStringLiteral("profile_path")).isString() || + !request.value(QStringLiteral("profile_sha256")).isString() || + !pathIsInsideSandbox(request.value(QStringLiteral("profile_path")).toString(), m_sandboxPaths.inputPath))) { - return handleOpen(id, request); + return makeErrorResponse(id, op, QStringLiteral("invalid-invocation"), + QStringLiteral("worker.invalid-request"), QStringLiteral("Invalid worker request.")); } - if (op == QLatin1String("preflight")) + if (!fileDigestMatches(request.value(QStringLiteral("input_path")).toString(), request.value(QStringLiteral("input_sha256")).toString()) || + (op == QLatin1String("preflight") && + !fileDigestMatches(request.value(QStringLiteral("profile_path")).toString(), request.value(QStringLiteral("profile_sha256")).toString()))) { - return handlePreflight(id, request); + return makeErrorResponse(id, op, QStringLiteral("input-error"), + QStringLiteral("worker.snapshot-mismatch"), QStringLiteral("Snapshot identity does not match.")); } - - return makeErrorResponse(id, op, QStringLiteral("invalid-invocation"), - QStringLiteral("worker.op-not-allowed"), - QStringLiteral("Operation is outside the worker allowlist.")); + return op == QLatin1String("open") ? handleOpen(id, request) : handlePreflight(id, request); } QJsonObject WorkerRuntime::handlePing(const QString& id) @@ -160,211 +134,71 @@ QJsonObject WorkerRuntime::handlePing(const QString& id) QJsonObject WorkerRuntime::handleCancel(const QString& id) { - requestCancel(); - return makeOkResponse(id, QStringLiteral("cancel"), QStringLiteral("success")); + return makeErrorResponse(id, QStringLiteral("cancel"), QStringLiteral("cancelled"), + QStringLiteral("worker.cancelled"), QStringLiteral("Cancellation requires supervisor termination.")); } QJsonObject WorkerRuntime::handleOpen(const QString& id, const QJsonObject& request) { - m_cancelRequested.store(false); - const QString inputPath = request.value(QStringLiteral("input_path")).toString(); - if (inputPath.isEmpty() || - !(pathIsInsideSandbox(inputPath, m_sandboxPaths.inputPath) || - QFileInfo(inputPath).absoluteFilePath() == QFileInfo(m_sandboxPaths.inputPath).absoluteFilePath())) - { - return makeErrorResponse(id, QStringLiteral("open"), QStringLiteral("invalid-invocation"), - QStringLiteral("worker.path-outside-sandbox"), - QStringLiteral("input_path is outside the sandbox input root.")); - } - - CancelFence fence(&m_cancelRequested); const QString password = request.value(QStringLiteral("password")).toString(); - const bool permissive = request.value(QStringLiteral("permissive")).toBool(true); - - pdf::PDFDocumentReader reader(nullptr, [&password](bool* ok) -> QString + bool firstAttempt = true; + pdf::PDFDocumentReader reader(nullptr, [&](bool* ok) { - *ok = true; - return password; }, permissive, false); - reader.setOperationControl(&fence); - - pdf::PDFDocument document = reader.readFromFile(inputPath); - if (fence.isOperationCancelled() || m_cancelRequested.load()) - { - return makeErrorResponse(id, QStringLiteral("open"), QStringLiteral("cancelled"), - QStringLiteral("worker.cancelled"), - QStringLiteral("Open was cancelled.")); - } - + *ok = firstAttempt; + firstAttempt = false; + return password; }, request.value(QStringLiteral("permissive")).toBool(), false); + const auto document = reader.readFromFile(request.value(QStringLiteral("input_path")).toString()); if (reader.getReadingResult() != pdf::PDFDocumentReader::Result::OK) - { - const QString status = reader.getReadingResult() == pdf::PDFDocumentReader::Result::Cancelled - ? QStringLiteral("cancelled") - : QStringLiteral("input-error"); - return makeErrorResponse(id, QStringLiteral("open"), status, - QStringLiteral("worker.open-failed"), - reader.getErrorMessage()); - } - - QFile file(inputPath); - if (!file.open(QIODevice::ReadOnly)) { return makeErrorResponse(id, QStringLiteral("open"), QStringLiteral("input-error"), - QStringLiteral("worker.open-failed"), - QStringLiteral("Failed to read input bytes for artifact identity.")); + QStringLiteral("worker.open-failed"), QStringLiteral("PDF could not be opened.")); } - const QByteArray sourceData = file.readAll(); - const pdf::PDFArtifactIdentity artifact = artifactFromBytes(sourceData, QFileInfo(inputPath).fileName()); - - QJsonArray warnings; - for (const QString& warning : reader.getWarnings()) + const auto artifact = artifactFromBytes(reader.getSource()); + if (artifact.sha256 != request.value(QStringLiteral("input_sha256")).toString()) { - warnings.append(warning); + return makeErrorResponse(id, QStringLiteral("open"), QStringLiteral("input-error"), + QStringLiteral("worker.snapshot-mismatch"), QStringLiteral("Snapshot identity does not match.")); } - - return makeOkResponse(id, QStringLiteral("open"), QStringLiteral("success"), QJsonObject{ - { QStringLiteral("artifact"), artifact.toJson() }, - { QStringLiteral("revision"), revisionJson(sourceData) }, - { QStringLiteral("page_count"), static_cast(document.getCatalog()->getPageCount()) }, - { QStringLiteral("warnings"), warnings }, - }); + return makeOkResponse(id, QStringLiteral("open"), QStringLiteral("success"), QJsonObject{ { QStringLiteral("artifact"), artifact.toJson() }, { QStringLiteral("input_sha256"), artifact.sha256 }, { QStringLiteral("page_count"), static_cast(document.getCatalog()->getPageCount()) } }); } QJsonObject WorkerRuntime::handlePreflight(const QString& id, const QJsonObject& request) { - m_cancelRequested.store(false); - const QString inputPath = request.value(QStringLiteral("input_path")).toString(); - const QString profilePath = request.value(QStringLiteral("profile_path")).toString(); - const QString outputDir = request.value(QStringLiteral("output_dir")).toString(); - - if (inputPath.isEmpty() || - !(pathIsInsideSandbox(inputPath, m_sandboxPaths.inputPath) || - QFileInfo(inputPath).absoluteFilePath() == QFileInfo(m_sandboxPaths.inputPath).absoluteFilePath())) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("invalid-invocation"), - QStringLiteral("worker.path-outside-sandbox"), - QStringLiteral("input_path is outside the sandbox input root.")); - } - if (!outputDir.isEmpty() && !pathIsInsideSandbox(outputDir, m_sandboxPaths.outputDir) && - QFileInfo(outputDir).absoluteFilePath() != QFileInfo(m_sandboxPaths.outputDir).absoluteFilePath()) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("invalid-invocation"), - QStringLiteral("worker.path-outside-sandbox"), - QStringLiteral("output_dir is outside the sandbox output root.")); - } - if (!profilePath.isEmpty() && - !pathIsInsideSandbox(profilePath, m_sandboxPaths.tempDir) && - !pathIsInsideSandbox(profilePath, m_sandboxPaths.inputPath) && - !pathIsInsideSandbox(profilePath, m_sandboxPaths.outputDir)) - { - // Profiles commonly live under the install share tree; allow absolute - // paths that the supervisor staged into temp, otherwise reject. - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("invalid-invocation"), - QStringLiteral("worker.path-outside-sandbox"), - QStringLiteral("profile_path is outside sandbox roots.")); - } - QJsonObject profile; - QString profileError; - const QString resolvedProfile = profilePath.isEmpty() - ? QString() - : profilePath; - if (resolvedProfile.isEmpty()) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("invalid-invocation"), - QStringLiteral("worker.profile-required"), - QStringLiteral("profile_path is required for preflight.")); - } - if (!pdf::PreflightEngine::loadProfile(resolvedProfile, profile, profileError)) + QString error; + QFile profileFile(request.value(QStringLiteral("profile_path")).toString()); + if (!profileFile.open(QIODevice::ReadOnly) || profileFile.size() > MAX_RESPONSE_BYTES || + !pdf::PreflightEngine::loadProfile(profileFile.fileName(), profile, error)) { return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("input-error"), - QStringLiteral("worker.profile-load-failed"), - profileError); + QStringLiteral("worker.profile-invalid"), QStringLiteral("Profile could not be loaded.")); } - - CancelFence fence(&m_cancelRequested); pdf::PreflightFileInspectionRequest inspectionRequest; - inspectionRequest.documentPath = inputPath; + inspectionRequest.documentPath = request.value(QStringLiteral("input_path")).toString(); + inspectionRequest.receiptDocumentId = id; + inspectionRequest.createReceipt = true; inspectionRequest.password = request.value(QStringLiteral("password")).toString(); - inspectionRequest.permissiveReading = request.value(QStringLiteral("permissive")).toBool(true); + inspectionRequest.permissiveReading = request.value(QStringLiteral("permissive")).toBool(); inspectionRequest.profile = profile; inspectionRequest.plan.full = true; inspectionRequest.plan.reason = QStringLiteral("worker-preflight"); - inspectionRequest.cancellation = &fence; - - const pdf::PreflightFileInspectionOutcome inspection = pdf::inspectPreflightFile(inspectionRequest); - if (fence.isOperationCancelled() || m_cancelRequested.load()) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("cancelled"), - QStringLiteral("worker.cancelled"), - QStringLiteral("Preflight was cancelled.")); - } - if (!inspection.documentReadOk) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("input-error"), - QStringLiteral("worker.open-failed"), - inspection.readErrorMessage); - } - - const pdf::PDFArtifactIdentity artifact = artifactFromBytes(inspection.sourceData, QFileInfo(inputPath).fileName()); - const pdf::PreflightVerdict verdict = pdf::reducePreflightVerdict(inspection.report); - QString status = QStringLiteral("success"); - switch (verdict.state) - { - case pdf::PreflightVerdictState::Pass: - status = QStringLiteral("success"); - break; - case pdf::PreflightVerdictState::Fail: - status = QStringLiteral("findings"); - break; - case pdf::PreflightVerdictState::Incomplete: - status = QStringLiteral("incomplete"); - break; - case pdf::PreflightVerdictState::Error: - status = QStringLiteral("preflight-error"); - break; - } - - QString publishedReport; - if (!outputDir.isEmpty() && inspection.inspectionRan && status != QLatin1String("incomplete") && - status != QLatin1String("preflight-error")) - { - const QString reportPath = QDir(outputDir).filePath(QStringLiteral("preflight-report.json")); - const QByteArray payload = QJsonDocument(inspection.report.toJson(inputPath)).toJson(QJsonDocument::Indented); - const pdf::PDFOperationResult writeResult = - pdf::PDFSafeFileWriter::writeData(reportPath, payload, pdf::PDFSafeFileWriter::OverwritePolicy::Overwrite); - if (!writeResult) - { - return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("incomplete"), - QStringLiteral("worker.report-write-failed"), - writeResult.getErrorMessage()); - } - publishedReport = reportPath; - } - else if (!outputDir.isEmpty() && (status == QLatin1String("incomplete") || status == QLatin1String("preflight-error"))) - { - // Fail-closed: never publish a partial report as the artifact. - publishedReport.clear(); - } - - QJsonObject data{ - { QStringLiteral("artifact"), artifact.toJson() }, - { QStringLiteral("revision"), revisionJson(inspection.sourceData) }, - { QStringLiteral("verdict"), pdf::preflightVerdictStateToString(verdict.state) }, - { QStringLiteral("report"), inspection.report.toJson(inputPath) }, - }; - if (!publishedReport.isEmpty()) - { - data.insert(QStringLiteral("report_path"), publishedReport); - } - - if (status == QLatin1String("incomplete")) + const auto inspection = pdf::inspectPreflightFile(inspectionRequest); + if (!inspection.documentReadOk || !inspection.receipt || + inspection.receipt->inputDigest != request.value(QStringLiteral("input_sha256")).toString()) { return makeErrorResponse(id, QStringLiteral("preflight"), QStringLiteral("incomplete"), - QStringLiteral("worker.incomplete"), - QStringLiteral("Preflight inspection was incomplete.")); - } - - return makeOkResponse(id, QStringLiteral("preflight"), status, data); + QStringLiteral("worker.inspection-incomplete"), QStringLiteral("Inspection did not complete.")); + } + auto receipt = *inspection.receipt; + const auto state = receipt.verdict.state; + const QString status = state == pdf::PreflightVerdictState::Pass ? QStringLiteral("success") : state == pdf::PreflightVerdictState::Fail ? QStringLiteral("findings") + : state == pdf::PreflightVerdictState::Error ? QStringLiteral("preflight-error") + : QStringLiteral("incomplete"); + receipt.verdict.reason = QStringLiteral("Inspection result: %1.").arg(pdf::preflightVerdictStateToString(state)); + QJsonObject response = (state == pdf::PreflightVerdictState::Pass || state == pdf::PreflightVerdictState::Fail) ? makeOkResponse(id, QStringLiteral("preflight"), status) : makeErrorResponse(id, QStringLiteral("preflight"), status, QStringLiteral("worker.incomplete"), QStringLiteral("Inspection did not complete.")); + response.insert(QStringLiteral("input_sha256"), receipt.inputDigest); + response.insert(QStringLiteral("profile_sha256"), request.value(QStringLiteral("profile_sha256"))); + response.insert(QStringLiteral("receipt"), receipt.toJson()); + return response; +} } - -} // namespace pdftool::worker diff --git a/PdfTool/pdfworkerruntime.h b/PdfTool/pdfworkerruntime.h index 568649235..44026baef 100644 --- a/PdfTool/pdfworkerruntime.h +++ b/PdfTool/pdfworkerruntime.h @@ -26,7 +26,6 @@ #include "pdfworkersandbox.h" #include -#include namespace pdftool::worker { @@ -34,10 +33,9 @@ namespace pdftool::worker class WorkerRuntime { public: - explicit WorkerRuntime(WorkerSandboxPaths sandboxPaths); + explicit WorkerRuntime(WorkerSandboxPaths sandboxPaths, QJsonObject sandboxStatus); QJsonObject handleRequest(const QJsonObject& request); - void requestCancel(); private: QJsonObject handlePing(const QString& id); @@ -49,7 +47,6 @@ class WorkerRuntime WorkerSandboxPaths m_sandboxPaths; QJsonObject m_sandboxStatus; - std::atomic_bool m_cancelRequested{ false }; }; } // namespace pdftool::worker diff --git a/PdfTool/pdfworkersandbox.cpp b/PdfTool/pdfworkersandbox.cpp index b44aa69d7..434b2c5f5 100644 --- a/PdfTool/pdfworkersandbox.cpp +++ b/PdfTool/pdfworkersandbox.cpp @@ -24,6 +24,7 @@ #include "pdfworkerprotocol.h" +#include #include #include @@ -41,6 +42,13 @@ #include #include +#include +#endif +#if defined(Q_OS_WIN) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include #include #endif @@ -83,6 +91,17 @@ bool setResourceLimits(const WorkerSandboxLimits& limits, QString* errorMessage) const qint64 rss = limits.rssBytes > 0 ? limits.rssBytes : DEFAULT_RSS_LIMIT_BYTES; const qint64 cpu = limits.cpuSeconds > 0 ? limits.cpuSeconds : DEFAULT_CPU_SECONDS; + const struct rlimit coreLimit + { + 0, 0 + }; + if (::setrlimit(RLIMIT_CORE, &coreLimit) != 0 || ::prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) != 0) + { + if (errorMessage) + *errorMessage = QStringLiteral("Failed to disable worker dumps."); + return false; + } + struct rlimit asLimit; asLimit.rlim_cur = static_cast(rss); asLimit.rlim_max = static_cast(rss); @@ -111,9 +130,20 @@ bool setResourceLimits(const WorkerSandboxLimits& limits, QString* errorMessage) bool denyNetworkWithSeccomp(QString* errorMessage) { - // Deny networking syscalls; everything else is allowed. Hand-rolled BPF so - // we do not introduce a libseccomp link dependency. const std::vector<__u32> denied = { + static_cast<__u32>(__NR_execve), +#ifdef __NR_execveat + static_cast<__u32>(__NR_execveat), +#endif +#ifdef __NR_fork + static_cast<__u32>(__NR_fork), +#endif +#ifdef __NR_vfork + static_cast<__u32>(__NR_vfork), +#endif +#ifdef __NR_io_uring_setup + static_cast<__u32>(__NR_io_uring_setup), +#endif static_cast<__u32>(__NR_socket), static_cast<__u32>(__NR_connect), static_cast<__u32>(__NR_accept), @@ -132,10 +162,34 @@ bool denyNetworkWithSeccomp(QString* errorMessage) std::vector filter; filter.push_back(BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch))); - filter.push_back(BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AUDIT_ARCH_X86_64, 1, 0)); + filter.push_back(BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, +#if defined(__x86_64__) + AUDIT_ARCH_X86_64, +#elif defined(__aarch64__) + AUDIT_ARCH_AARCH64, +#else +#error Unsupported worker seccomp architecture +#endif + 1, 0)); filter.push_back(BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS)); filter.push_back(BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr))); +#if defined(__x86_64__) + filter.push_back(BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x40000000, 0, 1)); + filter.push_back(BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS)); +#endif +#ifdef __NR_clone3 + // libc falls back to clone, whose flags can be checked for thread creation. + filter.push_back(BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_clone3, 0, 1)); + filter.push_back(BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | ENOSYS)); +#endif +#ifdef __NR_clone + filter.push_back(BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_clone, 0, 4)); + filter.push_back(BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0]))); + filter.push_back(BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x00010000 /* CLONE_THREAD */, 1, 0)); + filter.push_back(BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EACCES)); + filter.push_back(BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW)); +#endif for (const __u32 nr : denied) { filter.push_back(BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1)); @@ -156,7 +210,7 @@ bool denyNetworkWithSeccomp(QString* errorMessage) program.len = static_cast(filter.size()); program.filter = filter.data(); - if (::prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) + if (::syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, &program) != 0) { if (errorMessage) { @@ -172,7 +226,6 @@ bool pathAllowed(const QString& absolutePath, QString* errorMessage) const QFileInfo info(absolutePath); if (!info.exists()) { - // Create parent directory for temp/output if missing. if (errorMessage) { *errorMessage = QStringLiteral("Sandbox path does not exist: %1").arg(absolutePath); @@ -201,6 +254,9 @@ bool addLandlockPath(int rulesetFd, const QString& absolutePath, __u64 access, Q __u64 effectiveAccess = access; if (info.isFile()) { +#ifdef LANDLOCK_ACCESS_FS_REFER + effectiveAccess &= ~LANDLOCK_ACCESS_FS_REFER; +#endif effectiveAccess &= ~(LANDLOCK_ACCESS_FS_READ_DIR | LANDLOCK_ACCESS_FS_REMOVE_DIR | LANDLOCK_ACCESS_FS_MAKE_CHAR | @@ -233,7 +289,12 @@ bool addLandlockPath(int rulesetFd, const QString& absolutePath, __u64 access, Q bool applyLandlock(const WorkerSandboxPaths& paths, QString* errorMessage) { const int abi = landlock_create_ruleset(nullptr, 0, LANDLOCK_CREATE_RULESET_VERSION); - if (abi < 1) +#if !defined(LANDLOCK_ACCESS_FS_TRUNCATE) || !defined(LANDLOCK_ACCESS_FS_REFER) + if (errorMessage) + *errorMessage = QStringLiteral("Landlock headers do not support required filesystem rights."); + return false; +#else + if (abi < 3) { if (errorMessage) { @@ -259,6 +320,7 @@ bool applyLandlock(const WorkerSandboxPaths& paths, QString* errorMessage) LANDLOCK_ACCESS_FS_MAKE_BLOCK | LANDLOCK_ACCESS_FS_MAKE_SYM; + attr.handled_access_fs |= LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_TRUNCATE; const int rulesetFd = landlock_create_ruleset(&attr, sizeof(attr), 0); if (rulesetFd < 0) { @@ -291,7 +353,11 @@ bool applyLandlock(const WorkerSandboxPaths& paths, QString* errorMessage) LANDLOCK_ACCESS_FS_REMOVE_FILE | LANDLOCK_ACCESS_FS_MAKE_REG | LANDLOCK_ACCESS_FS_MAKE_DIR | - LANDLOCK_ACCESS_FS_REMOVE_DIR; + LANDLOCK_ACCESS_FS_REMOVE_DIR +#ifdef LANDLOCK_ACCESS_FS_TRUNCATE + | LANDLOCK_ACCESS_FS_TRUNCATE +#endif + ; // When input is a file, Landlock path_beneath on the file itself grants // access to that file; when it is a directory, the whole tree is readable. @@ -303,44 +369,20 @@ bool applyLandlock(const WorkerSandboxPaths& paths, QString* errorMessage) return false; } - // Allow reading the worker binary and shared libraries under /usr and /lib - // so Qt/LoopLibCore can continue to resolve after restriction. Without this - // the process dies on the next dlopen. Also allow common read-only system - // roots Qt and libc touch during startup (/etc, /dev, /proc). - for (const char* root : { "/usr", "/lib", "/lib64", "/opt", "/etc", "/dev", "/proc", "/sys" }) - { - if (::access(root, F_OK) != 0) - { - continue; - } - QString ignored; - if (!addLandlockPath(rulesetFd, QString::fromLatin1(root), - LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR, - &ignored)) - { - // Optional roots may be absent or unopenable; continue. - } - } - - // Allow the directory that contains the worker executable (build/install tree). + // Already mapped libraries remain available. These roots cover deferred + // font/locale/ICC data and runtime library loads; never grant /proc or /dev. + const QStringList runtimeRoots{ + QStringLiteral("/usr/lib"), QStringLiteral("/usr/lib64"), QStringLiteral("/lib"), QStringLiteral("/lib64"), + QStringLiteral("/usr/share/fonts"), QStringLiteral("/usr/share/fontconfig"), QStringLiteral("/etc/fonts"), + QStringLiteral("/usr/share/color"), QStringLiteral("/usr/share/locale"), QStringLiteral("/etc/ld.so.cache"), + QCoreApplication::applicationDirPath(), QDir(QCoreApplication::applicationDirPath()).absoluteFilePath(QStringLiteral("../lib")) + }; + for (const QString& path : runtimeRoots) { - char selfPath[4096] = {}; - const ssize_t length = ::readlink("/proc/self/exe", selfPath, sizeof(selfPath) - 1); - if (length > 0) + if (QFileInfo::exists(path) && !addLandlockPath(rulesetFd, path, readOnly, errorMessage)) { - selfPath[length] = '\0'; - const QString exeDir = QFileInfo(QString::fromLocal8Bit(selfPath, int(length))).absolutePath(); - QString ignored; - addLandlockPath(rulesetFd, exeDir, - LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR, - &ignored); - const QString libDir = QFileInfo(exeDir + QStringLiteral("/../lib")).absoluteFilePath(); - if (QFileInfo::exists(libDir)) - { - addLandlockPath(rulesetFd, libDir, - LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR, - &ignored); - } + ::close(rulesetFd); + return false; } } @@ -370,6 +412,7 @@ bool applyLandlock(const WorkerSandboxPaths& paths, QString* errorMessage) ::close(rulesetFd); return true; +#endif } #endif // Q_OS_LINUX @@ -412,6 +455,59 @@ bool applyWorkerSandbox(const WorkerSandboxPaths& paths, return false; } return true; +#elif defined(Q_OS_WIN) + Q_UNUSED(paths); + Q_UNUSED(limits); + HANDLE token = nullptr; + BOOL appContainer = FALSE; + DWORD size = 0; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) + return false; + const BOOL isolated = GetTokenInformation(token, TokenIsAppContainer, &appContainer, sizeof(appContainer), &size); + GetTokenInformation(token, TokenCapabilities, nullptr, 0, &size); + std::vector capabilities(size); + const BOOL queried = size > 0 && GetTokenInformation(token, TokenCapabilities, capabilities.data(), size, &size); + const bool noCapabilities = queried && reinterpret_cast(capabilities.data())->GroupCount == 0; + CloseHandle(token); + JOBOBJECT_EXTENDED_LIMIT_INFORMATION job{}; + PROCESS_MITIGATION_CHILD_PROCESS_POLICY children{}; + const DWORD required = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_ACTIVE_PROCESS | + JOB_OBJECT_LIMIT_PROCESS_MEMORY | JOB_OBJECT_LIMIT_PROCESS_TIME; + if (!isolated || !appContainer) + { + if (errorMessage) + *errorMessage = QStringLiteral("worker.sandbox.token"); + return false; + } + if (!noCapabilities) + { + if (errorMessage) + *errorMessage = QStringLiteral("worker.sandbox.capabilities"); + return false; + } + if (!QueryInformationJobObject(nullptr, JobObjectExtendedLimitInformation, &job, sizeof(job), nullptr)) + { + if (errorMessage) + *errorMessage = QStringLiteral("worker.sandbox.job-query"); + return false; + } + if ((job.BasicLimitInformation.LimitFlags & required) != required || + job.BasicLimitInformation.ActiveProcessLimit != 1 || + job.ProcessMemoryLimit != static_cast(limits.rssBytes > 0 ? limits.rssBytes : DEFAULT_RSS_LIMIT_BYTES) || + job.BasicLimitInformation.PerProcessUserTimeLimit.QuadPart != (limits.cpuSeconds > 0 ? limits.cpuSeconds : DEFAULT_CPU_SECONDS) * 10000000) + { + if (errorMessage) + *errorMessage = QStringLiteral("worker.sandbox.job-limits"); + return false; + } + if (!GetProcessMitigationPolicy(GetCurrentProcess(), ProcessChildProcessPolicy, &children, sizeof(children)) || + !children.NoChildProcessCreation) + { + if (errorMessage) + *errorMessage = QStringLiteral("worker.sandbox.children"); + return false; + } + return true; #else Q_UNUSED(paths); Q_UNUSED(limits); @@ -423,7 +519,7 @@ bool applyWorkerSandbox(const WorkerSandboxPaths& paths, #endif } -QJsonObject sandboxStatusJson(bool applied, const QString& detail) +QJsonObject sandboxStatusJson(bool applied, const QString& detail, WorkerSandboxLimits limits) { return QJsonObject{ { QStringLiteral("applied"), applied }, @@ -437,8 +533,8 @@ QJsonObject sandboxStatusJson(bool applied, const QString& detail) #endif }, { QStringLiteral("detail"), detail }, - { QStringLiteral("rss_limit_bytes"), DEFAULT_RSS_LIMIT_BYTES }, - { QStringLiteral("cpu_limit_seconds"), DEFAULT_CPU_SECONDS }, + { QStringLiteral("rss_limit_bytes"), (limits.rssBytes > 0 ? limits.rssBytes : DEFAULT_RSS_LIMIT_BYTES) }, + { QStringLiteral("cpu_limit_seconds"), (limits.cpuSeconds > 0 ? limits.cpuSeconds : DEFAULT_CPU_SECONDS) }, }; } diff --git a/PdfTool/pdfworkersandbox.h b/PdfTool/pdfworkersandbox.h index f98ca39a5..4d1e17ece 100644 --- a/PdfTool/pdfworkersandbox.h +++ b/PdfTool/pdfworkersandbox.h @@ -42,15 +42,12 @@ struct WorkerSandboxLimits qint64 cpuSeconds = 0; }; -/// Applies the Linux release-worker sandbox: no network (seccomp), Landlock FS -/// restriction to input/temp/output, and RLIMIT CPU/RSS. On non-Linux hosts this -/// returns false (Windows job-object hardening is a follow-on). -/// A missing sandbox for LOOP_PDF_WORKER_REQUIRE_SANDBOX builds is fatal. +/// Establishes or verifies containment before any document parsing. bool applyWorkerSandbox(const WorkerSandboxPaths& paths, const WorkerSandboxLimits& limits, QString* errorMessage); -QJsonObject sandboxStatusJson(bool applied, const QString& detail); +QJsonObject sandboxStatusJson(bool applied, const QString& detail, WorkerSandboxLimits limits = {}); } // namespace pdftool::worker diff --git a/PdfTool/worker-runtime.cmake b/PdfTool/worker-runtime.cmake new file mode 100644 index 000000000..c80ad06c3 --- /dev/null +++ b/PdfTool/worker-runtime.cmake @@ -0,0 +1,11 @@ +function(loop_stage_worker_runtime target) + if(WIN32) + target_compile_definitions(${target} PRIVATE _WIN32_WINNT=0x0A00) + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} + "-DWORKER=$" + "-DSEARCH_DIRS=${LOOP_QT_ROOT}/bin;${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin" + -P "${CMAKE_SOURCE_DIR}/PdfTool/write-worker-runtime.cmake" + VERBATIM) + endif() +endfunction() diff --git a/PdfTool/write-worker-runtime.cmake b/PdfTool/write-worker-runtime.cmake new file mode 100644 index 000000000..b46033dd4 --- /dev/null +++ b/PdfTool/write-worker-runtime.cmake @@ -0,0 +1,24 @@ +cmake_policy(VERSION 3.16) +if(POLICY CMP0207) + cmake_policy(SET CMP0207 NEW) +endif() +file(GET_RUNTIME_DEPENDENCIES + EXECUTABLES "${WORKER}" + DIRECTORIES ${SEARCH_DIRS} + RESOLVED_DEPENDENCIES_VAR dependencies + UNRESOLVED_DEPENDENCIES_VAR unresolved + CONFLICTING_DEPENDENCIES_PREFIX conflicts + PRE_EXCLUDE_REGEXES "api-ms-.*" "ext-ms-.*" + POST_EXCLUDE_REGEXES ".*[/\\\\][Ww][Ii][Nn][Dd][Oo][Ww][Ss][/\\\\].*") +if(unresolved OR conflicts_FILENAMES) + message(FATAL_ERROR "Worker dependency closure is unresolved or ambiguous: ${unresolved};${conflicts_FILENAMES}") +endif() +get_filename_component(directory "${WORKER}" DIRECTORY) +file(MAKE_DIRECTORY "${directory}/worker-runtime") +set(manifest "") +foreach(dependency IN LISTS dependencies) + get_filename_component(name "${dependency}" NAME) + file(COPY "${dependency}" DESTINATION "${directory}/worker-runtime") + string(APPEND manifest "worker-runtime/${name}\n") +endforeach() +file(WRITE "${WORKER}.runtime" "${manifest}") diff --git a/UnitTests/CMakeLists.txt b/UnitTests/CMakeLists.txt index 73ea5d3d9..90e74ff7f 100644 --- a/UnitTests/CMakeLists.txt +++ b/UnitTests/CMakeLists.txt @@ -639,12 +639,34 @@ set_target_properties(UnitTestsPdfToolContract PROPERTIES ) add_test(UnitTestsPdfToolContract "${CMAKE_BINARY_DIR}/${LOOP_INSTALL_BIN_DIR}/UnitTestsPdfToolContract") +add_executable(LoopPdfWorkerProbe + pdfworkerprobe.cpp + ${CMAKE_SOURCE_DIR}/PdfTool/pdfworkersandbox.cpp) +target_link_libraries(LoopPdfWorkerProbe PRIVATE Qt6::Core) +target_include_directories(LoopPdfWorkerProbe PRIVATE ${CMAKE_SOURCE_DIR}/PdfTool) +if(WIN32) + target_link_libraries(LoopPdfWorkerProbe PRIVATE ws2_32 advapi32) +endif() +set_target_properties(LoopPdfWorkerProbe PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${LOOP_INSTALL_BIN_DIR}) +loop_stage_worker_runtime(LoopPdfWorkerProbe) + add_executable(UnitTestsPdfWorkerIsolation tst_pdfworkerisolation.cpp -) -target_link_libraries(UnitTestsPdfWorkerIsolation PRIVATE Qt6::Core Qt6::Test) -add_dependencies(UnitTestsPdfWorkerIsolation PdfTool loop-pdf-worker) + ${CMAKE_SOURCE_DIR}/PdfTool/pdfworkerclient.cpp + ${CMAKE_SOURCE_DIR}/PdfTool/pdfworkerprocess.cpp + ${CMAKE_SOURCE_DIR}/PdfTool/pdfworkerprocess_win.cpp) +target_include_directories(UnitTestsPdfWorkerIsolation PRIVATE + ${CMAKE_SOURCE_DIR}/PdfTool ${CMAKE_SOURCE_DIR}/LoopLibCore/sources) +if(WIN32) + target_compile_definitions(UnitTestsPdfWorkerIsolation PRIVATE _WIN32_WINNT=0x0A00) + target_link_libraries(UnitTestsPdfWorkerIsolation PRIVATE userenv advapi32 ole32 ws2_32) +endif() +target_link_libraries(UnitTestsPdfWorkerIsolation PRIVATE LoopLibCore Qt6::Core Qt6::Gui Qt6::Test) +add_dependencies(UnitTestsPdfWorkerIsolation PdfTool loop-pdf-worker LoopPdfWorkerProbe) target_compile_definitions(UnitTestsPdfWorkerIsolation PRIVATE + PDFWORKER_EXECUTABLE_PATH="$" + PDFWORKER_PROBE_PATH="$" PDFTOOL_EXECUTABLE_PATH="$" LOOP_PREFLIGHT_SOURCE_DIR="${CMAKE_SOURCE_DIR}/loop-preflight" LOOP_SOURCE_DIR="${CMAKE_SOURCE_DIR}") @@ -732,7 +754,7 @@ endif() # Qt Quick/Qml, and linking the SHARED LoopLibQuick library from a plain # add_executable() test would be the first such link edge in this file -- # every other LoopLibQuick consumer uses qt_add_executable() plus -# qt_import_qml_plugins() (see ProductQuickAccessibilitySmoke/CMakeLists.txt), +# qt_import_qml_plugins() (see tools/ProductQuickAccessibilitySmoke/CMakeLists.txt), # neither of which this test needs. if(NOT LOOP_BUILD_ONLY_CORE_LIBRARY AND LOOP_BUILD_QUICK_CANVAS) add_executable(UnitTestsLoopStateVisual diff --git a/UnitTests/pdfworkerprobe.cpp b/UnitTests/pdfworkerprobe.cpp new file mode 100644 index 000000000..17648fd6a --- /dev/null +++ b/UnitTests/pdfworkerprobe.cpp @@ -0,0 +1,266 @@ +// MIT License +// +// Copyright (c) 2018-2025 Jakub Melka and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "pdfworkerprotocol.h" +#include "pdfworkersandbox.h" + +#if defined(Q_OS_WIN) +#include +#include +#else +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace +{ +void output(const QJsonObject& object) +{ + const QByteArray bytes = QJsonDocument(object).toJson(QJsonDocument::Compact) + '\n'; + fwrite(bytes.constData(), 1, size_t(bytes.size()), stdout); + fflush(stdout); +} + +int networkError = 0; + +bool networkDenied(quint16 port) +{ +#ifdef Q_OS_WIN + WSADATA data{}; + networkError = WSAStartup(MAKEWORD(2, 2), &data); + if (networkError) + return false; + SOCKET connection = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (connection == INVALID_SOCKET) + return WSAGetLastError() == WSAEACCES; + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = htons(port); + u_long nonblocking = 1; + if (ioctlsocket(connection, FIONBIO, &nonblocking)) + { + closesocket(connection); + WSACleanup(); + return false; + } + const int result = connect(connection, reinterpret_cast(&address), sizeof(address)); + int error = result == 0 ? 0 : WSAGetLastError(); + if (error == WSAEWOULDBLOCK) + { + fd_set writable, failed; + FD_ZERO(&writable); + FD_ZERO(&failed); + FD_SET(connection, &writable); + FD_SET(connection, &failed); + timeval deadline{ 2, 0 }; + const int ready = select(0, nullptr, &writable, &failed, &deadline); + if (ready == 0) + error = WSAETIMEDOUT; + else if (ready == SOCKET_ERROR) + error = WSAGetLastError(); + else + { + int length = sizeof(error); + if (getsockopt(connection, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &length)) + error = WSAGetLastError(); + } + } + networkError = error; + closesocket(connection); + WSACleanup(); + return result == SOCKET_ERROR && (error == WSAEACCES || error == WSAETIMEDOUT); +#else + Q_UNUSED(port); + const int connection = socket(AF_INET, SOCK_STREAM, 0); + if (connection >= 0) + close(connection); + return connection < 0 && errno == EACCES; +#endif +} + +bool childDenied() +{ +#ifdef Q_OS_WIN + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION child{}; + const auto executable = QDir::toNativeSeparators(QCoreApplication::applicationFilePath()).toStdWString(); + const BOOL created = CreateProcessW(executable.c_str(), nullptr, nullptr, nullptr, FALSE, CREATE_NO_WINDOW, + nullptr, nullptr, &startup, &child); + if (created) + { + TerminateProcess(child.hProcess, 1); + CloseHandle(child.hThread); + CloseHandle(child.hProcess); + } + const DWORD error = GetLastError(); + return !created && (error == ERROR_ACCESS_DENIED || error == ERROR_CHILD_PROCESS_BLOCKED); +#else + const pid_t child = fork(); + if (child == 0) + _exit(0); + return child < 0 && errno == EACCES; +#endif +} +} + +int main(int argc, char** argv) +{ +#ifdef Q_OS_WIN + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +#endif + QCoreApplication app(argc, argv); + const auto args = app.arguments(); + const auto option = [&](const QString& key) + { + const int index = args.indexOf(key); + return index >= 0 && index + 1 < args.size() ? args.at(index + 1) : QString(); + }; + const pdftool::worker::WorkerSandboxPaths paths{ option(QStringLiteral("--sandbox-input")), + option(QStringLiteral("--sandbox-temp")), option(QStringLiteral("--sandbox-output")) }; + QString error; + if (!pdftool::worker::applyWorkerSandbox(paths, {}, &error)) + return 3; + QFile input; + if (!input.open(stdin, QIODevice::ReadOnly)) + return 4; + for (;;) + { + const auto bytes = input.readLine(pdftool::worker::MAX_REQUEST_BYTES + 1); + if (bytes.isEmpty()) + break; + const auto request = QJsonDocument::fromJson(bytes).object(); + const QString id = request.value(QStringLiteral("id")).toString(); + const QString op = request.value(QStringLiteral("op")).toString(); + if (op == QLatin1String("ping")) + { + output(pdftool::worker::makeOkResponse(id, op, QStringLiteral("success"), + { { QStringLiteral("sandbox"), pdftool::worker::sandboxStatusJson(true, QStringLiteral("test-probe")) } })); + continue; + } + const QString mode = request.value(QStringLiteral("password")).toString(); + if (mode == QLatin1String("crash")) + std::abort(); + if (mode == QLatin1String("hang")) + for (;;) + QThread::msleep(10); + if (mode == QLatin1String("cpu")) + for (;;) + { + static volatile unsigned value = 0; + value = value + 1; + } + if (mode == QLatin1String("memory")) + { + std::vector allocations; + for (;;) + allocations.emplace_back(16 * 1024 * 1024, 'x'); + } + if (mode == QLatin1String("malformed")) + { + fputs("{invalid-json\n", stdout); + fflush(stdout); + continue; + } + if (mode == QLatin1String("oversized")) + { + const QByteArray chunk(65536, 'x'); + for (int i = 0; i < 1025; ++i) + fwrite(chunk.constData(), 1, size_t(chunk.size()), stdout); + fflush(stdout); + continue; + } + auto response = pdftool::worker::makeOkResponse(id, op, QStringLiteral("success")); + response.insert(QStringLiteral("input_sha256"), request.value(QStringLiteral("input_sha256"))); + response.insert(QStringLiteral("profile_sha256"), request.value(QStringLiteral("profile_sha256"))); + if (mode == QLatin1String("wrong-id")) + response.insert(QStringLiteral("id"), QStringLiteral("other-request")); + if (mode == QLatin1String("wrong-version")) + response.insert(QStringLiteral("v"), 1); + if (mode == QLatin1String("wrong-op")) + response.insert(QStringLiteral("op"), QStringLiteral("open")); + if (mode == QLatin1String("wrong-status")) + response.insert(QStringLiteral("ok"), false); + if (mode == QLatin1String("raw-error")) + { + fputs("LOOP_CUSTOMER_SECRET_20\n", stderr); + response = pdftool::worker::makeErrorResponse(id, op, QStringLiteral("input-error"), + QStringLiteral("LOOP_CUSTOMER_SECRET_20"), QStringLiteral("LOOP_CUSTOMER_SECRET_20")); + } + if (mode == QLatin1String("raw-success")) + { + response.insert(QStringLiteral("artifact"), QJsonObject{ + { QStringLiteral("sha256"), request.value(QStringLiteral("input_sha256")) }, + { QStringLiteral("size"), QFileInfo(request.value(QStringLiteral("input_path")).toString()).size() }, + { QStringLiteral("mediaType"), QStringLiteral("application/pdf") }, + { QStringLiteral("logicalName"), QStringLiteral("input.pdf") }, + { QStringLiteral("storageToken"), QStringLiteral("worker-input") }, + { QStringLiteral("extra"), QStringLiteral("LOOP_CUSTOMER_SECRET_20") } }); + response.insert(QStringLiteral("page_count"), 1); + response.insert(QStringLiteral("verdict"), QStringLiteral("PASS")); + response.insert(QStringLiteral("report_path"), QStringLiteral("LOOP_CUSTOMER_SECRET_20")); + } + if (mode.startsWith(QStringLiteral("probe:"))) + { + const int separator = mode.indexOf(QLatin1Char(':'), 6); + const quint16 port = mode.mid(6, separator - 6).toUShort(); + QFile outside(mode.mid(separator + 1)); + QFile runtime(QCoreApplication::applicationFilePath()); + QFile temporary(QDir(paths.tempDir).filePath(QStringLiteral("allowed.tmp"))); + const bool outsideDenied = !outside.open(QIODevice::ReadOnly); + const bool runtimeDenied = !runtime.open(QIODevice::WriteOnly | QIODevice::Append); + const bool tempAllowed = temporary.open(QIODevice::WriteOnly) && temporary.write("ok") == 2; + temporary.close(); + QFile snapshot(request.value(QStringLiteral("input_path")).toString()); + const bool snapshotDenied = !snapshot.open(QIODevice::WriteOnly | QIODevice::Append); +#ifdef Q_OS_WIN + QFile profile(QDir(qEnvironmentVariable("LOCALAPPDATA")).filePath(QStringLiteral("../outside-temp.tmp"))); + const bool profileDenied = !profile.open(QIODevice::WriteOnly); +#else + const bool profileDenied = true; +#endif + response.insert(QStringLiteral("artifact"), QJsonObject{ { QStringLiteral("sha256"), request.value(QStringLiteral("input_sha256")) } }); + response.insert(QStringLiteral("page_count"), 1); + response.insert(QStringLiteral("runtime_path"), QCoreApplication::applicationDirPath()); +#ifdef Q_OS_WIN + response.insert(QStringLiteral("profile_path"), QDir::cleanPath(QDir(qEnvironmentVariable("LOCALAPPDATA")).absoluteFilePath(QStringLiteral("..")))); +#endif + response.insert(QStringLiteral("probe"), QJsonObject{ + { QStringLiteral("network_denied"), networkDenied(port) }, { QStringLiteral("child_denied"), childDenied() }, { QStringLiteral("outside_denied"), outsideDenied }, { QStringLiteral("runtime_denied"), runtimeDenied }, { QStringLiteral("temp_allowed"), tempAllowed }, { QStringLiteral("snapshot_denied"), snapshotDenied }, { QStringLiteral("network_error"), networkError }, { QStringLiteral("profile_denied"), profileDenied } }); + } + output(response); + } + return 0; +} diff --git a/UnitTests/tst_documentfacadetest.cpp b/UnitTests/tst_documentfacadetest.cpp index 8d8a71f80..06d2dc060 100644 --- a/UnitTests/tst_documentfacadetest.cpp +++ b/UnitTests/tst_documentfacadetest.cpp @@ -74,6 +74,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter spec.jobId.isEmpty() ? QStringLiteral("job-%1").arg(++m_sequence) : spec.jobId; submittedSpecs.append(spec); + m_specs.insert(jobId, spec); m_status.insert(jobId, pdf::PDFJobStatus::Queued); if (runInline) @@ -116,6 +117,9 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter pdf::PDFJobSnapshot result; result.jobId = jobId; result.status = m_status.value(jobId, pdf::PDFJobStatus::Succeeded); + result.kind = m_specs.value(jobId).kind; + result.documentKey = m_specs.value(jobId).documentKey; + result.documentRevision = m_specs.value(jobId).documentRevision; return result; } @@ -150,6 +154,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter bool runInline = true; bool cancelStopsQueuedWork = true; + pdf::PDFJobStatus terminalStatus = pdf::PDFJobStatus::Succeeded; QList submittedSpecs; QStringList cancelledJobs; QStringList clearedKeys; @@ -170,11 +175,12 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter pdf::PDFProcessingLimits::conservativeDefaults(), [](int) {}); work(context); - m_status.insert(jobId, pdf::PDFJobStatus::Succeeded); + m_status.insert(jobId, terminalStatus); } quint64 m_sequence = 0; QHash m_status; + QHash m_specs; QHash m_deferred; }; @@ -274,6 +280,7 @@ private slots: void disabledCommandIsUnavailable(); void openAdmitsDocumentAndPublishesRevision(); + void workerLoadCannotPublishAfterSchedulerFailure(); void openFailureReportsTypedErrorAndBindsNoDocument(); void openCancellationIsTerminalAndNotSuccess(); void cancellingAQueuedOpenIsTerminal(); @@ -524,6 +531,17 @@ void DocumentFacadeTest::openAdmitsDocumentAndPublishesRevision() QVERIFY(harness.catalog.isEnabled(pdfinteraction::DocumentFacade::SaveCommandId)); } +void DocumentFacadeTest::workerLoadCannotPublishAfterSchedulerFailure() +{ + Harness harness; + harness.submitter.terminalStatus = pdf::PDFJobStatus::Failed; + harness.facade->open(QStringLiteral("/corpus/report.pdf")); + + QTRY_COMPARE(harness.facade->state(), pdfinteraction::DocumentState::Error); + QCOMPARE(harness.context.getDocument(), nullptr); + QCOMPARE(harness.facade->typedError(), QStringLiteral("document/job-not-admitted")); +} + void DocumentFacadeTest::openFailureReportsTypedErrorAndBindsNoDocument() { Harness harness; diff --git a/UnitTests/tst_evidencegraphtest.cpp b/UnitTests/tst_evidencegraphtest.cpp index 337bc5b7c..c667a37cb 100644 --- a/UnitTests/tst_evidencegraphtest.cpp +++ b/UnitTests/tst_evidencegraphtest.cpp @@ -44,6 +44,9 @@ class EvidenceGraphTest : public QObject private slots: void collectWithoutDocument_isIncomplete(); void emptyPage_isComplete(); + void cancelledCollection_isIncompleteAndCancelled(); + void cancelledPreflight_reportsCancelled(); + void pageScope_skipsUnselectedPages(); void incompleteGraphCannotPass(); void imageFamilyDualRunMatchesEngine(); void colorantsFamilyDualRunMatchesEngine(); @@ -125,6 +128,12 @@ void assertFindingCitesGraphRecord(const QList& findings, QFAIL(qPrintable(QStringLiteral("Expected finding type '%1' for check '%2'").arg(findingType, checkId))); } +class CancelledOperationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override { return true; } +}; + } // namespace void EvidenceGraphTest::collectWithoutDocument_isIncomplete() @@ -146,6 +155,62 @@ void EvidenceGraphTest::emptyPage_isComplete() QVERIFY(graph.incompleteReason.isEmpty()); } +void EvidenceGraphTest::cancelledCollection_isIncompleteAndCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + pdf::PDFEvidenceCollectSettings settings; + settings.operationControl = &cancelled; + + const pdf::PDFEvidenceGraph graph = pdf::PDFEvidenceCollector::collect(&session, pdf::pdfEvidenceAllDomains(), settings); + QVERIFY(!graph.isComplete()); + QCOMPARE(graph.incompleteReason, QStringLiteral("cancelled")); + QVERIFY(graph.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); +} + +void EvidenceGraphTest::cancelledPreflight_reportsCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + + pdf::PreflightEngine engine(&session); + engine.setOperationControl(&cancelled); + const QJsonObject profile{ + { QStringLiteral("name"), QStringLiteral("Color inventory") }, + { QStringLiteral("checks"), QJsonArray{ + QJsonObject{ + { QStringLiteral("id"), QStringLiteral("color-inventory") }, + { QStringLiteral("severity"), QStringLiteral("info") }, + { QStringLiteral("probe_dpi"), 150 }, + { QStringLiteral("rich_black_k_percent"), 10 } } } } + }; + const pdf::PreflightResult result = engine.run(profile); + QVERIFY(!result.inspectionComplete); + QCOMPARE(result.errorCode, QStringLiteral("cancelled")); +} + +void EvidenceGraphTest::pageScope_skipsUnselectedPages() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + + const pdf::PDFEvidenceGraph all = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants); + QVERIFY(!all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + pdf::PDFEvidenceCollectSettings settings; + settings.pageIndices = QSet(); + const pdf::PDFEvidenceGraph none = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QVERIFY(none.isComplete()); + QVERIFY(none.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + settings.pageIndices = QSet{ 0 }; + const pdf::PDFEvidenceGraph first = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QCOMPARE(first.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size(), + all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size()); +} + void EvidenceGraphTest::incompleteGraphCannotPass() { pdf::PreflightEngine engine(nullptr); diff --git a/UnitTests/tst_interactionboundarytest.cpp b/UnitTests/tst_interactionboundarytest.cpp index 7ce729cab..1b89c1090 100644 --- a/UnitTests/tst_interactionboundarytest.cpp +++ b/UnitTests/tst_interactionboundarytest.cpp @@ -232,13 +232,17 @@ void InteractionBoundaryTest::submitterCancellationIsTerminalAndNotSuccess() { pdf::PDFJobScheduler scheduler(1); pdfinteraction::PDFJobSchedulerSubmitter submitter(scheduler); + pdf::PDFDocumentContext context(nullptr); + const QString documentKey = QStringLiteral("doc-under-test"); + submitter.publishCurrentRevision(documentKey, context.getRevision()); std::atomic_bool started = false; pdf::PDFJobSpec spec; spec.jobId = QStringLiteral("interaction-cancel-me"); spec.kind = pdf::PDFJobKind::Rendering; spec.priority = pdf::PDFJobPriority::Interaction; - spec.documentKey = QStringLiteral("doc-under-test"); + spec.documentKey = documentKey; + spec.documentRevision = context.getRevision().toString(); // Cancel a running job cooperatively, as UnitTestsJobScheduler does. The work // exits on the cancellation token rather than on a flag this slot must live diff --git a/UnitTests/tst_jobschedulertest.cpp b/UnitTests/tst_jobschedulertest.cpp index 9358b9f09..80982daf8 100644 --- a/UnitTests/tst_jobschedulertest.cpp +++ b/UnitTests/tst_jobschedulertest.cpp @@ -44,6 +44,8 @@ private slots: void allWorkKindsUseOneSubmissionApi(); void cancellationIsTerminalAndMeasured(); void staleRevisionIsDiscardedBeforeWorkRuns(); + void missingFenceDoesNotRunDocumentWork(); + void supersededAndReopenedJobsNeverSucceed(); void progressAndOperationMetadataAreObservable(); void waitTimeoutCancelJoinsBeforeTerminalSnapshot(); void cancelledPreflightAndExportJobsAreNotSuccess(); @@ -267,9 +269,59 @@ void JobSchedulerTest::staleRevisionIsDiscardedBeforeWorkRuns() QVERIFY(!ran.load(std::memory_order_acquire)); } +void JobSchedulerTest::missingFenceDoesNotRunDocumentWork() +{ + pdf::PDFJobScheduler scheduler(1); + std::atomic_bool ran = false; + pdf::PDFJobSpec spec; + spec.documentKey = QStringLiteral("document-1"); + spec.documentRevision = QStringLiteral("revision-1"); + const QString jobId = scheduler.submit(spec, [&ran](pdf::PDFJobContext&) + { ran = true; }); + + QVERIFY(scheduler.waitForFinished(jobId, 1000)); + QCOMPARE(scheduler.snapshot(jobId).status, pdf::PDFJobStatus::Stale); + QVERIFY(!ran.load(std::memory_order_acquire)); +} + +void JobSchedulerTest::supersededAndReopenedJobsNeverSucceed() +{ + pdf::PDFJobScheduler scheduler(2); + const QString key = QStringLiteral("document-1"); + const QString revision = QStringLiteral("revision-1"); + scheduler.setCurrentRevision(key, revision); + + std::atomic_bool releaseOld = false; + std::atomic_bool oldStarted = false; + pdf::PDFJobSpec spec; + spec.documentKey = key; + spec.documentRevision = revision; + spec.priority = pdf::PDFJobPriority::VisiblePage; + const QString oldId = scheduler.submit(spec, [&releaseOld, &oldStarted](pdf::PDFJobContext&) + { + oldStarted = true; + while (!releaseOld.load(std::memory_order_acquire)) + { + std::this_thread::yield(); + } }); + QTRY_VERIFY_WITH_TIMEOUT(oldStarted.load(std::memory_order_acquire), 1000); + QVERIFY(!scheduler.waitForFinished(oldId, 10)); + + scheduler.clearCurrentRevision(key); + scheduler.setCurrentRevision(key, revision); + const QString retryId = scheduler.submit(spec, [](pdf::PDFJobContext&) {}); + QVERIFY(scheduler.waitForFinished(retryId, 1000)); + QCOMPARE(scheduler.snapshot(retryId).status, pdf::PDFJobStatus::Succeeded); + + releaseOld = true; + QVERIFY(scheduler.waitForFinished(oldId, 1000)); + QCOMPARE(scheduler.snapshot(oldId).status, pdf::PDFJobStatus::Stale); +} + void JobSchedulerTest::progressAndOperationMetadataAreObservable() { pdf::PDFJobScheduler scheduler(1); + scheduler.setCurrentRevision(QStringLiteral("document-2"), QStringLiteral("revision-4")); pdf::PDFJobSpec spec; spec.jobId = QStringLiteral("render-tile"); spec.kind = pdf::PDFJobKind::Rendering; @@ -367,6 +419,7 @@ void JobSchedulerTest::cancelledPreflightAndExportJobsAreNotSuccess() void JobSchedulerTest::test_finishedJobReleasesItsWorkClosure() { pdf::PDFJobScheduler scheduler(1); + scheduler.setCurrentRevision(QStringLiteral("doc-1"), QStringLiteral("1")); pdf::PDFJobSpec spec; spec.kind = pdf::PDFJobKind::Preflight; @@ -393,6 +446,7 @@ void JobSchedulerTest::test_finishedJobReleasesItsWorkClosure() void JobSchedulerTest::test_terminalJobRetentionIsBounded() { pdf::PDFJobScheduler scheduler(1); + scheduler.setCurrentRevision(QStringLiteral("doc-1"), QStringLiteral("1")); QList jobIds; for (int index = 0; index < 300; ++index) diff --git a/UnitTests/tst_pagesurfacetest.cpp b/UnitTests/tst_pagesurfacetest.cpp index 04a8088d0..e5fcfac7c 100644 --- a/UnitTests/tst_pagesurfacetest.cpp +++ b/UnitTests/tst_pagesurfacetest.cpp @@ -245,6 +245,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter bool runInline = true; bool refuseSubmission = false; bool cancelStopsQueuedWork = true; + pdf::PDFJobStatus terminalStatus = pdf::PDFJobStatus::Succeeded; QList submittedSpecs; QStringList cancelledJobs; QHash publishedRevisions; @@ -262,7 +263,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter pdf::PDFJobContext context(token, pdf::PDFProcessingLimits::conservativeDefaults(), [](int) {}); work(context); - m_status.insert(jobId, pdf::PDFJobStatus::Succeeded); + m_status.insert(jobId, terminalStatus); } quint64 m_sequence = 0; @@ -288,6 +289,10 @@ class FakePageSurfaceRenderer final : public pdfinteraction::IPageSurfaceRendere pdfinteraction::PageSurfaceResult result; result.key = request.key; result.token = request.token; + if (returnWrongPage) + { + ++result.key.pageIndex; + } if (jobContext.isCancellationRequested()) { @@ -317,6 +322,7 @@ class FakePageSurfaceRenderer final : public pdfinteraction::IPageSurfaceRendere int renderCount = 0; int shedCount = 0; bool reentered = false; + bool returnWrongPage = false; QList renderedKeys; pdfinteraction::SurfaceTerminalState nextState = pdfinteraction::SurfaceTerminalState::Complete; @@ -345,7 +351,10 @@ private slots: void supersededDemandIsCancelledBeforeNewWorkIsSubmitted(); void completionForASupersededRequestIsRejected(); void completionAgainstAnOldRevisionIsRejected(); + void workerSuccessWithWrongRequestIdentityIsRejected(); + void schedulerFailureCannotAdmitRenderedPixels(); void revisionReplacementDropsEveryStaleSurface(); + void documentKeyChangeDropsAdmittedSurfaces(); void cancellationIsTerminalAndNotSuccess(); void completionAfterDestructionReachesNobody(); void budgetExhaustionIsItsOwnTerminalState(); @@ -664,6 +673,30 @@ void PageSurfaceTest::completionAgainstAnOldRevisionIsRejected() QVERIFY(fixture.coordinator->snapshot().tiles.isEmpty()); } +void PageSurfaceTest::workerSuccessWithWrongRequestIdentityIsRejected() +{ + Fixture fixture; + fixture.renderer.returnWrongPage = true; + fixture.coordinator->requestSurfaces(); + Fixture::drain(); + + QCOMPARE(fixture.coordinator->counters().admitted, 0); + QVERIFY(fixture.coordinator->counters().rejectedSuperseded > 0); + QVERIFY(fixture.coordinator->snapshot().tiles.isEmpty()); +} + +void PageSurfaceTest::schedulerFailureCannotAdmitRenderedPixels() +{ + Fixture fixture; + fixture.submitter.terminalStatus = pdf::PDFJobStatus::Failed; + fixture.coordinator->requestSurfaces(); + Fixture::drain(); + + QCOMPARE(fixture.coordinator->counters().admitted, 0); + QVERIFY(fixture.coordinator->counters().failed > 0); + QVERIFY(fixture.coordinator->snapshot().tiles.isEmpty()); +} + void PageSurfaceTest::revisionReplacementDropsEveryStaleSurface() { Fixture fixture; @@ -681,6 +714,18 @@ void PageSurfaceTest::revisionReplacementDropsEveryStaleSurface() QVERIFY(fixture.coordinator->snapshot().tiles.isEmpty()); } +void PageSurfaceTest::documentKeyChangeDropsAdmittedSurfaces() +{ + Fixture fixture; + fixture.coordinator->requestSurfaces(); + Fixture::drain(); + QVERIFY(fixture.coordinator->counters().admittedBytes > 0); + + fixture.coordinator->setDocumentKey(QStringLiteral("doc-2")); + QCOMPARE(fixture.coordinator->counters().admittedBytes, qint64(0)); + QVERIFY(fixture.coordinator->snapshot().tiles.isEmpty()); +} + void PageSurfaceTest::cancellationIsTerminalAndNotSuccess() { Fixture fixture; diff --git a/UnitTests/tst_pdftoolcontract.cpp b/UnitTests/tst_pdftoolcontract.cpp index 74d5b6e41..08bf6224f 100644 --- a/UnitTests/tst_pdftoolcontract.cpp +++ b/UnitTests/tst_pdftoolcontract.cpp @@ -134,8 +134,49 @@ private slots: void rgbToCmykRefusesToWriteOverItsOwnInput(); void evidenceBundleExportVerifyPair(); void evidenceBundleRejectsNonJsonOutput(); + void benchmarkWithoutPreflightProfileIsIncomplete(); + void benchmarkWithPreflightProfileIsComplete(); }; +namespace +{ + +QJsonObject runBenchmarkEnvelope(const QStringList& extraArguments) +{ + const QString fixture = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/testdata/fixtures/image-dpi-low.pdf"); + QStringList arguments{ QStringLiteral("benchmark"), fixture, + QStringLiteral("--render-hw-accel"), QStringLiteral("0"), + QStringLiteral("--console-format"), QStringLiteral("json") }; + arguments << extraArguments; + const ToolRun run = runPdfTool(arguments); + verifyEnvelope(run, 0, QStringLiteral("benchmark")); + return run.json.value(QStringLiteral("data")).toObject().value(QStringLiteral("workload_envelope")).toObject(); +} + +} // namespace + +void PdfToolContractTest::benchmarkWithoutPreflightProfileIsIncomplete() +{ + const QJsonObject envelope = runBenchmarkEnvelope({}); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("incomplete")); + QCOMPARE(envelope.value(QStringLiteral("incomplete_reason")).toString(), QStringLiteral("preflight-measurement-unavailable")); + QCOMPARE(envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(), -1); +} + +void PdfToolContractTest::benchmarkWithPreflightProfileIsComplete() +{ + const QString profile = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/profiles/loop-default.json"); + const QJsonObject envelope = runBenchmarkEnvelope({ QStringLiteral("--profile"), profile }); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("complete")); + QVERIFY(envelope.value(QStringLiteral("incomplete_reason")).toString().isEmpty()); + const qint64 preflightHighWater = envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(); + QVERIFY2(preflightHighWater > 0, qPrintable(QString::number(preflightHighWater))); + QVERIFY(envelope.value(QStringLiteral("rss_high_water_bytes")).toInteger() >= preflightHighWater); + QCOMPARE(envelope.value(QStringLiteral("pages_materialized")).toInteger(), envelope.value(QStringLiteral("page_count")).toInteger()); +} + void PdfToolContractTest::helpIsWrapped() { const ToolRun run = runPdfTool({ QStringLiteral("help"), QStringLiteral("--console-format"), QStringLiteral("json") }); diff --git a/UnitTests/tst_pdfworkerisolation.cpp b/UnitTests/tst_pdfworkerisolation.cpp index 98d13f30a..0dc89cec0 100644 --- a/UnitTests/tst_pdfworkerisolation.cpp +++ b/UnitTests/tst_pdfworkerisolation.cpp @@ -22,6 +22,11 @@ #include "processoutputcapture.h" +#ifdef Q_OS_WIN +#include +#endif +#include + #include #include #include @@ -31,6 +36,15 @@ #include #include #include +#include +#include "pdfworkerclient.h" +#include "pdfworkerprocess.h" +#include +#include "pdfworkerprotocol.h" +#include "pdfpreflightverdict.h" +#include +#include +#include class PdfWorkerIsolationTest : public QObject { @@ -42,6 +56,13 @@ private slots: void workerPreflightRunsIsolated(); void crashingWorkerDoesNotKillSupervisor(); void workerSourcesOmitSentry(); + void supervisorFaults_data(); + void supervisorFaults(); + void supervisorCancellation(); + void oversizedRequestDoesNotPublish(); + void sandboxDenials(); + void workerExtrasAreNotPublished(); + void malformedPdfDoesNotLeak(); }; namespace @@ -100,9 +121,6 @@ QString defaultProfile() void PdfWorkerIsolationTest::workerPingSucceeds() { -#ifndef Q_OS_LINUX - QSKIP("Release-worker sandbox proof is Linux-first for #618."); -#endif const ToolRun run = runPdfTool({ QStringLiteral("worker-ping"), QStringLiteral("--console-format"), QStringLiteral("json") }); QCOMPARE(run.exitCode, 0); QCOMPARE(run.json.value(QStringLiteral("status")).toString(), QStringLiteral("success")); @@ -114,9 +132,6 @@ void PdfWorkerIsolationTest::workerPingSucceeds() void PdfWorkerIsolationTest::workerOpenReturnsArtifactIdentity() { -#ifndef Q_OS_LINUX - QSKIP("Release-worker sandbox proof is Linux-first for #618."); -#endif const QString pdf = fixturePdf(); QVERIFY(QFileInfo::exists(pdf)); const ToolRun run = runPdfTool({ QStringLiteral("worker-open"), pdf, QStringLiteral("--console-format"), QStringLiteral("json") }); @@ -130,9 +145,6 @@ void PdfWorkerIsolationTest::workerOpenReturnsArtifactIdentity() void PdfWorkerIsolationTest::workerPreflightRunsIsolated() { -#ifndef Q_OS_LINUX - QSKIP("Release-worker sandbox proof is Linux-first for #618."); -#endif const QString pdf = fixturePdf(); const QString profile = defaultProfile(); QVERIFY(QFileInfo::exists(pdf)); @@ -150,43 +162,234 @@ void PdfWorkerIsolationTest::workerPreflightRunsIsolated() const QString status = run.json.value(QStringLiteral("status")).toString(); QVERIFY(status == QLatin1String("success") || status == QLatin1String("findings") || status == QLatin1String("preflight-incomplete")); - // Never a silent PASS when the worker is unavailable. - QVERIFY(status != QLatin1String("unavailable") || run.exitCode != 0); + const auto data = run.json.value(QStringLiteral("data")).toObject(); + pdf::PreflightInspectionReceipt receipt; + QString error; + QVERIFY2(pdf::preflightInspectionReceiptFromJson(data.value(QStringLiteral("receipt")).toObject(), receipt, error), qPrintable(error)); + QFile input(pdf); + QVERIFY(input.open(QIODevice::ReadOnly)); + QCOMPARE(receipt.inputDigest, QString::fromLatin1(QCryptographicHash::hash(input.readAll(), QCryptographicHash::Sha256).toHex())); + QVERIFY(!receipt.checks.isEmpty()); + QCOMPARE(run.exitCode, pdf::preflightVerdictProcessExitCode(receipt.verdict.state)); } void PdfWorkerIsolationTest::crashingWorkerDoesNotKillSupervisor() { -#ifndef Q_OS_LINUX - QSKIP("Release-worker crash simulation uses a POSIX shell script."); + QTemporaryDir temp, output; + pdftool::PdfWorkerClient client; + QString error; + QVERIFY2(client.start(QStringLiteral(PDFWORKER_PROBE_PATH), temp.path(), temp.path(), output.path(), &error), qPrintable(error)); + const auto result = client.preflight(fixturePdf(), defaultProfile(), output.path(), QStringLiteral("crash")); + QCOMPARE(result.outcome, pdftool::WorkerClientOutcome::Incomplete); + QVERIFY(!client.isRunning()); + QCOMPARE(result.response.value(QStringLiteral("receipt")).toObject().value(QStringLiteral("verdict")).toObject().value(QStringLiteral("state")).toString(), QStringLiteral("incomplete")); + QVERIFY2(client.replaceWorker(&error), qPrintable(error)); + QCOMPARE(client.ping().outcome, pdftool::WorkerClientOutcome::Success); +} + +void PdfWorkerIsolationTest::supervisorFaults_data() +{ + QTest::addColumn("mode"); + for (const QString& mode : { QStringLiteral("hang"), QStringLiteral("memory"), QStringLiteral("malformed"), + QStringLiteral("oversized"), QStringLiteral("wrong-id"), QStringLiteral("wrong-op"), + QStringLiteral("wrong-version"), QStringLiteral("wrong-status"), QStringLiteral("raw-error"), + QStringLiteral("cpu") }) + { + QTest::newRow(qPrintable(mode)) << mode; + } +} + +void PdfWorkerIsolationTest::supervisorFaults() +{ + QFETCH(QString, mode); + QTemporaryDir temp, output; + pdftool::PdfWorkerClient client; + QString error; + QVERIFY2(client.start(QStringLiteral(PDFWORKER_PROBE_PATH), temp.path(), temp.path(), output.path(), &error), qPrintable(error)); + const int timeout = mode == QLatin1String("cpu") ? 135000 : mode == QLatin1String("hang") ? 150 + : 20000; + const auto result = client.preflight(fixturePdf(), defaultProfile(), output.path(), mode, false, timeout); + QCOMPARE(result.outcome, pdftool::WorkerClientOutcome::Incomplete); + const auto receipt = result.response.value(QStringLiteral("receipt")).toObject(); + QCOMPARE(receipt.value(QStringLiteral("verdict")).toObject().value(QStringLiteral("state")).toString(), QStringLiteral("incomplete")); + QCOMPARE(receipt.value(QStringLiteral("fidelity")).toString(), QStringLiteral("not-recorded")); + QVERIFY(!QJsonDocument(result.response).toJson().contains("LOOP_CUSTOMER_SECRET_20")); + QVERIFY(QDir(output.path()).entryList(QDir::Files).isEmpty()); + if (mode == QLatin1String("cpu") || mode == QLatin1String("memory")) + { + QCOMPARE(result.code, QStringLiteral("worker.exited")); + } + QVERIFY2(client.replaceWorker(&error), qPrintable(error)); + QCOMPARE(client.ping().outcome, pdftool::WorkerClientOutcome::Success); +} + +void PdfWorkerIsolationTest::oversizedRequestDoesNotPublish() +{ + QTemporaryDir temp, output; + pdftool::PdfWorkerClient client; + QString error; + QVERIFY2(client.start(QStringLiteral(PDFWORKER_EXECUTABLE_PATH), temp.path(), temp.path(), output.path(), &error), qPrintable(error)); + const auto result = client.preflight(fixturePdf(), defaultProfile(), output.path(), + QString(pdftool::worker::MAX_REQUEST_BYTES, QLatin1Char('x'))); + QCOMPARE(result.outcome, pdftool::WorkerClientOutcome::Incomplete); + QCOMPARE(result.code, QStringLiteral("worker.request-limit")); + pdf::PreflightInspectionReceipt receipt; + QVERIFY(pdf::preflightInspectionReceiptFromJson(result.response.value(QStringLiteral("receipt")).toObject(), receipt, error)); + QVERIFY(!receipt.verdict.isPass()); + QVERIFY(QDir(output.path()).entryList(QDir::Files).isEmpty()); + QVERIFY2(client.replaceWorker(&error), qPrintable(error)); + QCOMPARE(client.ping().outcome, pdftool::WorkerClientOutcome::Success); +} + +void PdfWorkerIsolationTest::supervisorCancellation() +{ + std::promise ready; + auto clientFuture = ready.get_future(); + auto result = std::async(std::launch::async, [&] + { + QTemporaryDir temp, output; + pdftool::PdfWorkerClient client; + QString error; + if (!client.start(QStringLiteral(PDFWORKER_PROBE_PATH), temp.path(), temp.path(), output.path(), &error)) + { + ready.set_value(nullptr); + return pdftool::WorkerClientResult{}; + } + ready.set_value(&client); + return client.preflight(fixturePdf(), defaultProfile(), output.path(), QStringLiteral("hang"), false, 60000); }); + auto* client = clientFuture.get(); + QVERIFY(client); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + client->cancel(); + const auto cancelled = result.get(); + QCOMPARE(cancelled.outcome, pdftool::WorkerClientOutcome::Cancelled); + QCOMPARE(cancelled.response.value(QStringLiteral("receipt")).toObject().value(QStringLiteral("verdict")).toObject().value(QStringLiteral("state")).toString(), QStringLiteral("incomplete")); +} + +void PdfWorkerIsolationTest::sandboxDenials() +{ + QTemporaryDir temp, output, outside; + quint16 port = 9; +#ifdef Q_OS_WIN + WSADATA winsock{}; + QCOMPARE(WSAStartup(MAKEWORD(2, 2), &winsock), 0); + const auto cleanupWinsock = qScopeGuard([] + { WSACleanup(); }); + const SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + QVERIFY(listener != INVALID_SOCKET); + const auto cleanupListener = qScopeGuard([&] + { closesocket(listener); }); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + QCOMPARE(bind(listener, reinterpret_cast(&address), sizeof(address)), 0); + QCOMPARE(listen(listener, 1), 0); + int addressSize = sizeof(address); + QCOMPARE(getsockname(listener, reinterpret_cast(&address), &addressSize), 0); + port = ntohs(address.sin_port); + const SOCKET control = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + QVERIFY(control != INVALID_SOCKET); + const auto cleanupControl = qScopeGuard([&] + { closesocket(control); }); + QCOMPARE(::connect(control, reinterpret_cast(&address), sizeof(address)), 0); + const SOCKET accepted = accept(listener, nullptr, nullptr); + QVERIFY(accepted != INVALID_SOCKET); + closesocket(accepted); #endif - QTemporaryDir temp; - QVERIFY(temp.isValid()); - const QString crashWorker = temp.filePath(QStringLiteral("crash-worker.sh")); + QFile secret(outside.filePath(QStringLiteral("secret.txt"))); + QVERIFY(secret.open(QIODevice::WriteOnly)); + secret.write("LOOP_CUSTOMER_SECRET_20"); + secret.close(); + QTemporaryDir inputDirectory; + const QString snapshot = inputDirectory.filePath(QStringLiteral("input.pdf")); + QVERIFY(QFile::copy(fixturePdf(), snapshot)); + QVERIFY(QFile::setPermissions(snapshot, QFileDevice::ReadOwner)); + const auto cleanupSnapshot = qScopeGuard([&] + { QFile::setPermissions(snapshot, QFileDevice::ReadOwner | QFileDevice::WriteOwner); }); + pdftool::WorkerProcess process; + QString error; + QVERIFY2(process.start(QStringLiteral(PDFWORKER_PROBE_PATH), + { QStringLiteral("--sandbox-input"), inputDirectory.path(), QStringLiteral("--sandbox-temp"), temp.path(), + QStringLiteral("--sandbox-output"), output.path() }, + inputDirectory.path(), temp.path(), output.path(), error), + qPrintable(error)); + QElapsedTimer timer; + timer.start(); + const std::atomic_bool cancelled{ false }; + const QJsonObject request{ { QStringLiteral("v"), pdftool::worker::PROTOCOL_VERSION }, + { QStringLiteral("id"), QUuid::createUuid().toString(QUuid::WithoutBraces) }, + { QStringLiteral("op"), QStringLiteral("open") }, + { QStringLiteral("input_path"), snapshot }, + { QStringLiteral("password"), QStringLiteral("probe:%1:").arg(port) + secret.fileName() } }; + QVERIFY(process.write(QJsonDocument(request).toJson(QJsonDocument::Compact) + '\n', timer, 10000, cancelled)); + QByteArray frame; + while (!frame.contains('\n') && timer.elapsed() < 10000 && frame.size() <= pdftool::worker::MAX_RESPONSE_BYTES) + { + frame.append(process.read(pdftool::worker::MAX_RESPONSE_BYTES + 1 - frame.size())); + QTest::qWait(10); + } + QVERIFY(frame.size() <= pdftool::worker::MAX_RESPONSE_BYTES); + QJsonParseError parseError; + const QJsonDocument document = QJsonDocument::fromJson(frame, &parseError); + QCOMPARE(parseError.error, QJsonParseError::NoError); + QVERIFY(document.isObject()); + const QJsonObject response = document.object(); + QCOMPARE(response.value(QStringLiteral("id")), request.value(QStringLiteral("id"))); + QVERIFY(response.value(QStringLiteral("ok")).toBool()); + const auto probe = response.value(QStringLiteral("probe")).toObject(); + for (const QString& key : { QStringLiteral("outside_denied"), QStringLiteral("network_denied"), QStringLiteral("child_denied"), + QStringLiteral("runtime_denied"), QStringLiteral("snapshot_denied"), QStringLiteral("profile_denied"), QStringLiteral("temp_allowed") }) { - QFile file(crashWorker); - QVERIFY(file.open(QIODevice::WriteOnly | QIODevice::Truncate)); - file.write("#!/bin/sh\n# Hostile stand-in for loop-pdf-worker.\nkill -SEGV $$\n"); - file.close(); + QVERIFY2(probe.value(key) == QJsonValue(true), qPrintable(key + QString::fromUtf8(QJsonDocument(probe).toJson(QJsonDocument::Compact)))); } - QVERIFY(QFile::setPermissions(crashWorker, - QFileDevice::ReadOwner | QFileDevice::WriteOwner | QFileDevice::ExeOwner | - QFileDevice::ReadGroup | QFileDevice::ExeGroup | QFileDevice::ReadOther | - QFileDevice::ExeOther)); - - QProcessEnvironment extra; - extra.insert(QStringLiteral("LOOP_PDF_WORKER_PATH"), crashWorker); - const ToolRun run = runPdfTool( - { QStringLiteral("worker-open"), fixturePdf(), QStringLiteral("--console-format"), QStringLiteral("json") }, - extra); - - // Supervisor must exit normally with a typed failure — never crash itself. +#ifdef Q_OS_WIN + const QString runtimePath = response.value(QStringLiteral("runtime_path")).toString(); + const QString profilePath = response.value(QStringLiteral("profile_path")).toString(); + QVERIFY(QFileInfo(runtimePath).isDir()); + QVERIFY2(QFileInfo(profilePath).isDir(), qPrintable(profilePath)); + process.stop(); + QVERIFY(!QFileInfo::exists(runtimePath)); + QVERIFY(!QFileInfo::exists(profilePath)); +#endif +} + +void PdfWorkerIsolationTest::workerExtrasAreNotPublished() +{ + QTemporaryDir temp, output; + pdftool::PdfWorkerClient client; + QString error; + QVERIFY2(client.start(QStringLiteral(PDFWORKER_PROBE_PATH), temp.path(), temp.path(), output.path(), &error), qPrintable(error)); + const auto result = client.openDocument(fixturePdf(), QStringLiteral("raw-success")); + QCOMPARE(result.outcome, pdftool::WorkerClientOutcome::Success); + QVERIFY(!result.response.contains(QStringLiteral("verdict"))); + QVERIFY(!result.response.contains(QStringLiteral("report_path"))); + QVERIFY(!QJsonDocument(result.response).toJson().contains("LOOP_CUSTOMER_SECRET_20")); + QVERIFY(QDir(output.path()).entryList(QDir::Files).isEmpty()); + QCOMPARE(client.ping().outcome, pdftool::WorkerClientOutcome::Success); +} + +void PdfWorkerIsolationTest::malformedPdfDoesNotLeak() +{ + QTemporaryDir fixture; + QFile malformed(fixture.filePath(QStringLiteral("malformed.pdf"))); + QVERIFY(malformed.open(QIODevice::WriteOnly)); + malformed.write("%PDF-1.7\nLOOP_CUSTOMER_SECRET_20\ninvalid-object\n%%EOF"); + malformed.close(); + QProcessEnvironment environment; + const QString logDirectory = fixture.filePath(QStringLiteral("logs")); + environment.insert(QStringLiteral("LOOP_LOG_DIR"), logDirectory); + environment.insert(QStringLiteral("LOOP_LOG_LEVEL"), QStringLiteral("debug")); + environment.insert(QStringLiteral("SENTRY_DSN"), QStringLiteral("https://public@127.0.0.1:1/1")); + environment.insert(QStringLiteral("SENTRY_DEBUG"), QStringLiteral("1")); + const auto run = runPdfTool({ QStringLiteral("worker-preflight"), malformed.fileName(), QStringLiteral("--profile"), + defaultProfile(), QStringLiteral("--console-format"), QStringLiteral("json") }, + environment); + QVERIFY(!QFileInfo::exists(logDirectory)); QVERIFY(run.exitCode != 0); - QCOMPARE(run.json.value(QStringLiteral("command")).toString(), QStringLiteral("worker-open")); - const QString status = run.json.value(QStringLiteral("status")).toString(); - QVERIFY(status == QLatin1String("processing-failure") || status == QLatin1String("preflight-incomplete")); - const QJsonObject data = run.json.value(QStringLiteral("data")).toObject(); - const QString code = data.value(QStringLiteral("code")).toString(data.value(QStringLiteral("worker_code")).toString()); - QVERIFY(code.contains(QStringLiteral("worker"))); + QVERIFY(!run.stdoutData.contains("LOOP_CUSTOMER_SECRET_20")); + QVERIFY(!run.stderrData.contains("LOOP_CUSTOMER_SECRET_20")); + const auto receipt = run.json.value(QStringLiteral("data")).toObject().value(QStringLiteral("receipt")).toObject(); + QCOMPARE(receipt.value(QStringLiteral("verdict")).toObject().value(QStringLiteral("state")).toString(), QStringLiteral("incomplete")); } void PdfWorkerIsolationTest::workerSourcesOmitSentry() diff --git a/UnitTests/tst_preflightinteraction.cpp b/UnitTests/tst_preflightinteraction.cpp index 3573efb5d..741cbb82b 100644 --- a/UnitTests/tst_preflightinteraction.cpp +++ b/UnitTests/tst_preflightinteraction.cpp @@ -311,6 +311,7 @@ void PreflightInteractionTest::controllerRepresentsIncompleteRun() void PreflightInteractionTest::controllerMarksAnInFlightRunStaleAndCancelsIt() { pdf::PDFJobScheduler scheduler(1); + scheduler.setCurrentRevision(QStringLiteral("doc"), QStringLiteral("rev-1")); PreflightController controller(&scheduler); std::atomic_bool started = false; diff --git a/UnitTests/tst_preflightverdicttest.cpp b/UnitTests/tst_preflightverdicttest.cpp index c46fcb804..f67e8754e 100644 --- a/UnitTests/tst_preflightverdicttest.cpp +++ b/UnitTests/tst_preflightverdicttest.cpp @@ -44,6 +44,7 @@ #include #include #include +#include class PreflightVerdictTest : public QObject { @@ -63,6 +64,13 @@ private slots: void incompleteInspectionWithoutFindings_isNotPass(); void cancellationMarkedIncomplete_isNotPass(); void requiredCheckMissingStatus_isIncomplete(); + void receiptIdentity_matchesGoldenVector(); + void receiptWireRoundTrip(); + void receiptWireRejectsTampering(); + void terminalReceiptPreservesUnknownIdentity(); + void receiptTerminalStates_data(); + void receiptTerminalStates(); + void receiptRejectsMismatchedProvenance(); void processExitCodes_matchPdfToolContract(); void budgetExceeded_neverAllowsCertificate(); void operatorSummary_distinguishesIncompleteFromPass(); @@ -136,6 +144,33 @@ pdf::PreflightResult budgetExceededResult() return result; } +struct ReceiptFixture +{ + pdf::PreflightResult result; + pdf::PreflightProfileData profile; + pdf::PDFRevisionIdentity revision; + pdf::PDFEvidenceGraph evidence; + + ReceiptFixture() + { + result.documentRevisionDigest = QString(64, QLatin1Char('a')); + result.effectiveProfileDigest = QString(64, QLatin1Char('b')); + result.coverageScope = QJsonObject{ + { QStringLiteral("claim"), QStringLiteral("Limited to enabled checks.") }, + { QStringLiteral("enabled_checks"), QJsonArray{ QStringLiteral("bleed") } } + }; + result.checkStatuses.append(makeCheckStatus(QStringLiteral("bleed"), QStringLiteral("ok"))); + profile.effectiveDigest = result.effectiveProfileDigest; + pdf::PreflightCheckConfig check; + check.id = QStringLiteral("bleed"); + check.enabled = true; + check.required = true; + profile.checks.append(check); + revision.document.documentId = QStringLiteral("receipt-fixture"); + revision.documentRevision = 1; + } +}; + /// A translator with no .qm file behind it: it answers one message in the /// Core verdict context, which is exactly what a shipped catalogue would do. class StubVerdictTranslator final : public QTranslator @@ -966,6 +1001,232 @@ void PreflightVerdictTest::requiredCheckMissingStatus_isIncomplete() QVERIFY(!verdict.isPass()); } +void PreflightVerdictTest::receiptIdentity_matchesGoldenVector() +{ + ReceiptFixture fixture; + pdf::PDFEvidenceRecord record; + record.id = QStringLiteral("evidence-2"); + record.fidelity = QStringLiteral("sampled"); + fixture.evidence.records.append(record); + pdf::PreflightFinding warning; + warning.evidenceIds.append(QStringLiteral("evidence-1")); + fixture.result.warnings.append(warning); + + pdf::PreflightInspectionReceipt receipt; + QString error; + QVERIFY2(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error), + qPrintable(error)); + QCOMPARE(receipt.identity, QStringLiteral("8e94a7fefac162eafa4a20516692fb281c28978788b9d91d4df3397061933a8b")); + QCOMPARE(receipt.verdict.state, pdf::PreflightVerdictState::Pass); + QCOMPARE(receipt.checks.size(), 1); + QVERIFY(receipt.checks.first().complete); + QCOMPARE(receipt.evidenceRefs, (QStringList{ QStringLiteral("evidence-1"), QStringLiteral("evidence-2") })); + QCOMPARE(receipt.fidelity, QStringLiteral("sampled")); + QCOMPARE(receipt.limitations, QStringList{ QStringLiteral("Limited to enabled checks.") }); + + fixture.revision.documentRevision = 2; + pdf::PreflightInspectionReceipt later; + QVERIFY2(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, later, error), + qPrintable(error)); + QCOMPARE(later.identity, receipt.identity); + QCOMPARE(later.revision.documentRevision, pdf::DocumentRevision(2)); + + fixture.profile.effectiveDigest = QString(64, QLatin1Char('c')); + fixture.result.effectiveProfileDigest = fixture.profile.effectiveDigest; + pdf::PreflightInspectionReceipt changedPolicy; + QVERIFY2(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, changedPolicy, error), + qPrintable(error)); + QCOMPARE(changedPolicy.identity, QStringLiteral("931952b4c72f56e67fa371c94eb01bec4383cab251286a3c675c7d8dcada11f8")); +} + +void PreflightVerdictTest::receiptWireRoundTrip() +{ + ReceiptFixture fixture; + fixture.revision.document.sourceDataHash = QByteArray::fromHex(fixture.result.documentRevisionDigest.toLatin1()); + pdf::PDFEvidenceRecord record; + record.id = QStringLiteral("wire-evidence"); + record.fidelity = QStringLiteral("exact"); + fixture.evidence.records.append(record); + fixture.revision.documentRevision = std::numeric_limits::max(); + fixture.revision.cacheGeneration = (quint64(1) << 54) + 3; + fixture.revision.effectiveProfileIdentity = QStringLiteral("effective-policy"); + pdf::PreflightInspectionReceipt receipt, decoded; + QString error; + QVERIFY(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error)); + QVERIFY2(pdf::preflightInspectionReceiptFromJson(receipt.toJson(), decoded, error), qPrintable(error)); + QCOMPARE(decoded.toJson(), receipt.toJson()); + QCOMPARE(decoded.revision, fixture.revision); + fixture.profile.profileIdentity = receipt.profileIdentity; + fixture.profile.coverageScope = receipt.coverageScope; + QVERIFY2(pdf::validatePreflightInspectionReceipt(decoded, decoded.inputDigest, fixture.revision, fixture.profile, error), qPrintable(error)); + auto changed = fixture.revision; + ++changed.cacheGeneration; + QVERIFY(!pdf::validatePreflightInspectionReceipt(decoded, decoded.inputDigest, changed, fixture.profile, error)); + QVERIFY(!pdf::validatePreflightInspectionReceipt(decoded, QString(64, QLatin1Char('e')), fixture.revision, fixture.profile, error)); + fixture.profile.effectiveDigest = QString(64, QLatin1Char('f')); + QVERIFY(!pdf::validatePreflightInspectionReceipt(decoded, decoded.inputDigest, fixture.revision, fixture.profile, error)); +} + +void PreflightVerdictTest::terminalReceiptPreservesUnknownIdentity() +{ + pdf::PDFRevisionIdentity revision; + revision.document.documentId = QStringLiteral("request-before-staging"); + const auto terminal = pdf::buildTerminalPreflightReceipt({}, revision, {}, QStringLiteral("worker.snapshot-failed")); + pdf::PreflightInspectionReceipt parsed; + QString error; + QVERIFY2(pdf::preflightInspectionReceiptFromJson(terminal.toJson(), parsed, error), qPrintable(error)); + QVERIFY(parsed.inputDigest.isEmpty()); + QVERIFY(parsed.effectiveProfileDigest.isEmpty()); + QVERIFY(!parsed.verdict.isPass()); + auto forged = terminal.toJson(); + auto verdict = forged.value(QStringLiteral("verdict")).toObject(); + verdict.insert(QStringLiteral("state"), QStringLiteral("pass")); + forged.insert(QStringLiteral("verdict"), verdict); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(forged, parsed, error)); +} + +void PreflightVerdictTest::receiptWireRejectsTampering() +{ + ReceiptFixture fixture; + fixture.revision.document.sourceDataHash = QByteArray::fromHex(fixture.result.documentRevisionDigest.toLatin1()); + pdf::PDFEvidenceRecord record; + record.id = QStringLiteral("wire-evidence"); + record.fidelity = QStringLiteral("exact"); + fixture.evidence.records.append(record); + pdf::PreflightInspectionReceipt receipt, decoded; + QString error; + QVERIFY(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error)); + auto json = receipt.toJson(); + json.insert(QStringLiteral("schema"), QStringLiteral("loop.inspection-receipt.v99")); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + json = receipt.toJson(); + json.insert(QStringLiteral("checks"), QJsonArray{}); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + json = receipt.toJson(); + auto checks = json.value(QStringLiteral("checks")).toArray(); + auto check = checks.first().toObject(); + check.insert(QStringLiteral("status"), QStringLiteral("unsupported")); + checks.replace(0, check); + json.insert(QStringLiteral("checks"), checks); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + json = receipt.toJson(); + json.insert(QStringLiteral("evidence_refs"), QJsonArray{}); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + json = receipt.toJson(); + checks = json.value(QStringLiteral("checks")).toArray(); + check = checks.first().toObject(); + check.insert(QStringLiteral("status"), QStringLiteral("invented-status")); + check.insert(QStringLiteral("complete"), false); + checks.replace(0, check); + json.insert(QStringLiteral("checks"), checks); + auto incompleteVerdict = json.value(QStringLiteral("verdict")).toObject(); + incompleteVerdict.insert(QStringLiteral("state"), QStringLiteral("incomplete")); + json.insert(QStringLiteral("verdict"), incompleteVerdict); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + json = receipt.toJson(); + auto revision = json.value(QStringLiteral("revision")).toObject(); + revision.insert(QStringLiteral("cache_generation"), 0); + json.insert(QStringLiteral("revision"), revision); + QVERIFY(!pdf::preflightInspectionReceiptFromJson(json, decoded, error)); + const auto failure = pdf::buildTerminalPreflightReceipt(receipt.inputDigest, fixture.revision, + receipt.effectiveProfileDigest, QStringLiteral("worker.crashed")); + QVERIFY(pdf::preflightInspectionReceiptFromJson(failure.toJson(), decoded, error)); + QVERIFY(!decoded.verdict.isPass()); +} + +void PreflightVerdictTest::receiptTerminalStates_data() +{ + QTest::addColumn("caseName"); + QTest::addColumn("expected"); + QTest::newRow("pass") << QStringLiteral("pass") << pdf::PreflightVerdictState::Pass; + QTest::newRow("no-evidence") << QStringLiteral("no-evidence") << pdf::PreflightVerdictState::Incomplete; + QTest::newRow("fail") << QStringLiteral("fail") << pdf::PreflightVerdictState::Fail; + QTest::newRow("missing-required") << QStringLiteral("missing-required") << pdf::PreflightVerdictState::Incomplete; + QTest::newRow("unsupported") << QStringLiteral("unsupported") << pdf::PreflightVerdictState::Incomplete; + QTest::newRow("budget-limited") << QStringLiteral("budget-limited") << pdf::PreflightVerdictState::Incomplete; + QTest::newRow("cancelled") << QStringLiteral("cancelled") << pdf::PreflightVerdictState::Incomplete; + QTest::newRow("parser-error") << QStringLiteral("parser-error") << pdf::PreflightVerdictState::Error; +} + +void PreflightVerdictTest::receiptTerminalStates() +{ + QFETCH(QString, caseName); + QFETCH(pdf::PreflightVerdictState, expected); + ReceiptFixture fixture; + if (caseName != QLatin1String("no-evidence")) + { + pdf::PDFEvidenceRecord record; + record.id = QStringLiteral("evidence-1"); + record.fidelity = QStringLiteral("exact"); + fixture.evidence.records.append(record); + } + if (caseName == QLatin1String("fail")) + { + fixture.result.errors.append(blockingFinding()); + } + else if (caseName == QLatin1String("missing-required")) + { + fixture.result.checkStatuses.clear(); + } + else if (caseName == QLatin1String("unsupported")) + { + fixture.result.checkStatuses.first().status = QStringLiteral("unsupported"); + } + else if (caseName == QLatin1String("budget-limited")) + { + fixture.result.checkStatuses.first().budgetKind = QStringLiteral("raster-pixels"); + } + else if (caseName == QLatin1String("cancelled")) + { + fixture.result.errorCode = QStringLiteral("cancelled"); + } + else if (caseName == QLatin1String("parser-error")) + { + fixture.result.errorCode = QStringLiteral("parser-error"); + } + + pdf::PreflightInspectionReceipt receipt; + QString error; + QVERIFY2(pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error), + qPrintable(error)); + QCOMPARE(receipt.verdict.state, expected); + QCOMPARE(receipt.verdict.isPass(), expected == pdf::PreflightVerdictState::Pass); + if (caseName == QLatin1String("no-evidence")) + { + pdf::PreflightInspectionReceipt parsed; + QVERIFY2(pdf::preflightInspectionReceiptFromJson(receipt.toJson(), parsed, error), qPrintable(error)); + } + if (expected == pdf::PreflightVerdictState::Incomplete) + { + QVERIFY(!receipt.verdict.allowsCertificateIssuance()); + } +} + +void PreflightVerdictTest::receiptRejectsMismatchedProvenance() +{ + ReceiptFixture fixture; + fixture.evidence.artifact.sha256 = QString(64, QLatin1Char('c')); + pdf::PreflightInspectionReceipt receipt; + QString error; + QVERIFY(!pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error)); + QVERIFY(!error.isEmpty()); + QVERIFY(receipt.identity.isEmpty()); + + fixture.evidence.artifact.sha256.clear(); + fixture.evidence.revision = fixture.revision; + fixture.evidence.revision.documentRevision = 2; + QVERIFY(!pdf::buildPreflightInspectionReceipt(fixture.result, fixture.profile, fixture.revision, + fixture.evidence, receipt, error)); + QVERIFY(receipt.identity.isEmpty()); +} + void PreflightVerdictTest::processExitCodes_matchPdfToolContract() { QCOMPARE(pdf::preflightVerdictProcessExitCode(pdf::PreflightVerdictState::Pass), 0); diff --git a/UnitTests/tst_resourcebudgettest.cpp b/UnitTests/tst_resourcebudgettest.cpp index de22587bd..42f9178e1 100644 --- a/UnitTests/tst_resourcebudgettest.cpp +++ b/UnitTests/tst_resourcebudgettest.cpp @@ -25,7 +25,7 @@ void ResourceBudgetTest::conservativeDefaultsExposeAllPools() { const pdf::PDFResourceBudgetConfig config = pdf::PDFResourceBudgetConfig::conservativeDefaults(); QCOMPARE(config.residentLimitBytes, 768 * pdf::PDFResourceBudgetConfig::MiB); - QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 256 * pdf::PDFResourceBudgetConfig::MiB); + QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 640 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::CompiledEvidenceCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::RasterTileCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::GpuTextureCache), 128 * pdf::PDFResourceBudgetConfig::MiB); diff --git a/UnitTests/tst_viewportcommandbridgetest.cpp b/UnitTests/tst_viewportcommandbridgetest.cpp index ab8840883..ca26d60af 100644 --- a/UnitTests/tst_viewportcommandbridgetest.cpp +++ b/UnitTests/tst_viewportcommandbridgetest.cpp @@ -67,6 +67,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter { const QString jobId = spec.jobId.isEmpty() ? QStringLiteral("job-%1").arg(++m_sequence) : spec.jobId; + m_specs.insert(jobId, spec); m_status.insert(jobId, pdf::PDFJobStatus::Queued); if (runInline) @@ -92,6 +93,9 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter pdf::PDFJobSnapshot result; result.jobId = jobId; result.status = m_status.value(jobId, pdf::PDFJobStatus::Succeeded); + result.kind = m_specs.value(jobId).kind; + result.documentKey = m_specs.value(jobId).documentKey; + result.documentRevision = m_specs.value(jobId).documentRevision; return result; } @@ -111,6 +115,7 @@ class FakeJobSubmitter final : public pdfinteraction::IJobSubmitter private: quint64 m_sequence = 0; QHash m_status; + QHash m_specs; }; class FakeDocumentLoader final : public pdfinteraction::IDocumentLoader diff --git a/UnitTests/tst_workloadenvelopetest.cpp b/UnitTests/tst_workloadenvelopetest.cpp index 721a0649f..f73e115c0 100644 --- a/UnitTests/tst_workloadenvelopetest.cpp +++ b/UnitTests/tst_workloadenvelopetest.cpp @@ -38,11 +38,21 @@ class WorkloadEnvelopeTest : public QObject private slots: void identityFieldsArePresent(); + void rssHighWaterIsMeasuredOnSupportedPlatforms(); void shedPrefetchAndQualityBeforeInteraction(); void pageHeavyEnvelopeRecordsIdentity(); void interactionSlotRunsWhenBackgroundIsSaturated(); }; +void WorkloadEnvelopeTest::rssHighWaterIsMeasuredOnSupportedPlatforms() +{ +#if defined(Q_OS_LINUX) || defined(Q_OS_WIN) + QVERIFY(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes() > 0); +#else + QCOMPARE(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(), qint64(-1)); +#endif +} + void WorkloadEnvelopeTest::identityFieldsArePresent() { qputenv("GIT_COMMIT", QByteArrayLiteral("0123456789abcdef0123456789abcdef01234567")); diff --git a/agent-policy.json b/agent-policy.json index e06ffff2a..1728177d9 100644 --- a/agent-policy.json +++ b/agent-policy.json @@ -63,7 +63,7 @@ ], "preserve": "Preserve required validation, security, cancellation, provenance, and failure handling.", "closing": [ - "Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence anti-slop summary in the handoff.", + "Re-run the owning issue's normal verification after the pass, and record a 1-3 sentence quality summary in the handoff.", "The pass is review judgment on the diff; do not add brittle automated style metrics.", "Canonical policy: Berry Studio BSP-002 §4.2 and §4.3 (https://app.notion.com/p/3b39cb079ddb811d8527ed129c3ac511)." ] @@ -229,17 +229,17 @@ "UnitTests/tst_quickaccessibilitytest.cpp", "UnitTests/tst_shellkeyboardtest.cpp", "UnitTests/tst_loopstatevisualtest.cpp", - "ProductQuickAccessibilitySmoke/**" + "tools/ProductQuickAccessibilitySmoke/**" ], "targets": ["LoopLibQuick", "LoopEditor", "ProductQuickAccessibilitySmoke"], "tests": ["UnitTestsQuickCanvas", "UnitTestsCanvasParity", "UnitTestsEditorHost", "UnitTestsDocumentViewSession", "UnitTestsProductOperatorLoop", "UnitTestsQuickAccessibility", "UnitTestsShellKeyboard", "UnitTestsP4S9Interaction", "UnitTestsLoopStateVisual", "UnitTestsQuickDocumentModel"] }, "developer_widgets": { "paths": [ - "CodeGenerator/**", - "JBIG2_Viewer/**", - "PdfExampleGenerator/**", - "CanvasBenchmark/**" + "tools/CodeGenerator/**", + "tools/JBIG2_Viewer/**", + "tools/PdfExampleGenerator/**", + "tools/CanvasBenchmark/**" ], "targets": ["CodeGenerator", "JBIG2_VIEWER", "PdfExampleGenerator", "CanvasBenchmark"], "tests": [] diff --git a/architecture/boundaries.yaml b/architecture/boundaries.yaml index 160fa620b..9b3da0ecf 100644 --- a/architecture/boundaries.yaml +++ b/architecture/boundaries.yaml @@ -5,10 +5,10 @@ widgets_link_tokens: - Qt6::Widgets - Qt6::QuickWidgets widgets_link_allow: - - CanvasBenchmark/CMakeLists.txt - - CodeGenerator/CMakeLists.txt - - JBIG2_Viewer/CMakeLists.txt - - PdfExampleGenerator/CMakeLists.txt + - tools/CanvasBenchmark/CMakeLists.txt + - tools/CodeGenerator/CMakeLists.txt + - tools/JBIG2_Viewer/CMakeLists.txt + - tools/PdfExampleGenerator/CMakeLists.txt layers: - id: core cmake: LoopLibCore/CMakeLists.txt @@ -90,6 +90,8 @@ layers: - Qt6::Xml - ole32 - sapi + - userenv + - advapi32 forbidden_includes: - "^QtWidgets(/|$)" - "^QWidget$" @@ -151,6 +153,9 @@ separations: - PdfTool/pdfworkersandbox.cpp - PdfTool/pdfworkersandbox.h - PdfTool/pdfworkerprotocol.h + - PdfTool/pdfworkerprocess.cpp + - PdfTool/pdfworkerprocess_win.cpp + - PdfTool/pdfworkerprocess.h forbidden_includes: - "^pdfsentry\\.h$" - "^sentry\\.h$" diff --git a/architecture/proof-lanes.yaml b/architecture/proof-lanes.yaml index 0d27efdf6..daf3180d7 100644 --- a/architecture/proof-lanes.yaml +++ b/architecture/proof-lanes.yaml @@ -167,6 +167,8 @@ subsystems: summary: Preflight engine, corpus, and profile identity. owns_targets: true paths_from: agent-policy:preflight + paths: + - LoopLibCore/sources/pdfpreflightreceipt.cpp required: - kind: unit bind: policy @@ -234,6 +236,29 @@ subsystems: bind: catalog id: docs/generated/preflight-corpus-coverage.json executes: binding + - id: core-qualification + summary: Internal L01 CI and package provenance checks; module admission remains reviewed. + paths: + - scripts/qualification/check_core_qualification.py + - scripts/qualification/test_check_core_qualification.py + - scripts/ci/compare_package_boundary_evidence.py + - scripts/ci/test_compare_package_boundary_evidence.py + - docs/CORE_QUALIFICATION.md + required: + - kind: unit + bind: script + id: scripts/qualification/test_check_core_qualification.py + executes: binding + - kind: unit + bind: script + id: scripts/ci/test_compare_package_boundary_evidence.py + executes: binding + - kind: unit + bind: workflow + id: core-qualification-ci + ref: .github/workflows/ci.yml + contains: scripts.qualification.test_check_core_qualification + executes: binding - id: packaging summary: Install and release-gate workflows. paths: @@ -339,7 +364,7 @@ subsystems: id: agent-policy:pdftool executes: binding - id: pdf-worker-isolation - summary: "PdfTool supervisor plus loop-pdf-worker isolation for untrusted open/preflight (#618)." + summary: "Linux and Windows worker containment, bounded IPC, receipt admission and privacy (#20)." owns_targets: true paths: - PdfTool/loop-pdf-worker-main.cpp @@ -350,6 +375,12 @@ subsystems: - PdfTool/pdfworkerprotocol.h - PdfTool/pdfworkerclient.cpp - PdfTool/pdfworkerclient.h + - PdfTool/pdfworkerprocess.cpp + - PdfTool/pdfworkerprocess_win.cpp + - PdfTool/pdfworkerprocess.h + - PdfTool/worker-runtime.cmake + - PdfTool/write-worker-runtime.cmake + - UnitTests/pdfworkerprobe.cpp - PdfTool/pdftoolworker.cpp - PdfTool/pdftoolworker.h - UnitTests/tst_pdfworkerisolation.cpp diff --git a/changes/cc-code-review-ebc857.evidence.yaml b/changes/cc-code-review-ebc857.evidence.yaml new file mode 100644 index 000000000..fc9edfdf8 --- /dev/null +++ b/changes/cc-code-review-ebc857.evidence.yaml @@ -0,0 +1,17 @@ +format_version: 1 +kind: evidence +claims: + - id: retention-failure-keeps-published-output + evidence: + - unit:agent-policy:pdftool + - unit:agent-policy:pagemaster + - unit:agent-policy:quick + - id: core-coverage-claim + evidence: + - unit:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle +unresolved: + - core:scripts/ci/check_independent_validation_gate.py + - Focused Qt builds and tests (UnitTestsEditorHost, UnitTestsRepairOperatorAcceptance, UnitTestsPreflightEngine) require a configured build directory; this worktree has none. diff --git a/changes/cc-code-review-ebc857.md b/changes/cc-code-review-ebc857.md new file mode 100644 index 000000000..6b54e819b --- /dev/null +++ b/changes/cc-code-review-ebc857.md @@ -0,0 +1,4 @@ +Category: fixed +Audience: users managing PDF operation history and reading preflight reports +Breaking-Change: no +Summary: Keep the restored revision open and the repair report written when history retention fails after an accepted rollback or repair, and restore the general "no formal GWG conformance" statement in every preflight report's coverage claim. diff --git a/changes/cc-g00-legacy-reference-reconcile.evidence.yaml b/changes/cc-g00-legacy-reference-reconcile.evidence.yaml new file mode 100644 index 000000000..6c04b5c51 --- /dev/null +++ b/changes/cc-g00-legacy-reference-reconcile.evidence.yaml @@ -0,0 +1,11 @@ +format_version: 1 +kind: evidence +claims: + - id: overlay-records-separate-live-and-legacy-issues + evidence: + - unit:scripts/ci/test_preflight_check_catalog.py + - architecture:scripts/generate-architecture-catalogs.py + - id: backlog-rows-point-at-live-trackers + evidence: + - architecture:scripts/generate-architecture-catalogs.py +unresolved: [] diff --git a/changes/cc-g00-legacy-reference-reconcile.md b/changes/cc-g00-legacy-reference-reconcile.md new file mode 100644 index 000000000..69a7a4009 --- /dev/null +++ b/changes/cc-g00-legacy-reference-reconcile.md @@ -0,0 +1,4 @@ +Category: internal +Audience: developers +Breaking-Change: no +Summary: Separate live studio-berry/loop issue records from frozen legacy ones in the preflight catalog overlay (#111) — github_issues entries now carry `repository`, legacy snapshots are keyed `legacy#` and can never be an open row's `closed_by`, the generator adds an opt-in `--verify-github` read-back that rejects a missing issue, a pull request, or a changed title, state, or milestone, sixteen open backlog rows re-point to the reset issues filed for them, and 25 of the 28 legacy issue and pull-request links under docs/ become `legacy #` text (the three in docs/GOVERNED_EXECUTION.md and ADR-011 wait for a change that can carry the governed-execution proof lanes), with the convention recorded in docs/LEGACY_ISSUE_PROVENANCE.md. diff --git a/changes/cc-issue-19-resource-envelopes.evidence.yaml b/changes/cc-issue-19-resource-envelopes.evidence.yaml new file mode 100644 index 000000000..927e5a335 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.evidence.yaml @@ -0,0 +1,29 @@ +format_version: 1 +kind: evidence +claims: + - id: benchmark-preflight-phase-and-cancellation + evidence: + - unit:agent-policy:pdftool + - unit:agent-policy:pagemaster + - packaging:linux-build + - packaging:windows-build + - id: hosted-qualification-workflow + evidence: + - packaging:linux-build + - packaging:windows-build + - security:codeql + - id: agent-policy-bindings + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - id: linux-rss-high-water-reads-procfs + evidence: + - unit:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle +unresolved: + - core:scripts/ci/check_independent_validation_gate.py + - Native PdfTool and UnitTestsPdfToolContract were not built locally (no configured build directory for this worktree); the linux-build and windows-build CI lanes compile and run them. + - Hosted qualification measurements come from the resource-envelope-qualification workflow run on this PR and are recorded under docs/evidence/issue-19-resource-envelope/ after that run. diff --git a/changes/cc-issue-19-resource-envelopes.md b/changes/cc-issue-19-resource-envelopes.md new file mode 100644 index 000000000..3fc93b1f7 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.md @@ -0,0 +1,4 @@ +Category: added +Audience: developers and release qualifiers +Breaking-Change: no +Summary: Qualify hostile and production resource envelopes on hosted Linux and Windows runners (#19). `PdfTool benchmark --profile` adds a measured preflight phase, so a clean run reports a complete envelope; rendering stops within one page slice of an interrupt, and Windows honours CTRL_BREAK. A new resource-envelope qualification workflow generates a deterministic synthetic fixture bundle, runs the strict matrix with separate cancellation and reopen-after-cancel recovery probes and a hostile budget-exhaustion lane, and builds schema-2 evidence carrying the CI run id. Crashes, timeouts, and skipped workloads can never count as a passing envelope. diff --git a/changes/cc-pr-template-quality-pass.evidence.yaml b/changes/cc-pr-template-quality-pass.evidence.yaml new file mode 100644 index 000000000..7bf863783 --- /dev/null +++ b/changes/cc-pr-template-quality-pass.evidence.yaml @@ -0,0 +1,12 @@ +format_version: 1 +kind: evidence +claims: + - id: policy-handoff-quality-summary + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - id: pr-template-quality-pass + evidence: + - architecture:scripts/generate-architecture-catalogs.py +unresolved: [] diff --git a/changes/cc-pr-template-quality-pass.md b/changes/cc-pr-template-quality-pass.md new file mode 100644 index 000000000..a79e02408 --- /dev/null +++ b/changes/cc-pr-template-quality-pass.md @@ -0,0 +1,4 @@ +Category: internal +Audience: contributors +Breaking-Change: no +Summary: Rename the pull request template's "Anti-slop pass" section and summary to "Quality pass" and "Quality summary", and ask for a "quality summary" in the agent policy handoff; checklist items and the agent-policy.json key names are unchanged. diff --git a/changes/cc-unstable-receipt-fix.evidence.yaml b/changes/cc-unstable-receipt-fix.evidence.yaml new file mode 100644 index 000000000..302a50d82 --- /dev/null +++ b/changes/cc-unstable-receipt-fix.evidence.yaml @@ -0,0 +1,26 @@ +format_version: 1 +kind: evidence +claims: + - id: receipt-no-evidence-is-incomplete + evidence: + - unit:agent-policy:core +unresolved: + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle + - core:scripts/ci/check_independent_validation_gate.py + - unit:agent-policy:preflight + - unit:UnitTestsPreflightChecks + - unit:UnitTestsPreflightEngine + - unit:UnitTestsPreflightInteraction + - unit:UnitTestsPreflightVerdict + - unit:UnitTestsPreflightProfileResolver + - unit:UnitTestsProfileIdentity + - unit:UnitTestsOperatorAcceptance + - integration:UnitTestsPreflightCorpus + - integration:UnitTestsPreflightWorkflowAcceptance + - differential:UnitTestsStandardOracle + - architecture:loop-preflight/testdata/fixtures + - architecture:docs/generated/preflight-check-catalog.json + - architecture:docs/generated/preflight-corpus-coverage.json + - "Not run locally - this worktree has no configured build directory, so UnitTestsPreflightVerdict was neither built nor run. The listed lanes are left for hosted CI." diff --git a/changes/cc-unstable-receipt-fix.md b/changes/cc-unstable-receipt-fix.md new file mode 100644 index 000000000..736bd2ceb --- /dev/null +++ b/changes/cc-unstable-receipt-fix.md @@ -0,0 +1,4 @@ +Category: fixed +Audience: developers +Breaking-Change: no +Summary: An inspection receipt built from a run with no recorded evidence is now Incomplete instead of PASS, matching the receipt parser, which rejects PASS without evidence references. Previously Core could emit a PASS receipt that its own validation refused. diff --git a/changes/chore-simplify-tool-tree.evidence.yaml b/changes/chore-simplify-tool-tree.evidence.yaml new file mode 100644 index 000000000..82c8a57d3 --- /dev/null +++ b/changes/chore-simplify-tool-tree.evidence.yaml @@ -0,0 +1,22 @@ +format_version: 1 +kind: evidence +claims: + - id: tool-tree-path-contracts + evidence: + - unit:agent-policy:developer_widgets + - unit:agent-policy:quick + - unit:agent-policy:build_policy + - unit:agent-policy:plugins + - unit:scripts/agent/test_architecture_contracts.py + - architecture:docs/generated/architecture-catalog.json + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - id: migrated-source-contracts + evidence: + - unit:scripts/ci/test_check_preflight_truth_source.py + - unit:scripts/ci/test_check_qml_mirror_parity.py + - packaging:linux-build + - packaging:windows-build +unresolved: + - CMake configure and native build were not run. + - Focused Python contract tests were not run. diff --git a/changes/chore-simplify-tool-tree.md b/changes/chore-simplify-tool-tree.md new file mode 100644 index 000000000..b472814a9 --- /dev/null +++ b/changes/chore-simplify-tool-tree.md @@ -0,0 +1,4 @@ +Category: internal +Audience: maintainers +Breaking-Change: no +Summary: Grouped the standalone developer and qualification applications under `tools/` and updated their CMake, source, and evidence paths. diff --git a/changes/codex-issue-15-evidence-core-reset.md b/changes/codex-issue-15-evidence-core-reset.md new file mode 100644 index 000000000..24e8974c2 --- /dev/null +++ b/changes/codex-issue-15-evidence-core-reset.md @@ -0,0 +1,4 @@ +Category: internal +Audience: contributors +Breaking-Change: no +Summary: Pin the loop2 Evidence Core source SHA, inherited contracts, catalog coverage, legacy gap dispositions, and proof limits for L01-01. diff --git a/changes/codex-issue-16-inspection-receipt.evidence.yaml b/changes/codex-issue-16-inspection-receipt.evidence.yaml new file mode 100644 index 000000000..2a7cc4b8a --- /dev/null +++ b/changes/codex-issue-16-inspection-receipt.evidence.yaml @@ -0,0 +1,36 @@ +format_version: 1 +kind: evidence +claims: + - id: core-inspection-receipt + evidence: + - unit:agent-policy:core + - unit:UnitTestsPreflightVerdict + - architecture:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle + - packaging:linux-build + - packaging:windows-build + - id: preflight-coverage-and-verdict + evidence: + - unit:agent-policy:preflight + - unit:UnitTestsPreflightChecks + - unit:UnitTestsPreflightEngine + - unit:UnitTestsPreflightInteraction + - unit:UnitTestsPreflightVerdict + - unit:UnitTestsPreflightProfileResolver + - unit:UnitTestsProfileIdentity + - unit:UnitTestsOperatorAcceptance + - integration:UnitTestsPreflightCorpus + - integration:UnitTestsPreflightWorkflowAcceptance + - differential:UnitTestsStandardOracle + - architecture:loop-preflight/testdata/fixtures + - architecture:docs/generated/preflight-check-catalog.json + - architecture:docs/generated/preflight-corpus-coverage.json + - id: agent-policy-gate + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py +unresolved: + - core:scripts/ci/check_independent_validation_gate.py diff --git a/changes/codex-issue-16-inspection-receipt.md b/changes/codex-issue-16-inspection-receipt.md new file mode 100644 index 000000000..8afef67f7 --- /dev/null +++ b/changes/codex-issue-16-inspection-receipt.md @@ -0,0 +1,4 @@ +Category: added +Audience: integrators +Breaking-Change: no +Summary: Add a revision-bound typed Core inspection receipt with stable identity and fail-closed verdicts, and fix the mapped CTest gate for multi-config builds. diff --git a/changes/codex-issue-17-fence-scheduled-results.evidence.yaml b/changes/codex-issue-17-fence-scheduled-results.evidence.yaml new file mode 100644 index 000000000..9a6cbfbe8 --- /dev/null +++ b/changes/codex-issue-17-fence-scheduled-results.evidence.yaml @@ -0,0 +1,34 @@ +format_version: 1 +kind: evidence +claims: + - id: scheduled-result-fencing + evidence: + - unit:agent-policy:core + - architecture:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle + - id: page-surface-admission + evidence: + - unit:agent-policy:interaction + - id: editor-result-admission + evidence: + - unit:agent-policy:quick + - id: preflight-controller-fence + evidence: + - unit:agent-policy:preflight + - unit:UnitTestsPreflightChecks + - unit:UnitTestsPreflightEngine + - unit:UnitTestsPreflightInteraction + - unit:UnitTestsPreflightVerdict + - unit:UnitTestsPreflightProfileResolver + - unit:UnitTestsProfileIdentity + - unit:UnitTestsOperatorAcceptance + - integration:UnitTestsPreflightCorpus + - integration:UnitTestsPreflightWorkflowAcceptance + - differential:UnitTestsStandardOracle + - architecture:loop-preflight/testdata/fixtures + - architecture:docs/generated/preflight-check-catalog.json + - architecture:docs/generated/preflight-corpus-coverage.json +unresolved: + - core:scripts/ci/check_independent_validation_gate.py diff --git a/changes/codex-issue-17-fence-scheduled-results.md b/changes/codex-issue-17-fence-scheduled-results.md new file mode 100644 index 000000000..29b2f09df --- /dev/null +++ b/changes/codex-issue-17-fence-scheduled-results.md @@ -0,0 +1,4 @@ +Category: fixed +Audience: integrators +Breaking-Change: no +Summary: Fence scheduled document work across close and reopen, and admit rendered and Editor results only under their current request identities. diff --git a/changes/codex-issue-20-worker-isolation.evidence.yaml b/changes/codex-issue-20-worker-isolation.evidence.yaml new file mode 100644 index 000000000..cd27be277 --- /dev/null +++ b/changes/codex-issue-20-worker-isolation.evidence.yaml @@ -0,0 +1,41 @@ +format_version: 1 +kind: evidence +claims: + - id: isolated-worker-boundary + evidence: + - unit:UnitTestsPdfWorkerIsolation + - unit:UnitTestsPreflightVerdict + - unit:UnitTestsPreflightChecks + - unit:UnitTestsPreflightEngine + - unit:UnitTestsPreflightInteraction + - unit:UnitTestsPreflightProfileResolver + - unit:UnitTestsProfileIdentity + - unit:UnitTestsOperatorAcceptance + - unit:agent-policy:pagemaster + - integration:UnitTestsPreflightCorpus + - integration:UnitTestsPreflightWorkflowAcceptance + - differential:UnitTestsStandardOracle + - architecture:loop-preflight/testdata/fixtures + - architecture:docs/generated/preflight-check-catalog.json + - architecture:docs/generated/preflight-corpus-coverage.json + - unit:agent-policy:preflight + - unit:agent-policy:core + - unit:agent-policy:pdftool + - unit:agent-policy:build_policy + - unit:scripts/agent/test_architecture_contracts.py + - security:scripts/ci/check_pdf_worker_isolation.py + - security:scripts/ci/check_source_integrity.py + - architecture:agent-policy:pdftool + - architecture:docs/generated/architecture-catalog.json + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - architecture:scripts/generate_phase5_widgets_evidence.py + - unit:scripts/ci/test_verify_phase5_widgets_contract.py + - differential:UnitTestsConversionOracle +unresolved: + - core:scripts/ci/check_independent_validation_gate.py + - Linux runtime qualification of this source revision + - Windows installed-product qualification and externally configured dump collectors are not admitted by local worker tests + - Remaining in-process Editor, ordinary PdfTool and PageMaster paths + - External OS or administrator dump collectors require explicit qualification disposition + - Module release admission remains issue 21 diff --git a/changes/codex-issue-20-worker-isolation.md b/changes/codex-issue-20-worker-isolation.md new file mode 100644 index 000000000..157f17929 --- /dev/null +++ b/changes/codex-issue-20-worker-isolation.md @@ -0,0 +1,4 @@ +Category: security +Audience: operators, developers +Breaking-Change: yes +Summary: Require Linux or Windows containment for isolated PDF open/preflight, admit bounded protocol v2 responses through Core inspection receipts, and keep worker faults and document content out of supervisor diagnostics. Worker protocol v1 peers are rejected; remaining in-process routes require separate release disposition. diff --git a/changes/codex-issue-21-core-qualification.evidence.yaml b/changes/codex-issue-21-core-qualification.evidence.yaml new file mode 100644 index 000000000..8a256d66e --- /dev/null +++ b/changes/codex-issue-21-core-qualification.evidence.yaml @@ -0,0 +1,22 @@ +format_version: 1 +kind: evidence +claims: + - id: core-qualification-preparation + evidence: + - unit:scripts/qualification/test_check_core_qualification.py + - unit:scripts/ci/test_compare_package_boundary_evidence.py + - unit:core-qualification-ci + - id: qualification-ci-contract + evidence: + - packaging:linux-build + - packaging:windows-build + - security:codeql + - id: qualification-proof-map + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - id: qualification-runbook + evidence: + - architecture:scripts/generate-architecture-catalogs.py +unresolved: [] diff --git a/changes/codex-issue-21-core-qualification.md b/changes/codex-issue-21-core-qualification.md new file mode 100644 index 000000000..2538c0d37 --- /dev/null +++ b/changes/codex-issue-21-core-qualification.md @@ -0,0 +1,4 @@ +Category: internal +Audience: maintainers +Breaking-Change: no +Summary: Prepare L01 Core qualification with an exact-SHA CI/package provenance checker, focused Linux/Windows script tests, and an issue-dossier runbook that keeps final module admission pending full L01 evidence and reviewer acceptance. diff --git a/changes/codex-promotion-stage-field.md b/changes/codex-promotion-stage-field.md new file mode 100644 index 000000000..955ca1f40 --- /dev/null +++ b/changes/codex-promotion-stage-field.md @@ -0,0 +1,4 @@ +Category: internal +Audience: maintainers +Breaking-Change: no +Summary: Track linked issue work through dev, unstable, and stable in the organization Promotion stage field, retiring the legacy queue label and stable-branch auto-close behavior. diff --git a/changes/docs-quick-canvas-editor-gaps.evidence.yaml b/changes/docs-quick-canvas-editor-gaps.evidence.yaml new file mode 100644 index 000000000..7a80895ce --- /dev/null +++ b/changes/docs-quick-canvas-editor-gaps.evidence.yaml @@ -0,0 +1,10 @@ +format_version: 1 +kind: evidence +claims: + - id: quick-canvas-editor-gap-record + evidence: + - architecture:agent-policy:documentation + - architecture:scripts/generate-architecture-catalogs.py +unresolved: + - mapped quick and interaction test lanes were not run; build-local does not exist and .local-vcpkg was removed, so configure and dependency restore are both approval-required under agent-policy.json + - ProductQuickAccessibilitySmoke was not run for the same reason diff --git a/changes/docs-quick-canvas-editor-gaps.md b/changes/docs-quick-canvas-editor-gaps.md new file mode 100644 index 000000000..f354a68e9 --- /dev/null +++ b/changes/docs-quick-canvas-editor-gaps.md @@ -0,0 +1,6 @@ +# Quick canvas-editor gap record + +Category: added +Audience: developers +Breaking-Change: no +Summary: Record two unimplemented Quick canvas-editor gaps, the inert Select/Hand toolbar controls and the discarded DragSession commit, with the boundary and contract constraints that make each a decision rather than a patch. diff --git a/docs/CORE_QUALIFICATION.md b/docs/CORE_QUALIFICATION.md new file mode 100644 index 000000000..cb3f7b5cd --- /dev/null +++ b/docs/CORE_QUALIFICATION.md @@ -0,0 +1,155 @@ +# L01-07 Core qualification runbook + +[Issue #21](https://github.com/studio-berry/loop/issues/21) prepares and admits one +exact source SHA for the [L01 Evidence Core gate](https://github.com/studio-berry/loop/issues/2). +The preparation PR may merge while admission is pending. Its tooling verifies +CI/package provenance; the complete issue dossier and reviewer decision establish +module admission. Release promotion remains a separate R00 decision. + +## Prerequisites and candidate freeze + +Review the acceptance evidence for every other L01 child below. An issue closure, +merged PR, or legacy qualification claim alone does not satisfy a row. + +| Issues | Required acceptance evidence | +| --- | --- | +| #15 | Reviewed reset inventory, capability/gap dispositions, and current catalog diff. | +| #16 | Receipt identity and golden vectors; missing coverage, unsupported, budget-limited, cancelled, and parser-error paths stay non-PASS. | +| #17 | Scheduler/result fencing under reorder, timeout, retry, cancel, and reopen. | +| #18 | Independent standards, signature, conversion, and fidelity reports bound to exact output bytes, validator versions, and visible limitations. | +| #19 | Strict Linux/Windows resource matrix, declared budgets, measured memory/time, cancellation/recovery, and hostile workload dispositions. | +| #20 | Worker crash/timeout/resource fault injection, host recovery, no silent in-process fallback, and telemetry/content inspection. | +| #113–120 | Accepted regression fixtures and before/after reports, unchanged golden corpus, current catalog dispositions, and no clean false PASS on the named defects. | + +After these outcomes and the preparation PR are accepted and integrated into +`dev`, select candidate `C` as a full 40-character SHA. Verify the live `dev` ref +against the local ref before selection; the current workspace or old branch head +is not an implicit candidate. Record the comparison base, committed catalog, +profile/corpus manifest digests, and each child acceptance link. + +Use a qualification branch pinned to `C` for dispatch and keep it fixed through +review. Follow repository approval requirements for upstream sync, pushes, +packaging/installation, and external writes. A source repair creates a new +candidate and new evidence packet; do not combine checks from multiple SHAs. +Route repairs to the owning child issue and reuse its outcome PR when one exists. + +## Run and inspect the required lanes + +Dispatch full `CI` on the pinned qualification ref. Ordinary PR CI skips the full +Linux/Windows jobs, so its green aggregate is insufficient. Record the explicit +run ID and verify its `headSha == C`; never select evidence solely by latest run +or branch name. Inspect `source_integrity`, `linux / build`, and `windows / build`, +including the Widgets-absent and normal builds/tests and preflight corpus gate. + +```sh +gh workflow run ci.yml --repo studio-berry/loop --ref "$QUALIFICATION_REF" +gh run list --repo studio-berry/loop --workflow ci.yml \ + --branch "$QUALIFICATION_REF" --event workflow_dispatch \ + --json databaseId,headSha,status,conclusion,url +``` + +Check test registration, counts, output, and individual skips, not just the step +exit status. Run the repository's mapped Core/preflight proofs, generated-catalog +and architecture checks. For any implementation PR, run `check-change.py` against +its accepted base and retain the exact report. An incomplete report is not proof. + +Reproduce #18's accepted independent claim matrix, #19's strict resource workflow, +and #20's process-isolation faults on `C` on both required platforms. Retain +fixture/profile/output digests, commands, tool versions, budgets, measurements, +run IDs, and terminal dispositions. Source guards and self-validation do not +substitute for the external oracle or installed-runtime results. A negative PDF +fixture may correctly yield Fail/Incomplete while its behavioral test passes; +an incomplete inspection must never be relabelled as a passing PDF. + +Dispatch `Linux_AppImage` and `Windows_MSI` with `source_sha=C` on the pinned ref. +Inspect their exact-checkout guards, installed/relocated PdfTool preflight and +runtime smokes, dependency inspection, and package lifecycle results. Capture +package run IDs and input/check-out SHA, artifact IDs, names, hashes, and expiry +dates. A package workflow's run SHA alone does not establish its checked-out +`source_sha`. Record every skip, including hosted Windows operator-launch skips, +and decide whether it omitted a required Core lane. A qualifying skip blocks +admission; unrelated conditional skips need an explicit disposition. + +```sh +gh workflow run LinuxInstall.yml --repo studio-berry/loop \ + --ref "$QUALIFICATION_REF" --field source_sha="$C" +gh workflow run WindowsInstall.yml --repo studio-berry/loop \ + --ref "$QUALIFICATION_REF" --field source_sha="$C" +``` + +## Check the collected provenance + +Keep downloads, transcripts, and generated reports outside the repository. Download +the package-boundary evidence and the actual AppImage/MSI bytes from the recorded +package runs. Unpack the GitHub artifact archives before checking: the inspector +hashes package files, not GitHub's enclosing ZIP archives. + +```sh +gh run download "$LINUX_PACKAGE_RUN_ID" --repo studio-berry/loop \ + --name loop-package-boundary-linux-evidence --dir "$EVIDENCE_DIR/linux" +gh run download "$WINDOWS_PACKAGE_RUN_ID" --repo studio-berry/loop \ + --name loop-package-boundary-windows-evidence --dir "$EVIDENCE_DIR/windows" +gh run download "$LINUX_PACKAGE_RUN_ID" --repo studio-berry/loop \ + --name "$APPIMAGE_NAME" --dir "$EVIDENCE_DIR/packages" +gh run download "$WINDOWS_PACKAGE_RUN_ID" --repo studio-berry/loop \ + --name "$MSI_NAME" --dir "$EVIDENCE_DIR/packages" +``` + +From the repository root, capture the selected full CI snapshot and run the checker +(the example uses a POSIX shell; PowerShell accepts the same CLI arguments): + +```sh +gh run view "$CI_RUN_ID" --repo studio-berry/loop \ + --json databaseId,url,headSha,status,conclusion,jobs > "$EVIDENCE_DIR/ci-run.json" +python scripts/qualification/check_core_qualification.py \ + --candidate-sha "$C" --ci-run "$EVIDENCE_DIR/ci-run.json" \ + --linux-evidence "$EVIDENCE_DIR/linux/evidence.json" \ + --windows-evidence "$EVIDENCE_DIR/windows/evidence.json" \ + --linux-package "$EVIDENCE_DIR/packages/$APPIMAGE_NAME" \ + --windows-package "$EVIDENCE_DIR/packages/$MSI_NAME" \ + > "$EVIDENCE_DIR/core-provenance.txt" +``` + +The checker reads existing evidence formats and makes no network calls or issue +changes. Exit 0 means the CI/package provenance subset was verified; its output +explicitly keeps module admission **PENDING REVIEW**. Invalid input, missing or +duplicate required jobs/steps, nonterminal/failed/skipped required lanes, mixed +SHAs, incomplete package inspection, and mismatched package names/sizes/digests +return a nonzero exit with a reason and no successful provenance report. +Unrelated conditional skips, such as the fast-lane steps in a full build, are +allowed. Required step names follow the current reusable workflows; renamed lanes +must be reconciled deliberately rather than silently guessed. + +Snapshots are supplied evidence, not independently authenticated records. The +reviewer must inspect the live run/job links and actual test/smoke/oracle output. +The checker does not establish test counts, per-test skips, installed smoke +success, external oracle agreement, resource metrics, or worker containment. + +## Issue dossier and reviewer decision + +Post the complete dossier on #21 and link that exact comment from parent #2. +Do not commit a dossier into the frozen candidate or treat this preparation PR +as closing #21. Retain the full sanitized evidence outside the source tree; record +important measured results and identities in the issue before expiring Actions +artifacts are lost. Refresh expired or unavailable required evidence before review. + +Use this review checklist in the issue comment: + +| Field/row | Record | +| --- | --- | +| Candidate | Full `C`, comparison base, qualification ref, catalog/profile/corpus digests. | +| Child acceptance | One row per #15–20 and #113–120 with acceptance evidence and reviewer disposition. | +| Source/tests | Run/job IDs and links, commands, platform/toolchain, test counts, per-test skips, expected terminal behavior. | +| Independent claims | Claim scope, exact PDF/output digests, validator/version, result, supported limits. | +| Resources/isolation | Budget/workload identities, measured maxima, cancellation/recovery and fault/telemetry results. | +| Installed runtime | Exact package and fixture identities, commands, smoke/lifecycle transcripts, skipped/unavailable lanes. | +| Artifacts | Package/run/artifact IDs, byte counts, SHA-256, source SHA, retention/expiry, provenance-check result. | +| Deviations | Every limitation/skip, owner, evidence, and explicit qualifying/nonqualifying disposition. | +| Decision | Named reviewer, timestamp, exact `C`, admit/blocked decision, remaining risk. | + +Admission requires all required L01 outcomes and lanes to be supported on `C` +without qualifying skips. Missing, stale, unavailable, cancelled, failed, or +unreviewed proof keeps admission blocked. Only after the named reviewer accepts +that packet may #21 be closed and the parent gate updated. The handoff records +changed files, observed verification, remaining risks, and a 1–3 sentence quality +review summary. diff --git a/docs/CORRECTION_COVERAGE_MATRIX.md b/docs/CORRECTION_COVERAGE_MATRIX.md index 6ebfd2c50..5b0bfd264 100644 --- a/docs/CORRECTION_COVERAGE_MATRIX.md +++ b/docs/CORRECTION_COVERAGE_MATRIX.md @@ -42,7 +42,7 @@ Save-mode semantics for the source artifact are defined once in the generated catalog under `save_modes` and referenced per operation. Target scopes describe the current implicit selector behaviour. The shared -selector AST tracked in GitHub #587 is not yet wired into repair plans; until it +selector AST tracked in legacy #587 is not yet wired into repair plans; until it lands, operations declare whole-document, page, resource, or production-geometry scopes resolved during `analyze()`. diff --git a/docs/EDITOR_RECOVERY.md b/docs/EDITOR_RECOVERY.md index 5d724ddfe..86ee3a3f6 100644 --- a/docs/EDITOR_RECOVERY.md +++ b/docs/EDITOR_RECOVERY.md @@ -26,7 +26,7 @@ only half reachable today: the approval half is pinned by `UnitTestsOperationHistory::noSavePathProducesAnApprovedOutputRecord` (no save path records an approval or an approved output, so a recovered file cannot be presented as approved), and the restore half is tracked by -[#575](https://github.com/studio-berry/loop/issues/575). +legacy #575. ## Safety contract diff --git a/docs/EVIDENCE_CORE_RESET_INVENTORY.md b/docs/EVIDENCE_CORE_RESET_INVENTORY.md new file mode 100644 index 000000000..f2e9a27b3 --- /dev/null +++ b/docs/EVIDENCE_CORE_RESET_INVENTORY.md @@ -0,0 +1,83 @@ +# L01-01 Evidence Core reset inventory + +This is the source-to-contract disposition for [loop2 #15](https://github.com/studio-berry/loop2/issues/15), not a release qualification. The audited source is `origin/dev` at `5c8366a33e07597f5213be8da594e3120f73ed69` (2026-09-24); `git ls-remote origin refs/heads/dev` matched the local tracking ref before the topic branch was made. The [L01 parent](https://github.com/studio-berry/loop2/issues/2) owns the module gate. Legacy Loop status is intake evidence, not a loop2 completion claim. + +## Authority and catalog diff + +The binding local inventory comes from [the generated architecture catalog](generated/architecture-catalog.json), [check catalog](generated/preflight-check-catalog.json), [coverage backlog](generated/preflight-coverage-backlog.json), [corpus map](generated/preflight-corpus-coverage.json), [proof lanes](../architecture/proof-lanes.yaml), and source at the SHA above. `python scripts/generate-architecture-catalogs.py --check` passed. Regeneration is therefore a zero-diff check against the committed catalogs at this SHA; this PR makes no catalog or check-registry change. SHA-256 identifies the exact catalog artifacts: + +| Artifact | SHA-256 | +| --- | --- | +| `architecture-catalog.json` | `cfce8290a035aa36888949bd55d0446113ef8b2206e6907d8b47ac7688c8b3f0` | +| `preflight-check-catalog.json` | `0c1f3d09734a1d09250c907855aa89cff869b2bb5ed6569fc3a6e9051299ee46` | +| `preflight-coverage-backlog.json` | `de68c626b180905d59aa73e94b41d04ecff6ca3b761fb912d812ba007a36162d` | +| `preflight-corpus-coverage.json` | `aec3acc671cf8ccb5611e2977b03b1268b5eb4fdf09adca9bbad230ffe65ee91` | + +The catalog has **22 registered checks**: 5 `covered`, 17 `partial`, and no `not_covered` check row. Its separate backlog has 26 rows: 18 `open`, 7 `landed`, and 1 `closed`. `covered` is limited to the catalog's named check and corpus scope; it is not a standards certificate. The [coverage matrix](PREFLIGHT_COVERAGE_MATRIX.md) defines the claim and the fixture rule. + +## Source-to-contract dispositions + +`Prove` means keep the primitive and obtain current exact-SHA behavioral evidence. `Reuse` means the source contract is already present and no replacement is justified. `Repair` names a bounded next gate. `Defer` keeps a known limitation visible without treating a legacy issue as an implementation mandate. Every row's owner is LoopLibCore unless another owner is named. + +| Inherited capability or boundary | Disposition and owner | Source, contract, and proof | +| --- | --- | --- | +| Profile import, variable binding, check registry, per-check status, and coverage scope | **Reuse; prove** under L01-02. Core preflight. | [`preflightengine.cpp`](../LoopLibCore/sources/preflightengine.cpp), [ADR-002](adr/adr-002-preflight-engine-orchestrator.md), [check catalog](generated/preflight-check-catalog.json); `UnitTestsPreflightEngine`, `UnitTestsPreflightChecks`, `UnitTestsPreflightProfileResolver`, `UnitTestsPreflightCorpus`. | +| Evidence graph, report JSON, and portable bundle | **Reuse** existing evidence fields; **repair** complete revision-bound receipt and limitation admission in [#16](https://github.com/studio-berry/loop2/issues/16). Core evidence. | [`pdfevidencegraph.cpp`](../LoopLibCore/sources/pdfevidencegraph.cpp), [`pdfpreflightevidencebundle.cpp`](../LoopLibCore/sources/pdfpreflightevidencebundle.cpp), [bundle contract](PREFLIGHT_EVIDENCE_BUNDLE.md); `UnitTestsEvidenceGraph`, `UnitTestsPreflightEngine`. A bundle is not itself proof that every required inspection ran. | +| Artifact, document revision, and profile identity | **Reuse; prove** exact request/result binding under [#16](https://github.com/studio-berry/loop2/issues/16) and [#17](https://github.com/studio-berry/loop2/issues/17). Core identity. | [`pdfartifactidentity.h`](../LoopLibCore/sources/pdfartifactidentity.h), [revision contract](REVISION_CONTEXT.md), [ADR-001](adr/adr-001-pdf-document-session.md); `UnitTestsIdentitySeparation`, `UnitTestsDocumentSession`, `UnitTestsRevisionStress`. | +| Canonical Pass/Fail/Incomplete/Error reducer and certificate gate | **Reuse; prove** all four states and no zero-finding budget PASS under [#16](https://github.com/studio-berry/loop2/issues/16). Core verdict. | [`pdfpreflightverdict.cpp`](../LoopLibCore/sources/pdfpreflightverdict.cpp), [verdict contract](PREFLIGHT_VERDICT.md); `UnitTestsPreflightVerdict`, `UnitTestsPreflightEngine`. `PreflightResult::pass` is derived compatibility data. | +| Fixed-capacity job scheduler, cancellation, stale-result discard | **Reuse** the existing scheduler; **repair** producer/result fencing under [#17](https://github.com/studio-berry/loop2/issues/17). Core scheduling. | [`pdfjobscheduler.cpp`](../LoopLibCore/sources/pdfjobscheduler.cpp), [scheduler contract](JOB_SCHEDULER.md); `UnitTestsJobScheduler`, `UnitTestsRevisionStress`, `scripts/ci/check_unmanaged_async.py`. Caller coverage is not complete merely because the scheduler exists. | +| Parser, reader, renderer, session, processing and resource budgets | **Reuse** Core primitives; **prove** hostile and production envelopes under [#19](https://github.com/studio-berry/loop2/issues/19). Core PDF. | [`pdfdocumentreader.cpp`](../LoopLibCore/sources/pdfdocumentreader.cpp) calls [`pdfparser.cpp`](../LoopLibCore/sources/pdfparser.cpp); [`pdfrenderer.cpp`](../LoopLibCore/sources/pdfrenderer.cpp) and [budget contract](RESOURCE_BUDGETS.md) bound work. `UnitTestsProcessingBudget`, `UnitTestsResourceBudget`, `UnitTestsBudgetExhaustion`, `UnitTestsBudgetCorpus` are mapped tests. The unbudgeted cumulative `PDFFunction::createFunction()` path remains an explicit deferred contract-level gap in that document. | +| PdfTool open/preflight process boundary | **Reuse** the Linux-first worker proof from legacy #618; **repair/audit** remaining privileged-host paths under [#20](https://github.com/studio-berry/loop2/issues/20). PdfTool supervisor and Core. | [`pdfworkerprotocol.h`](../PdfTool/pdfworkerprotocol.h) allowlists `ping`, `open`, `preflight`, `cancel`; [`pdfworkerclient.cpp`](../PdfTool/pdfworkerclient.cpp) maps worker failure/timeout to unavailable/incomplete; [`pdfworkersandbox.cpp`](../PdfTool/pdfworkersandbox.cpp), `UnitTestsPdfWorkerIsolation`, `scripts/ci/check_pdf_worker_isolation.py`. Windows runtime tests skip the Linux sandbox proof; [`editorhost.cpp`](../LoopEditor/editorhost.cpp) still constructs an in-process `PreflightEngine`. The open legacy #619 does not justify a replacement worker primitive. | +| Independent standards/rendering validation | **Reuse** the validation harness; **prove** independent oracle outputs and fidelity claims under [#18](https://github.com/studio-berry/loop2/issues/18). Core qualification. | [`check_independent_validation_gate.py`](../scripts/ci/check_independent_validation_gate.py), [independent evidence schema](schemas/independent-validation-evidence.schema.json), [coverage matrix](PREFLIGHT_COVERAGE_MATRIX.md), `UnitTestsConversionOracle`. The source gate checks presence/guards; it is not a current installed-runtime oracle result. | +| Cross-platform exact-SHA admission | **Defer** release admission to [#21](https://github.com/studio-berry/loop2/issues/21). Core qualification with CI owners. | [Proof lanes](../architecture/proof-lanes.yaml) bind `linux-build` and `windows-build`; [parent exit gate](https://github.com/studio-berry/loop2/issues/2) requires one exact-SHA packet. No such packet is asserted by this inventory. | + +The accepted/implemented **inherited** ADR coverage relevant to this slice is [ADR-001](adr/adr-001-pdf-document-session.md) for session/revision authority, [ADR-002](adr/adr-002-preflight-engine-orchestrator.md) for the Core registry, and [ADR-011](adr/adr-011-architecture-contracts-d1-d5.md) for canonical digests and governed output identity. Their `Last-verified` SHAs belong to the copied Loop history; current loop2 behavior still needs the proof above. The [worker-isolation ADR](https://app.notion.com/p/3dc9cb079ddb812b9f1fd668196818c6) is marked **proposed**, while legacy #618 is the narrower accepted gate. The [master roadmap](https://app.notion.com/p/3bb9cb079ddb80c4a15feaa98f963f4c) is planning context; its L01 receipt sketch does not create a new public schema. + +## Registered check disposition + +Each ID below is present in [`PreflightEngine::registerBuiltInChecks()`](../LoopLibCore/sources/preflightengine.cpp) and the [generated check catalog](generated/preflight-check-catalog.json). Core preflight owns every row. **Reuse; prove** means retain the check and its catalog limitation, then run the mapped engine/check/corpus tests on an exact candidate SHA. A partial row remains partial even if those tests pass; the open gaps below govern work beyond that measured scope. + +| Catalog coverage | Check IDs | Disposition | +| --- | --- | --- | +| `covered` | `embedded-fonts`, `image-resolution`, `output-intent`, `page-size`, `trim` | **Reuse; prove** each named scope and its fixture evidence. | +| `partial` | `bleed`, `color-inventory`, `color-mode`, `conformance-claims`, `content-bleed`, `dieline`, `font-integrity`, `hidden-layers`, `ink-coverage`, `invisible-content`, `obscured-content`, `off-page-content`, `processing-steps`, `thin-parts`, `thin-strokes`, `transparency-risk`, `white-overprint` | **Reuse; prove** the bounded detection and preserve each catalog limitation. | + +## Open legacy gap disposition + +These are **all 18 `open` rows** in the [generated coverage backlog](generated/preflight-coverage-backlog.json) at the pinned SHA. Core preflight owns each. **Defer** means keep its current backlog row and issue reference, if any; prioritize only after [#18](https://github.com/studio-berry/loop2/issues/18) defines the independent claim and [#16](https://github.com/studio-berry/loop2/issues/16) makes missing coverage visible in receipts. An `unfiled` row is intentionally not a request to file a replacement primitive. + +| Open gap ID | Priority | Disposition; evidence/legacy owner | +| --- | --- | --- | +| `barcode-slug-braille` | P1 | **Defer**; backlog row, legacy #604. | +| `devicen-per-colorant-ink-limit` | P1 | **Defer**; backlog row, legacy #600. | +| `gwg-2022-2024-certificates` | P1 | **Defer**; backlog row, legacy #664. | +| `imposition-and-reader-spreads` | P1 | **Defer**; backlog row, legacy #603. | +| `pdfvt-variable-data` | P1 | **Defer**; backlog row, legacy #605. | +| `bleed-raster-strip-depth` | P2 | **Defer**; backlog row, legacy #47. | +| `color-mode-icc-alternate` | P2 | **Defer**; backlog row, unfiled. | +| `dieline-geometry` | P2 | **Defer**; backlog row, legacy #604. | +| `font-glyph-coverage` | P2 | **Defer**; backlog row, unfiled. | +| `hidden-layers-ocmd` | P2 | **Defer**; backlog row, unfiled. | +| `ink-coverage-raster-tac` | P2 | **Defer**; backlog row, unfiled. | +| `invisible-content-breadth` | P2 | **Defer**; backlog row, unfiled. | +| `obscured-content-occlusion` | P2 | **Defer**; backlog row, unfiled. | +| `off-page-content-clipping` | P2 | **Defer**; backlog row, unfiled. | +| `transparency-rip-interaction` | P2 | **Defer**; backlog row, unfiled. | +| `white-overprint-renderer` | P2 | **Defer**; backlog row, legacy #49. | +| `color-inventory-probe-depth` | P3 | **Defer**; backlog row, unfiled. | +| `thin-parts-raster-budget` | P3 | **Defer**; backlog row, unfiled; current failure is incomplete rather than a silent PASS. | + +The seven `landed` and one `closed` backlog rows remain in the generated source and are **reuse/prove**, not new work: `corrupt-embedded-fonts`, `devicen-dieline-detection`, `hairline-and-thin-stroke-widths`, `nested-font-resources`, `output-intent-identity`, `thin-filled-parts`, `bleed-box-rewrite-only`, and `pdfx5-pdfa3-output`. Their exact state and `closed_by` are in the [backlog](generated/preflight-coverage-backlog.json). + +## Proof record and limits + +Source SHA: `5c8366a33e07597f5213be8da594e3120f73ed69`. Fixture identity is the committed [preflight corpus map](generated/preflight-corpus-coverage.json) plus its `manifest` and `snapshot_dir` fields; no fixture or sealed output changed in this PR. These source commands passed on the pinned tree using the workspace Python runtime: + +```text +python scripts/generate-architecture-catalogs.py --check +python -m unittest scripts.ci.test_preflight_check_catalog scripts.ci.test_preflight_corpus_coverage scripts.ci.test_check_independent_validation_gate -q # 44 passed +python scripts/ci/check_pdf_worker_isolation.py +python scripts/ci/check_independent_validation_gate.py +``` + +The worker and independent-validation scripts are static contract checks. `ctest --test-dir build -N` found the focused C++ test registrations but no executables in the existing build, so native execution was unavailable without a build/configure step. This inventory does not claim runtime, installed-package, Linux sandbox, independent oracle, or release admission proof. The [L01 parent](https://github.com/studio-berry/loop2/issues/2) and [#21](https://github.com/studio-berry/loop2/issues/21) retain those gates. diff --git a/docs/ISSUE_PROMOTION.md b/docs/ISSUE_PROMOTION.md index 70819f8ed..f49c4c801 100644 --- a/docs/ISSUE_PROMOTION.md +++ b/docs/ISSUE_PROMOTION.md @@ -1,52 +1,54 @@ # Issue promotion tracking -`.github/workflows/issue-promotion.yml` keeps GitHub issue state aligned with -the repository's promotion lines: - -- A linked issue whose change reaches `dev` is given the existing - `in promotion queue` label. -- A linked issue that already has that label is closed with GitHub's - `completed` state reason when the corresponding change reaches `stable`. - The queue label is then removed. - -The workflow listens to `push` on `dev` and `stable`, not to pull-request close -events. That covers topic-branch merges, squash merges, regular merge commits, -fast-forward promotion, and direct commits with one consistent evidence path. -The repository may use `unstable` as an intermediate release-candidate line; -it is intentionally not a closure boundary. Issues remain queued until the -work reaches `stable`. - -For each push the workflow asks GitHub for the exact `before...after` -comparison. It also reads merged pull-request titles/bodies and source commits, -which preserves multiple issue links when a squash commit does not retain the -whole PR body. +`.github/workflows/issue-promotion.yml` records the highest promotion branch +known to contain linked issue work. It updates the organization-wide +`Promotion stage` single-select issue field (ID `47367010`): + +| Branch reached | Field value | +| --- | --- | +| `dev` | `Dev present` | +| `unstable` | `Unstable present` | +| `stable` | `Stable present` | + +Branch presence is not issue acceptance. The workflow never changes an issue's +open/closed state. A module gate or sub-issue closes only after its own acceptance +evidence is reviewed. The legacy `in promotion queue` label and its automatic +stable-branch closure rule are retired in loop2. + +The workflow listens to pushes on all three branches. For each push it compares +the exact `before...after` range, then reads merged PR titles, bodies, and source +commits. This preserves issue links across squash promotion. It updates a field +only if the new branch is later in the promotion chain than the current value; +back-merges and delayed runs cannot intentionally downgrade it. An issue can +move directly to `Stable present` if the stable push provides the first usable +link. The field records observed branch membership, not a required path. ## Linking convention Use same-repository references in a commit subject or PR title, such as -`fix: handle bleed (#123)`, or explicit linking language in a PR/commit body, +`fix: handle bleed (#123)`, or explicit linking language in a PR or commit body, such as `Closes #123`, `Fixes #124`, `Resolves #125`, `Implements #126`, or -`Related to #127`. Qualified references (`studio-berry/loop#123`) and issue URLs -for this repository are also accepted. References to another repository are -ignored, and pull-request numbers are ignored after GitHub identifies them as -PRs rather than issues. - -## Safety rules - -- Stable promotion is range-based: only links found in the new push range and - its associated merged PR/source-commit evidence are considered. -- Stable closure requires the issue to still be open and to already carry - `in promotion queue`. A direct or partial stable push cannot close an issue - that never reached `dev` through this state. -- Closed issues are never reopened or relabeled by a back-merge into `dev`. -- API reads complete before any label or close mutation. A truncated, - non-forward, or failed comparison stops the job without guessing. The - mutations are idempotent, so rerunning a failed workflow is safe. -- The workflow serializes runs per branch so concurrent pushes do not race - issue updates. It does not make `issue-promotion` a required branch check; - the existing `agent-fast / build` and `release_ok` protections remain the - code/release gates. - -If an API outage causes a run to fail, rerun that workflow run. A later push -also compares from its recorded predecessor, so the exact range remains -auditable in the run log. +`Related to #127`. Qualified references (`studio-berry/loop2#123`) and issue +URLs for this repository are also accepted. References to other repositories +are ignored, and PR numbers are ignored after GitHub identifies them as PRs. + +## Safety and recovery + +- Every issue and its current field value is read before any field is changed. + Truncated, non-forward, or failed comparisons stop the job without guessing. +- Existing field values are preserved by the additive issue-field API call. + Unknown promotion options fail explicitly rather than being overwritten. +- Runs are serialized across promotion branches. Re-running a failed workflow + is safe because equal or later field values are left alone. +- The workflow does not create or remove labels and does not close or reopen + issues. GitHub's separate linked-PR auto-close setting still applies to PRs + merged into the default branch; avoid closing keywords on acceptance-gated + issues unless that behavior is intended. +- If an API outage causes a run to fail, rerun that workflow run. A later push + compares from its own recorded predecessor, so the original range remains + auditable in the failed run. + +`Promotion stage` is an organization issue field, so changing or recreating it +requires updating `PROMOTION_FIELD_ID` in `scripts/github/issue_promotion.py`. +The workflow's `issues: write` permission is needed to set values. It is not a +release check; the existing code and release gates remain authoritative. diff --git a/docs/LEGACY_ISSUE_PROVENANCE.md b/docs/LEGACY_ISSUE_PROVENANCE.md new file mode 100644 index 000000000..5acad59e7 --- /dev/null +++ b/docs/LEGACY_ISSUE_PROVENANCE.md @@ -0,0 +1,94 @@ +# Legacy issue provenance + +Loop's planning history predates the reset repository. This page records what a bare +issue number means in this tree, where the retired repository's content can still be +recovered, and how references are written from now on. + +## What happened + +`studio-berry/loop2` was created on 2026-09-24 as the reset codebase and now serves as +`studio-berry/loop`. The repository that held `studio-berry/loop` before the rename is +not reachable from either the `studio-berry` or the `mberrys` account as of 2026-09-28: +`gh repo view` finds no `mberrys/loop`, `mberrys/loop2`, `mberrys/Loop-pdf`, or +`studio-berry/Loop-pdf`, and `studio-berry` lists no repository with that history. Its +issue and pull-request numbers (at least through #686) overlap the reset repository's +sequence, which started again at 1. + +Consequences: + +- A link to `github.com/studio-berry/loop/issues/` written before the rename either + returns 404 or opens an unrelated reset issue. Neither is the issue the author meant. +- A bare `#` in a file dated before 2026-09-24 means the legacy issue or pull request + unless the file says otherwise (`loop2 #15`, `#15`, and every issue in the roadmap's + L01–L12 suite are reset issues). +- Issue and pull-request numbers share one sequence, so the collision surface grows with + every new issue or PR. + +## Where legacy content is recoverable + +The retired repository itself cannot be recovered from GitHub here. The specifications +and dispositions survive in Notion: + +- the *Loop Issues* ledger and the *Sessions* ledger, linked from the master roadmap + (§2) — the source of record for legacy titles, bodies, and status; +- `docs/ROADMAP_0.5.0-0.8.0.md`, `docs/github-milestones/`, and the handoff documents + under `docs/`, which quote legacy numbers as they were written. + +Do not treat a legacy issue's status as reset-repository status. Reconcile against code, +tests, and exact-SHA evidence, and write a new issue for a demonstrated remaining gap. + +## How references are written + +| Reference | Meaning | +| --- | --- | +| `#` in an issue, PR, or code comment | An issue or PR in `studio-berry/loop` (the reset repository), resolved by GitHub. | +| `legacy #` in prose, `legacy#` in machine-read files | A retired-repository issue. Never linked; never a live tracker. | +| `studio-berry/loop#` or the full URL | A reset issue, when a file could be read outside the repository. | + +Rules: + +1. **Never link a legacy number.** No `github.com/studio-berry/loop/issues/` URL may + point at a legacy issue. Existing ones were rewritten to `legacy #` text, except + the links to legacy #656 and #675 in `docs/GOVERNED_EXECUTION.md` and + `docs/adr/adr-011-architecture-contracts-d1-d5.md`: both belong to the + governed-execution subsystem, whose binding proof lanes (build, packaging, unit) + cannot be produced by a documentation change, so they are rewritten with the next + change that carries them. +2. **Machine-read records name their repository.** `github_issues` entries in + `docs/preflight-check-catalog-overlay.json` carry `repository`. Live records are + `#` with `studio-berry/loop`; frozen snapshots are `legacy#` with `legacy`. +3. **A legacy record can document closed work, never open a gap.** The catalog generator + refuses an open legacy issue as a backlog row's `closed_by`. Re-point the row to a + live issue. +4. **Live records are read back before promotion.** + `python3 scripts/generate-architecture-catalogs.py --check --verify-github` compares + title, state, and milestone with GitHub and rejects a number that now resolves to a + pull request or another issue. + +## Preflight backlog re-pointing + +Open backlog rows used to cite legacy trackers. Each now cites the reset issue filed for +the gap; the legacy number is kept here as provenance. + +| Backlog row | Legacy tracker | Live tracker | +| --- | --- | --- | +| `barcode-slug-braille`, `dieline-geometry` | legacy #604 | #143 (X00-04) | +| `devicen-per-colorant-ink-limit` | legacy #600 | #142 (X00-03) | +| `gwg-2022-2024-certificates` | legacy #664 | #144 (X00-05) | +| `imposition-and-reader-spreads`, `pdfvt-variable-data` | legacy #603, legacy #605 | #141 (X00-02) | +| `bleed-raster-strip-depth` | legacy #47 | #120 (L01-15) | +| `white-overprint-renderer` | legacy #49 | #119 (L01-14) | +| `transparency-rip-interaction` | unfiled | #119 (L01-14) | +| `color-mode-icc-alternate` | unfiled | #113 (L01-08) | +| `font-glyph-coverage` | unfiled | #114 (L01-09) | +| `hidden-layers-ocmd` | unfiled | #115 (L01-10) | +| `ink-coverage-raster-tac` | unfiled | #116 (L01-11) | +| `invisible-content-breadth` | unfiled | #117 (L01-12) | +| `obscured-content-occlusion`, `off-page-content-clipping` | unfiled | #118 (L01-13) | + +Rows that landed (`corrupt-embedded-fonts`, `nested-font-resources`, +`output-intent-identity`, `thin-filled-parts`, `pdfx5-pdfa3-output`) and the closed +`devicen-dieline-detection` row keep their legacy references as `legacy#` snapshots, +as does the `invisible-content-breadth` gap text for its earlier detector. +`color-inventory-probe-depth` and `thin-parts-raster-budget` stay register-only with a +reviewed deferral. diff --git a/docs/PDF_WORKER_ISOLATION_AUDIT.md b/docs/PDF_WORKER_ISOLATION_AUDIT.md new file mode 100644 index 000000000..f79d94b80 --- /dev/null +++ b/docs/PDF_WORKER_ISOLATION_AUDIT.md @@ -0,0 +1,56 @@ +# Untrusted PDF worker audit (#20) + +## Scope and contract + +This change hardens worker-open and worker-preflight. Global untrusted-PDF isolation is **not admitted** while the in-process entrypoints below remain. + +Protocol v2 is newline-delimited JSON. Requests are at most 64 KiB including the newline; responses at most 64 MiB. Each frame binds version, UUID request ID, allowlisted operation, booleans and status. Admitted responses are reconstructed from validated operation fields; extra verdicts, report paths and nested artifact fields are discarded. A preflight request additionally binds the SHA-256 of immutable staged PDF and profile bytes. The supervisor resolves the staged profile through Core, then requires the returned Core receipt to match input, full revision, effective profile identity, coverage and enabled check set. Revision counters are decimal strings to preserve all 64 bits. + +Receipt schema loop.inspection-receipt.v1 serializes the existing Core receipt. Its identity continues to use loop.inspection-receipt-identity.v1; it is not an attestation that independently verifies a compromised parser's findings. Session document IDs are request UUIDs rather than memory addresses. Open returns only an artifact identification result. Terminal preflight failures have incomplete fidelity and no admitted coverage. If snapshot staging fails before a digest is known, that digest is explicitly empty; only an incomplete receipt with no checks, evidence or profile coverage can carry an unknown identity. + +One request deadline includes write and read. Cancellation signals the active supervisor operation, which terminates the worker. A crash, resource fault, malformed response or identity mismatch produces an incomplete receipt. Replacement is explicit and never replays the failed PDF. There is no in-process fallback. + +## Containment + +Linux requires Landlock ABI 3 or newer and seccomp on x86-64 or AArch64. It restricts input to a separate read-only snapshot directory, permits temporary writes, disables core dumps and dumpability, and limits address space and CPU. Seccomp denies networking, process creation, execution and io_uring. Thread creation remains available. Read-only runtime roots are library trees, font/ICC/locale data, font configuration, the loader cache, and the worker binary/library directories. /proc, /sys, /dev, /opt and general /etc or /usr access are not granted. These declared runtime roots still need distribution-specific qualification. + +Windows requires Windows 10 process creation attributes. The launcher creates a unique AppContainer profile with zero capabilities, a private runtime copy, restricted inherited pipe handles, child-process denial and an atomically assigned Job Object. The detached worker uses inherited pipes without allocating a console host; console allocation would conflict with child-process denial. The job enforces one process, 768 MiB committed memory, 120 CPU seconds and termination on close. The worker verifies its AppContainer token, zero capabilities, job limits and child-process policy before parsing. The complete AppContainer package directory, private runtime and snapshots receive protected read/execute ACLs; temporary data receives low-integrity writable ACLs. No installed-directory ACL is modified. + +The build discovers the worker's DLL closure with CMake runtime dependency discovery, rejects unresolved or ambiguous dependencies, stages declared non-system DLLs, and emits a runtime manifest. The launcher accepts only manifest files beneath worker-runtime/, copies them into a per-session directory and grants no write access there. System DLLs remain supplied by Windows. Windows reroutes the whitelisted base profile environment to the new AppContainer; the launcher does not reroute it twice. Native process, job and pipe references close before profile deletion. Qt's worker is a QCoreApplication and does not use a GUI platform plugin. + +References: [Microsoft AppContainer launch](https://learn.microsoft.com/en-us/windows/win32/secauthz/implementing-an-appcontainer), [process attributes](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-updateprocthreadattribute), [error-mode suppression](https://learn.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-seterrormode). + +## Customer-content handling + +The isolated PdfTool route creates neither product logging nor a product crash-reporting session. Worker and launcher sources are covered by the no-telemetry boundary. Worker stderr is discarded at process creation. Parser errors are replaced by fixed messages, and supervisor-generated diagnostic messages never forward raw worker text. Successful user-visible receipts are inspection output; they must not be sent as telemetry. + +Product-generated dumps are disabled. Administrator-installed debuggers, endpoint agents, OS dump collectors, Linux privileged tracing and Windows LocalDumps policies are external collection channels. Qualification must inspect the actual host policy and captured artifacts and explicitly disposition these channels; this source change cannot certify their absence. + +## Remaining privileged entrypoints + +| Entry point | Existing privileged behavior | Disposition | +| --- | --- | --- | +| PdfTool/pdftoolpreflight.cpp | Ordinary preflight calls inspectPreflightFile in the host | Release blocker; migrate or exclude from untrusted admission | +| PdfTool/pdftoolabstractapplication.cpp | Shared commands call readFromFile / readFromFileOnHeap | Release blocker; audit all parser consumers | +| PdfTool/pdftooldiff.cpp and PdfTool/pdftoolverifyredaction.cpp | Each command parses multiple PDFs directly in the host | Release blocker; separate migration | +| PdfTool/pdftoolrepairdiff.cpp | Repair comparison reopens input directly | Release blocker; separate migration | +| PdfTool/pdftoolunite.cpp | Batch unification parses each selected input in the host | Release blocker; separate migration | +| PdfTool/pdftoolverifysignatures.cpp | Signature verification parses document input in the host | Release blocker; separate migration | +| LoopLibCore/sources/pdfgovernedexecution.cpp and pdfrepairdiff.cpp | Governed validation and repair comparison reopen candidates in-process | Release blocker; migrate callers or exclude from untrusted admission | +| LoopLibCore/sources/pdfdocumentbuilder.cpp | In-memory document reconstruction reparses generated bytes | Release blocker for pipelines containing untrusted content | +| PdfTool/pdftoolactionlist.cpp | Action-list input parsing and candidate execution remain in-process | Release blocker; separate migration | +| PdfTool/pdftoolrepair.cpp | Repair input, candidate reopening and final validation remain in-process | Release blocker; separate migration | +| LoopEditor/editorhost.cpp and LoopLibInteraction/sources/documentloader.cpp | Editor holds parsed state and runs preflight in-process | Release blocker; separate migration | +| LoopLibCore/sources/pdfpagemasterexport.cpp and render/transform consumers | Core batch export and document transformations remain in-process | Release blocker; outside this isolated open/preflight slice | + +Issue #21 owns module release admission. Merging #20 does not clear these blockers. + +## Verification + +UnitTestsPdfWorkerIsolation executes the actual worker open/preflight route on Windows and Linux and has no qualifying platform skips. Its separate, non-installed LoopPdfWorkerProbe injects crashes, hangs, resource faults, malformed/oversized frames, protocol and identity mismatches, and privacy markers. It also probes outside-file reads, network connection, child creation, runtime/snapshot/package writes, private runtime/profile cleanup and temporary writes. Core tests exercise lossless receipt round-trip, unsupported schemas, coverage omissions, inconsistent states and stale identities. + +Exact source identity, commands, fixture digests and observed results are recorded in the implementation handoff and external build evidence. Source checks alone do not qualify sandbox enforcement, telemetry or installed runtime behavior. + +Local Windows focused verification executed the isolation and Core verdict suites with zero failures and zero skips. The mapped change gate ran all 58 owning test executables and passed; exact snapshots and subsequent admission checks are recorded in the external evidence. Network denial uses a live loopback listener with a supervisor connection as its control; an AppContainer connection must return access denied or remain blocked until the bounded probe deadline. Linux executable qualification and installed-product qualification remain unavailable in this run. + +The parser inventory used PDFDocumentReader, readFromFileOnHeap and readFromBuffer searches across PdfTool, LoopEditor, LoopLibInteraction, LoopLibCore/sources, LoopLibQuick and desktop/smoke tools. Reader implementation methods are the shared primitive rather than additional product entrypoints. Renderer and transformation consumers retain the broader blocker above; this inventory does not grant them untrusted-content admission. diff --git a/docs/PREFLIGHT_COVERAGE_MATRIX.md b/docs/PREFLIGHT_COVERAGE_MATRIX.md index 0c2fba35c..9c9c5adf4 100644 --- a/docs/PREFLIGHT_COVERAGE_MATRIX.md +++ b/docs/PREFLIGHT_COVERAGE_MATRIX.md @@ -143,12 +143,34 @@ State and closure (`state_rule` in the generated file): `closed_by` is a verified GitHub issue (`#`), a registered check id, or the literal `unfiled` when neither exists. Issue numbers are never inferred: the -overlay records each one in `github_issues` with the number, title, state, and -milestone read back from `gh issue view`, the generator refuses a reference with -no verified record, and it refuses a row whose `state` disagrees with the -recorded issue state — so a closed issue forces a row to be re-triaged rather -than left stale. Every `not_covered` class must appear in a P1 row's `gap`, and -no P1 row may invent a class the matrix does not list. +overlay records each one in `github_issues` with the number, title, state, +milestone, and `repository` read back from `gh issue view`, the generator +refuses a reference with no verified record, and it refuses a row whose `state` +disagrees with the recorded issue state — so a closed issue forces a row to be +re-triaged rather than left stale. Every `not_covered` class must appear in a P1 +row's `gap`, and no P1 row may invent a class the matrix does not list. + +`github_issues` holds two kinds of record, keyed by how they are written: + +- `#` is an issue in `studio-berry/loop`. It may be a row's `closed_by` + and can be read back at any time. +- `legacy#` is a frozen snapshot from the retired repository, whose + numbers overlap the live ones and can no longer be read back (see + [`LEGACY_ISSUE_PROVENANCE.md`](LEGACY_ISSUE_PROVENANCE.md)). It may document a + row that was closed there, in `closed_by` or in `gap` text, but the generator + refuses a legacy record that is still `OPEN` as a row's `closed_by`, and a + bare `#` in `gap` text never satisfies a `legacy#` record. + +Issue and pull-request numbers share one sequence, so a live record can drift or +collide as the repository grows. `--check` stays offline; run + +```text +python3 scripts/generate-architecture-catalogs.py --check --verify-github +``` + +to read every live record back with `gh` and fail on a missing issue, a pull +request, or a changed title, state, or milestone. Run it before a promotion and +after retitling or closing a cited issue. A row that is not filed carries a `deferral` reason instead, and the generator refuses both an unfiled row without one and a filed row that still carries one — diff --git a/docs/PREFLIGHT_VERDICT.md b/docs/PREFLIGHT_VERDICT.md index d8bdee88c..5cad6a70e 100644 --- a/docs/PREFLIGHT_VERDICT.md +++ b/docs/PREFLIGHT_VERDICT.md @@ -40,3 +40,22 @@ postflight, Action List step results, the Editor controller, and the certificate gate consume this same contract. New surfaces must call the Core reducer or consume the normalized `verdict` object; they must not infer status from `errors.isEmpty()` or `findings.isEmpty()`. + +## Typed inspection receipt + +`buildPreflightInspectionReceipt()` projects a Core result, its effective +profile, the current `PDFRevisionIdentity`, and the evidence graph into one +`PreflightInspectionReceipt`. The receipt carries the exact input digest, +revision, profile identity and digest, coverage scope and per-check completion, +sorted evidence IDs, weakest recorded evidence fidelity, explicit limitations, +and the canonical four-state verdict. It is an in-memory Core contract; the +preflight report and evidence-bundle schemas are unchanged. + +The receipt identity is SHA-256 over canonical JSON containing the input digest, +effective profile digest, and evaluated coverage scope, with a versioned kind. +It is stable when the same bytes and policy are inspected again, even if the +session revision counter changes. A missing, duplicate, unsupported, skipped, +or budget-limited check status, incomplete evidence, or unsupported fidelity +prevents a PASS receipt. A definite blocking finding remains FAIL, with any +coverage limitation still visible. Mismatched input, profile, or revision +provenance rejects receipt construction before publication. diff --git a/docs/QUICK_CANVAS_EDITOR_GAPS.md b/docs/QUICK_CANVAS_EDITOR_GAPS.md new file mode 100644 index 000000000..5434152c0 --- /dev/null +++ b/docs/QUICK_CANVAS_EDITOR_GAPS.md @@ -0,0 +1,117 @@ +# Quick canvas-editor gaps: Select/Hand tools and drag commit + +Two gaps found while planning the canvas-editor GUI work, recorded here so the +next session does not have to rediscover them. Neither is fixed; both are +tracked as issues #103 and #104. No production source is changed by this +document. + +Verified against `origin/dev` at `e9953734`. + +## 1. The Select and Hand toolbar buttons do nothing + +`LoopEditor/qml/ShellToolBar.qml:121-135` presents `Select` and `Hand` as +checkable tool buttons. Neither has an `onClicked` handler, neither is bound to +`activeTool`, and the two can be checked independently. + +### The obvious fix is wrong + +`InteractionController::m_activeTool` is currently **write-only**. Across +`LoopLibInteraction` and `LoopLibQuick`, every reference is: + +| Location | Use | +| --- | --- | +| `interactioncontroller.cpp:106` | equality guard on set | +| `interactioncontroller.cpp:111` | assignment | +| `interactioncontroller.h:113` | inline getter | +| `interactioncontroller.h:208` | declaration | +| `loopcanvasitem.cpp:190-197` | getter, and an equality guard on set | +| `loopcanvasitem.h:100,141,162` | property, declaration, notify signal | + +Nothing reads the value to branch behavior. `handlePointerPress` +(`interactioncontroller.cpp:226-268`) routes pan to `m_panButton`, which +defaults to middle mouse, and routes selection and drag off pointer button, +modifiers, and `InteractionTargetKind`. The active tool is not consulted. + +`LoopCanvasItem::activeTool` is also declared `READ` only +(`loopcanvasitem.h:100`), so the C++ setter has never been reachable from QML. + +Wiring the buttons to the existing setter would make two dead controls look +live while changing no observable behavior. That is a worse defect than the +current honest-but-inert one, because the operator loses the ability to tell +the feature is absent. + +### What a real fix needs + +1. A tool vocabulary defined in a contract, not implied by control labels. Tool + IDs are free-form today; the only test uses `"measure"` + (`UnitTests/tst_interactioncontrollertest.cpp:448`), which no production path + sets. +2. Hand/select pointer semantics: Hand claims left-drag for panning and + suppresses selection and drag initiation; Select retains today's behavior. +3. Cancel-on-change stays as it is. `setActiveTool` already calls + `cancelActive(InteractionCancelReason::ToolChanged)` (issue #141 AC3), so a + tool change must not leave a half-applied transform. + +This is a missing feature rather than a defect, and the behavior change lands in +the `interaction` boundary. + +## 2. Completed drags are discarded + +`InteractionController` emits exactly one `dragCompleted(DragSession)` per +completed drag and deliberately leaves the commit to its owner. The Quick host +is that owner, and it drops the session. + +`LoopEditor/editorhost.cpp:3579-3586`: + +```cpp +void EditorHost::onDragCompleted(pdfinteraction::DragSession session) +{ + Q_UNUSED(session); + if (m_session->interaction()) + { + m_session->interaction()->refreshOverlay(); + } +} +``` + +An operator can select a finding or object, drag it, watch the preview move, +and have nothing committed. + +### Why it is a contract change, not a patch + +`docs/INTERACTION_CONTRACT.md:33-34` states that the owner routes the session +through `CommandCatalog`, "which stays the only mutation path." The command it +names was never added. All 107 IDs in `docs/loop-shell-actions.json` are +navigation, create, color, bookmark, or render actions; none is a move or +translate. + +- `docs/loop-shell-actions.json` is validated by the protected schema + `docs/schemas/loop-shell-actions.schema.json` and declares + `expected_action_count: 107`, which the action list must match. +- The mutation itself would land in `LoopLibCore/sources/**`, listed under + `protected_paths` in `agent-policy.json`. + +Per `AGENTS.md`, a required change to a protected schema or central type is +reported rather than invented. + +## Why no code accompanies this document + +Both gaps need a decision that a patch cannot make. The tool semantics are a +feature design; the drag command is a new public contract entry with undo, +revision-fencing, and payload questions still open. The plumbing that would +carry the tool choice — an `activeTool` property and a `setActiveTool` invokable +on `EditorHost`, routing through the attached canvas to the existing controller +— was prototyped and reverted, because on its own it is a no-op. It is +straightforward to re-derive once the semantics exist. + +## Local verification limit + +The mapped test lanes cannot be run in this checkout. `build-local/` does not +exist, and `C:/.dev/repos/loop/.local-vcpkg/` — the `CMAKE_TOOLCHAIN_FILE` +referenced by every build cache under `C:/.dev/build/` — has been removed. +Restoring vcpkg and configuring are both approval-required under +`agent-policy.json`. This document is therefore unproven by build or test; it +records findings from source reading and search, and each claim above cites the +file and line it was read from. + +Refs #103, #104 diff --git a/docs/REPO_MAP.md b/docs/REPO_MAP.md index 534583be0..89e630ab1 100644 --- a/docs/REPO_MAP.md +++ b/docs/REPO_MAP.md @@ -26,7 +26,7 @@ tooling. Do not infer Loop branch policy from upstream's `master` branch. The reviewed machine-readable policy is [`branch-policy.json`](branch-policy.json). The current factual branch and -workflow audit is tracked in GitHub issue [#232](https://github.com/studio-berry/loop/issues/232). +workflow audit is tracked in legacy issue #232. ## Versioning diff --git a/docs/RESOURCE_BUDGETS.md b/docs/RESOURCE_BUDGETS.md index 6c1541a1e..1911acb60 100644 --- a/docs/RESOURCE_BUDGETS.md +++ b/docs/RESOURCE_BUDGETS.md @@ -22,7 +22,7 @@ the following conservative defaults: | Pool | Limit | Admission rule | |------|-------|----------------| -| active document model | 256 MiB | interaction-priority hard boundary | +| active document model | 640 MiB | interaction-priority hard boundary | | compiled/evidence cache | 128 MiB | insertion-order eviction, then reject | | raster/tile cache | 128 MiB | prefetch shed, then visible admission reject | | GPU texture cache | 128 MiB | source-image byte proxy; physical GPU bytes are unavailable (`-1`) | diff --git a/docs/RESOURCE_ENVELOPE.md b/docs/RESOURCE_ENVELOPE.md index 3d5362e16..a525afc56 100644 --- a/docs/RESOURCE_ENVELOPE.md +++ b/docs/RESOURCE_ENVELOPE.md @@ -60,10 +60,15 @@ $env:QT_QPA_PLATFORM = "offscreen" $env:QT_PLUGIN_PATH = "C:\path\to\qt\plugins" PdfTool.exe benchmark C:\temp\loop-div2k-10000-pages.pdf ` --render-hw-accel 0 ` - --render-rasterizers 8 ` + --render-rasterizers 3 ` --console-format json ``` +Rasterizers are pinned to 3 because the benchmark renders at the default 300 DPI: a Letter page +image is 33.7 MB, each rasterizer holds one at a time, and the 128 MiB `raster-tile-cache` pool +admits three. A fourth concurrent page is rejected as budget-exceeded and the run exits with +`PartialOutput`. + The JSON result includes the `workload_envelope` object. A successful Windows software-renderer run on the local 0.2.0 candidate rendered all 10,000 pages in 48,513 ms and recorded a peak RSS of 380,985,344 bytes. @@ -84,9 +89,15 @@ unsupported, budget-exceeded, or incomplete. The envelope is schema version 2. `resources` is produced by the shared `PDFResourceBudget` authority and contains the resident ceiling plus all seven named pool records. `pages_materialized` reports pages actually processed by a -runner; it is not the catalog page count. `preflight_high_water_bytes` remains -`-1` until a run includes the preflight phase, and such a record is explicitly -`incomplete` rather than being promoted to a passing result. The deterministic +runner; it is not the catalog page count. `preflight_high_water_bytes` is the process high-water when the +`benchmark --profile ` preflight phase ends; without `--profile` +it stays `-1` and the record is explicitly `incomplete` rather than being +promoted to a passing result. `--preflight-page-last ` limits that phase to +pages 1 through `n` while rendering still covers every selected page; the +runner passes it (256) for fixtures above 1,000 pages, because preflight costs +roughly 0.5-0.7 s per page on a hosted runner, and records it as +`profile.preflight_page_last`. A sampled record's `preflight_high_water_bytes` +covers the sampled pages, not the whole document. The deterministic pathological and transparency/spot fixtures can be generated without the external DIV2K corpus: @@ -154,7 +165,7 @@ recommended cold-process timing/RSS sample: python scripts/resource_envelope/run_matrix.py ` --pdf-tool C:\path\to\PdfTool.exe ` --manifest C:\temp\resource-envelope-fixtures.json ` - --repetitions 3 --rasterizers 8 --strict ` + --repetitions 3 --rasterizers 3 --strict ` --output C:\temp\resource-envelope-matrix.json ``` @@ -166,6 +177,12 @@ reported as a passing complete run. Add `--baseline C:\previous\resource-envelope-matrix.json` to compare matching fixture digests and platform/toolchain identities. The default regression margin is `2.0`; use a narrower margin only after collecting stable platform -baselines. Add `--cancel-fixture pathological-vector ---cancel-after-seconds 1` to send an interrupt to one controlled probe and -record the application's cancellation latency. +baselines. The runner passes `--profile` (default +`loop-preflight/profiles/loop-default.json`) so every run measures the +preflight phase. Add `--cancel-fixture ten-thousand-page +--cancel-after-seconds 3` for the separate cancellation probe, which interrupts +one extra run and then times a fresh process reopening the fixture and +rendering its first page (`recovery_ms`). `--strict` requires that probe and +also runs the hostile lane over `UnitTests/testdata/budget_exhaustion/`. For +the hosted, synthetic-fixture version of this run, see +`docs/RESOURCE_ENVELOPE_QUALIFICATION.md`. diff --git a/docs/RESOURCE_ENVELOPE_BUDGETS.json b/docs/RESOURCE_ENVELOPE_BUDGETS.json index fed9176c2..6d11f8a5a 100644 --- a/docs/RESOURCE_ENVELOPE_BUDGETS.json +++ b/docs/RESOURCE_ENVELOPE_BUDGETS.json @@ -7,7 +7,7 @@ "resource_budget": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -26,6 +26,19 @@ "cancellation_latency_ms": 5000, "recovery_ms": 30000 }, + "synthetic-image-heavy": { + "page_count": 10000, + "wall_time_ms": 600000, + "rss_high_water_bytes": 805306368, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, + "large-document-500mb": { + "wall_time_ms": 120000, + "rss_high_water_bytes": 2147483648, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, "pathological-vector": { "page_count": 256, "wall_time_ms": 120000, diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index f29358e59..cadc4644a 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -14,23 +14,68 @@ PdfTool benchmark output and integrated document-session output are separate evidence records. A Quick first-view record remains `incomplete` until the Quick product path is implemented in Phase 4. -## Qualification sequence +## Hosted qualification (issue #19) + +`.github/workflows/resource-envelope-qualification.yml` is the qualifying run. +It triggers on `workflow_dispatch` and on pull requests that touch the +benchmark, the envelope scripts, or the budget contract. Each Linux and Windows +job: + +1. Builds `PdfTool` from the candidate SHA. +2. Generates the fixture bundle with + `scripts/resource_envelope/synthetic_workload.py`. The office, 500 MB + image-heavy, and 10,000-page fixtures are deterministic synthetic PDFs + (SHAKE-256 noise images stored with FlateDecode), so hosted runners need + no external corpus. The 10,000-page fixture uses the + `synthetic-image-heavy` workload caps, which equal the DIV2K caps except + `wall_time_ms` (600 s): three rasterizers on a hosted runner render the + 10,000 pages in about 350 s (Linux) to 510 s (Windows). +3. Runs `run_matrix.py --strict --repetitions 3` with: + - a measured preflight phase (`benchmark --profile`, default + `loop-preflight/profiles/loop-default.json`), so a clean run reports + `status: complete` with a real `preflight_high_water_bytes`. Fixtures + above 1,000 pages (the 10,000-page one) preflight their first 256 pages + only (`--preflight-page-last 256`, recorded as + `profile.preflight_page_last`); rendering still covers every page; + - a `large-document-500mb` workload for the 500 MB fixture, whose process + RSS cap is 2 GiB instead of the 768 MiB resident limit: the reader holds + the whole file plus its object model in memory, so the peak is about 3.3 + times the file size (1.7 GB measured on both platforms), while the + accounted pools stay under the resident limit; + - a cancellation probe on `ten-thousand-page`, which interrupts a render-only + run (no preflight phase, whose document-wide setup does not poll for + cancellation) + and requires a `cancelled` envelope within the workload's + `cancellation_latency_ms`; + - a recovery probe, which times a fresh process reopening the same + fixture and rendering its first page (`recovery_ms`, within the + workload's `recovery_ms`); + - a hostile lane over `UnitTests/testdata/budget_exhaustion/`, where each + PDF must end in a contained PdfTool exit code (rejection is fine) within + the hostile timeout, without breaching the resident ceiling. + +The `evidence` job combines both matrices with +`scripts/qualification/build_resource_envelope_evidence.py` into a schema +version 2 record that carries the run id and URL, and validates it with +`validate_resource_envelope_evidence.py`. The disposition is `passed` only when +both platforms passed strictly on the same candidate SHA. + +A crash (an exit code outside PdfTool's defined codes, or `InternalError`), a +timeout, a non-success exit, or a missing envelope never produces a +`measured` fixture. Crashes and timeouts fail the record outright. + +## External DIV2K sequence + +The original DIV2K qualification remains available for local runs: 1. Validate the external DIV2K corpus and generate one canonical manifest with `--hash-all`. 2. Build the deterministic 10,000-page image-heavy PDF and record its digest. 3. Create an external fixture manifest using the schema at `docs/schemas/resource-envelope-fixtures.schema.json`, then run - `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the 2 MB office, - image-heavy, 10,000-page, pathological-vector, and transparency/spot - fixtures. Supply the multi-GB fixture when platform addressability permits. - The strict job is expected to remain non-passing until the native benchmark - also supplies preflight and recovery measurements; unavailable fields must - not be promoted to zero. -4. Run PdfTool benchmark profiles on Linux and Windows with the same manifest. -5. Run the integrated session/scheduler harness with the same workload identity. -6. Replay the bounded lifecycle trace corpus on both platforms. -7. Attach JSON results, digests, platform identities, and dispositions to the - candidate-SHA evidence dossier. + `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the + same probe and hostile options as the hosted workflow. +4. Run the integrated session/scheduler harness with the same workload identity. +5. Replay the bounded lifecycle trace corpus on both platforms. No unavailable measurement may be converted to zero or treated as a pass. diff --git a/docs/SESSION_09_HANDOFF.md b/docs/SESSION_09_HANDOFF.md index d2f171898..4ec38d87f 100644 --- a/docs/SESSION_09_HANDOFF.md +++ b/docs/SESSION_09_HANDOFF.md @@ -49,9 +49,9 @@ Deleted Phase 5 identities remain recorded in `docs/product-surface.json` with packaging and budget work beyond this ledger-closeout diff; qualification lanes should not treat it as the Session 09 ledger baseline. -**Dev integration:** merged via [PR #535](https://github.com/studio-berry/loop/pull/535) @ +**Dev integration:** merged via legacy PR #535 @ `1f69bdf8bff037e5cae2d37e3c2e3eae8b2ca6b5`. Session 13 scaffolding landed on -`dev` via [PR #539](https://github.com/studio-berry/loop/pull/539) @ +`dev` via legacy PR #539 @ `ebde8661bff037e5cae2d37e3c2e3eae8b2ca6b5` (current qualification `candidate_sha`). diff --git a/docs/adr/adr-007-qt-quick-controls-shell.md b/docs/adr/adr-007-qt-quick-controls-shell.md index 437abdc09..31b463aa7 100644 --- a/docs/adr/adr-007-qt-quick-controls-shell.md +++ b/docs/adr/adr-007-qt-quick-controls-shell.md @@ -169,4 +169,4 @@ adoption preserves feature delivery while those risks are measured. - [Qt Quick Controls](https://doc.qt.io/qt-6/qtquickcontrols-index.html) - [Qt 6.11 changes to Qt Quick](https://doc.qt.io/qt-6/quick-changes-qt6.html) - [QQuickWindow scene-graph backend selection](https://doc.qt.io/qt-6/qquickwindow.html) -- [Loop issue #178](https://github.com/studio-berry/loop/issues/178) +- legacy issue #178 diff --git a/docs/adr/adr-008-generated-history-rewrite.md b/docs/adr/adr-008-generated-history-rewrite.md index 10dc3fe1d..89296b667 100644 --- a/docs/adr/adr-008-generated-history-rewrite.md +++ b/docs/adr/adr-008-generated-history-rewrite.md @@ -9,18 +9,18 @@ ## Context -[#265](https://github.com/studio-berry/loop/issues/265) asked to decide +legacy #265 asked to decide whether to rewrite or retain generated dependency and build blobs already present in the 195 unreleased `dev` commits, and originally recommended a `dev`-only rewrite because `stable` had not yet received that history. -That window closed when [PR #188](https://github.com/studio-berry/loop/pull/188) +That window closed when legacy PR #188 merged to `stable` on 2026-08-13. After the merge, both `origin/dev` and `origin/stable` still contained the same 982 blobs (~400.5 MiB): `.docker-vcpkg*`, `build-fuzz-docker/` (including a 45 MiB `libLoopLibCore.so`), `debug-b0e75b.log`, `scripts/debug-pr188.*`, and stray -`loop-ocr` bytecode. Branch tips were already clean ([#249](https://github.com/studio-berry/loop/pull/249), -[#258](https://github.com/studio-berry/loop/pull/258)); only history held the +`loop-ocr` bytecode. Branch tips were already clean (legacy #249, +legacy #258); only history held the blobs. Rewriting only `dev` would not reclaim GitHub storage. Rewriting `stable` diff --git a/docs/adr/adr-009-canvas-hosting-benchmark.md b/docs/adr/adr-009-canvas-hosting-benchmark.md index 3f5e7a029..1eee1dde6 100644 --- a/docs/adr/adr-009-canvas-hosting-benchmark.md +++ b/docs/adr/adr-009-canvas-hosting-benchmark.md @@ -147,4 +147,4 @@ Those remain explicit later gates in ADR-007 and ADR-010. - [Quick-root admission](adr-010-quick-root-admission.md) - [Qt Quick Controls shell](adr-007-qt-quick-controls-shell.md) - [Quick composition contract](../QUICK_COMPOSITION.md) -- [Issue #247](https://github.com/studio-berry/loop/issues/247) +- legacy issue #247 diff --git a/docs/generated/architecture-catalog.json b/docs/generated/architecture-catalog.json index 8cf0e202f..1943b210d 100644 --- a/docs/generated/architecture-catalog.json +++ b/docs/generated/architecture-catalog.json @@ -509,7 +509,8 @@ ], ".github/workflows/issue-promotion.yml": [ "dev", - "stable" + "stable", + "unstable" ], ".github/workflows/release-gate.yml": [ "stable" diff --git a/docs/generated/huge-document-envelope.json b/docs/generated/huge-document-envelope.json index 9b9fd6b90..0a78d2d2b 100644 --- a/docs/generated/huge-document-envelope.json +++ b/docs/generated/huge-document-envelope.json @@ -32,7 +32,7 @@ "config": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -45,7 +45,7 @@ "resident_high_water_bytes": 0, "pressure": "normal", "pools": { - "active-document-model": { "limit_bytes": 268435456, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, + "active-document-model": { "limit_bytes": 671088640, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "compiled-evidence-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "raster-tile-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "gpu-texture-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, diff --git a/docs/generated/phase5-widgets-disposition.json b/docs/generated/phase5-widgets-disposition.json index 0a9eda4ae..499eb35ee 100644 --- a/docs/generated/phase5-widgets-disposition.json +++ b/docs/generated/phase5-widgets-disposition.json @@ -64,9 +64,9 @@ "follow_up_issue": null }, { - "id": "ui:CodeGenerator/generatormainwindow.ui", + "id": "ui:tools/CodeGenerator/generatormainwindow.ui", "kind": "ui-form", - "path": "CodeGenerator/generatormainwindow.ui", + "path": "tools/CodeGenerator/generatormainwindow.ui", "owner_target": "CodeGenerator", "consumer": "loop-cli", "rationale": "Developer fixture generator; not a product surface.", @@ -77,9 +77,9 @@ "replacement_target": "loop-cli" }, { - "id": "ui:JBIG2_Viewer/mainwindow.ui", + "id": "ui:tools/JBIG2_Viewer/mainwindow.ui", "kind": "ui-form", - "path": "JBIG2_Viewer/mainwindow.ui", + "path": "tools/JBIG2_Viewer/mainwindow.ui", "owner_target": "JBIG2_VIEWER", "consumer": "loop-inspect", "rationale": "Developer JBIG2 probe; not a product surface.", @@ -363,14 +363,14 @@ ], "legacy_surface_disposition": [ { - "path": "CodeGenerator/generatormainwindow.ui", - "surface_id": "ui:CodeGenerator/generatormainwindow.ui", + "path": "tools/CodeGenerator/generatormainwindow.ui", + "surface_id": "ui:tools/CodeGenerator/generatormainwindow.ui", "status": "matched", "shell_disposition": "HEADLESS" }, { - "path": "JBIG2_Viewer/mainwindow.ui", - "surface_id": "ui:JBIG2_Viewer/mainwindow.ui", + "path": "tools/JBIG2_Viewer/mainwindow.ui", + "surface_id": "ui:tools/JBIG2_Viewer/mainwindow.ui", "status": "matched", "shell_disposition": "HEADLESS" } diff --git a/docs/generated/phase5-widgets-inventory.json b/docs/generated/phase5-widgets-inventory.json index 9d4ade75f..58f59ce93 100644 --- a/docs/generated/phase5-widgets-inventory.json +++ b/docs/generated/phase5-widgets-inventory.json @@ -17,18 +17,12 @@ }, "inputs": { "cmake_files": [ - "CanvasBenchmark/CMakeLists.txt", - "CodeGenerator/CMakeLists.txt", - "JBIG2_Viewer/CMakeLists.txt", "LoopEditor/CMakeLists.txt", "LoopLibCore/CMakeLists.txt", "LoopLibInteraction/CMakeLists.txt", "LoopLibQuick/CMakeLists.txt", - "PdfExampleGenerator/CMakeLists.txt", "PdfTool/CMakeLists.txt", "PdfTool/CMakeLists.txt", - "ProductQuickAccessibilitySmoke/CMakeLists.txt", - "QuickShellSmoke/CMakeLists.txt", "UnitTests/CMakeLists.txt", "UnitTests/CMakeLists.txt", "UnitTests/CMakeLists.txt", @@ -98,7 +92,14 @@ "UnitTests/CMakeLists.txt", "UnitTests/CMakeLists.txt", "UnitTests/CMakeLists.txt", - "loop-preflight/tools/CMakeLists.txt" + "UnitTests/CMakeLists.txt", + "loop-preflight/tools/CMakeLists.txt", + "tools/CanvasBenchmark/CMakeLists.txt", + "tools/CodeGenerator/CMakeLists.txt", + "tools/JBIG2_Viewer/CMakeLists.txt", + "tools/PdfExampleGenerator/CMakeLists.txt", + "tools/ProductQuickAccessibilitySmoke/CMakeLists.txt", + "tools/QuickShellSmoke/CMakeLists.txt" ], "shell_ledger": "docs/loop-shell.json", "product_ledger": "docs/product-surface.json", @@ -108,7 +109,7 @@ { "id": "CanvasBenchmark", "kind": "executable", - "cmake": "CanvasBenchmark/CMakeLists.txt", + "cmake": "tools/CanvasBenchmark/CMakeLists.txt", "profile_enabled": false, "profile_condition": "qualification target excluded from the product-surface manifest", "install_rule": false, @@ -136,7 +137,11 @@ "transitive_targets": [ "LoopLibCore" ], - "transitive_qt_modules": [], + "transitive_qt_modules": [ + "Sql", + "Svg", + "Xml" + ], "qt_modules": [ "Core", "Gui", @@ -144,7 +149,10 @@ "Quick", "QuickControls2", "QuickWidgets", - "Widgets" + "Sql", + "Svg", + "Widgets", + "Xml" ], "widgets_linkage": "direct", "widgets_paths": [ @@ -158,7 +166,7 @@ { "id": "CodeGenerator", "kind": "executable", - "cmake": "CodeGenerator/CMakeLists.txt", + "cmake": "tools/CodeGenerator/CMakeLists.txt", "profile_enabled": false, "profile_condition": "LOOP_BUILD_CODE_GENERATOR=OFF from docs/product-surface.json", "install_rule": false, @@ -180,10 +188,15 @@ "transitive_targets": [ "LoopLibCore" ], - "transitive_qt_modules": [], + "transitive_qt_modules": [ + "Sql", + "Svg" + ], "qt_modules": [ "Core", "Gui", + "Sql", + "Svg", "Widgets", "Xml" ], @@ -199,7 +212,7 @@ { "id": "JBIG2_VIEWER", "kind": "executable", - "cmake": "JBIG2_Viewer/CMakeLists.txt", + "cmake": "tools/JBIG2_Viewer/CMakeLists.txt", "profile_enabled": false, "profile_condition": "LOOP_BUILD_JBIG2_VIEWER=OFF from docs/product-surface.json", "install_rule": false, @@ -219,11 +232,18 @@ "transitive_targets": [ "LoopLibCore" ], - "transitive_qt_modules": [], + "transitive_qt_modules": [ + "Sql", + "Svg", + "Xml" + ], "qt_modules": [ "Core", "Gui", - "Widgets" + "Sql", + "Svg", + "Widgets", + "Xml" ], "widgets_linkage": "direct", "widgets_paths": [ @@ -422,6 +442,7 @@ "UnitTestsPageBoxCorpus", "UnitTestsPageMasterExport", "UnitTestsPdfToolContract", + "UnitTestsPdfWorkerIsolation", "UnitTestsPluginAbi", "UnitTestsPreflightEngine", "UnitTestsPreflightInteraction", @@ -540,10 +561,36 @@ "ProductQuickAccessibilitySmoke" ] }, + { + "id": "LoopPdfWorkerProbe", + "kind": "executable", + "cmake": "UnitTests/CMakeLists.txt", + "profile_enabled": false, + "profile_condition": "qualification target excluded from the product-surface manifest", + "install_rule": false, + "installed_in_profile": false, + "build_only_in_profile": false, + "direct_links": [ + "Qt6::Core", + "advapi32", + "ws2_32" + ], + "direct_qt_modules": [ + "Core" + ], + "transitive_targets": [], + "transitive_qt_modules": [], + "qt_modules": [ + "Core" + ], + "widgets_linkage": "none", + "widgets_paths": [], + "consumers": [] + }, { "id": "PdfExampleGenerator", "kind": "executable", - "cmake": "PdfExampleGenerator/CMakeLists.txt", + "cmake": "tools/PdfExampleGenerator/CMakeLists.txt", "profile_enabled": false, "profile_condition": "LOOP_BUILD_EXAMPLE_GENERATOR=OFF from docs/product-surface.json", "install_rule": false, @@ -599,8 +646,10 @@ "Qt6::Core", "Qt6::Gui", "Qt6::Xml", + "advapi32", "ole32", - "sapi" + "sapi", + "userenv" ], "direct_qt_modules": [ "Core", @@ -628,7 +677,7 @@ { "id": "ProductQuickAccessibilitySmoke", "kind": "executable", - "cmake": "ProductQuickAccessibilitySmoke/CMakeLists.txt", + "cmake": "tools/ProductQuickAccessibilitySmoke/CMakeLists.txt", "profile_enabled": false, "profile_condition": "qualification target excluded from the product-surface manifest", "install_rule": false, @@ -681,7 +730,7 @@ { "id": "QuickShellSmoke", "kind": "executable", - "cmake": "QuickShellSmoke/CMakeLists.txt", + "cmake": "tools/QuickShellSmoke/CMakeLists.txt", "profile_enabled": false, "profile_condition": "qualification target excluded from the product-surface manifest", "install_rule": false, @@ -2462,18 +2511,35 @@ "installed_in_profile": false, "build_only_in_profile": false, "direct_links": [ + "LoopLibCore", "Qt6::Core", - "Qt6::Test" + "Qt6::Gui", + "Qt6::Test", + "advapi32", + "ole32", + "userenv", + "ws2_32" ], "direct_qt_modules": [ "Core", + "Gui", "Test" ], - "transitive_targets": [], - "transitive_qt_modules": [], + "transitive_targets": [ + "LoopLibCore" + ], + "transitive_qt_modules": [ + "Sql", + "Svg", + "Xml" + ], "qt_modules": [ "Core", - "Test" + "Gui", + "Sql", + "Svg", + "Test", + "Xml" ], "widgets_linkage": "none", "widgets_paths": [], @@ -3464,7 +3530,7 @@ "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", - "cmake": "CanvasBenchmark/CMakeLists.txt" + "cmake": "tools/CanvasBenchmark/CMakeLists.txt" }, { "target": "CodeGenerator", @@ -3472,7 +3538,7 @@ "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", - "cmake": "CodeGenerator/CMakeLists.txt" + "cmake": "tools/CodeGenerator/CMakeLists.txt" }, { "target": "JBIG2_VIEWER", @@ -3480,7 +3546,7 @@ "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", - "cmake": "JBIG2_Viewer/CMakeLists.txt" + "cmake": "tools/JBIG2_Viewer/CMakeLists.txt" }, { "target": "PdfExampleGenerator", @@ -3488,21 +3554,21 @@ "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", - "cmake": "PdfExampleGenerator/CMakeLists.txt" + "cmake": "tools/PdfExampleGenerator/CMakeLists.txt" } ], "plugin_ui": [], "ui_forms": [ { - "id": "ui:CodeGenerator/generatormainwindow.ui", - "path": "CodeGenerator/generatormainwindow.ui", + "id": "ui:tools/CodeGenerator/generatormainwindow.ui", + "path": "tools/CodeGenerator/generatormainwindow.ui", "owner_target": "CodeGenerator", "plugin": null, "widgets_related": true }, { - "id": "ui:JBIG2_Viewer/mainwindow.ui", - "path": "JBIG2_Viewer/mainwindow.ui", + "id": "ui:tools/JBIG2_Viewer/mainwindow.ui", + "path": "tools/JBIG2_Viewer/mainwindow.ui", "owner_target": "JBIG2_VIEWER", "plugin": null, "widgets_related": true @@ -3513,7 +3579,7 @@ "id": "target:CanvasBenchmark", "kind": "developer-tool", "target": "CanvasBenchmark", - "cmake": "CanvasBenchmark/CMakeLists.txt", + "cmake": "tools/CanvasBenchmark/CMakeLists.txt", "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", @@ -3523,7 +3589,7 @@ "id": "target:CodeGenerator", "kind": "developer-tool", "target": "CodeGenerator", - "cmake": "CodeGenerator/CMakeLists.txt", + "cmake": "tools/CodeGenerator/CMakeLists.txt", "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", @@ -3533,7 +3599,7 @@ "id": "target:JBIG2_VIEWER", "kind": "developer-tool", "target": "JBIG2_VIEWER", - "cmake": "JBIG2_Viewer/CMakeLists.txt", + "cmake": "tools/JBIG2_Viewer/CMakeLists.txt", "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", @@ -3543,16 +3609,16 @@ "id": "target:PdfExampleGenerator", "kind": "developer-tool", "target": "PdfExampleGenerator", - "cmake": "PdfExampleGenerator/CMakeLists.txt", + "cmake": "tools/PdfExampleGenerator/CMakeLists.txt", "profile_enabled": false, "installed_in_profile": false, "widgets_linkage": "direct", "consumers": [] }, { - "id": "ui:CodeGenerator/generatormainwindow.ui", + "id": "ui:tools/CodeGenerator/generatormainwindow.ui", "kind": "ui-form", - "path": "CodeGenerator/generatormainwindow.ui", + "path": "tools/CodeGenerator/generatormainwindow.ui", "target": "CodeGenerator", "plugin": null, "profile_enabled": false, @@ -3563,9 +3629,9 @@ ] }, { - "id": "ui:JBIG2_Viewer/mainwindow.ui", + "id": "ui:tools/JBIG2_Viewer/mainwindow.ui", "kind": "ui-form", - "path": "JBIG2_Viewer/mainwindow.ui", + "path": "tools/JBIG2_Viewer/mainwindow.ui", "target": "JBIG2_VIEWER", "plugin": null, "profile_enabled": false, @@ -3577,7 +3643,7 @@ } ], "counts": { - "targets": 82, + "targets": 83, "installed_in_profile": 5, "build_only_in_profile": 2, "widgets_surfaces": 4, diff --git a/docs/generated/preflight-coverage-backlog.json b/docs/generated/preflight-coverage-backlog.json index 4813743e6..8c428da56 100644 --- a/docs/generated/preflight-coverage-backlog.json +++ b/docs/generated/preflight-coverage-backlog.json @@ -15,87 +15,136 @@ "format_version": 1, "generated_by": "scripts/generate-architecture-catalogs.py", "github_issues": { - "#12": { - "milestone": "0.0.1", - "number": 12, - "state": "CLOSED", - "title": "Validate output-intent and ICC-profile presence, identity, and color-space consistency" + "#113": { + "milestone": "L01 - Evidence Core", + "number": 113, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-08 \u2014 Classify ICC-based color by its painted space" }, - "#124": { - "milestone": null, - "number": 124, - "state": "CLOSED", - "title": "Detect hidden, invisible, and off-page content in preflight" + "#114": { + "milestone": "L01 - Evidence Core", + "number": 114, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-09 \u2014 Audit glyph coverage in embedded fonts" }, - "#130": { - "milestone": null, - "number": 130, - "state": "CLOSED", - "title": "Detect corrupt fonts, not just embedded/unembedded status" + "#115": { + "milestone": "L01 - Evidence Core", + "number": 115, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-10 \u2014 Evaluate optional-content membership for hidden content" }, - "#131": { - "milestone": null, - "number": 131, - "state": "CLOSED", - "title": "Extend thin-part detection beyond simple strokes (expand #23)" + "#116": { + "milestone": "L01 - Evidence Core", + "number": 116, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-11 \u2014 Stop isolated over-limit ink regions passing ink coverage" }, - "#47": { - "milestone": null, - "number": 47, + "#117": { + "milestone": "L01 - Evidence Core", + "number": 117, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-12 \u2014 Broaden invisible-content detection" + }, + "#118": { + "milestone": "L01 - Evidence Core", + "number": 118, + "repository": "studio-berry/loop", + "state": "OPEN", + "title": "L01-13 \u2014 Make off-page and obscured-content geometry clip-aware" + }, + "#119": { + "milestone": "L01 - Evidence Core", + "number": 119, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "[Residual] Tier-2 bleed raster golden corpus + strip calibration" + "title": "L01-14 \u2014 Judge white overprint and transparency interaction on the authoritative renderer" }, - "#49": { - "milestone": "0.8.0", - "number": 49, + "#120": { + "milestone": "L01 - Evidence Core", + "number": 120, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "[0.8.0] Overprint-correct rendering in standard and advanced renderers" + "title": "L01-15 \u2014 Close the sparse-mark bleed raster false pass" }, - "#600": { + "#141": { "milestone": null, - "number": 600, + "number": 141, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "Ink Manager: separation management and spot-color library" + "title": "X00-02 \u2014 Re-file parked legacy expansion candidates with value cases" }, - "#603": { + "#142": { "milestone": null, - "number": 603, + "number": 142, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "Imposition: n-up, gang-up, booklets, creep, and sheet furniture" + "title": "X00-03 \u2014 Per-named-colorant ink limits for DeviceN and spot separations" }, - "#604": { + "#143": { "milestone": null, - "number": 604, + "number": 143, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "Packaging production semantics: ISO 19593 Processing Steps, dielines, varnish/foil/white, barcode validation" + "title": "X00-04 \u2014 Barcode, slug, Braille, and dieline geometry validation" }, - "#605": { + "#144": { "milestone": null, - "number": 605, + "number": 144, + "repository": "studio-berry/loop", "state": "OPEN", - "title": "PDF/VT variable-data production: understanding, validation, and preservation" + "title": "X00-05 \u2014 Standards currency: PDF/X-6, PDF 2.0 output intents, and GWG 2022 check sets" + }, + "legacy#12": { + "milestone": "0.0.1", + "number": 12, + "repository": "legacy", + "state": "CLOSED", + "title": "Validate output-intent and ICC-profile presence, identity, and color-space consistency" + }, + "legacy#124": { + "milestone": null, + "number": 124, + "repository": "legacy", + "state": "CLOSED", + "title": "Detect hidden, invisible, and off-page content in preflight" + }, + "legacy#130": { + "milestone": null, + "number": 130, + "repository": "legacy", + "state": "CLOSED", + "title": "Detect corrupt fonts, not just embedded/unembedded status" }, - "#628": { + "legacy#131": { + "milestone": null, + "number": 131, + "repository": "legacy", + "state": "CLOSED", + "title": "Extend thin-part detection beyond simple strokes (expand #23)" + }, + "legacy#628": { "milestone": "0.3.0", "number": 628, + "repository": "legacy", "state": "CLOSED", "title": "DeviceN dieline / spot geometry not detected (Separation-only collector)" }, - "#664": { - "milestone": null, - "number": 664, - "state": "OPEN", - "title": "[0.3.0] No check inspects formal GWG 2022/2024 sheetfed or packaging conformance" - }, - "#665": { + "legacy#665": { "milestone": null, "number": 665, + "repository": "legacy", "state": "OPEN", "title": "[0.3.0] No check validates or produces PDF/X-5 and PDF/A-3 output" }, - "#70": { + "legacy#70": { "milestone": null, "number": 70, + "repository": "legacy", "state": "CLOSED", "title": "Traverse nested Form XObjects, inherited resources, and appearance streams for embedded fonts" } @@ -111,7 +160,7 @@ "priority_rule": "P1 - no registered check inspects the defect class at all (a matrix not_covered entry), so a clean run is silent about it; P2 - a registered check inspects the class but its named limitation can suppress or misclassify a finding on defective content, which would be a false clean pass; P3 - a registered check inspects the class and the named limitation only narrows reported detail or fails closed as incomplete, so it cannot turn a defect into a silent pass.", "rows": [ { - "closed_by": "#604", + "closed_by": "#143", "deferral": null, "families": [ "packaging", @@ -124,7 +173,7 @@ "state": "open" }, { - "closed_by": "#600", + "closed_by": "#142", "deferral": null, "families": [ "packaging", @@ -138,7 +187,7 @@ "state": "open" }, { - "closed_by": "#664", + "closed_by": "#144", "deferral": null, "families": [ "sheetfed-offset", @@ -150,7 +199,7 @@ "state": "open" }, { - "closed_by": "#603", + "closed_by": "#141", "deferral": null, "families": [ "sheetfed-offset", @@ -163,7 +212,7 @@ "state": "open" }, { - "closed_by": "#605", + "closed_by": "#141", "deferral": null, "families": [ "web-offset", @@ -176,7 +225,7 @@ "state": "open" }, { - "closed_by": "#47", + "closed_by": "#120", "deferral": null, "families": [ "sheetfed-offset", @@ -189,8 +238,8 @@ "state": "open" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture whose ICC space resolves to an allowed alternate, so the false pass is reproducible rather than argued.", + "closed_by": "#113", + "deferral": null, "families": [ "sheetfed-offset", "web-offset", @@ -210,13 +259,13 @@ "sheetfed-offset", "digital" ], - "gap": "embedded font programs that fail to parse, or lose their cmap, were unchecked; font-integrity now reports them (filed as #130)", + "gap": "embedded font programs that fail to parse, or lose their cmap, were unchecked; font-integrity now reports them (filed as legacy#130)", "id": "corrupt-embedded-fonts", "priority": "P2", "state": "landed" }, { - "closed_by": "#628", + "closed_by": "legacy#628", "deferral": null, "families": [ "packaging" @@ -227,7 +276,7 @@ "state": "closed" }, { - "closed_by": "#604", + "closed_by": "#143", "deferral": null, "families": [ "packaging" @@ -238,8 +287,8 @@ "state": "open" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with a parsable embedded program that is missing a used glyph.", + "closed_by": "#114", + "deferral": null, "families": [ "sheetfed-offset", "digital" @@ -262,8 +311,8 @@ "state": "landed" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with an optional-content membership configuration that hides print content.", + "closed_by": "#115", + "deferral": null, "families": [ "packaging" ], @@ -273,8 +322,8 @@ "state": "open" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with an isolated over-limit element below min_region_area_pct.", + "closed_by": "#116", + "deferral": null, "families": [ "sheetfed-offset", "web-offset", @@ -286,13 +335,13 @@ "state": "open" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture painting an invisible class outside text render mode 3 and zero-alpha graphics state.", + "closed_by": "#117", + "deferral": null, "families": [ "sheetfed-offset", "packaging" ], - "gap": "invisible-content covers text render mode 3 and zero-alpha graphics state only, so other invisible classes pass clean (the wider detector is filed as #124)", + "gap": "invisible-content covers text render mode 3 and zero-alpha graphics state only, so other invisible classes pass clean (the wider detector is filed as legacy#124)", "id": "invisible-content-breadth", "priority": "P2", "state": "open" @@ -307,14 +356,14 @@ "newspaper", "digital" ], - "gap": "fonts used only inside nested Form XObjects, inherited resources, or annotation appearance streams were unchecked; embedded-fonts now traverses them (filed as #70)", + "gap": "fonts used only inside nested Form XObjects, inherited resources, or annotation appearance streams were unchecked; embedded-fonts now traverses them (filed as legacy#70)", "id": "nested-font-resources", "priority": "P2", "state": "landed" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture where a bounding-box heuristic cannot see the occlusion a RIP would.", + "closed_by": "#118", + "deferral": null, "families": [ "sheetfed-offset" ], @@ -324,8 +373,8 @@ "state": "open" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture whose clipped mark exposes the pre-clip bounds comparison.", + "closed_by": "#118", + "deferral": null, "families": [ "sheetfed-offset", "digital" @@ -344,7 +393,7 @@ "packaging", "digital" ], - "gap": "output intent presence, identity, subtype, and ICC payload were unchecked; output-intent now validates them (filed as #12)", + "gap": "output intent presence, identity, subtype, and ICC payload were unchecked; output-intent now validates them (filed as legacy#12)", "id": "output-intent-identity", "priority": "P2", "state": "landed" @@ -355,14 +404,14 @@ "families": [ "packaging" ], - "gap": "thin filled parts, not only strokes, were unchecked for a minimum width; thin-parts now measures them (filed as #131)", + "gap": "thin filled parts, not only strokes, were unchecked for a minimum width; thin-parts now measures them (filed as legacy#131)", "id": "thin-filled-parts", "priority": "P2", "state": "landed" }, { - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture reproducing an overprint-plus-transparency interaction only a RIP flattens.", + "closed_by": "#119", + "deferral": null, "families": [ "sheetfed-offset", "packaging", @@ -374,7 +423,7 @@ "state": "open" }, { - "closed_by": "#49", + "closed_by": "#119", "deferral": null, "families": [ "sheetfed-offset", @@ -419,7 +468,7 @@ "packaging", "digital" ], - "gap": "PDF/X-5n, PDF/X-5g, and PDF/A-3 are not produced or validated; conformance-claims reports a declared level as unsupported instead of a silent pass (filed as #665)", + "gap": "PDF/X-5n, PDF/X-5g, and PDF/A-3 are not produced or validated; conformance-claims reports a declared level as unsupported instead of a silent pass (filed as legacy#665)", "id": "pdfx5-pdfa3-output", "priority": "P3", "state": "landed" diff --git a/docs/handoffs/0.2.0-gh143-parity-checklist.md b/docs/handoffs/0.2.0-gh143-parity-checklist.md index aa7edf41e..00d87fa9a 100644 --- a/docs/handoffs/0.2.0-gh143-parity-checklist.md +++ b/docs/handoffs/0.2.0-gh143-parity-checklist.md @@ -4,16 +4,16 @@ Status: tracks harvest from PR #358 / `gh-143` into the Qt Quick architecture on `dev`. Widgets implementation code from that branch is superseded; this checklist maps each gh-143 test and contract to its Quick/interaction target. -Parent epic: [#356](https://github.com/studio-berry/loop/issues/356). +Parent epic: legacy #356. ## Notion mapping (0.2.0–0.4.0) | GitHub | Notion anchor | Release | | --- | --- | --- | -| [#361](https://github.com/studio-berry/loop/issues/361) | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → P4-S8 | 0.2.0 | -| [#362](https://github.com/studio-berry/loop/issues/362) | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → harvest / DenyExtraGraphics | 0.2.0 | -| [#360](https://github.com/studio-berry/loop/issues/360) | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → cache budget harvest; [migration handoff](https://app.notion.com/p/3c39cb079ddb8123a4defd5740d4815b) | 0.2.0 | -| [#363](https://github.com/studio-berry/loop/issues/363) | [P4-S7 handoff](https://app.notion.com/p/3c69cb079ddb814e85d6e571a29740ab); [0.2.0](https://app.notion.com/p/3c09cb079ddb80dfa2f9d6d5a15f2d8e) harvest index | 0.2.0 | +| legacy #361 | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → P4-S8 | 0.2.0 | +| legacy #362 | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → harvest / DenyExtraGraphics | 0.2.0 | +| legacy #360 | [0.2.0 QT Quick](https://app.notion.com/p/3c39cb079ddb80c29549dfe9abace836) → cache budget harvest; [migration handoff](https://app.notion.com/p/3c39cb079ddb8123a4defd5740d4815b) | 0.2.0 | +| legacy #363 | [P4-S7 handoff](https://app.notion.com/p/3c69cb079ddb814e85d6e571a29740ab); [0.2.0](https://app.notion.com/p/3c09cb079ddb80dfa2f9d6d5a15f2d8e) harvest index | 0.2.0 | **0.3.0** ([Governed Corrections](https://app.notion.com/p/3c39cb079ddb8152b9f1f16d2fa2bacd)): lists **#361** as upstream P4-S8 dependency for detect→pinpoint, not 0.3.0 implementation. diff --git a/docs/loop-shell.json b/docs/loop-shell.json index e0898ee89..5a7a64236 100644 --- a/docs/loop-shell.json +++ b/docs/loop-shell.json @@ -233,7 +233,7 @@ ], "legacy_surface_disposition": [ { - "path": "CodeGenerator/generatormainwindow.ui", + "path": "tools/CodeGenerator/generatormainwindow.ui", "disposition": "HEADLESS", "owner": "m.berry", "replacement_target": "loop-cli", @@ -243,7 +243,7 @@ "rationale": "Developer fixture generator; not a product surface." }, { - "path": "JBIG2_Viewer/mainwindow.ui", + "path": "tools/JBIG2_Viewer/mainwindow.ui", "disposition": "HEADLESS", "owner": "m.berry", "replacement_target": "loop-inspect", diff --git a/docs/preflight-check-catalog-overlay.json b/docs/preflight-check-catalog-overlay.json index db2905a89..916f07a9d 100644 --- a/docs/preflight-check-catalog-overlay.json +++ b/docs/preflight-check-catalog-overlay.json @@ -17,89 +17,138 @@ "backlog_priority_rule": "P1 - no registered check inspects the defect class at all (a matrix not_covered entry), so a clean run is silent about it; P2 - a registered check inspects the class but its named limitation can suppress or misclassify a finding on defective content, which would be a false clean pass; P3 - a registered check inspects the class and the named limitation only narrows reported detail or fails closed as incomplete, so it cannot turn a defect into a silent pass.", "backlog_state_rule": "open - the gap is still present; landed - a registered check now covers the class and closed_by names that check id; closed - the filed issue that tracked the gap is closed.", "github_issues": { - "#12": { + "legacy#12": { "number": 12, "title": "Validate output-intent and ICC-profile presence, identity, and color-space consistency", "state": "CLOSED", - "milestone": "0.0.1" + "milestone": "0.0.1", + "repository": "legacy" }, - "#47": { - "number": 47, - "title": "[Residual] Tier-2 bleed raster golden corpus + strip calibration", - "state": "OPEN", - "milestone": null - }, - "#49": { - "number": 49, - "title": "[0.8.0] Overprint-correct rendering in standard and advanced renderers", - "state": "OPEN", - "milestone": "0.8.0" - }, - "#70": { + "legacy#70": { "number": 70, "title": "Traverse nested Form XObjects, inherited resources, and appearance streams for embedded fonts", "state": "CLOSED", - "milestone": null + "milestone": null, + "repository": "legacy" }, - "#124": { + "legacy#124": { "number": 124, "title": "Detect hidden, invisible, and off-page content in preflight", "state": "CLOSED", - "milestone": null + "milestone": null, + "repository": "legacy" }, - "#130": { + "legacy#130": { "number": 130, "title": "Detect corrupt fonts, not just embedded/unembedded status", "state": "CLOSED", - "milestone": null + "milestone": null, + "repository": "legacy" }, - "#131": { + "legacy#131": { "number": 131, "title": "Extend thin-part detection beyond simple strokes (expand #23)", "state": "CLOSED", - "milestone": null + "milestone": null, + "repository": "legacy" }, - "#600": { - "number": 600, - "title": "Ink Manager: separation management and spot-color library", + "legacy#628": { + "number": 628, + "title": "DeviceN dieline / spot geometry not detected (Separation-only collector)", + "state": "CLOSED", + "milestone": "0.3.0", + "repository": "legacy" + }, + "legacy#665": { + "number": 665, + "title": "[0.3.0] No check validates or produces PDF/X-5 and PDF/A-3 output", "state": "OPEN", - "milestone": null + "milestone": null, + "repository": "legacy" }, - "#603": { - "number": 603, - "title": "Imposition: n-up, gang-up, booklets, creep, and sheet furniture", + "#113": { + "number": 113, + "title": "L01-08 \u2014 Classify ICC-based color by its painted space", "state": "OPEN", - "milestone": null + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" }, - "#604": { - "number": 604, - "title": "Packaging production semantics: ISO 19593 Processing Steps, dielines, varnish/foil/white, barcode validation", + "#114": { + "number": 114, + "title": "L01-09 \u2014 Audit glyph coverage in embedded fonts", "state": "OPEN", - "milestone": null + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" }, - "#605": { - "number": 605, - "title": "PDF/VT variable-data production: understanding, validation, and preservation", + "#115": { + "number": 115, + "title": "L01-10 \u2014 Evaluate optional-content membership for hidden content", "state": "OPEN", - "milestone": null + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" }, - "#628": { - "number": 628, - "title": "DeviceN dieline / spot geometry not detected (Separation-only collector)", - "state": "CLOSED", - "milestone": "0.3.0" + "#116": { + "number": 116, + "title": "L01-11 \u2014 Stop isolated over-limit ink regions passing ink coverage", + "state": "OPEN", + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" }, - "#664": { - "number": 664, - "title": "[0.3.0] No check inspects formal GWG 2022/2024 sheetfed or packaging conformance", + "#117": { + "number": 117, + "title": "L01-12 \u2014 Broaden invisible-content detection", "state": "OPEN", - "milestone": null + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" }, - "#665": { - "number": 665, - "title": "[0.3.0] No check validates or produces PDF/X-5 and PDF/A-3 output", + "#118": { + "number": 118, + "title": "L01-13 \u2014 Make off-page and obscured-content geometry clip-aware", + "state": "OPEN", + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" + }, + "#119": { + "number": 119, + "title": "L01-14 \u2014 Judge white overprint and transparency interaction on the authoritative renderer", + "state": "OPEN", + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" + }, + "#120": { + "number": 120, + "title": "L01-15 \u2014 Close the sparse-mark bleed raster false pass", + "state": "OPEN", + "milestone": "L01 - Evidence Core", + "repository": "studio-berry/loop" + }, + "#141": { + "number": 141, + "title": "X00-02 \u2014 Re-file parked legacy expansion candidates with value cases", "state": "OPEN", - "milestone": null + "milestone": null, + "repository": "studio-berry/loop" + }, + "#142": { + "number": 142, + "title": "X00-03 \u2014 Per-named-colorant ink limits for DeviceN and spot separations", + "state": "OPEN", + "milestone": null, + "repository": "studio-berry/loop" + }, + "#143": { + "number": 143, + "title": "X00-04 \u2014 Barcode, slug, Braille, and dieline geometry validation", + "state": "OPEN", + "milestone": null, + "repository": "studio-berry/loop" + }, + "#144": { + "number": 144, + "title": "X00-05 \u2014 Standards currency: PDF/X-6, PDF 2.0 output intents, and GWG 2022 check sets", + "state": "OPEN", + "milestone": null, + "repository": "studio-berry/loop" } }, "checks": { @@ -1323,7 +1372,7 @@ "newspaper" ], "state": "open", - "closed_by": "#604", + "closed_by": "#143", "deferral": null }, { @@ -1337,7 +1386,7 @@ "newspaper" ], "state": "open", - "closed_by": "#600", + "closed_by": "#142", "deferral": null }, { @@ -1349,7 +1398,7 @@ "packaging" ], "state": "open", - "closed_by": "#664", + "closed_by": "#144", "deferral": null }, { @@ -1362,7 +1411,7 @@ "newspaper" ], "state": "open", - "closed_by": "#603", + "closed_by": "#141", "deferral": null }, { @@ -1375,13 +1424,13 @@ "digital" ], "state": "open", - "closed_by": "#605", + "closed_by": "#141", "deferral": null }, { "id": "pdfx5-pdfa3-output", "priority": "P3", - "gap": "PDF/X-5n, PDF/X-5g, and PDF/A-3 are not produced or validated; conformance-claims reports a declared level as unsupported instead of a silent pass (filed as #665)", + "gap": "PDF/X-5n, PDF/X-5g, and PDF/A-3 are not produced or validated; conformance-claims reports a declared level as unsupported instead of a silent pass (filed as legacy#665)", "families": [ "sheetfed-offset", "packaging", @@ -1401,7 +1450,7 @@ "digital" ], "state": "open", - "closed_by": "#47", + "closed_by": "#120", "deferral": null }, { @@ -1416,8 +1465,8 @@ "digital" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture whose ICC space resolves to an allowed alternate, so the false pass is reproducible rather than argued." + "closed_by": "#113", + "deferral": null }, { "id": "dieline-geometry", @@ -1427,7 +1476,7 @@ "packaging" ], "state": "open", - "closed_by": "#604", + "closed_by": "#143", "deferral": null }, { @@ -1439,8 +1488,8 @@ "digital" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with a parsable embedded program that is missing a used glyph." + "closed_by": "#114", + "deferral": null }, { "id": "hidden-layers-ocmd", @@ -1450,8 +1499,8 @@ "packaging" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with an optional-content membership configuration that hides print content." + "closed_by": "#115", + "deferral": null }, { "id": "ink-coverage-raster-tac", @@ -1463,20 +1512,20 @@ "newspaper" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture with an isolated over-limit element below min_region_area_pct." + "closed_by": "#116", + "deferral": null }, { "id": "invisible-content-breadth", "priority": "P2", - "gap": "invisible-content covers text render mode 3 and zero-alpha graphics state only, so other invisible classes pass clean (the wider detector is filed as #124)", + "gap": "invisible-content covers text render mode 3 and zero-alpha graphics state only, so other invisible classes pass clean (the wider detector is filed as legacy#124)", "families": [ "sheetfed-offset", "packaging" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture painting an invisible class outside text render mode 3 and zero-alpha graphics state." + "closed_by": "#117", + "deferral": null }, { "id": "obscured-content-occlusion", @@ -1486,8 +1535,8 @@ "sheetfed-offset" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture where a bounding-box heuristic cannot see the occlusion a RIP would." + "closed_by": "#118", + "deferral": null }, { "id": "off-page-content-clipping", @@ -1498,8 +1547,8 @@ "digital" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture whose clipped mark exposes the pre-clip bounds comparison." + "closed_by": "#118", + "deferral": null }, { "id": "transparency-rip-interaction", @@ -1511,8 +1560,8 @@ "digital" ], "state": "open", - "closed_by": "unfiled", - "deferral": "Deferred: filing needs a fixture reproducing an overprint-plus-transparency interaction only a RIP flattens." + "closed_by": "#119", + "deferral": null }, { "id": "white-overprint-renderer", @@ -1523,13 +1572,13 @@ "packaging" ], "state": "open", - "closed_by": "#49", + "closed_by": "#119", "deferral": null }, { "id": "corrupt-embedded-fonts", "priority": "P2", - "gap": "embedded font programs that fail to parse, or lose their cmap, were unchecked; font-integrity now reports them (filed as #130)", + "gap": "embedded font programs that fail to parse, or lose their cmap, were unchecked; font-integrity now reports them (filed as legacy#130)", "families": [ "sheetfed-offset", "digital" @@ -1546,7 +1595,7 @@ "packaging" ], "state": "closed", - "closed_by": "#628", + "closed_by": "legacy#628", "deferral": null }, { @@ -1564,7 +1613,7 @@ { "id": "nested-font-resources", "priority": "P2", - "gap": "fonts used only inside nested Form XObjects, inherited resources, or annotation appearance streams were unchecked; embedded-fonts now traverses them (filed as #70)", + "gap": "fonts used only inside nested Form XObjects, inherited resources, or annotation appearance streams were unchecked; embedded-fonts now traverses them (filed as legacy#70)", "families": [ "sheetfed-offset", "web-offset", @@ -1579,7 +1628,7 @@ { "id": "output-intent-identity", "priority": "P2", - "gap": "output intent presence, identity, subtype, and ICC payload were unchecked; output-intent now validates them (filed as #12)", + "gap": "output intent presence, identity, subtype, and ICC payload were unchecked; output-intent now validates them (filed as legacy#12)", "families": [ "sheetfed-offset", "web-offset", @@ -1593,7 +1642,7 @@ { "id": "thin-filled-parts", "priority": "P2", - "gap": "thin filled parts, not only strokes, were unchecked for a minimum width; thin-parts now measures them (filed as #131)", + "gap": "thin filled parts, not only strokes, were unchecked for a minimum width; thin-parts now measures them (filed as legacy#131)", "families": [ "packaging" ], diff --git a/docs/product-surface.json b/docs/product-surface.json index a1834adae..4c31b1988 100644 --- a/docs/product-surface.json +++ b/docs/product-surface.json @@ -631,7 +631,7 @@ "ui": { "contract": "docs/loop-shell.json", "entrypoint_surface": "loop-editor", - "legacy_forms": ["CodeGenerator/generatormainwindow.ui", "JBIG2_Viewer/mainwindow.ui"] + "legacy_forms": ["tools/CodeGenerator/generatormainwindow.ui", "tools/JBIG2_Viewer/mainwindow.ui"] }, "shell_contract": "docs/loop-shell.json" } diff --git a/docs/quick-runtime-manifest.json b/docs/quick-runtime-manifest.json index b2c7c93af..a36418ac3 100644 --- a/docs/quick-runtime-manifest.json +++ b/docs/quick-runtime-manifest.json @@ -20,8 +20,8 @@ "name": "QuickShellSmoke", "cmake_option": "LOOP_BUILD_QUICK_SHELL_SMOKE", "qml_uri": "Loop.QuickShellSmoke", - "qml_file": "QuickShellSmoke/QuickShellSmoke.qml", - "cmake_file": "QuickShellSmoke/CMakeLists.txt", + "qml_file": "tools/QuickShellSmoke/QuickShellSmoke.qml", + "cmake_file": "tools/QuickShellSmoke/CMakeLists.txt", "install": false, "qt_modules": [ "Qt6::Qml", @@ -33,8 +33,8 @@ "name": "CanvasBenchmark", "cmake_option": "LOOP_BUILD_CANVAS_BENCHMARK", "qml_uri": "Loop.CanvasBenchmark", - "qml_file": "CanvasBenchmark/CanvasBenchmark.qml", - "cmake_file": "CanvasBenchmark/CMakeLists.txt", + "qml_file": "tools/CanvasBenchmark/CanvasBenchmark.qml", + "cmake_file": "tools/CanvasBenchmark/CMakeLists.txt", "install": false, "qt_modules": [ "Qt6::Qml", diff --git a/scripts/agent/check-change.py b/scripts/agent/check-change.py index aa9698c3f..8140733d3 100644 --- a/scripts/agent/check-change.py +++ b/scripts/agent/check-change.py @@ -488,7 +488,7 @@ def main() -> int: add_clang_tidy_checks(evidence, sources, build_dir, dry_run=args.dry_run) if tests: expression = "^(" + "|".join(re.escape(test) for test in tests) + ")$" - add_result(evidence, "focused_tests", ["ctest", "--test-dir", str(build_dir), "--output-on-failure", "-R", expression], ROOT, args.dry_run) + add_result(evidence, "focused_tests", ["ctest", "--test-dir", str(build_dir), "-C", "Release", "--output-on-failure", "-R", expression], ROOT, args.dry_run) report = { "format_version": 1, diff --git a/scripts/ci/check_loop_identity.py b/scripts/ci/check_loop_identity.py index befbbe163..7261aecc6 100644 --- a/scripts/ci/check_loop_identity.py +++ b/scripts/ci/check_loop_identity.py @@ -55,13 +55,13 @@ "LoopEditor/main.cpp": "LoopEditor", "PdfTool/main.cpp": "PdfTool", "PdfTool/loop-pdf-worker-main.cpp": "LoopPdfWorker", - "CodeGenerator/main.cpp": "CodeGenerator", - "JBIG2_Viewer/main.cpp": "Jbig2Viewer", - "PdfExampleGenerator/main.cpp": "PdfExampleGenerator", + "tools/CodeGenerator/main.cpp": "CodeGenerator", + "tools/JBIG2_Viewer/main.cpp": "Jbig2Viewer", + "tools/PdfExampleGenerator/main.cpp": "PdfExampleGenerator", "loop-preflight/tools/generate_fixtures.cpp": "LoopPreflightFixtureGenerator", - "QuickShellSmoke/main.cpp": "QuickShellSmoke", - "ProductQuickAccessibilitySmoke/main.cpp": "ProductQuickAccessibilitySmoke", - "CanvasBenchmark/main.cpp": "CanvasBenchmark", + "tools/QuickShellSmoke/main.cpp": "QuickShellSmoke", + "tools/ProductQuickAccessibilitySmoke/main.cpp": "ProductQuickAccessibilitySmoke", + "tools/CanvasBenchmark/main.cpp": "CanvasBenchmark", } diff --git a/scripts/ci/check_pdf_worker_isolation.py b/scripts/ci/check_pdf_worker_isolation.py index 39ea15f55..83012b63d 100755 --- a/scripts/ci/check_pdf_worker_isolation.py +++ b/scripts/ci/check_pdf_worker_isolation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Static isolation checks for loop-pdf-worker (#618). +"""Static isolation checks for loop-pdf-worker (#20). Fails if the worker target sources initialize Sentry/crashpad, or if the IPC allowlist drifts away from open/preflight/cancel/ping. @@ -20,6 +20,9 @@ ROOT / "PdfTool/pdfworkersandbox.cpp", ROOT / "PdfTool/pdfworkersandbox.h", ROOT / "PdfTool/pdfworkerprotocol.h", + ROOT / "PdfTool/pdfworkerprocess.cpp", + ROOT / "PdfTool/pdfworkerprocess_win.cpp", + ROOT / "PdfTool/pdfworkerprocess.h", ] FORBIDDEN = ( diff --git a/scripts/ci/check_preflight_truth_source.py b/scripts/ci/check_preflight_truth_source.py index fa0ba824f..b480a3e25 100644 --- a/scripts/ci/check_preflight_truth_source.py +++ b/scripts/ci/check_preflight_truth_source.py @@ -41,7 +41,7 @@ contents are kept, because a string literal is exactly where user-visible verdict copy lives and rendering it is the violation regardless of quoting. -Scope: `ProductQuickAccessibilitySmoke/qml/**`, `ProductQuickAccessibilitySmoke/ +Scope: `tools/ProductQuickAccessibilitySmoke/qml/**`, `tools/ProductQuickAccessibilitySmoke/ main.cpp`, `LoopEditor/qml/**` and `LoopEditor/editorhost.cpp`. The whole of editorhost.cpp is in scope on purpose: it is the QML-facing host, and it holds no Core reducer call (`grep -n reducePreflightVerdict LoopEditor/editorhost.cpp` is @@ -68,17 +68,17 @@ # The GUI layers. `*` also covers a nested directory under either qml root. SCOPE_PATTERNS = ( - "ProductQuickAccessibilitySmoke/qml/*.qml", - "ProductQuickAccessibilitySmoke/main.cpp", + "tools/ProductQuickAccessibilitySmoke/qml/*.qml", + "tools/ProductQuickAccessibilitySmoke/main.cpp", "LoopEditor/qml/*.qml", "LoopEditor/editorhost.cpp", ) SCOPE_DIRECTORIES = ( - "ProductQuickAccessibilitySmoke/qml", + "tools/ProductQuickAccessibilitySmoke/qml", "LoopEditor/qml", ) SCOPE_FILES = ( - "ProductQuickAccessibilitySmoke/main.cpp", + "tools/ProductQuickAccessibilitySmoke/main.cpp", "LoopEditor/editorhost.cpp", ) @@ -164,7 +164,7 @@ def format(self) -> str: # preflight Q_INVOKABLE commands; they ask Core, they do not decide # # Controller-owned and bound read-only by the shipped pane -# (ProductQuickAccessibilitySmoke/qml/PreflightPane.qml and its LoopEditor/qml mirror): +# (tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml and its LoopEditor/qml mirror): # findingsModel the findings list model (:11, used at :148) -- render its # rows and navigate them; do not count it to pick a verdict # progress the job's own progress value (:135), already Core's diff --git a/scripts/ci/check_qml_mirror_parity.py b/scripts/ci/check_qml_mirror_parity.py index 8734a38a0..12ae104e2 100644 --- a/scripts/ci/check_qml_mirror_parity.py +++ b/scripts/ci/check_qml_mirror_parity.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Guard that the accessibility-smoke QML mirrors stay byte-identical. -`ProductQuickAccessibilitySmoke/CMakeLists.txt` states the contract: +`tools/ProductQuickAccessibilitySmoke/CMakeLists.txt` states the contract: LoopEditor/qml/ remains the single source of truth; keep every mirror byte-identical when either side changes. @@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[2] SOURCE_ROOT = "LoopEditor/qml" -MIRROR_ROOT = "ProductQuickAccessibilitySmoke/qml" +MIRROR_ROOT = "tools/ProductQuickAccessibilitySmoke/qml" @dataclass(frozen=True) diff --git a/scripts/ci/compare_package_boundary_evidence.py b/scripts/ci/compare_package_boundary_evidence.py index 149f48abf..b173df4e3 100644 --- a/scripts/ci/compare_package_boundary_evidence.py +++ b/scripts/ci/compare_package_boundary_evidence.py @@ -23,6 +23,8 @@ def load(path: Path, expected_platform: str) -> dict[str, Any]: evidence = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise PairError(f"unable to read evidence {path}: {exc}") from exc + if not isinstance(evidence, dict): + raise PairError(f"package evidence must be a JSON object: {path}") if evidence.get("schema_version") != 1 or evidence.get("kind") != "loop-package-boundary-evidence": raise PairError(f"unsupported evidence schema: {path}") if evidence.get("platform") != expected_platform: @@ -42,14 +44,16 @@ def load(path: Path, expected_platform: str) -> dict[str, Any]: ) if not isinstance(checks, dict) or any(checks.get(name) is not True for name in required_checks): raise PairError(f"package evidence checks are incomplete: {path}") - if not FULL_SHA.fullmatch(str(evidence.get("source_sha", ""))): + source_sha = evidence.get("source_sha") + if not isinstance(source_sha, str) or not FULL_SHA.fullmatch(source_sha): raise PairError(f"package evidence source SHA is not full length: {path}") package = evidence.get("package") expected_format = "AppImage" if expected_platform == "linux" else "MSI" if ( not isinstance(package, dict) or package.get("format") != expected_format - or not re.fullmatch(r"[0-9a-fA-F]{64}", str(package.get("sha256", ""))) + or not isinstance(package.get("sha256"), str) + or not re.fullmatch(r"[0-9a-fA-F]{64}", package["sha256"]) ): raise PairError(f"package identity is incomplete: {path}") return evidence diff --git a/scripts/ci/test_check_preflight_truth_source.py b/scripts/ci/test_check_preflight_truth_source.py index ca33138a6..84683da9f 100644 --- a/scripts/ci/test_check_preflight_truth_source.py +++ b/scripts/ci/test_check_preflight_truth_source.py @@ -35,10 +35,10 @@ ) -PREFLIGHT_PANE = "ProductQuickAccessibilitySmoke/qml/PreflightPane.qml" +PREFLIGHT_PANE = "tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml" MAIN_QML = "LoopEditor/qml/Main.qml" EDITOR_HOST = "LoopEditor/editorhost.cpp" -SMOKE_MAIN = "ProductQuickAccessibilitySmoke/main.cpp" +SMOKE_MAIN = "tools/ProductQuickAccessibilitySmoke/main.cpp" # Every file the guard is expected to scan in this tree. The guard discovers # them (`*.qml` under the two QML roots, plus the two host files), so adding a @@ -63,22 +63,22 @@ "LoopEditor/qml/StateBadge.qml", "LoopEditor/qml/Workspace.qml", "LoopEditor/qml/WorkspacePlaceholderPane.qml", - "ProductQuickAccessibilitySmoke/main.cpp", - "ProductQuickAccessibilitySmoke/qml/ActionListPane.qml", - "ProductQuickAccessibilitySmoke/qml/CanvasPane.qml", - "ProductQuickAccessibilitySmoke/qml/DocumentPane.qml", - "ProductQuickAccessibilitySmoke/qml/InspectPane.qml", - "ProductQuickAccessibilitySmoke/qml/InspectorPane.qml", - "ProductQuickAccessibilitySmoke/qml/Main.qml", - "ProductQuickAccessibilitySmoke/qml/MenuModel.qml", - "ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml", - "ProductQuickAccessibilitySmoke/qml/PreflightPane.qml", - "ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml", - "ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml", - "ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml", - "ProductQuickAccessibilitySmoke/qml/StateBadge.qml", - "ProductQuickAccessibilitySmoke/qml/Workspace.qml", - "ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml", + "tools/ProductQuickAccessibilitySmoke/main.cpp", + "tools/ProductQuickAccessibilitySmoke/qml/ActionListPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/CanvasPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/DocumentPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/InspectPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/InspectorPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/Main.qml", + "tools/ProductQuickAccessibilitySmoke/qml/MenuModel.qml", + "tools/ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml", + "tools/ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml", + "tools/ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml", + "tools/ProductQuickAccessibilitySmoke/qml/StateBadge.qml", + "tools/ProductQuickAccessibilitySmoke/qml/Workspace.qml", + "tools/ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml", } ) @@ -253,8 +253,8 @@ def test_scope_covers_the_gui_layers_only(self) -> None: def test_scope_file_list_is_exactly_the_expected_set(self) -> None: """The discovered file list equals the expected names, not a count. - `gui_sources()` globs `*.qml` under `ProductQuickAccessibilitySmoke/qml` - and `LoopEditor/qml` and adds `ProductQuickAccessibilitySmoke/main.cpp` + `gui_sources()` globs `*.qml` under `tools/ProductQuickAccessibilitySmoke/qml` + and `LoopEditor/qml` and adds `tools/ProductQuickAccessibilitySmoke/main.cpp` and `LoopEditor/editorhost.cpp`. A new pane under either root is picked up by discovery, which makes this assertion fail until the name is added to `EXPECTED_GUI_SOURCES` -- deliberate, so the new pane is placed in diff --git a/scripts/ci/test_check_qml_mirror_parity.py b/scripts/ci/test_check_qml_mirror_parity.py index d46e6f037..91402da05 100644 --- a/scripts/ci/test_check_qml_mirror_parity.py +++ b/scripts/ci/test_check_qml_mirror_parity.py @@ -34,7 +34,7 @@ def test_flags_a_drifted_pair(self) -> None: self.assertEqual( violations[0].format(), "LoopEditor/qml/PreflightPane.qml:2: mirror-drift vs " - "ProductQuickAccessibilitySmoke/qml/PreflightPane.qml (source 2 lines, mirror 1 lines)", + "tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml (source 2 lines, mirror 1 lines)", ) def test_flags_a_mirror_that_only_lost_its_line_endings(self) -> None: @@ -48,7 +48,7 @@ def test_flags_a_source_without_a_mirror(self) -> None: self.assertEqual( violations[0].format(), "LoopEditor/qml/NewPane.qml:1: mirror-missing " - "(no ProductQuickAccessibilitySmoke/qml/NewPane.qml twin)", + "(no tools/ProductQuickAccessibilitySmoke/qml/NewPane.qml twin)", ) def test_flags_a_mirror_without_a_source(self) -> None: @@ -56,7 +56,7 @@ def test_flags_a_mirror_without_a_source(self) -> None: self.assertEqual([violation.rule for violation in violations], ["mirror-orphan"]) self.assertEqual( violations[0].format(), - "ProductQuickAccessibilitySmoke/qml/RetiredPane.qml:1: mirror-orphan " + "tools/ProductQuickAccessibilitySmoke/qml/RetiredPane.qml:1: mirror-orphan " "(no LoopEditor/qml/RetiredPane.qml source)", ) @@ -92,7 +92,7 @@ def test_custom_roots_are_reported(self) -> None: # -- the real tree ---------------------------------------------------------- def test_the_two_roots_are_the_documented_contract_ends(self) -> None: self.assertEqual(SOURCE_ROOT, "LoopEditor/qml") - self.assertEqual(MIRROR_ROOT, "ProductQuickAccessibilitySmoke/qml") + self.assertEqual(MIRROR_ROOT, "tools/ProductQuickAccessibilitySmoke/qml") def test_both_roots_are_populated_with_the_same_names(self) -> None: sources = source_texts() diff --git a/scripts/ci/test_compare_package_boundary_evidence.py b/scripts/ci/test_compare_package_boundary_evidence.py index 0e9fd6869..79d22f89b 100644 --- a/scripts/ci/test_compare_package_boundary_evidence.py +++ b/scripts/ci/test_compare_package_boundary_evidence.py @@ -54,6 +54,19 @@ def test_linux_and_windows_evidence_share_the_expected_sha(self): self.assertEqual(pair["status"], "passed") self.assertEqual(pair["source_sha"], source_sha) + def test_malformed_identity_is_rejected_without_coercion(self): + invalid_records = [[], evidence("linux", 1)] + numeric_digest = evidence("linux", "d" * 40) + numeric_digest["package"]["sha256"] = int("1" * 64) + invalid_records.append(numeric_digest) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "linux.json" + for record in invalid_records: + with self.subTest(record=record): + path.write_text(json.dumps(record), encoding="utf-8") + with self.assertRaises(PAIR.PairError): + PAIR.load(path, "linux") + def test_pair_rejects_mismatched_sha_and_failed_evidence(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/scripts/ci/test_preflight_check_catalog.py b/scripts/ci/test_preflight_check_catalog.py index 0fdd967cc..b1c065371 100644 --- a/scripts/ci/test_preflight_check_catalog.py +++ b/scripts/ci/test_preflight_check_catalog.py @@ -165,11 +165,103 @@ def invent(overlay: dict) -> None: def test_state_disagreeing_with_issue_state_fails(self) -> None: def mismatch(overlay: dict) -> None: for row in overlay["backlog"]: - if row["closed_by"] == "#605": + if row["closed_by"] == "#141": row["state"] = "closed" break - self.assert_overlay_fails(mismatch, "disagrees with #605", target="backlog") + self.assert_overlay_fails(mismatch, "disagrees with #141", target="backlog") + + def test_record_without_a_repository_fails(self) -> None: + self.assert_overlay_fails( + lambda overlay: overlay["github_issues"]["#141"].pop("repository"), + "github_issues entry '#141' missing repository", + target="backlog", + ) + + def test_live_record_naming_another_repository_fails(self) -> None: + self.assert_overlay_fails( + lambda overlay: overlay["github_issues"]["#141"].__setitem__("repository", "legacy"), + "'#141' must record repository 'studio-berry/loop'", + target="backlog", + ) + + def test_legacy_record_naming_the_live_repository_fails(self) -> None: + self.assert_overlay_fails( + lambda overlay: overlay["github_issues"]["legacy#124"].__setitem__( + "repository", "studio-berry/loop" + ), + "'legacy#124' must record repository 'legacy'", + target="backlog", + ) + + def test_open_legacy_issue_cannot_track_a_gap_fails(self) -> None: + def reopen(overlay: dict) -> None: + overlay["github_issues"]["legacy#124"]["state"] = "OPEN" + for row in overlay["backlog"]: + if row["id"] == "devicen-dieline-detection": + row["closed_by"] = "legacy#124" + row["state"] = "open" + + self.assert_overlay_fails(reopen, "cites open legacy issue legacy#124", target="backlog") + + def test_bare_number_does_not_satisfy_a_legacy_record_fails(self) -> None: + def collide(overlay: dict) -> None: + for row in overlay["backlog"]: + if row["id"] == "invisible-content-breadth": + row["gap"] += " (see #124)" + + self.assert_overlay_fails(collide, "cite issues with no verified record: #124", target="backlog") + + def test_gap_text_citing_an_unrecorded_issue_fails(self) -> None: + def cite(overlay: dict) -> None: + overlay["backlog"][0]["gap"] += " (filed as #98765)" + + self.assert_overlay_fails(cite, "cite issues with no verified record: #98765", target="backlog") + + def test_committed_overlay_records_the_live_repository_only_for_live_numbers(self) -> None: + for reference, entry in self.overlay()["github_issues"].items(): + expected = "legacy" if reference.startswith("legacy#") else "studio-berry/loop" + self.assertEqual(entry["repository"], expected, reference) + + def test_github_read_back_accepts_matching_records(self) -> None: + live = {"title": "t", "state": "OPEN", "milestone": None, "is_pull_request": False} + records = {"#7": {"number": 7, "title": "t", "state": "OPEN", "milestone": None, "repository": "studio-berry/loop"}} + self.assertEqual(generator.verify_issue_records_against_github(records, lambda number: live), []) + + def test_github_read_back_skips_legacy_snapshots(self) -> None: + def unreachable(number: int) -> dict: + raise AssertionError("a legacy snapshot must not be read back") + + records = { + "legacy#7": {"number": 7, "title": "t", "state": "CLOSED", "milestone": None, "repository": "legacy"} + } + self.assertEqual(generator.verify_issue_records_against_github(records, unreachable), []) + + def test_github_read_back_reports_every_collision_kind(self) -> None: + live = { + 1: {"title": "other", "state": "OPEN", "milestone": None, "is_pull_request": False}, + 2: {"title": "t", "state": "CLOSED", "milestone": None, "is_pull_request": False}, + 3: {"title": "t", "state": "OPEN", "milestone": "L01", "is_pull_request": False}, + 4: {"title": "t", "state": "OPEN", "milestone": None, "is_pull_request": True}, + 5: None, + } + records = { + f"#{number}": { + "number": number, + "title": "t", + "state": "OPEN", + "milestone": None, + "repository": "studio-berry/loop", + } + for number in live + } + problems = generator.verify_issue_records_against_github(records, live.__getitem__) + self.assertEqual(len(problems), 5) + self.assertIn("#1: title is 'other'", problems[0]) + self.assertIn("#2: state is 'CLOSED'", problems[1]) + self.assertIn("#3: milestone is 'L01'", problems[2]) + self.assertIn("#4: resolves to a pull request", problems[3]) + self.assertIn("#5: no such issue", problems[4]) def test_uncovered_class_missing_from_backlog_fails(self) -> None: def drop(overlay: dict) -> None: diff --git a/scripts/ci/test_product_quick_accessibility_smoke.py b/scripts/ci/test_product_quick_accessibility_smoke.py index bea672143..da7b223a9 100644 --- a/scripts/ci/test_product_quick_accessibility_smoke.py +++ b/scripts/ci/test_product_quick_accessibility_smoke.py @@ -9,8 +9,8 @@ ROOT = Path(__file__).resolve().parents[2] SOURCE = ROOT / "LoopEditor" / "qml" -MIRROR = ROOT / "ProductQuickAccessibilitySmoke" / "qml" -SMOKE_MAIN = ROOT / "ProductQuickAccessibilitySmoke" / "main.cpp" +MIRROR = ROOT / "tools" / "ProductQuickAccessibilitySmoke" / "qml" +SMOKE_MAIN = ROOT / "tools" / "ProductQuickAccessibilitySmoke" / "main.cpp" class ProductQuickAccessibilitySmokeContractTests(unittest.TestCase): diff --git a/scripts/ci/test_verify_plugin_form_accounting.py b/scripts/ci/test_verify_plugin_form_accounting.py index b928ea953..b9330fe0c 100644 --- a/scripts/ci/test_verify_plugin_form_accounting.py +++ b/scripts/ci/test_verify_plugin_form_accounting.py @@ -27,7 +27,7 @@ def test_valid_accounting_passes(self) -> None: self.assertEqual(completed.returncode, 0, completed.stderr + completed.stdout) def test_unledgered_repo_ui_fails(self) -> None: - orphan = ROOT / "CodeGenerator" / "orphan-form.ui" + orphan = ROOT / "tools" / "CodeGenerator" / "orphan-form.ui" original_shell = SHELL_PATH.read_text(encoding="utf-8") try: orphan.write_text('\n', encoding="utf-8") diff --git a/scripts/ci/test_workflow_contracts.py b/scripts/ci/test_workflow_contracts.py index 37df97abf..5c0fadfb8 100644 --- a/scripts/ci/test_workflow_contracts.py +++ b/scripts/ci/test_workflow_contracts.py @@ -15,17 +15,18 @@ def test_packaging_dispatch_permissions_have_unique_keys(self): permissions = workflow.split("permissions:\n", 1)[1].split("\njobs:", 1)[0] self.assertEqual(permissions.count("actions:"), 1) - def test_issue_promotion_workflow_tracks_only_protected_promotion_pushes(self): + def test_issue_promotion_workflow_tracks_all_promotion_pushes(self): workflow = (ROOT / ".github/workflows/issue-promotion.yml").read_text( encoding="utf-8" ) - self.assertIn("branches:\n - dev\n - stable", workflow) + self.assertIn("branches:\n - dev\n - unstable\n - stable", workflow) self.assertNotIn("pull_request:", workflow) self.assertIn("issues: write", workflow) self.assertIn("pull-requests: read", workflow) self.assertIn("python3 scripts/github/issue_promotion.py", workflow) self.assertIn("python3 -m scripts.github.test_issue_promotion", workflow) self.assertIn("cancel-in-progress: false", workflow) + self.assertIn("queue: max", workflow) def test_agent_fast_runs_its_dedicated_policy_tests(self): workflow = (ROOT / ".github/workflows/reusable-linux.yml").read_text(encoding="utf-8") diff --git a/scripts/generate-architecture-catalogs.py b/scripts/generate-architecture-catalogs.py index 4ee804409..c342493c5 100644 --- a/scripts/generate-architecture-catalogs.py +++ b/scripts/generate-architecture-catalogs.py @@ -13,6 +13,7 @@ import difflib import json import re +import subprocess import sys from pathlib import Path from typing import Any, Iterable @@ -167,7 +168,12 @@ def parse_engine_matrix_id() -> str: PREFLIGHT_BACKLOG_PRIORITIES = {"P1", "P2", "P3"} PREFLIGHT_BACKLOG_STATES = {"open", "landed", "closed"} PREFLIGHT_BACKLOG_UNFILED = "unfiled" +GITHUB_ISSUE_REPOSITORY = "studio-berry/loop" +LEGACY_ISSUE_REPOSITORY = "legacy" GITHUB_ISSUE_REF = re.compile(r"#[1-9][0-9]*") +LEGACY_ISSUE_REF = re.compile(r"legacy#[1-9][0-9]*") +ISSUE_RECORD_KEY = re.compile(r"(?:legacy)?#[1-9][0-9]*") +ISSUE_REF_IN_TEXT = re.compile(r"(?`` is an issue in + ``studio-berry/loop`` that ``--verify-github`` can read back. ``legacy#`` + is a frozen snapshot from the retired repository, whose numbers overlap the + live ones and can no longer be read back: it may document a row that was + closed there, but it can never be the open tracker of a gap. """ issues = overlay.get("github_issues") if not isinstance(issues, dict) or not issues: raise ValueError("preflight catalog overlay is missing github_issues") verified: dict[str, dict[str, Any]] = {} for reference, entry in issues.items(): - if not isinstance(reference, str) or not GITHUB_ISSUE_REF.fullmatch(reference): - raise ValueError(f"github_issues key '{reference}' is not a '#' reference") + if not isinstance(reference, str) or not ISSUE_RECORD_KEY.fullmatch(reference): + raise ValueError( + f"github_issues key '{reference}' is not a '#' or 'legacy#' reference" + ) if not isinstance(entry, dict): raise ValueError(f"github_issues entry '{reference}' must be an object") - absent = sorted({"number", "title", "state", "milestone"} - set(entry)) + absent = sorted({"number", "title", "state", "milestone", "repository"} - set(entry)) if absent: raise ValueError(f"github_issues entry '{reference}' missing {', '.join(absent)}") - if entry["number"] != int(reference[1:]): + expected_repository = ( + LEGACY_ISSUE_REPOSITORY if LEGACY_ISSUE_REF.fullmatch(reference) else GITHUB_ISSUE_REPOSITORY + ) + if entry["repository"] != expected_repository: + raise ValueError( + f"github_issues entry '{reference}' must record repository '{expected_repository}'" + ) + if entry["number"] != int(reference.split("#")[1]): raise ValueError(f"github_issues entry '{reference}' records a different number") if entry["state"] not in {"OPEN", "CLOSED"}: raise ValueError(f"github_issues entry '{reference}' must record the GitHub state verbatim") @@ -434,12 +455,17 @@ def build_preflight_backlog( if closed_by == PREFLIGHT_BACKLOG_UNFILED: if state != "open": raise ValueError(f"backlog row '{identifier}' is unfiled but its state is '{state}'") - elif isinstance(closed_by, str) and GITHUB_ISSUE_REF.fullmatch(closed_by): + elif isinstance(closed_by, str) and ISSUE_RECORD_KEY.fullmatch(closed_by): if closed_by not in verified: raise ValueError( f"backlog row '{identifier}' references unverified issue {closed_by}; " "confirm it with 'gh issue view' and record it in github_issues first" ) + if LEGACY_ISSUE_REF.fullmatch(closed_by) and verified[closed_by]["state"] != "CLOSED": + raise ValueError( + f"backlog row '{identifier}' cites open legacy issue {closed_by}, which can no " + "longer be read back; re-point it to a live issue or unfile it with a deferral" + ) if state == "landed": raise ValueError(f"backlog row '{identifier}' is landed by a check, not by {closed_by}") expected = "open" if verified[closed_by]["state"] == "OPEN" else "closed" @@ -487,9 +513,9 @@ def build_preflight_backlog( referenced: set[str] = set() for row in parsed: - if GITHUB_ISSUE_REF.fullmatch(row["closed_by"]): + if ISSUE_RECORD_KEY.fullmatch(row["closed_by"]): referenced.add(row["closed_by"]) - referenced.update(re.findall(GITHUB_ISSUE_REF, row["gap"])) + referenced.update(ISSUE_REF_IN_TEXT.findall(row["gap"])) unused = sorted(reference for reference in verified if reference not in referenced) if unused: raise ValueError("github_issues entries cited by no backlog row: " + ", ".join(unused)) @@ -1170,6 +1196,57 @@ def serialized_preflight_catalog() -> str: return json.dumps(build_preflight_check_catalog(registry, parse_repair_operations()), indent=2, sort_keys=True) + "\n" +def fetch_github_issue(number: int) -> dict[str, Any] | None: + """Read one issue back from GitHub, or None when the number does not exist.""" + result = subprocess.run( + ["gh", "api", f"repos/{GITHUB_ISSUE_REPOSITORY}/issues/{number}"], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + if result.returncode != 0: + if "Not Found" in result.stdout or "Not Found" in result.stderr: + return None + raise OSError(f"gh api issue {number} failed: {result.stderr.strip()}") + live = json.loads(result.stdout) + return { + "title": live["title"], + "state": live["state"].upper(), + "milestone": (live.get("milestone") or {}).get("title"), + "is_pull_request": "pull_request" in live, + } + + +def verify_issue_records_against_github( + records: dict[str, dict[str, Any]], fetch: Any = fetch_github_issue +) -> list[str]: + """Compare every live ``#`` record with GitHub; legacy snapshots are not re-read. + + Issue and pull-request numbers share one sequence, so a record that resolves + to a pull request, or to an issue with another title, is a collision rather + than a verified reference. + """ + problems: list[str] = [] + live_records = [ + (reference, entry) + for reference, entry in records.items() + if entry["repository"] == GITHUB_ISSUE_REPOSITORY + ] + for reference, entry in sorted(live_records, key=lambda item: item[1]["number"]): + live = fetch(entry["number"]) + if live is None: + problems.append(f"{reference}: no such issue in {GITHUB_ISSUE_REPOSITORY}") + continue + if live["is_pull_request"]: + problems.append(f"{reference}: resolves to a pull request, not an issue") + continue + for field in ("title", "state", "milestone"): + if live[field] != entry[field]: + problems.append(f"{reference}: {field} is {live[field]!r} on GitHub, recorded {entry[field]!r}") + return problems + + def serialized_preflight_backlog() -> str: overlay = json.loads(read(PREFLIGHT_OVERLAY_PATH)) registry = parse_preflight_checks() @@ -1212,6 +1289,11 @@ def main() -> int: mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--check", action="store_true", help="validate ADRs and the committed catalog") mode.add_argument("--write", action="store_true", help="validate ADRs and write the catalog") + parser.add_argument( + "--verify-github", + action="store_true", + help="read every live github_issues record back from GitHub (needs gh; not run in CI)", + ) args = parser.parse_args() errors = validate_adrs() @@ -1230,6 +1312,19 @@ def main() -> int: print(f"error: cannot generate architecture catalog: {error}", file=sys.stderr) return 1 + if args.verify_github: + try: + problems = verify_issue_records_against_github( + json.loads(read(PREFLIGHT_OVERLAY_PATH))["github_issues"] + ) + except OSError as error: + print(f"error: cannot read issues back from GitHub: {error}", file=sys.stderr) + return 1 + if problems: + for problem in problems: + print(f"error: github_issues {problem}", file=sys.stderr) + return 1 + if args.write: CATALOG_PATH.parent.mkdir(parents=True, exist_ok=True) CATALOG_PATH.write_text(expected, encoding="utf-8", newline="\n") diff --git a/scripts/generate_phase5_widgets_evidence.py b/scripts/generate_phase5_widgets_evidence.py index b8312d2d1..22eb12e39 100644 --- a/scripts/generate_phase5_widgets_evidence.py +++ b/scripts/generate_phase5_widgets_evidence.py @@ -248,11 +248,11 @@ def _profile_for_target( return False, "qualification target excluded from the product-surface manifest" if normalized.startswith("Fuzz/"): return False, "qualification target excluded from the product-surface manifest" - if normalized.startswith("QuickShellSmoke/"): + if normalized.startswith("tools/QuickShellSmoke/"): return False, "qualification target excluded from the product-surface manifest" - if normalized.startswith("CanvasBenchmark/"): + if normalized.startswith("tools/CanvasBenchmark/"): return False, "qualification target excluded from the product-surface manifest" - if normalized.startswith("ProductQuickAccessibilitySmoke/"): + if normalized.startswith("tools/ProductQuickAccessibilitySmoke/"): return False, "qualification target excluded from the product-surface manifest" return True, "target is reachable from the maintained product graph" diff --git a/scripts/github/issue_promotion.py b/scripts/github/issue_promotion.py index d55492fab..de2fd7477 100644 --- a/scripts/github/issue_promotion.py +++ b/scripts/github/issue_promotion.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -"""Track GitHub issues as changes move through the dev and stable branches. +"""Track the highest promotion branch containing linked issue work. -The workflow that invokes this module runs on push events for both protected -promotion branches. Evidence is collected from the exact ``before...after`` -push range, together with merged pull-request metadata and source commits for -promotion PRs. Mutations happen only after the complete evidence set has been -collected so a partial API read cannot cause a partial promotion claim. +The workflow runs on pushes to dev, unstable, and stable. Evidence is collected +from the exact ``before...after`` push range, merged pull-request metadata, +and source commits for promotion PRs. Mutations happen after all issue and +field reads complete, so a partial API read cannot cause a partial promotion +claim. Issue state is never changed by this workflow. """ from __future__ import annotations @@ -19,14 +19,21 @@ from pathlib import Path from typing import Any, Iterable, Mapping from urllib.error import HTTPError, URLError -from urllib.parse import quote, urlencode +from urllib.parse import urlencode from urllib.request import Request, urlopen -QUEUE_LABEL = "in promotion queue" +PROMOTION_FIELD_ID = 47367010 DEV_BRANCH = "dev" +UNSTABLE_BRANCH = "unstable" STABLE_BRANCH = "stable" -SUPPORTED_BRANCHES = frozenset({DEV_BRANCH, STABLE_BRANCH}) +STAGE_BY_BRANCH = { + DEV_BRANCH: "Dev present", + UNSTABLE_BRANCH: "Unstable present", + STABLE_BRANCH: "Stable present", +} +STAGE_RANK = {stage: rank for rank, stage in enumerate(STAGE_BY_BRANCH.values())} +SUPPORTED_BRANCHES = frozenset(STAGE_BY_BRANCH) ZERO_SHA = "0" * 40 FULL_SHA = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) @@ -120,7 +127,7 @@ def __init__(self, token: str, *, api_url: str = "https://api.github.com") -> No "Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}", "User-Agent": "loop-issue-promotion", - "X-GitHub-Api-Version": "2022-11-28", + "X-GitHub-Api-Version": "2026-03-10", } def request( @@ -234,25 +241,29 @@ def issue(self, repository: str, number: int) -> dict[str, Any]: raise GitHubApiError(f"issue #{number} returned a non-object response") return response - def add_label(self, repository: str, number: int, label: str) -> None: + def promotion_stage(self, repository: str, number: int) -> str | None: + path = f"repos/{repository}/issues/{number}/issue-field-values" + values = self.request("GET", path) + if not isinstance(values, list): + raise GitHubApiError(f"issue #{number} field values returned a non-list response") + matches = [ + value for value in values + if isinstance(value, dict) and value.get("issue_field_id") == PROMOTION_FIELD_ID + ] + if not matches: + return None + if len(matches) != 1: + raise GitHubApiError(f"issue #{number} has duplicate promotion field values") + option = matches[0].get("single_select_option") + if not isinstance(option, dict) or not isinstance(option.get("name"), str): + raise GitHubApiError(f"issue #{number} has an invalid promotion field value") + return option["name"] + + def set_promotion_stage(self, repository: str, number: int, stage: str) -> None: self.request( "POST", - f"repos/{repository}/issues/{number}/labels", - payload={"labels": [label]}, - ) - - def close_issue(self, repository: str, number: int) -> None: - self.request( - "PATCH", - f"repos/{repository}/issues/{number}", - payload={"state": "closed", "state_reason": "completed"}, - ) - - def remove_label(self, repository: str, number: int, label: str) -> None: - encoded_label = quote(label, safe="") - self.request( - "DELETE", - f"repos/{repository}/issues/{number}/labels/{encoded_label}", + f"repos/{repository}/issues/{number}/issue-field-values", + payload={"issue_field_values": [{"field_id": PROMOTION_FIELD_ID, "value": stage}]}, ) @@ -300,11 +311,11 @@ def _collect_pull_request(self, number: int) -> None: ) self.issue_numbers.update(pull_refs) - # A stable promotion may be merged as a squash commit. Its push range + # A promotion may be merged as a squash commit. Its push range # then contains only the new squash SHA, so inspect the promotion PR's # source commits and their merged topic PRs as well. A dev squash PR # is expanded only when its title/body had no usable issue link. - expand_source = self.target_branch == STABLE_BRANCH or not pull_refs + expand_source = self.target_branch != DEV_BRANCH or not pull_refs if not expand_source: return for commit in self.client.pull_request_commits(self.repository, number): @@ -314,45 +325,32 @@ def _collect_pull_request(self, number: int) -> None: @dataclass(frozen=True) class IssueAction: number: int - kind: str - reason: str - - -def _label_names(issue: Mapping[str, Any]) -> set[str]: - labels = issue.get("labels", []) - if not isinstance(labels, list): - return set() - names: set[str] = set() - for label in labels: - if isinstance(label, dict) and isinstance(label.get("name"), str): - names.add(label["name"]) - elif isinstance(label, str): - names.add(label) - return names + stage: str def plan_issue_actions( - target_branch: str, issues: Mapping[int, Mapping[str, Any]] + target_branch: str, + issues: Mapping[int, Mapping[str, Any]], + current_stages: Mapping[int, str | None], ) -> tuple[IssueAction, ...]: - """Plan idempotent actions while enforcing the stable queue guard.""" + """Plan monotonic field updates without changing issue state.""" if target_branch not in SUPPORTED_BRANCHES: raise ValueError(f"unsupported promotion branch: {target_branch!r}") actions: list[IssueAction] = [] + target_stage = STAGE_BY_BRANCH[target_branch] for number in sorted(issues): issue = issues[number] if "pull_request" in issue: continue - if issue.get("state") != "open": - continue - labels = _label_names(issue) - if target_branch == DEV_BRANCH: - if QUEUE_LABEL not in labels: - actions.append(IssueAction(number, "label", "linked work reached dev")) - continue - if QUEUE_LABEL in labels: - actions.append(IssueAction(number, "close", "queued work reached stable")) + current_stage = current_stages[number] + if current_stage is not None and current_stage not in STAGE_RANK: + raise GitHubApiError( + f"issue #{number} has unknown promotion stage {current_stage!r}" + ) + if current_stage is None or STAGE_RANK[current_stage] < STAGE_RANK[target_stage]: + actions.append(IssueAction(number, target_stage)) return tuple(actions) @@ -362,19 +360,8 @@ def apply_issue_actions( actions: Iterable[IssueAction], ) -> None: for action in actions: - if action.kind == "label": - client.add_label(repository, action.number, QUEUE_LABEL) - print(f"issue #{action.number}: added {QUEUE_LABEL!r}") - elif action.kind == "close": - client.close_issue(repository, action.number) - try: - client.remove_label(repository, action.number, QUEUE_LABEL) - except GitHubApiError as exc: - if exc.status_code != 404: - raise - print(f"issue #{action.number}: closed as completed") - else: - raise ValueError(f"unknown issue action: {action.kind!r}") + client.set_promotion_stage(repository, action.number, action.stage) + print(f"issue #{action.number}: promotion stage set to {action.stage!r}") def process_push( @@ -403,6 +390,7 @@ def process_push( return 0 issues: dict[int, Mapping[str, Any]] = {} + current_stages: dict[int, str | None] = {} for number in sorted(numbers): try: issues[number] = client.issue(repository, number) @@ -411,8 +399,10 @@ def process_push( print(f"issue #{number}: not found; skipped", file=sys.stderr) continue raise + if "pull_request" not in issues[number]: + current_stages[number] = client.promotion_stage(repository, number) - actions = plan_issue_actions(target_branch, issues) + actions = plan_issue_actions(target_branch, issues, current_stages) print( f"Found {len(numbers)} issue link(s); planned {len(actions)} action(s) " f"for {target_branch}." diff --git a/scripts/github/test_issue_promotion.py b/scripts/github/test_issue_promotion.py index f39af59ee..f38e57a2d 100644 --- a/scripts/github/test_issue_promotion.py +++ b/scripts/github/test_issue_promotion.py @@ -4,6 +4,7 @@ from __future__ import annotations import unittest +from unittest.mock import patch from typing import Any from scripts.github import issue_promotion as module @@ -42,6 +43,9 @@ def __init__(self) -> None: ], 901: [], } + self.issues: dict[int, dict[str, Any]] = {} + self.stages: dict[int, str | None] = {} + self.writes: list[tuple[int, str]] = [] def compare_commits(self, repository: str, before: str, after: str) -> list[dict[str, Any]]: return self.commits @@ -55,14 +59,23 @@ def pull_request(self, repository: str, number: int) -> dict[str, Any]: def pull_request_commits(self, repository: str, number: int) -> list[dict[str, Any]]: return self.pull_commits[number] + def issue(self, repository: str, number: int) -> dict[str, Any]: + return self.issues[number] + + def promotion_stage(self, repository: str, number: int) -> str | None: + return self.stages.get(number) + + def set_promotion_stage(self, repository: str, number: int, stage: str) -> None: + self.writes.append((number, stage)) + class IssuePromotionTests(unittest.TestCase): - repository = "studio-berry/loop" + repository = "studio-berry/loop2" def test_extracts_multiple_same_repository_refs_and_ignores_external_refs(self) -> None: text = ( - "Closes #12, fixes studio-berry/loop#13, skips other/repo#14, " - "and see https://github.com/studio-berry/loop/issues/15." + "Closes #12, fixes studio-berry/loop2#13, skips other/repo#14, " + "and see https://github.com/studio-berry/loop2/issues/15." ) self.assertEqual( module.extract_issue_references(text, self.repository), {12, 13, 15} @@ -101,41 +114,137 @@ def test_collects_direct_push_commit_refs_without_a_pull_request(self) -> None: self.assertEqual(evidence.collect("e" * 40, "f" * 40), {104, 105}) - def test_dev_labels_only_open_unqueued_issues(self) -> None: + def test_unstable_expands_squashed_source_commits(self) -> None: + evidence = module.PromotionEvidence( + FakeGitHubClient(), self.repository, module.UNSTABLE_BRANCH + ) + self.assertEqual(evidence.collect("c" * 40, "d" * 40), {101, 102, 103}) + + def test_dev_sets_stage_without_changing_issue_state(self) -> None: issues = { - 1: {"state": "open", "labels": []}, - 2: { - "state": "open", - "labels": [{"name": module.QUEUE_LABEL}], - }, - 3: {"state": "closed", "labels": []}, - 4: {"state": "open", "labels": [], "pull_request": {}}, + 1: {"state": "open"}, + 2: {"state": "open"}, + 3: {"state": "closed"}, + 4: {"state": "open", "pull_request": {}}, } self.assertEqual( - module.plan_issue_actions(module.DEV_BRANCH, issues), - (module.IssueAction(1, "label", "linked work reached dev"),), + module.plan_issue_actions( + module.DEV_BRANCH, + issues, + {1: None, 2: "Dev present", 3: None}, + ), + ( + module.IssueAction(1, "Dev present"), + module.IssueAction(3, "Dev present"), + ), ) - def test_stable_closes_only_open_queued_issues(self) -> None: + def test_promotion_is_monotonic_and_stable_never_closes(self) -> None: issues = { - 5: { - "state": "open", - "labels": [{"name": module.QUEUE_LABEL}], - }, - 6: {"state": "open", "labels": []}, - 7: { - "state": "closed", - "labels": [{"name": module.QUEUE_LABEL}], - }, - 8: { - "state": "open", - "labels": [{"name": module.QUEUE_LABEL}], - "pull_request": {}, - }, + 5: {"state": "open"}, + 6: {"state": "open"}, + 7: {"state": "closed"}, + 8: {"state": "open", "pull_request": {}}, } self.assertEqual( - module.plan_issue_actions(module.STABLE_BRANCH, issues), - (module.IssueAction(5, "close", "queued work reached stable"),), + module.plan_issue_actions( + module.STABLE_BRANCH, + issues, + {5: "Dev present", 6: None, 7: "Stable present"}, + ), + ( + module.IssueAction(5, "Stable present"), + module.IssueAction(6, "Stable present"), + ), + ) + self.assertEqual( + module.plan_issue_actions( + module.DEV_BRANCH, {5: issues[5]}, {5: "Stable present"} + ), + (), + ) + + def test_unknown_stage_fails_before_mutation(self) -> None: + with self.assertRaisesRegex(module.GitHubApiError, "unknown promotion stage"): + module.plan_issue_actions( + module.STABLE_BRANCH, + {1: {"state": "open"}}, + {1: "Needs review"}, + ) + + def test_stable_push_sets_field_without_closing_issue(self) -> None: + client = FakeGitHubClient() + client.commits = [ + {"sha": "a" * 40, "commit": {"message": "fix: linked work (#15)"}} + ] + client.commit_pulls = {"a" * 40: []} + client.issues = {15: {"state": "open"}} + client.stages = {15: "Unstable present"} + + self.assertEqual( + module.process_push( + client=client, + repository=self.repository, + target_branch=module.STABLE_BRANCH, + before="b" * 40, + after="c" * 40, + ), + 0, + ) + self.assertEqual(client.writes, [(15, "Stable present")]) + self.assertEqual(client.issues[15]["state"], "open") + + def test_failed_field_read_prevents_all_writes(self) -> None: + client = FakeGitHubClient() + client.commits = [ + {"sha": "a" * 40, "commit": {"message": "fix: linked work (#15, #16)"}} + ] + client.commit_pulls = {"a" * 40: []} + client.issues = {15: {"state": "open"}, 16: {"state": "open"}} + + def read_stage(repository: str, number: int) -> str | None: + if number == 16: + raise module.GitHubApiError("field API unavailable") + return None + + with patch.object(client, "promotion_stage", side_effect=read_stage): + with self.assertRaisesRegex(module.GitHubApiError, "field API unavailable"): + module.process_push( + client=client, + repository=self.repository, + target_branch=module.DEV_BRANCH, + before="b" * 40, + after="c" * 40, + ) + self.assertEqual(client.writes, []) + + def test_field_api_uses_additive_endpoint(self) -> None: + client = module.GitHubClient("fake-token") + calls: list[tuple[str, str, dict[str, Any] | None]] = [] + + def request(method: str, path: str, *, payload: dict[str, Any] | None = None) -> Any: + calls.append((method, path, payload)) + if method == "GET": + return [ + {"issue_field_id": 1, "value": "High"}, + { + "issue_field_id": module.PROMOTION_FIELD_ID, + "single_select_option": {"name": "Dev present"}, + }, + ] + return {} + + with patch.object(client, "request", side_effect=request): + self.assertEqual(client.promotion_stage(self.repository, 15), "Dev present") + client.set_promotion_stage(self.repository, 15, "Unstable present") + self.assertEqual(calls[-1][0], "POST") + self.assertEqual( + calls[-1][2], + { + "issue_field_values": [ + {"field_id": module.PROMOTION_FIELD_ID, "value": "Unstable present"} + ] + }, ) diff --git a/scripts/qualification/build_resource_envelope_evidence.py b/scripts/qualification/build_resource_envelope_evidence.py new file mode 100644 index 000000000..2ee3477a1 --- /dev/null +++ b/scripts/qualification/build_resource_envelope_evidence.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Build hosted resource-envelope qualification evidence from matrix results. + +Each ``--matrix PLATFORM=PATH`` is one strict ``run_matrix.py`` output from the +hosted qualification workflow. The evidence records the exact CI run, candidate +SHA, fixture manifest digest, and per-fixture measurements for every platform. +It claims ``passed`` only when every required platform passed strict +qualification on the same candidate SHA. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.resource_envelope.run_matrix import matrix_passes + + +EVIDENCE_KIND = "loop-resource-envelope-qualification-evidence" +REQUIRED_PLATFORMS = ("linux", "windows") +MEASUREMENT_FIELDS = ("status", "rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms", "pages_materialized", "page_count") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _platform_record(matrix: Mapping[str, Any], matrix_sha256: str) -> dict[str, Any]: + fixtures: dict[str, Any] = {} + for record in matrix["fixtures"]: + result = record.get("result") if isinstance(record.get("result"), Mapping) else {} + fixtures[record["fixture_id"]] = { + "status": record["status"], + "required": record.get("required", False), + "fixture_sha256": record.get("fixture_sha256"), + "workload": record.get("workload"), + "envelope": {field: result.get(field) for field in MEASUREMENT_FIELDS} if result else None, + "statistics": record.get("statistics"), + "validation_errors": record.get("validation_errors", []), + } + probe = matrix.get("cancellation_recovery_probe") or {} + hostile = matrix.get("hostile") or {} + identity = next((record["identity"] for record in matrix["fixtures"] if isinstance(record.get("identity"), Mapping) and record["identity"]), {}) + return { + "matrix_sha256": matrix_sha256, + "candidate_sha": matrix["candidate_sha"], + "strict_passed": matrix_passes(matrix, strict=True), + "summary": matrix["summary"], + "runtime": {key: identity.get(key) for key in ("os", "qt", "compiler", "cpu", "renderer", "build")}, + "fixtures": fixtures, + "cancellation_recovery_probe": { + "status": probe.get("status", "not-run"), + "fixture_id": probe.get("fixture_id"), + "cancellation_latency_ms": probe.get("cancellation", {}).get("cancellation_latency_ms", -1), + "recovery_ms": probe.get("recovery", {}).get("recovery_ms", -1), + "validation_errors": probe.get("validation_errors", []), + }, + "hostile": { + "summary": hostile.get("summary", {"total": 0, "contained": 0}), + "failed_cases": [case["case_id"] for case in hostile.get("cases", []) if case.get("status") != "contained"], + }, + } + + +def build_evidence( + matrices: Mapping[str, Path], + fixture_manifest: Path, + run_id: str, + run_url: str, +) -> dict[str, Any]: + platforms: dict[str, Any] = {} + for platform, path in sorted(matrices.items()): + matrix = json.loads(path.read_text(encoding="utf-8")) + platforms[platform] = _platform_record(matrix, _sha256(path)) + + candidates = {record["candidate_sha"] for record in platforms.values()} + reasons: list[str] = [] + missing = [platform for platform in REQUIRED_PLATFORMS if platform not in platforms] + reasons.extend(f"platform {platform} produced no matrix" for platform in missing) + if len(candidates) > 1: + reasons.append(f"platforms measured different candidate SHAs: {sorted(candidates)}") + for platform, record in platforms.items(): + if not record["strict_passed"]: + reasons.append(f"platform {platform} did not pass strict qualification") + + rejected = any( + record["summary"]["failed"] or record["cancellation_recovery_probe"]["status"] == "failed" or record["hostile"]["failed_cases"] + for record in platforms.values() + ) + disposition = "passed" if not reasons else ("rejected" if rejected else "incomplete") + manifest = json.loads(fixture_manifest.read_text(encoding="utf-8")) + return { + "schema_kind": EVIDENCE_KIND, + "schema_version": 2, + "issue": 19, + "candidate_sha": next(iter(candidates)) if len(candidates) == 1 else "", + "disposition": disposition, + "disposition_reasons": reasons, + "fixture_manifest_sha256": _sha256(fixture_manifest), + "fixture_generator": manifest.get("generator"), + "ci_runs": [{"platform": platform, "run_id": run_id, "run_url": run_url, "candidate_sha": record["candidate_sha"]} for platform, record in platforms.items()], + "platforms": platforms, + } + + +def _matrix_args(values: Sequence[str]) -> dict[str, Path]: + result: dict[str, Path] = {} + for value in values: + platform, separator, path = value.partition("=") + if not separator or not platform or not path: + raise ValueError("--matrix must be PLATFORM=PATH") + result[platform] = Path(path) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", action="append", default=[], metavar="PLATFORM=PATH") + parser.add_argument("--fixture-manifest", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + evidence = build_evidence(_matrix_args(args.matrix), args.fixture_manifest, args.run_id, args.run_url) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"resource-envelope evidence error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({"disposition": evidence["disposition"], "reasons": evidence["disposition_reasons"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualification/check_core_qualification.py b/scripts/qualification/check_core_qualification.py new file mode 100644 index 000000000..e70853b8e --- /dev/null +++ b/scripts/qualification/check_core_qualification.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Check L01 CI and package provenance without granting module admission.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Sequence + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.ci.compare_package_boundary_evidence import FULL_SHA, compare + + +BUILD_STEPS = ( + "Build Widgets-absent release profile", + "Test Widgets-absent release profile", + "Build project", + "Run unit tests", + "Run preflight corpus gate", +) +REQUIRED_JOBS = { + "source_integrity": ("Verify tracked source integrity",), + "linux / build": BUILD_STEPS, + "windows / build": BUILD_STEPS, +} + + +def load_run(path: Path) -> dict[str, Any]: + snapshot = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(snapshot, dict): + raise ValueError("CI snapshot must be a JSON object") + return snapshot + + +def require_success(record: dict[str, Any], label: str) -> None: + if record.get("status") != "completed" or record.get("conclusion") != "success": + raise ValueError(f"{label} must be completed and successful") + + +def unique_records(records: Any, required: Sequence[str], label: str) -> dict[str, dict[str, Any]]: + if not isinstance(records, list) or any(not isinstance(item, dict) for item in records): + raise ValueError(f"{label} must be an array of objects") + selected = {} + for name in required: + matches = [item for item in records if item.get("name") == name] + if len(matches) != 1: + raise ValueError(f"{label}: expected exactly one {name!r}; found {len(matches)}") + selected[name] = matches[0] + return selected + + +def validate_run(snapshot: dict[str, Any], candidate_sha: str) -> dict[str, dict[str, Any]]: + if snapshot.get("headSha") != candidate_sha: + raise ValueError("CI headSha must equal candidate SHA") + run_id = snapshot.get("databaseId") + if type(run_id) is not int or run_id <= 0: + raise ValueError("CI databaseId must be a positive integer") + if snapshot.get("url") not in ( + f"https://github.com/studio-berry/loop/actions/runs/{run_id}", + f"https://github.com/studio-berry/loop2/actions/runs/{run_id}", + ): + raise ValueError("CI URL must identify this repository and run ID") + require_success(snapshot, "CI run") + jobs = unique_records(snapshot.get("jobs"), tuple(REQUIRED_JOBS), "CI jobs") + job_ids = set() + for name, job in jobs.items(): + require_success(job, name) + job_id = job.get("databaseId") + if type(job_id) is not int or job_id <= 0 or job_id in job_ids: + raise ValueError(f"{name}: expected a unique positive job ID") + job_ids.add(job_id) + steps = unique_records(job.get("steps"), REQUIRED_JOBS[name], f"{name} steps") + for step_name, step in steps.items(): + require_success(step, f"{name}: {step_name}") + return jobs + + +def verify_package(path: Path, identity: dict[str, Any]) -> None: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"package must be a regular file: {path.name}") + if identity.get("name") != path.name: + raise ValueError(f"package name differs from evidence: {path.name}") + expected_size = identity.get("size") + if type(expected_size) is not int or expected_size <= 0: + raise ValueError(f"package evidence needs a positive byte count: {path.name}") + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + if size != expected_size: + raise ValueError(f"package byte count differs from evidence: {path.name}") + if digest.hexdigest() != identity["sha256"].lower(): + raise ValueError(f"package SHA-256 differs from evidence: {path.name}") + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--ci-run", type=Path, required=True, help="gh run view JSON snapshot") + parser.add_argument("--linux-evidence", type=Path, required=True) + parser.add_argument("--windows-evidence", type=Path, required=True) + parser.add_argument("--linux-package", type=Path, required=True) + parser.add_argument("--windows-package", type=Path, required=True) + args = parser.parse_args(argv) + try: + if not FULL_SHA.fullmatch(args.candidate_sha): + raise ValueError("candidate SHA must be a full 40-character Git SHA") + candidate_sha = args.candidate_sha.lower() + snapshot = load_run(args.ci_run) + jobs = validate_run(snapshot, candidate_sha) + pair = compare(args.linux_evidence, args.windows_evidence, candidate_sha) + verify_package(args.linux_package, pair["packages"]["linux"]) + verify_package(args.windows_package, pair["packages"]["windows"]) + except (OSError, ValueError) as error: + print(f"Core qualification provenance rejected: {error}", file=sys.stderr) + return 1 + + print(f"Core qualification provenance verified for {candidate_sha}") + print(f"CI run: {snapshot['url']}") + for name, job in jobs.items(): + print(f"{name}: {snapshot['url']}/job/{job['databaseId']}") + for platform, package in pair["packages"].items(): + print(f"{platform}: {package['name']} bytes={package['size']} sha256={package['sha256']}") + print("Module admission: PENDING REVIEW. Inspect test details, all L01 child evidence, and reviewer decision.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/qualification/test_build_resource_envelope_evidence.py b/scripts/qualification/test_build_resource_envelope_evidence.py new file mode 100644 index 000000000..859728bcb --- /dev/null +++ b/scripts/qualification/test_build_resource_envelope_evidence.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.qualification.build_resource_envelope_evidence import build_evidence +from scripts.qualification.validate_resource_envelope_evidence import validate_evidence +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + +CANDIDATE = "a" * 40 + + +def _matrix(failed_fixture: str | None = None) -> dict: + fixtures = [] + for fixture_id, spec in FIXTURE_SPECS.items(): + if not spec["required"]: + fixtures.append({"fixture_id": fixture_id, "status": "unavailable", "required": False}) + continue + status = "failed" if fixture_id == failed_fixture else "measured" + fixtures.append({ + "fixture_id": fixture_id, + "status": status, + "required": True, + "fixture_sha256": "b" * 64, + "identity": {"os": "test-os", "qt": "6.11.1"}, + "result": {"status": "complete", "rss_high_water_bytes": 100, "preflight_high_water_bytes": 90, "elapsed_ms": 10, "pages_materialized": 1, "page_count": 1}, + }) + failed = int(failed_fixture is not None) + return { + "candidate_sha": CANDIDATE, + "fixtures": fixtures, + "cancellation_recovery_probe": {"status": "measured", "fixture_id": "ten-thousand-page", "cancellation": {"cancellation_latency_ms": 40}, "recovery": {"recovery_ms": 900}}, + "hostile": {"summary": {"total": 7, "contained": 7}, "cases": []}, + "summary": {"total": len(fixtures), "measured": len(fixtures) - 1 - failed, "flagged": 0, "skipped": 1, "failed": failed, "candidate_sha_verified": True}, + } + + +class BuildResourceEnvelopeEvidenceTest(unittest.TestCase): + def _build(self, matrices: dict[str, dict]) -> dict: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = {} + for platform, matrix in matrices.items(): + paths[platform] = root / f"{platform}.json" + paths[platform].write_text(json.dumps(matrix), encoding="utf-8") + manifest = root / "fixtures.json" + manifest.write_text(json.dumps({"generator": {"version": 1}}), encoding="utf-8") + return build_evidence(paths, manifest, "123456", "https://github.com/studio-berry/loop/actions/runs/123456") + + def test_both_platforms_passing_is_valid_passed_evidence(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + self.assertEqual(evidence["disposition"], "passed", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_missing_platform_is_incomplete(self) -> None: + evidence = self._build({"linux": _matrix()}) + self.assertEqual(evidence["disposition"], "incomplete") + self.assertIn("platform windows produced no matrix", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_failed_fixture_is_rejected(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix(failed_fixture="image-heavy-500mb")}) + self.assertEqual(evidence["disposition"], "rejected") + self.assertEqual(validate_evidence(evidence), []) + + def test_passed_claim_with_unavailable_measurement_is_invalid(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + tampered = copy.deepcopy(evidence) + tampered["platforms"]["linux"]["fixtures"]["office-2mb"]["envelope"]["preflight_high_water_bytes"] = -1 + tampered["platforms"]["windows"]["cancellation_recovery_probe"]["recovery_ms"] = -1 + errors = "\n".join(validate_evidence(tampered)) + self.assertIn("platforms.linux.fixtures.office-2mb.preflight_high_water_bytes is unavailable", errors) + self.assertIn("platforms.windows.cancellation_recovery_probe.recovery_ms is unavailable", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/qualification/test_check_core_qualification.py b/scripts/qualification/test_check_core_qualification.py new file mode 100644 index 000000000..22c5d788a --- /dev/null +++ b/scripts/qualification/test_check_core_qualification.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import contextlib +import copy +import hashlib +import io +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from scripts.qualification import check_core_qualification as checker + + +class CoreQualificationTest(unittest.TestCase): + candidate_sha = "a" * 40 + + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.run_path = self.root / "run.json" + self.snapshot = { + "databaseId": 100, + "url": "https://github.com/studio-berry/loop/actions/runs/100", + "headSha": self.candidate_sha, + "status": "completed", + "conclusion": "success", + "jobs": [ + self.job("source_integrity", 1, ["Verify tracked source integrity"]), + self.job("linux / build", 2, self.build_steps()), + self.job("windows / build", 3, self.build_steps()), + ], + } + self.records = {} + for platform, name, package_format in ( + ("linux", "Loop.AppImage", "AppImage"), + ("windows", "Loop.msi", "MSI"), + ): + package = self.root / name + payload = f"test package for {platform}".encode() + package.write_bytes(payload) + self.records[platform] = { + "schema_version": 1, + "kind": "loop-package-boundary-evidence", + "platform": platform, + "status": "passed", + "source_sha": self.candidate_sha, + "forbidden_findings": [], + "checks": { + "all_payload_files_hashed": True, + "all_binary_files_inspected": True, + "target_architecture_matches": True, + "qt6widgets_absent": True, + "qt6widgets_surface_absent": True, + "unresolved_non_system_dependencies_absent": True, + }, + "package": { + "name": name, + "format": package_format, + "size": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + }, + } + self.arguments = [ + "--candidate-sha", self.candidate_sha, + "--ci-run", str(self.run_path), + "--linux-evidence", str(self.root / "linux.json"), + "--windows-evidence", str(self.root / "windows.json"), + "--linux-package", str(self.root / "Loop.AppImage"), + "--windows-package", str(self.root / "Loop.msi"), + ] + + @staticmethod + def build_steps() -> list[str]: + return [ + "Build Widgets-absent release profile", + "Test Widgets-absent release profile", + "Build project", + "Run unit tests", + "Run preflight corpus gate", + ] + + @staticmethod + def job(name: str, identifier: int, steps: list[str]) -> dict: + return { + "name": name, + "databaseId": identifier, + "status": "completed", + "conclusion": "success", + "steps": [ + {"name": step, "status": "completed", "conclusion": "success"} + for step in steps + ], + } + + def write_inputs(self) -> None: + self.run_path.write_text(json.dumps(self.snapshot), encoding="utf-8") + for platform, record in self.records.items(): + (self.root / f"{platform}.json").write_text(json.dumps(record), encoding="utf-8") + + def run_checker(self) -> tuple[int, str, str]: + self.write_inputs() + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + code = checker.main(self.arguments) + return code, output.getvalue(), errors.getvalue() + + def assert_rejected(self, reason: str) -> None: + code, output, errors = self.run_checker() + self.assertEqual(code, 1) + self.assertEqual(output, "") + self.assertIn(reason, errors) + + def test_valid_evidence_verifies_provenance_and_keeps_admission_pending(self) -> None: + code, output, errors = self.run_checker() + self.assertEqual(code, 0, errors) + self.assertIn(self.candidate_sha, output) + self.assertIn("Module admission: PENDING REVIEW", output) + self.assertIn("/job/3", output) + self.assertIn(self.records["linux"]["package"]["sha256"], output) + + def test_direct_script_invocation(self) -> None: + self.write_inputs() + result = subprocess.run( + [sys.executable, str(Path(checker.__file__)), *self.arguments], + cwd=self.root, capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("PENDING REVIEW", result.stdout) + + def test_candidate_must_be_a_full_sha(self) -> None: + self.arguments[1] = "dev" + self.assert_rejected("full 40-character") + + def test_mixed_ci_and_package_shas_are_rejected(self) -> None: + self.snapshot["headSha"] = "b" * 40 + self.assert_rejected("CI headSha") + self.snapshot["headSha"] = self.candidate_sha + self.records["windows"]["source_sha"] = "b" * 40 + self.assert_rejected("source SHA mismatch") + + def test_aggregate_success_does_not_hide_bad_required_jobs(self) -> None: + for conclusion in ("skipped", "cancelled", "failure", None): + with self.subTest(conclusion=conclusion): + self.snapshot["jobs"][2]["conclusion"] = conclusion + self.assert_rejected("windows / build must be completed and successful") + self.snapshot["jobs"][2]["conclusion"] = "success" + self.snapshot["jobs"][2]["status"] = "in_progress" + self.assert_rejected("windows / build must be completed and successful") + + def test_required_jobs_and_steps_cannot_be_missing_or_duplicated(self) -> None: + original = copy.deepcopy(self.snapshot) + for missing in (True, False): + with self.subTest(job_missing=missing): + self.snapshot = copy.deepcopy(original) + if missing: + self.snapshot["jobs"].pop() + else: + self.snapshot["jobs"].append(copy.deepcopy(self.snapshot["jobs"][2])) + self.assert_rejected("expected exactly one 'windows / build'") + with self.subTest(step_missing=missing): + self.snapshot = copy.deepcopy(original) + steps = self.snapshot["jobs"][1]["steps"] + if missing: + steps.pop() + else: + steps.append(copy.deepcopy(steps[-1])) + self.assert_rejected("expected exactly one 'Run preflight corpus gate'") + + def test_skipped_required_step_is_rejected_but_unrelated_skip_is_allowed(self) -> None: + self.snapshot["jobs"][1]["steps"][3]["conclusion"] = "skipped" + self.assert_rejected("Run unit tests must be completed and successful") + self.snapshot["jobs"][1]["steps"][3]["conclusion"] = "success" + self.snapshot["jobs"][1]["steps"].append( + {"name": "Prove change with agent-fast", "status": "completed", "conclusion": "skipped"} + ) + self.snapshot["jobs"].append( + {"name": "agent-fast / build", "status": "completed", "conclusion": "skipped"} + ) + self.assertEqual(self.run_checker()[0], 0) + + def test_each_required_step_must_complete_successfully(self) -> None: + for job in self.snapshot["jobs"]: + for step in job["steps"]: + with self.subTest(job=job["name"], step=step["name"]): + step["status"] = "in_progress" + self.assert_rejected(f"{step['name']} must be completed and successful") + step["status"] = "completed" + + def test_run_must_be_terminal_and_from_this_repository(self) -> None: + self.snapshot["conclusion"] = "failure" + self.assert_rejected("CI run must be completed and successful") + self.snapshot["conclusion"] = "success" + self.snapshot["url"] = "https://github.com/other/repo/actions/runs/100" + self.assert_rejected("CI URL") + + def test_malformed_run_records_fail_explicitly(self) -> None: + original = copy.deepcopy(self.snapshot) + for malformed in ([], None, "jobs", [None]): + with self.subTest(jobs=malformed): + self.snapshot = copy.deepcopy(original) + self.snapshot["jobs"] = malformed + self.assert_rejected("CI jobs") + self.snapshot = copy.deepcopy(original) + self.snapshot["jobs"][1]["steps"] = [None] + self.assert_rejected("steps must be an array of objects") + self.snapshot = [] + self.assert_rejected("CI snapshot must be a JSON object") + + def test_incomplete_and_malformed_package_evidence_are_rejected(self) -> None: + self.records["linux"]["checks"]["all_payload_files_hashed"] = False + self.assert_rejected("checks are incomplete") + self.records["linux"] = [] + self.assert_rejected("package evidence must be a JSON object") + + def test_package_identity_requires_name_size_and_actual_bytes(self) -> None: + identity = self.records["linux"]["package"] + original = copy.deepcopy(identity) + for size in (None, True, -1, original["size"] + 1): + with self.subTest(size=size): + identity["size"] = size + self.assert_rejected("byte count") + identity.update(original) + identity["name"] = "Other.AppImage" + self.assert_rejected("name differs") + identity.update(original) + (self.root / "Loop.AppImage").write_bytes(b"x" * original["size"]) + self.assert_rejected("SHA-256 differs") + + def test_invalid_json_or_absent_package_is_rejected(self) -> None: + self.write_inputs() + self.run_path.write_text("{", encoding="utf-8") + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + self.assertEqual(checker.main(self.arguments), 1) + self.assertIn("rejected", errors.getvalue()) + (self.root / "Loop.msi").unlink() + self.assert_rejected("package must be a regular file") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/qualification/validate_resource_envelope_evidence.py b/scripts/qualification/validate_resource_envelope_evidence.py index 9f1a9478c..b4922d09c 100644 --- a/scripts/qualification/validate_resource_envelope_evidence.py +++ b/scripts/qualification/validate_resource_envelope_evidence.py @@ -1,5 +1,12 @@ #!/usr/bin/env python3 -"""Validate frozen Session 11 resource-envelope qualification evidence.""" +"""Validate resource-envelope qualification evidence. + +Schema version 1 is the frozen Session 11 record and can never claim +``passed``. Schema version 2 is built from the hosted qualification workflow +(``build_resource_envelope_evidence.py``) and may claim ``passed`` only when +every required platform measured every required fixture, the cancellation +and recovery probe, and the hostile corpus on one candidate SHA. +""" from __future__ import annotations @@ -24,6 +31,7 @@ REQUIRED_FIXTURES = tuple( fixture_id for fixture_id, spec in FIXTURE_SPECS.items() if spec["required"] ) +REQUIRED_PLATFORMS = ("linux", "windows") def validate_evidence( @@ -31,11 +39,100 @@ def validate_evidence( *, manifest: dict[str, Any] | None = None, ) -> list[str]: - errors: list[str] = [] if evidence.get("schema_kind") != "loop-resource-envelope-qualification-evidence": - errors.append("schema_kind must be loop-resource-envelope-qualification-evidence") + return ["schema_kind must be loop-resource-envelope-qualification-evidence"] + if evidence.get("schema_version") == 2: + return _validate_hosted_evidence(evidence) if evidence.get("schema_version") != 1: - errors.append("schema_version must be 1") + return ["schema_version must be 1 or 2"] + return _validate_session_11_evidence(evidence, manifest) + + +def _is_measured(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _validate_platform(platform: str, record: Any, candidate_sha: Any) -> list[str]: + if not isinstance(record, dict): + return [f"platforms.{platform} must be an object"] + errors: list[str] = [] + if record.get("candidate_sha") != candidate_sha: + errors.append(f"platforms.{platform}.candidate_sha must equal candidate_sha") + if record.get("strict_passed") is not True: + errors.append(f"platforms.{platform} did not pass strict qualification") + if not isinstance(record.get("matrix_sha256"), str) or not SHA256_RE.fullmatch(record["matrix_sha256"]): + errors.append(f"platforms.{platform}.matrix_sha256 must be a SHA-256 digest") + fixtures = record.get("fixtures") if isinstance(record.get("fixtures"), dict) else {} + for fixture_id in REQUIRED_FIXTURES: + entry = fixtures.get(fixture_id) + envelope = entry.get("envelope") if isinstance(entry, dict) else None + if not isinstance(entry, dict) or entry.get("status") != "measured" or not isinstance(envelope, dict): + errors.append(f"platforms.{platform}.fixtures.{fixture_id} is not measured") + continue + if envelope.get("status") != "complete": + errors.append(f"platforms.{platform}.fixtures.{fixture_id} envelope is not complete") + for field in ("rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms"): + if not _is_measured(envelope.get(field)): + errors.append(f"platforms.{platform}.fixtures.{fixture_id}.{field} is unavailable") + probe = record.get("cancellation_recovery_probe") if isinstance(record.get("cancellation_recovery_probe"), dict) else {} + if probe.get("status") != "measured": + errors.append(f"platforms.{platform} cancellation/recovery probe is not measured") + for field in ("cancellation_latency_ms", "recovery_ms"): + if not _is_measured(probe.get(field)): + errors.append(f"platforms.{platform}.cancellation_recovery_probe.{field} is unavailable") + hostile = record.get("hostile") if isinstance(record.get("hostile"), dict) else {} + summary = hostile.get("summary") if isinstance(hostile.get("summary"), dict) else {} + if not _is_measured(summary.get("total")) or summary.get("total") == 0 or summary.get("contained") != summary.get("total"): + errors.append(f"platforms.{platform} hostile corpus was not fully contained") + return errors + + +def _validate_hosted_evidence(evidence: dict[str, Any]) -> list[str]: + errors: list[str] = [] + candidate_sha = evidence.get("candidate_sha") + if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): + errors.append("candidate_sha must be a 40-character lowercase commit SHA") + if not isinstance(evidence.get("fixture_manifest_sha256"), str) or not SHA256_RE.fullmatch(evidence["fixture_manifest_sha256"]): + errors.append("fixture_manifest_sha256 must be a SHA-256 digest") + disposition = evidence.get("disposition") + if disposition not in {"incomplete", "passed", "rejected"}: + errors.append("disposition must be incomplete, passed, or rejected") + reasons = evidence.get("disposition_reasons") + if not isinstance(reasons, list): + errors.append("disposition_reasons must be an array") + elif disposition == "passed" and reasons: + errors.append("passed evidence cannot carry disposition_reasons") + elif disposition != "passed" and not reasons: + errors.append("non-passed evidence must state its disposition_reasons") + + runs = evidence.get("ci_runs") + if not isinstance(runs, list) or not runs: + errors.append("ci_runs must be a non-empty array") + else: + for index, run in enumerate(runs): + if not isinstance(run, dict) or not str(run.get("run_id", "")).isdigit(): + errors.append(f"ci_runs[{index}].run_id must be a GitHub Actions run id") + continue + if not str(run.get("run_url", "")).startswith("https://github.com/"): + errors.append(f"ci_runs[{index}].run_url must be a GitHub Actions run URL") + if run.get("candidate_sha") != candidate_sha: + errors.append(f"ci_runs[{index}].candidate_sha must equal candidate_sha") + + platforms = evidence.get("platforms") + if not isinstance(platforms, dict) or not platforms: + errors.append("platforms must be a non-empty object") + return errors + if disposition == "passed": + for platform in REQUIRED_PLATFORMS: + if platform not in platforms: + errors.append(f"passed evidence is missing platform {platform}") + for platform, record in platforms.items(): + errors.extend(_validate_platform(platform, record, candidate_sha)) + return errors + + +def _validate_session_11_evidence(evidence: dict[str, Any], manifest: dict[str, Any] | None) -> list[str]: + errors: list[str] = [] candidate_sha = evidence.get("candidate_sha") if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index ae5eeab31..5d732e683 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -3,7 +3,13 @@ Large PDFs stay outside the repository. A qualification run should use a manifest with exact fixture digests and sizes; the legacy ``--fixture`` form is -kept for exploratory runs and is intentionally not sufficient for ``--strict``. +intentionally not sufficient for ``--strict``. + +Besides the measured fixtures, a strict run carries a cancellation probe that +interrupts one fixture, a recovery probe that times a fresh process reopening +it, and a hostile lane that feeds the checked-in budget-exhaustion PDFs to +PdfTool. A crash, timeout, or skipped workload never counts as a passing +envelope. """ from __future__ import annotations @@ -26,20 +32,44 @@ ROOT = Path(__file__).resolve().parents[2] DEFAULT_BUDGETS = ROOT / "docs" / "RESOURCE_ENVELOPE_BUDGETS.json" +DEFAULT_PREFLIGHT_PROFILE = ROOT / "loop-preflight" / "profiles" / "loop-default.json" +DEFAULT_HOSTILE_CORPUS = ROOT / "UnitTests" / "testdata" / "budget_exhaustion" MATRIX_KIND = "loop-resource-envelope-matrix" -DEFAULT_RASTERIZERS = 8 +# Each rasterizer holds one page image at a time. A 300 DPI Letter page is +# 33.7 MB, so three fit the 128 MiB raster-tile-cache pool and a fourth is +# rejected as budget-exceeded, leaving pages unrendered (exit code 5). +DEFAULT_RASTERIZERS = 3 +# Preflight renders and walks every page it covers (about 0.5-0.7 s per page on +# a hosted runner), so a full pass over a very large document outlives any +# practical timeout. Documents above the threshold get a preflight phase over +# their first pages only; rendering still covers every page. +PREFLIGHT_SAMPLE_THRESHOLD_PAGES = 1000 +PREFLIGHT_SAMPLE_PAGES = 256 +TIMEOUT_REASON = "benchmark-timeout" +# PdfTool's defined terminal exit codes (pdftoolresult.h) except InternalError +# (7). Anything else, including a negative POSIX signal or a Windows exception +# status, means the process did not reach a controlled disposition. +CONTAINED_EXIT_CODES = frozenset({0, 1, 2, 3, 4, 5, 6, 8, 9}) +EXIT_SUCCESS = 0 +EXIT_CANCELLED = 6 +# Validation errors carrying one of these markers fail the record outright; +# every other error only flags it. +HARD_ERROR_MARKERS = ("does not match", "exceeds", "identity", "fixture SHA", "manifest", "timeout", "crashed") # These names mirror issue #242. multi-gb is optional because platform # addressability and available disk are environment-dependent. FIXTURE_SPECS: dict[str, dict[str, Any]] = { "office-2mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 1_500_000, "max_bytes": 2_500_000}, - "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 450_000_000, "max_bytes": 550_000_000}, + "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": "large-document-500mb", "min_bytes": 450_000_000, "max_bytes": 550_000_000}, "multi-gb": {"required": False, "expected_page_count": None, "workload": None, "min_bytes": 1_000_000_000, "max_bytes": None}, "ten-thousand-page": {"required": True, "expected_page_count": 10000, "workload": "div2k-image-heavy", "min_bytes": None, "max_bytes": None}, "pathological-vector": {"required": True, "expected_page_count": 256, "workload": "pathological-vector", "min_bytes": None, "max_bytes": None}, "transparency-spots": {"required": True, "expected_page_count": 256, "workload": None, "min_bytes": None, "max_bytes": None}, } +Runner = Callable[..., subprocess.CompletedProcess[str]] +CancelRunner = Callable[[list[str], float, float | None], subprocess.CompletedProcess[str]] + def _sha256(path: Path) -> str: digest = hashlib.sha256() @@ -73,6 +103,40 @@ def _envelope_from_output(payload: Mapping[str, Any]) -> dict[str, Any] | None: return None +def _envelope_from_process(completed: subprocess.CompletedProcess[str]) -> dict[str, Any] | None: + payload = _extract_json(completed.stdout or "") + return _envelope_from_output(payload) if payload else None + + +def _find_key(value: Any, key: str) -> Any: + if isinstance(value, Mapping): + if key in value: + return value[key] + children: Sequence[Any] = list(value.values()) + elif isinstance(value, list): + children = value + else: + return None + for child in children: + found = _find_key(child, key) + if found is not None: + return found + return None + + +def _failure_detail(completed: subprocess.CompletedProcess[str]) -> str: + """Short, log-sized account of why a PdfTool run did not succeed.""" + payload = _extract_json(completed.stdout or "") + errors = _find_key(payload, "rendering-errors") if payload else None + parts = [] + if errors is not None: + parts.append("rendering-errors=" + json.dumps(errors)[:800]) + stderr = (completed.stderr or "").strip() + if stderr: + parts.append("stderr=" + stderr[-400:]) + return "; ".join(parts) + + def _git_head() -> str: try: return subprocess.run( @@ -97,6 +161,48 @@ def _candidate_identity() -> dict[str, Any]: } +def _benchmark_command( + pdf_tool: Path, + fixture_path: Path, + rasterizers: int, + preflight_profile: Path | None, + first_page_only: bool = False, + preflight_page_last: int | None = None, +) -> list[str]: + # Pin rasterizers to a fixed value so the same code and fixtures produce + # comparable RSS and elapsed time across hosts with different CPU counts. + # The value is recorded in the result profile. + command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + if preflight_profile is not None: + command += ["--profile", str(preflight_profile)] + if preflight_page_last is not None: + command += ["--preflight-page-last", str(preflight_page_last)] + if first_page_only: + command += ["--page-first", "1", "--page-last", "1"] + return command + + +def _preflight_page_last(page_count: int | None) -> int | None: + """Last page of the preflight phase, or None when it covers the whole document.""" + if page_count is not None and page_count > PREFLIGHT_SAMPLE_THRESHOLD_PAGES: + return PREFLIGHT_SAMPLE_PAGES + return None + + +def _identity_errors(envelope: Mapping[str, Any], candidate_sha: str, fixture_sha256: str) -> list[str]: + identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} + errors: list[str] = [] + if identity.get("commit") != candidate_sha: + errors.append(f"identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") + if identity.get("fixture_digest") != fixture_sha256: + errors.append(f"identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {fixture_sha256!r}") + return errors + + +def _is_hard(error: str) -> bool: + return any(marker in error for marker in HARD_ERROR_MARKERS) + + def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_after_seconds: float | None) -> subprocess.CompletedProcess[str]: creationflags = 0 popen_kwargs: dict[str, Any] = {} @@ -127,9 +233,25 @@ def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_af except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired(command, timeout_seconds, stdout, stderr) return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) +def _timed_run(runner: Runner, command: list[str], timeout_seconds: float) -> tuple[subprocess.CompletedProcess[str] | None, str, int]: + """Runs one PdfTool process; returns (completed, failure reason, wall ms).""" + started = time.monotonic() + try: + completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + return None, TIMEOUT_REASON, int((time.monotonic() - started) * 1000) + except OSError as exc: + return None, f"benchmark-launch-failed:{exc}", -1 + wall_ms = int((time.monotonic() - started) * 1000) + if completed.returncode not in CONTAINED_EXIT_CODES: + return completed, f"process-crashed:{completed.returncode}", wall_ms + return completed, "", wall_ms + + def _empty_result(fixture_id: str, reason: str) -> dict[str, Any]: return { "fixture_id": fixture_id, @@ -208,6 +330,25 @@ def _fixture_metadata(fixture_id: str, fixture_path: Path, metadata: Mapping[str return details, errors +def _fixture_workload(fixture_id: str, metadata: Mapping[str, Any] | None) -> str | None: + if metadata is not None and "workload" in metadata: + return str(metadata["workload"]) + return FIXTURE_SPECS[fixture_id]["workload"] + + +def _rss_limit(budgets: Mapping[str, Any], workload: str | None) -> int | None: + """Process RSS ceiling: the workload's own cap when it declares one, else the resident limit. + + The reader holds a whole document in memory, so a very large document's + process RSS is a multiple of its file size and cannot fit the resident + limit that governs the accounted pools. + """ + workload_limit = budgets.get("workloads", {}).get(workload, {}).get("rss_high_water_bytes") if workload else None + if isinstance(workload_limit, int): + return workload_limit + return budgets.get("resource_budget", {}).get("resident_limit_bytes") + + def _aggregate_envelopes(envelopes: list[Mapping[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: # Use the highest-RSS run as the safety representative and the median # elapsed time. This keeps peak-memory validation conservative while @@ -236,13 +377,13 @@ def run_fixture( timeout_seconds: float, baseline: Mapping[str, Any] | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, metadata: Mapping[str, Any] | None = None, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, require_provenance: bool = False, - cancel_after_seconds: float | None = None, candidate_sha: str | None = None, + preflight_profile: Path | None = None, ) -> dict[str, Any]: if repetitions < 1 or rasterizers < 1: raise ValueError("repetitions and rasterizers must be positive") @@ -254,17 +395,17 @@ def run_fixture( pdf_tool = Path(pdf_tool).resolve() fixture_path = Path(fixture_path).resolve() spec = FIXTURE_SPECS[fixture_id] + workload = _fixture_workload(fixture_id, metadata) fixture_details, provenance_errors = _fixture_metadata(fixture_id, fixture_path, metadata, require_provenance) - # Pin rasterizers to a fixed value (8) so the same code and fixtures - # produce comparable RSS and elapsed time across hosts with different - # CPU counts. The value is recorded in the result profile. - command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + expected_page_count = metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"] + preflight_page_last = _preflight_page_last(expected_page_count) + command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile, preflight_page_last=preflight_page_last) record: dict[str, Any] = { "fixture_id": fixture_id, - "path": str(fixture_path.resolve()), - "expected_page_count": metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"], - "workload": spec["workload"], - "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers}, + "path": str(fixture_path), + "expected_page_count": expected_page_count, + "workload": workload, + "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers, "preflight_profile": str(preflight_profile) if preflight_profile else None, "preflight_page_last": preflight_page_last if preflight_profile else None}, "command": command, **fixture_details, } @@ -274,78 +415,216 @@ def run_fixture( runs: list[dict[str, Any]] = [] envelopes: list[Mapping[str, Any]] = [] - for index in range(repetitions): - try: - if runner is subprocess.run and cancel_after_seconds is not None: - completed = _run_benchmark_process(command, timeout_seconds, cancel_after_seconds) - else: - completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) - except subprocess.TimeoutExpired: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-timeout", "process_exit_code": None}) - continue - except OSError as exc: - runs.append({"run": index + 1, "status": "unavailable", "reason": f"benchmark-launch-failed:{exc}", "process_exit_code": None}) - continue - payload = _extract_json(completed.stdout) - envelope = _envelope_from_output(payload) if payload else None - if envelope is None: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-envelope-missing", "process_exit_code": completed.returncode, "stderr": completed.stderr[-2000:]}) + validation_errors: list[str] = [] + for index in range(1, repetitions + 1): + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + envelope = _envelope_from_process(completed) if completed is not None else None + exit_code = completed.returncode if completed is not None else None + if failure or envelope is None: + reason = failure or "benchmark-envelope-missing" + run: dict[str, Any] = {"run": index, "status": "unavailable", "reason": reason, "process_exit_code": exit_code, "process_wall_ms": wall_ms} + if completed is not None: + run["stderr"] = (completed.stderr or "")[-2000:] + run["detail"] = _failure_detail(completed) + runs.append(run) + validation_errors.append(f"run {index}: {reason}") + if failure == TIMEOUT_REASON: + break continue envelopes.append(envelope) - runs.append({"run": index + 1, "status": "recorded", "process_exit_code": completed.returncode, "result": envelope}) - - if not envelopes: - record.update({"status": "unavailable", "result": None, "runs": runs, "validation_errors": [], "regressions": []}) - return record - - representative, stats = _aggregate_envelopes(envelopes) - validation_errors: list[str] = [] - for index, envelope in enumerate(envelopes, start=1): - for error in validate_envelope(envelope, budgets, spec["workload"]): + runs.append({"run": index, "status": "recorded", "process_exit_code": exit_code, "process_wall_ms": wall_ms, "result": envelope}) + if exit_code != EXIT_SUCCESS: + runs[-1]["detail"] = _failure_detail(completed) + validation_errors.append(f"run {index}: process exit code {exit_code} is not success") + for error in validate_envelope(envelope, budgets, workload): validation_errors.append(f"run {index}: {error}") expected_page_count = record["expected_page_count"] if expected_page_count is not None and envelope.get("page_count") != expected_page_count: validation_errors.append(f"run {index}: page_count {envelope.get('page_count')} does not match expected {expected_page_count}") rss = envelope.get("rss_high_water_bytes") - resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") - if isinstance(rss, int) and rss >= 0 and isinstance(resident_limit, int) and rss > resident_limit: - validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {resident_limit}") - identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} - if identity.get("commit") != candidate_sha: - validation_errors.append(f"run {index}: identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") - if identity.get("fixture_digest") != record.get("fixture_sha256"): - validation_errors.append(f"run {index}: identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {record.get('fixture_sha256')!r}") + rss_limit = _rss_limit(budgets, workload) + if isinstance(rss, int) and rss >= 0 and isinstance(rss_limit, int) and rss > rss_limit: + validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {rss_limit}") + validation_errors.extend(f"run {index}: {error}" for error in _identity_errors(envelope, candidate_sha, record["fixture_sha256"])) + + record["runs"] = runs + record["validation_errors"] = sorted(set(validation_errors)) + record["regressions"] = [] + if not envelopes: + record["result"] = None + record["status"] = "failed" if any(_is_hard(error) for error in record["validation_errors"]) else "unavailable" + return record + + representative, stats = _aggregate_envelopes(envelopes) record["identity"] = representative.get("identity", {}) record["result"] = representative record["statistics"] = stats - record["runs"] = runs - unavailable_runs = [run for run in runs if run["status"] != "recorded"] - validation_errors.extend( - f"run {run['run']}: {run['reason']}" for run in unavailable_runs - ) - record["validation_errors"] = sorted(set(validation_errors)) comparison = dict(representative) comparison["fixture_sha256"] = record["fixture_sha256"] comparison["identity"] = record["identity"] record["regressions"] = _regressions(comparison, baseline, margin) - if cancel_after_seconds is not None: - if representative.get("status") != "cancelled": - validation_errors.append("cancellation probe did not produce a cancelled envelope") - if not isinstance(representative.get("cancellation_latency_ms"), int) or representative["cancellation_latency_ms"] < 0: - validation_errors.append("cancellation probe did not report cancellation latency") - record["cancellation_probe"] = {"requested_after_seconds": cancel_after_seconds} - record["validation_errors"] = sorted(set(validation_errors)) - hard_error_markers = ("does not match", "exceeds", "identity", "fixture SHA", "manifest") - hard_errors = [error for error in record["validation_errors"] if any(marker in error for marker in hard_error_markers)] - if record["regressions"] or hard_errors: + if record["regressions"] or any(_is_hard(error) for error in record["validation_errors"]): record["status"] = "failed" - elif record["validation_errors"] or representative.get("status") != "complete": + elif record["validation_errors"] or any(envelope.get("status") != "complete" for envelope in envelopes): record["status"] = "flagged" else: record["status"] = "measured" return record +def run_cancellation_probe( + pdf_tool: Path, + fixture_id: str, + fixture_path: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + cancel_after_seconds: float, + candidate_sha: str, + workload: str | None = None, + rasterizers: int = DEFAULT_RASTERIZERS, + cancel_runner: CancelRunner = _run_benchmark_process, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Interrupts one render run, then times a fresh process reopening the fixture. + + The interrupted run has no preflight phase: preflight setup (fonts, ink + mapper, resource scan) is document-wide and does not poll for cancellation, + so on a very large document it would dominate the latency this probe + measures. Preflight cancellation is covered by the evidence-graph tests. + + ``recovery_ms`` is the wall time from launching that fresh process until it + exits having rendered the first page: the time an operator waits to get the + document back after abandoning a run. + """ + pdf_tool = Path(pdf_tool).resolve() + fixture_path = Path(fixture_path).resolve() + fixture_sha256 = _sha256(fixture_path) + limits = budgets.get("workloads", {}).get(workload, {}) if workload else {} + errors: list[str] = [] + + cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None) + cancellation: dict[str, Any] = {"command": cancel_command, "requested_after_seconds": cancel_after_seconds, "cancellation_latency_ms": -1} + try: + completed = cancel_runner(cancel_command, timeout_seconds, cancel_after_seconds) + except subprocess.TimeoutExpired: + completed = None + errors.append("cancellation probe timeout: the process did not stop after the interrupt") + except OSError as exc: + completed = None + errors.append(f"cancellation probe launch failed: {exc}") + if completed is not None: + cancellation["process_exit_code"] = completed.returncode + envelope = _envelope_from_process(completed) + if completed.returncode not in CONTAINED_EXIT_CODES: + errors.append(f"cancellation probe crashed with exit code {completed.returncode}") + elif envelope is None: + errors.append("cancellation probe produced no envelope") + else: + cancellation["result"] = envelope + latency = envelope.get("cancellation_latency_ms") + if envelope.get("status") != "cancelled": + errors.append(f"cancellation probe ended {envelope.get('status')!r}; the interrupt arrived after the run finished or was ignored") + if completed.returncode != EXIT_CANCELLED: + errors.append(f"cancellation probe exit code {completed.returncode} is not Cancelled ({EXIT_CANCELLED})") + if not isinstance(latency, int) or isinstance(latency, bool) or latency < 0: + errors.append("cancellation probe did not report cancellation latency") + else: + cancellation["cancellation_latency_ms"] = latency + limit = limits.get("cancellation_latency_ms") + if isinstance(limit, int) and latency > limit: + errors.append(f"cancellation_latency_ms {latency} exceeds workload policy {limit}") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + + recovery_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None, first_page_only=True) + recovery: dict[str, Any] = {"command": recovery_command, "recovery_ms": -1} + completed, failure, wall_ms = _timed_run(runner, recovery_command, timeout_seconds) + if completed is not None: + recovery["process_exit_code"] = completed.returncode + if failure: + errors.append(f"recovery probe {failure}") + elif completed is not None: + envelope = _envelope_from_process(completed) + if envelope is None: + errors.append("recovery probe produced no envelope") + else: + recovery["result"] = envelope + if completed.returncode != EXIT_SUCCESS: + errors.append(f"recovery probe exit code {completed.returncode} is not success") + if envelope.get("pages_materialized") != 1: + errors.append(f"recovery probe materialized {envelope.get('pages_materialized')!r} pages, expected 1") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + recovery["recovery_ms"] = wall_ms + limit = limits.get("recovery_ms") + if isinstance(limit, int) and wall_ms > limit: + errors.append(f"recovery_ms {wall_ms} exceeds workload policy {limit}") + + return { + "fixture_id": fixture_id, + "workload": workload, + "fixture_sha256": fixture_sha256, + "status": "failed" if errors else "measured", + "cancellation": cancellation, + "recovery": recovery, + "validation_errors": errors, + } + + +def run_hostile_corpus( + pdf_tool: Path, + corpus_dir: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + rasterizers: int = DEFAULT_RASTERIZERS, + preflight_profile: Path | None = None, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Requires every hostile PDF to end in a contained PdfTool disposition. + + Rejecting the input (InputError, ProcessingFailure, budget-exceeded) is a + correct outcome here; crashing, hanging, or breaching the resident ceiling + is not. + """ + pdf_tool = Path(pdf_tool).resolve() + corpus_dir = Path(corpus_dir).resolve() + manifest = json.loads((corpus_dir / "manifest.json").read_text(encoding="utf-8")) + cases = manifest.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError(f"hostile corpus manifest has no cases: {corpus_dir}") + resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") + records: list[dict[str, Any]] = [] + for case in cases: + path = corpus_dir / str(case["pdf"]) + record: dict[str, Any] = {"case_id": case["id"], "path": str(path), "expected": case.get("expected")} + errors: list[str] = [] + if not path.is_file() or _sha256(path) != case.get("sha256"): + errors.append("hostile fixture SHA-256 does not match manifest") + else: + command = _benchmark_command(pdf_tool, path, rasterizers, preflight_profile) + record["command"] = command + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + record["process_wall_ms"] = wall_ms + if completed is not None: + record["process_exit_code"] = completed.returncode + if failure: + errors.append(failure) + elif completed is not None: + envelope = _envelope_from_process(completed) + record["disposition"] = envelope.get("status") if envelope else "rejected" + if envelope is not None: + record["result"] = envelope + rss = envelope.get("rss_high_water_bytes") + if isinstance(rss, int) and isinstance(resident_limit, int) and rss > resident_limit: + errors.append(f"RSS {rss} exceeds resident policy {resident_limit}") + record["validation_errors"] = errors + record["status"] = "failed" if errors else "contained" + records.append(record) + return { + "corpus": str(corpus_dir), + "cases": records, + "summary": {"total": len(records), "contained": sum(record["status"] == "contained" for record in records)}, + } + + def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("schema_kind") != "loop-resource-envelope-fixtures" or payload.get("schema_version") != 1: @@ -371,6 +650,8 @@ def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: raise ValueError(f"fixture manifest provenance missing: {fixture_id}") if "page_count" in record and (not isinstance(record["page_count"], int) or record["page_count"] < 1): raise ValueError(f"fixture manifest page_count is invalid: {fixture_id}") + if "workload" in record and (not isinstance(record["workload"], str) or not record["workload"]): + raise ValueError(f"fixture manifest workload is invalid: {fixture_id}") normalized = dict(record) fixture_path = Path(str(record["path"])) if not fixture_path.is_absolute(): @@ -398,17 +679,22 @@ def run_matrix( timeout_seconds: float, baseline: Mapping[str, Any] | Path | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, cancel_fixture: str | None = None, cancel_after_seconds: float | None = None, + preflight_profile: Path | None = None, + hostile_corpus: Path | None = None, + hostile_timeout_seconds: float | None = None, + cancel_runner: CancelRunner = _run_benchmark_process, ) -> dict[str, Any]: pdf_tool = Path(pdf_tool).resolve() baseline_by_fixture = _baseline_records(baseline) if isinstance(baseline, Path) else (baseline or {}) identity = _candidate_identity() candidate_sha = identity["candidate_sha"] records: list[dict[str, Any]] = [] + resolved: dict[str, tuple[Path, Mapping[str, Any] | None]] = {} for fixture_id, spec in FIXTURE_SPECS.items(): supplied = fixtures.get(fixture_id) if supplied is None: @@ -417,40 +703,42 @@ def run_matrix( records.append(record) continue metadata = dict(supplied) if isinstance(supplied, Mapping) else None - fixture_path = Path(metadata["path"]) if metadata else Path(supplied) - fixture_path = fixture_path.resolve() + fixture_path = (Path(metadata["path"]) if metadata else Path(supplied)).resolve() if not fixture_path.is_file(): record = _empty_result(fixture_id, "fixture-not-found") record["required"] = spec["required"] records.append(record) continue - record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), cancel_after_seconds if fixture_id == cancel_fixture else None, candidate_sha) + resolved[fixture_id] = (fixture_path, metadata) + record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), candidate_sha, preflight_profile) record["required"] = spec["required"] - result = record.get("result") - if isinstance(result, Mapping): - result_identity = result.get("identity") - if isinstance(result_identity, Mapping): - commit = result_identity.get("commit") - if commit != candidate_sha: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity commit does not match checkout HEAD"])) - record["status"] = "failed" - expected_digest = record.get("fixture_sha256") - fixture_digest = result_identity.get("fixture_digest") - if expected_digest and fixture_digest != expected_digest: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity fixture digest does not match input SHA-256"])) - record["status"] = "failed" records.append(record) + probe: dict[str, Any] | None = None + if cancel_fixture is not None: + if cancel_fixture not in resolved: + probe = {"fixture_id": cancel_fixture, "status": "unavailable", "validation_errors": ["cancellation fixture not supplied"]} + else: + fixture_path, metadata = resolved[cancel_fixture] + probe = run_cancellation_probe(pdf_tool, cancel_fixture, fixture_path, budgets, timeout_seconds, cancel_after_seconds or 1.0, candidate_sha, + _fixture_workload(cancel_fixture, metadata), rasterizers, cancel_runner, runner) + + hostile = None + if hostile_corpus is not None: + hostile = run_hostile_corpus(pdf_tool, hostile_corpus, budgets, hostile_timeout_seconds or timeout_seconds, rasterizers, preflight_profile, runner) + failed = sum(record["status"] == "failed" for record in records) flagged = sum(record["required"] and record["status"] in {"flagged", "unavailable"} for record in records) skipped = sum(not record["required"] and record["status"] == "unavailable" for record in records) return { "schema_kind": MATRIX_KIND, - "schema_version": 2, + "schema_version": 3, "candidate_sha": identity["candidate_sha"], "candidate_identity": identity, "generated_at_utc": datetime.now(timezone.utc).isoformat(), "fixtures": records, + "cancellation_recovery_probe": probe, + "hostile": hostile, "summary": { "total": len(records), "measured": sum(record["status"] == "measured" for record in records), @@ -458,10 +746,48 @@ def run_matrix( "skipped": skipped, "failed": failed, "candidate_sha_verified": identity["verified"], + "cancellation_recovery_probe": probe["status"] if probe else "not-run", + "hostile_contained": f"{hostile['summary']['contained']}/{hostile['summary']['total']}" if hostile else "not-run", }, } +def matrix_passes(matrix: Mapping[str, Any], strict: bool) -> bool: + summary = matrix["summary"] + probe = matrix.get("cancellation_recovery_probe") + hostile = matrix.get("hostile") + hostile_failed = bool(hostile) and hostile["summary"]["contained"] != hostile["summary"]["total"] + if summary["failed"] or hostile_failed or (probe is not None and probe["status"] == "failed"): + return False + if not strict: + return True + return ( + not summary["flagged"] + and summary["candidate_sha_verified"] + and probe is not None and probe["status"] == "measured" + and hostile is not None + ) + + +def matrix_failure_reasons(matrix: Mapping[str, Any]) -> list[str]: + reasons: list[str] = [] + for record in matrix["fixtures"]: + if record["status"] in {"failed", "flagged"} or (record.get("required") and record["status"] == "unavailable"): + details = record["validation_errors"] or [record.get("reason", "no detail recorded")] + reasons.extend(f"fixture {record['fixture_id']} {record['status']}: {error}" for error in details) + detailed = next((run for run in record.get("runs", []) if run.get("detail")), None) + if detailed is not None: + reasons.append(f"fixture {record['fixture_id']} run {detailed['run']} detail: {detailed['detail']}") + probe = matrix.get("cancellation_recovery_probe") + if probe is not None and probe["status"] != "measured": + reasons.extend(f"probe {probe['status']}: {error}" for error in probe["validation_errors"]) + for case in (matrix.get("hostile") or {}).get("cases", []): + if case["status"] != "contained": + exit_code = case.get("process_exit_code") + reasons.extend(f"hostile {case['case_id']} (exit {exit_code}): {error}" for error in case["validation_errors"]) + return reasons + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pdf-tool", type=Path, required=True) @@ -475,24 +801,39 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--repetitions", type=int, default=1) parser.add_argument("--rasterizers", type=int, default=DEFAULT_RASTERIZERS) parser.add_argument("--timeout-seconds", type=float, default=120.0) + parser.add_argument("--preflight-profile", type=Path, default=DEFAULT_PREFLIGHT_PROFILE, + help="profile for the benchmark's measured preflight phase") + parser.add_argument("--no-preflight", action="store_true", help="omit the preflight phase (records stay incomplete)") parser.add_argument("--cancel-fixture", choices=tuple(FIXTURE_SPECS)) parser.add_argument("--cancel-after-seconds", type=float) - parser.add_argument("--strict", action="store_true", help="fail when required fixtures, provenance, or measurements are unavailable") + parser.add_argument("--hostile-corpus", type=Path, help=f"budget-exhaustion corpus directory (strict default: {DEFAULT_HOSTILE_CORPUS.relative_to(ROOT)})") + parser.add_argument("--hostile-timeout-seconds", type=float, default=60.0) + parser.add_argument("--strict", action="store_true", help="fail when required fixtures, probes, provenance, or measurements are unavailable") args = parser.parse_args(argv) try: - if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1: - raise ValueError("margin, timeout-seconds, repetitions, and rasterizers must be positive") + if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1 or args.hostile_timeout_seconds <= 0: + raise ValueError("margin, timeouts, repetitions, and rasterizers must be positive") if args.strict and args.manifest is None: raise ValueError("--strict requires a fixture --manifest with exact digests and sizes") + if args.strict and args.cancel_fixture is None: + raise ValueError("--strict requires --cancel-fixture for the cancellation and recovery probes") + if args.strict and args.no_preflight: + raise ValueError("--strict cannot omit the preflight phase") if (args.cancel_fixture is None) != (args.cancel_after_seconds is None): raise ValueError("--cancel-fixture and --cancel-after-seconds must be supplied together") if args.cancel_after_seconds is not None and args.cancel_after_seconds <= 0: raise ValueError("cancel-after-seconds must be positive") - if args.cancel_fixture is not None and args.repetitions != 1: - raise ValueError("cancellation probes require --repetitions 1") + preflight_profile = None if args.no_preflight else args.preflight_profile.resolve() + if preflight_profile is not None and not preflight_profile.is_file(): + raise ValueError(f"preflight profile not found: {preflight_profile}") + hostile_corpus = args.hostile_corpus or (DEFAULT_HOSTILE_CORPUS if args.strict else None) fixtures: Mapping[str, Path | Mapping[str, Any]] = _load_fixture_manifest(args.manifest) if args.manifest else _fixture_args(args.fixture) budgets = json.loads(args.budgets.read_text(encoding="utf-8")) - matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, repetitions=args.repetitions, rasterizers=args.rasterizers, cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds) + matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, + repetitions=args.repetitions, rasterizers=args.rasterizers, + cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds, + preflight_profile=preflight_profile, hostile_corpus=hostile_corpus, + hostile_timeout_seconds=args.hostile_timeout_seconds) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(matrix, indent=2) + "\n", encoding="utf-8") except (OSError, ValueError, json.JSONDecodeError) as exc: @@ -500,7 +841,9 @@ def main(argv: Sequence[str] | None = None) -> int: return 2 print(json.dumps(matrix["summary"], indent=2)) - return 1 if matrix["summary"]["failed"] or args.strict and (matrix["summary"]["flagged"] or not matrix["summary"]["candidate_sha_verified"]) else 0 + for reason in matrix_failure_reasons(matrix): + print(reason, file=sys.stderr) + return 0 if matrix_passes(matrix, args.strict) else 1 if __name__ == "__main__": diff --git a/scripts/resource_envelope/synthetic_workload.py b/scripts/resource_envelope/synthetic_workload.py new file mode 100644 index 000000000..0c0eb23ef --- /dev/null +++ b/scripts/resource_envelope/synthetic_workload.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Build the deterministic synthetic resource-envelope fixture bundle. + +Hosted qualification cannot reach the external DIV2K corpus or a private +fixture bundle, so the office, image-heavy, and 10,000-page fixtures are +generated here. Every pixel derives from SHAKE-256 over a fixed label and the +PDF structure is fixed, so the same generator version produces byte-identical +fixtures on every platform. Image data is incompressible noise stored with +FlateDecode: file size tracks decoded size and every page pays a real decode. + + python scripts/resource_envelope/synthetic_workload.py \\ + --output-dir /tmp/loop-envelope --manifest /tmp/loop-envelope/fixtures.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import zlib +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Sequence + +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +from scripts.resource_envelope.create_fixture_manifest import create_manifest +from scripts.resource_envelope.pathological_workload import build_pathological_pdf +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + + +GENERATOR_VERSION = 1 +PAGE_WIDTH = 612 +PAGE_HEIGHT = 792 +IMAGE_HEAVY_WORKLOAD = "synthetic-image-heavy" +_WORDS = ( + "press", "proof", "bleed", "ink", "plate", "sheet", "trim", "spot", "cyan", "magenta", + "yellow", "black", "overprint", "separation", "imposition", "signature", "gutter", "margin", + "raster", "vector", "profile", "output", "intent", "coverage", "density", "register", +) + + +@dataclass(frozen=True) +class ImageSpec: + width: int + height: int + + @property + def decoded_bytes(self) -> int: + return self.width * self.height * 3 + + +# Sized so each fixture lands inside run_matrix.FIXTURE_SPECS byte bounds. +OFFICE_PAGES = 24 +OFFICE_IMAGE_EVERY = 4 +OFFICE_IMAGE = ImageSpec(380, 280) +IMAGE_HEAVY_PAGES = 60 +IMAGE_HEAVY_IMAGE = ImageSpec(2048, 1360) +TEN_THOUSAND_PAGES = 10000 +TEN_THOUSAND_UNIQUE_IMAGES = 64 +TEN_THOUSAND_IMAGE = ImageSpec(640, 480) + + +def _noise(label: str, size: int) -> bytes: + return hashlib.shake_256(f"loop-resource-envelope/v{GENERATOR_VERSION}/{label}".encode("ascii")).digest(size) + + +def _stream(dictionary: bytes, payload: bytes) -> bytes: + return dictionary[:-2] + b" /Length " + str(len(payload)).encode("ascii") + b" >>\nstream\n" + payload + b"\nendstream" + + +def _image_object(label: str, image: ImageSpec) -> bytes: + dictionary = ( + f"<< /Type /XObject /Subtype /Image /Width {image.width} /Height {image.height}" + " /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /FlateDecode >>" + ).encode("ascii") + return _stream(dictionary, zlib.compress(_noise(label, image.decoded_bytes), level=1)) + + +def _image_placement(image: ImageSpec, name: str) -> bytes: + scale = min(PAGE_WIDTH / image.width, PAGE_HEIGHT / image.height) + width = image.width * scale + height = image.height * scale + x = (PAGE_WIDTH - width) / 2 + y = (PAGE_HEIGHT - height) / 2 + return f"q {width:.4f} 0 0 {height:.4f} {x:.4f} {y:.4f} cm /{name} Do Q\n".encode("ascii") + + +def _page_object(parent: int, contents: int, resources: bytes) -> bytes: + return ( + f"<< /Type /Page /Parent {parent} 0 R /MediaBox [0 0 {PAGE_WIDTH} {PAGE_HEIGHT}] /Resources ".encode("ascii") + + resources + + f" /Contents {contents} 0 R >>".encode("ascii") + ) + + +class _PdfWriter: + """Writes numbered objects straight to disk so a 500 MB fixture never sits in memory.""" + + def __init__(self, handle: BinaryIO, object_count: int) -> None: + self._handle = handle + self._offsets = [0] * (object_count + 1) + self._written = 0 + self._write(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + + def _write(self, data: bytes) -> None: + self._handle.write(data) + self._written += len(data) + + def add(self, number: int, body: bytes) -> None: + if self._offsets[number]: + raise ValueError(f"object {number} written twice") + self._offsets[number] = self._written + self._write(f"{number} 0 obj\n".encode("ascii") + body + b"\nendobj\n") + + def finish(self, file_id: str) -> None: + missing = [number for number, offset in enumerate(self._offsets) if number and not offset] + if missing: + raise ValueError(f"objects never written: {missing[:5]}") + xref = self._written + self._write(f"xref\n0 {len(self._offsets)}\n0000000000 65535 f \n".encode("ascii")) + for offset in self._offsets[1:]: + self._write(f"{offset:010d} 00000 n \n".encode("ascii")) + self._write( + f"trailer\n<< /Size {len(self._offsets)} /Root 1 0 R /ID [<{file_id}> <{file_id}>] >>\n" + f"startxref\n{xref}\n%%EOF\n".encode("ascii") + ) + + +def _file_id(fixture_id: str) -> str: + return hashlib.sha256(f"{fixture_id}/v{GENERATOR_VERSION}".encode("ascii")).hexdigest()[:32] + + +def _office_text(page_index: int) -> bytes: + selector = _noise(f"office/text/{page_index}", 40 * 12) + lines = [b"BT", b"/F1 10 Tf", b"12 TL", f"54 {PAGE_HEIGHT - 60} Td".encode("ascii")] + for line in range(40): + words = " ".join(_WORDS[value % len(_WORDS)] for value in selector[line * 12 : line * 12 + 12]) + lines.append(f"({words}) '".encode("ascii")) + lines.append(b"ET") + for row in range(8): + y = 80 + row * 18 + lines.append(f"0.2 w 54 {y} m {PAGE_WIDTH - 54} {y} l S".encode("ascii")) + return b"\n".join(lines) + b"\n" + + +def build_office(output: Path, pages: int = OFFICE_PAGES, image: ImageSpec = OFFICE_IMAGE) -> None: + image_pages = [index for index in range(pages) if index % OFFICE_IMAGE_EVERY == 0] + font = 3 + first_image = 4 + first_page = first_image + len(image_pages) + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + writer.add(font, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>") + for slot in range(len(image_pages)): + writer.add(first_image + slot, _image_object(f"office/image/{slot}", image)) + for index in range(pages): + content = _office_text(index) + xobjects = b"" + if index in image_pages: + slot = image_pages.index(index) + content += f"q {image.width / 2:.4f} 0 0 {image.height / 2:.4f} 54 {PAGE_HEIGHT - 620} cm /Im0 Do Q\n".encode("ascii") + xobjects = f" /XObject << /Im0 {first_image + slot} 0 R >>".encode("ascii") + resources = f"<< /Font << /F1 {font} 0 R >>".encode("ascii") + xobjects + b" >>" + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", content)) + writer.finish(_file_id("office-2mb")) + + +def build_image_pages(output: Path, fixture_id: str, pages: int, unique_images: int, image: ImageSpec) -> None: + if pages < 1 or unique_images < 1: + raise ValueError("pages and unique_images must be positive") + first_image = 3 + first_page = first_image + unique_images + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + for slot in range(unique_images): + writer.add(first_image + slot, _image_object(f"{fixture_id}/image/{slot}", image)) + placement = _image_placement(image, "Im0") + for index in range(pages): + resources = f"<< /XObject << /Im0 {first_image + index % unique_images} 0 R >> >>".encode("ascii") + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", placement)) + writer.finish(_file_id(fixture_id)) + + +def _provenance(fixture_id: str) -> str: + return f"scripts/resource_envelope/synthetic_workload.py generator v{GENERATOR_VERSION} ({fixture_id})" + + +def build_bundle(output_dir: Path) -> dict[str, object]: + output_dir.mkdir(parents=True, exist_ok=True) + paths = {fixture_id: output_dir / f"{fixture_id}.pdf" for fixture_id in FIXTURE_SPECS if fixture_id != "multi-gb"} + build_office(paths["office-2mb"]) + build_image_pages(paths["image-heavy-500mb"], "image-heavy-500mb", IMAGE_HEAVY_PAGES, IMAGE_HEAVY_PAGES, IMAGE_HEAVY_IMAGE) + build_image_pages(paths["ten-thousand-page"], "ten-thousand-page", TEN_THOUSAND_PAGES, TEN_THOUSAND_UNIQUE_IMAGES, TEN_THOUSAND_IMAGE) + build_pathological_pdf(paths["pathological-vector"], 256, 512, "pathological-vector") + build_pathological_pdf(paths["transparency-spots"], 256, 256, "transparency-spots") + + manifest = create_manifest(paths, "synthetic") + for record in manifest["fixtures"]: + fixture_id = str(record["fixture_id"]) + record["provenance"] = _provenance(fixture_id) + record["path"] = paths[fixture_id].name + if fixture_id == "ten-thousand-page": + record["workload"] = IMAGE_HEAVY_WORKLOAD + spec = FIXTURE_SPECS[fixture_id] + size = int(record["size_bytes"]) + if (spec["min_bytes"] is not None and size < spec["min_bytes"]) or (spec["max_bytes"] is not None and size > spec["max_bytes"]): + raise ValueError(f"{fixture_id} generated {size} bytes, outside its fixture bounds") + manifest["generator"] = {"script": "scripts/resource_envelope/synthetic_workload.py", "version": GENERATOR_VERSION} + return manifest + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True, help="fixture manifest to write; paths are relative to it") + args = parser.parse_args(argv) + try: + if args.manifest.resolve().parent != args.output_dir.resolve(): + raise ValueError("--manifest must be written inside --output-dir so its relative paths resolve") + manifest = build_bundle(args.output_dir) + args.manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"synthetic workload error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({record["fixture_id"]: {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} for record in manifest["fixtures"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py new file mode 100644 index 000000000..70197ec41 --- /dev/null +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import _failure_detail, matrix_failure_reasons, matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus +from scripts.resource_envelope.validate_envelope import POOL_NAMES + +CANDIDATE = "candidate-sha" + + +def _policy() -> dict: + return { + "resource_budget": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "workloads": { + "pathological-vector": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200, "cancellation_latency_ms": 50, "recovery_ms": 60000}, + "synthetic-image-heavy": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200}, + "large-document": {"wall_time_ms": 100, "rss_high_water_bytes": 2000}, + }, + } + + +def _envelope(digest: str, status: str = "complete", preflight: int = 5, rss: int = 10, materialized: int = 256, latency: int = -1) -> dict: + return { + "identity": {"commit": CANDIDATE, "fixture_digest": digest}, + "family": "benchmark-render", + "status": status, + "page_count": 256, + "rss_high_water_bytes": rss, + "preflight_high_water_bytes": preflight, + "pages_materialized": materialized, + "elapsed_ms": 10, + "cancellation_latency_ms": latency, + "prefetch_shed": False, + "interaction_slot_held": False, + "resources": { + "config": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "resident_bytes": 0, + "resident_high_water_bytes": 0, + "pressure": "normal", + "pools": {pool: {"limit_bytes": 100, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0} for pool in POOL_NAMES}, + }, + } + + +def _process(command: list[str], returncode: int, envelope: dict | None) -> subprocess.CompletedProcess[str]: + stdout = json.dumps({"data": {"workload_envelope": envelope}}) if envelope else "" + return subprocess.CompletedProcess(command, returncode, stdout, "") + + +class _Fixture: + def __enter__(self) -> "_Fixture": + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "fixture.pdf" + self.path.write_bytes(b"fixture") + self.digest = hashlib.sha256(b"fixture").hexdigest() + self.metadata = {"path": str(self.path), "sha256": self.digest, "size_bytes": 7, "provenance": "unit-test", "page_count": 256} + return self + + def __exit__(self, *exc: object) -> None: + self._directory.cleanup() + + def measure(self, runner, **kwargs) -> dict: + return run_fixture(Path("PdfTool"), "pathological-vector", self.path, _policy(), 1, runner=runner, metadata=self.metadata, candidate_sha=CANDIDATE, **kwargs) + + +class MeasuredRunTest(unittest.TestCase): + def test_complete_envelope_with_preflight_is_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + self.assertGreaterEqual(record["runs"][0]["process_wall_ms"], 0) + + def test_preflight_profile_reaches_the_command(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-2:], ["--profile", "profile.json"]) + + def test_small_document_preflight_covers_every_page(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + + def test_large_document_preflight_is_sampled(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-4:], ["--profile", "profile.json", "--preflight-page-last", "256"]) + self.assertEqual(record["profile"]["preflight_page_last"], 256) + + def test_sampling_needs_a_preflight_profile(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + + def test_workload_rss_cap_replaces_the_resident_limit(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=1000))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + def test_workload_rss_cap_still_binds(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=2500))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: RSS 2500 exceeds resident policy 2000", record["validation_errors"]) + + def test_crashed_process_fails_even_with_an_envelope(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, -11, _envelope(fixture.digest))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: process-crashed:-11", record["validation_errors"]) + + def test_timeout_fails(self) -> None: + def runner(command, **kwargs): + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: benchmark-timeout", record["validation_errors"]) + + def test_timeout_is_not_repeated(self) -> None: + calls = [] + + def runner(command, **kwargs): + calls.append(command) + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner, repetitions=3) + self.assertEqual(len(calls), 1) + self.assertEqual(len(record["runs"]), 1) + + def test_partial_output_exit_is_flagged_not_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 5, _envelope(fixture.digest))) + self.assertEqual(record["status"], "flagged") + + def test_manifest_workload_overrides_the_default(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "synthetic-image-heavy" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["workload"], "synthetic-image-heavy") + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + +class CancellationProbeTest(unittest.TestCase): + def _probe(self, fixture: _Fixture, cancel_runner, runner) -> dict: + return run_cancellation_probe(Path("PdfTool"), "pathological-vector", fixture.path, _policy(), 5, 0.5, CANDIDATE, + "pathological-vector", cancel_runner=cancel_runner, runner=runner) + + def test_cancelled_run_and_reopen_are_measured(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=20, materialized=40)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "measured", probe["validation_errors"]) + self.assertEqual(probe["cancellation"]["cancellation_latency_ms"], 20) + self.assertNotIn("--profile", probe["cancellation"]["command"]) + self.assertGreaterEqual(probe["recovery"]["recovery_ms"], 0) + self.assertEqual(probe["recovery"]["command"][-4:], ["--page-first", "1", "--page-last", "1"]) + + def test_run_that_finished_before_the_interrupt_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 0, _envelope(fixture.digest)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "failed") + self.assertTrue(any("not cancelled" in error or "ended 'complete'" in error for error in probe["validation_errors"])) + + def test_latency_over_policy_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=51)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertIn("cancellation_latency_ms 51 exceeds workload policy 50", probe["validation_errors"]) + + def test_hung_interrupt_fails(self) -> None: + def cancel_runner(command, timeout, cancel_after): + raise subprocess.TimeoutExpired(command, timeout) + + with _Fixture() as fixture: + probe = self._probe(fixture, cancel_runner, lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1))) + self.assertEqual(probe["status"], "failed") + + +class HostileCorpusTest(unittest.TestCase): + def test_rejection_is_contained_and_crash_is_not(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + cases = [] + for case_id in ("rejected", "crashing"): + (corpus / f"{case_id}.pdf").write_bytes(case_id.encode()) + cases.append({"id": case_id, "pdf": f"{case_id}.pdf", "sha256": hashlib.sha256(case_id.encode()).hexdigest()}) + (corpus / "manifest.json").write_text(json.dumps({"cases": cases}), encoding="utf-8") + + def runner(command, **_): + return _process(command, 3 if "rejected.pdf" in command[2] else -6, None) + + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=runner) + by_id = {case["case_id"]: case for case in hostile["cases"]} + self.assertEqual(by_id["rejected"]["status"], "contained") + self.assertEqual(by_id["rejected"]["disposition"], "rejected") + self.assertEqual(by_id["crashing"]["status"], "failed") + self.assertEqual(hostile["summary"], {"total": 2, "contained": 1}) + + def test_tampered_fixture_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + (corpus / "case.pdf").write_bytes(b"tampered") + (corpus / "manifest.json").write_text(json.dumps({"cases": [{"id": "case", "pdf": "case.pdf", "sha256": "0" * 64}]}), encoding="utf-8") + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=lambda command, **_: _process(command, 0, None)) + self.assertEqual(hostile["cases"][0]["status"], "failed") + + +class MatrixPassTest(unittest.TestCase): + def _matrix(self, probe_status: str | None = "measured", contained: int = 2, flagged: int = 0) -> dict: + return { + "summary": {"failed": 0, "flagged": flagged, "candidate_sha_verified": True}, + "cancellation_recovery_probe": {"status": probe_status} if probe_status else None, + "hostile": {"summary": {"total": 2, "contained": contained}}, + } + + def test_strict_requires_probe_hostile_and_no_flags(self) -> None: + self.assertTrue(matrix_passes(self._matrix(), strict=True)) + self.assertFalse(matrix_passes(self._matrix(probe_status=None), strict=True)) + self.assertFalse(matrix_passes(self._matrix(flagged=1), strict=True)) + self.assertTrue(matrix_passes(self._matrix(flagged=1), strict=False)) + + def test_uncontained_hostile_case_fails_even_without_strict(self) -> None: + self.assertFalse(matrix_passes(self._matrix(contained=1), strict=False)) + + +class FailureDetailTest(unittest.TestCase): + def test_keeps_render_errors_and_stderr_tail_of_a_partial_run(self) -> None: + stdout = json.dumps({"data": {"nested": [{"rendering-errors": [{"page-no": 3, "message": "bad image"}]}]}}) + completed = subprocess.CompletedProcess(["PdfTool"], 5, stdout, "warning: x\n") + detail = _failure_detail(completed) + self.assertIn('"bad image"', detail) + self.assertIn("stderr=warning: x", detail) + + def test_is_empty_when_nothing_was_reported(self) -> None: + self.assertEqual(_failure_detail(subprocess.CompletedProcess(["PdfTool"], 5, "", "")), "") + + +class FailureReasonTest(unittest.TestCase): + def test_names_every_failing_record_probe_and_hostile_case(self) -> None: + matrix = { + "fixtures": [ + {"fixture_id": "office-2mb", "status": "measured", "required": True, "validation_errors": []}, + {"fixture_id": "image-heavy-500mb", "status": "flagged", "required": True, "validation_errors": ["run 1: process exit code 1 is not success"], + "runs": [{"run": 1, "detail": "stderr=render failed"}]}, + {"fixture_id": "ten-thousand-page", "status": "unavailable", "required": True, "validation_errors": [], "reason": "fixture-not-found"}, + {"fixture_id": "multi-gb", "status": "unavailable", "required": False, "validation_errors": [], "reason": "fixture-not-supplied-optional"}, + ], + "cancellation_recovery_probe": {"status": "failed", "validation_errors": ["recovery probe process-crashed:-11"]}, + "hostile": {"cases": [ + {"case_id": "ok", "status": "contained", "validation_errors": []}, + {"case_id": "deep-tree", "status": "failed", "process_exit_code": 0, "validation_errors": ["RSS 9 exceeds resident policy 4"]}, + ]}, + } + self.assertEqual(matrix_failure_reasons(matrix), [ + "fixture image-heavy-500mb flagged: run 1: process exit code 1 is not success", + "fixture image-heavy-500mb run 1 detail: stderr=render failed", + "fixture ten-thousand-page unavailable: fixture-not-found", + "probe failed: recovery probe process-crashed:-11", + "hostile deep-tree (exit 0): RSS 9 exceeds resident policy 4", + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/resource_envelope/test_synthetic_workload.py b/scripts/resource_envelope/test_synthetic_workload.py new file mode 100644 index 000000000..7473b6b06 --- /dev/null +++ b/scripts/resource_envelope/test_synthetic_workload.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import hashlib +import re +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS +from scripts.resource_envelope.synthetic_workload import ImageSpec, _PdfWriter, build_image_pages, build_office + + +def _xref_problems(pdf: bytes) -> list[str]: + start = int(re.search(rb"startxref\n(\d+)\n%%EOF\n$", pdf).group(1)) + count = int(re.match(rb"xref\n0 (\d+)\n", pdf[start:]).group(1)) + rows = pdf[start:].split(b"\n")[2 : 2 + count] + problems = [f"object {number}" for number, row in enumerate(rows[1:], start=1) if not pdf[int(row[:10]) :].startswith(f"{number} 0 obj".encode())] + for match in re.finditer(rb"/Length (\d+) >>\nstream\n", pdf): + if pdf[match.end() + int(match.group(1)) :][:10] != b"\nendstream": + problems.append(f"stream at {match.start()}") + return problems + + +class SyntheticWorkloadTest(unittest.TestCase): + def test_image_pages_are_deterministic_and_well_formed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.pdf" + second = Path(directory) / "second.pdf" + build_image_pages(first, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + build_image_pages(second, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + pdf = first.read_bytes() + self.assertEqual(hashlib.sha256(pdf).digest(), hashlib.sha256(second.read_bytes()).digest()) + self.assertEqual(pdf.count(b"/Type /Page "), 30) + self.assertEqual(pdf.count(b"/Subtype /Image"), 4) + self.assertIn(b"/Count 30", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_fixture_identity_changes_the_pixels(self) -> None: + with tempfile.TemporaryDirectory() as directory: + one = Path(directory) / "one.pdf" + two = Path(directory) / "two.pdf" + build_image_pages(one, "ten-thousand-page", 2, 1, ImageSpec(8, 8)) + build_image_pages(two, "image-heavy-500mb", 2, 1, ImageSpec(8, 8)) + self.assertNotEqual(one.read_bytes(), two.read_bytes()) + + def test_office_fixture_fits_its_size_bounds(self) -> None: + spec = FIXTURE_SPECS["office-2mb"] + with tempfile.TemporaryDirectory() as directory: + office = Path(directory) / "office.pdf" + build_office(office) + pdf = office.read_bytes() + self.assertGreaterEqual(len(pdf), spec["min_bytes"]) + self.assertLessEqual(len(pdf), spec["max_bytes"]) + self.assertIn(b"/BaseFont /Helvetica", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_writer_rejects_unwritten_objects(self) -> None: + with tempfile.TemporaryFile() as handle: + writer = _PdfWriter(handle, 2) + writer.add(1, b"<< >>") + with self.assertRaises(ValueError): + writer.finish("0" * 32) + + +if __name__ == "__main__": + unittest.main() diff --git a/CanvasBenchmark/CMakeLists.txt b/tools/CanvasBenchmark/CMakeLists.txt similarity index 100% rename from CanvasBenchmark/CMakeLists.txt rename to tools/CanvasBenchmark/CMakeLists.txt diff --git a/CanvasBenchmark/CanvasBenchmark.qml b/tools/CanvasBenchmark/CanvasBenchmark.qml similarity index 100% rename from CanvasBenchmark/CanvasBenchmark.qml rename to tools/CanvasBenchmark/CanvasBenchmark.qml diff --git a/CanvasBenchmark/main.cpp b/tools/CanvasBenchmark/main.cpp similarity index 100% rename from CanvasBenchmark/main.cpp rename to tools/CanvasBenchmark/main.cpp diff --git a/CodeGenerator/CMakeLists.txt b/tools/CodeGenerator/CMakeLists.txt similarity index 100% rename from CodeGenerator/CMakeLists.txt rename to tools/CodeGenerator/CMakeLists.txt diff --git a/CodeGenerator/codegenerator.cpp b/tools/CodeGenerator/codegenerator.cpp similarity index 97% rename from CodeGenerator/codegenerator.cpp rename to tools/CodeGenerator/codegenerator.cpp index 0288a638c..a3b439859 100644 --- a/CodeGenerator/codegenerator.cpp +++ b/tools/CodeGenerator/codegenerator.cpp @@ -34,7 +34,6 @@ namespace codegen GeneratedCodeStorage::GeneratedCodeStorage(QObject* parent) : BaseClass(parent) { - } QObjectList GeneratedCodeStorage::getFunctions() const @@ -78,13 +77,15 @@ void GeneratedCodeStorage::removeFunction(GeneratedFunction* function) void GeneratedCodeStorage::generateCode(QTextStream& stream, CodeGeneratorParameters& parameters) const { - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; for (const QObject* object : m_functions) { const GeneratedFunction* generatedFunction = qobject_cast(object); generatedFunction->generateCode(stream, parameters); - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; } } @@ -345,7 +346,6 @@ QString CodeGenerator::generateSource(QString className, int indent) const GeneratedFunction::GeneratedFunction(QObject* parent) : BaseClass(parent) { - } QString GeneratedFunction::getFunctionTypeString() const @@ -397,7 +397,7 @@ void GeneratedFunction::generateCode(QTextStream& stream, CodeGeneratorParameter { QStringList parameterCaptions; QStringList parameterTexts; - std::function gatherParameters = [&](const GeneratedBase* object, Pass pass) + std::function gatherParameters = [&](const GeneratedBase* object, Pass pass) { if (pass != Pass::Enter) { @@ -475,7 +475,8 @@ void GeneratedFunction::generateCode(QTextStream& stream, CodeGeneratorParameter QString indent(parameters.indent, QChar(QChar::Space)); stream << "{" << Qt::endl; - stream << indent << "PDFObjectFactory objectBuilder;" << Qt::endl << Qt::endl; + stream << indent << "PDFObjectFactory objectBuilder;" << Qt::endl + << Qt::endl; generateSourceCode(stream, parameters); @@ -596,7 +597,6 @@ GeneratedAction::GeneratedAction(QObject* parent) : BaseClass(parent), m_actionType(CreateObject) { - } bool GeneratedAction::hasField(FieldType fieldType) const @@ -828,7 +828,7 @@ void GeneratedBase::generateSourceCode(QTextStream& stream, CodeGeneratorParamet generateSourceCodeImpl(stream, parameters, Pass::Leave); } -void GeneratedBase::applyFunctor(std::function& functor) const +void GeneratedBase::applyFunctor(std::function& functor) const { functor(this, Pass::Enter); @@ -1038,7 +1038,6 @@ QStringList GeneratedBase::getFormattedTextBlock(QString firstPrefix, QString pr GeneratedPDFObject::GeneratedPDFObject(QObject* parent) : BaseClass(parent) { - } bool GeneratedPDFObject::hasField(GeneratedBase::FieldType fieldType) const @@ -1289,7 +1288,6 @@ void GeneratedPDFObject::generateSourceCodeImpl(QTextStream& stream, CodeGenerat GeneratedParameter::GeneratedParameter(QObject* parent) : BaseClass(parent) { - } bool GeneratedParameter::hasField(GeneratedBase::FieldType fieldType) const @@ -1655,9 +1653,11 @@ QString XFACodeGenerator::generateSource() const stream.setRealNumberPrecision(3); stream.setRealNumberNotation(QTextStream::FixedNotation); - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; stream << "namespace xfa" << Qt::endl; - stream << "{" << Qt::endl << Qt::endl; + stream << "{" << Qt::endl + << Qt::endl; // Forward declarations for (const Class& myClass : m_classes) @@ -1672,7 +1672,8 @@ QString XFACodeGenerator::generateSource() const stream << "{" << Qt::endl; stream << "public:" << Qt::endl; stream << " XFA_AbstractVisitor() = default;" << Qt::endl; - stream << " virtual ~XFA_AbstractVisitor() = default;" << Qt::endl << Qt::endl; + stream << " virtual ~XFA_AbstractVisitor() = default;" << Qt::endl + << Qt::endl; for (const Class& myClass : m_classes) { stream << QString(" virtual void visit(const XFA_%1* node) { Q_UNUSED(node); }").arg(myClass.className) << Qt::endl; @@ -1702,7 +1703,8 @@ QString XFACodeGenerator::generateSource() const { stream << " " << getEnumValueName(enumValue) << "," << Qt::endl; } - stream << " };" << Qt::endl << Qt::endl; + stream << " };" << Qt::endl + << Qt::endl; } for (const auto& typeItem : m_types) @@ -1725,10 +1727,12 @@ QString XFACodeGenerator::generateSource() const } stream << QString(" };") << Qt::endl; stream << QString(" parseEnumAttribute(element, attributeFieldName, attribute, defaultValue, enumValues);") << Qt::endl; - stream << QString(" }") << Qt::endl << Qt::endl; + stream << QString(" }") << Qt::endl + << Qt::endl; } - stream << "};" << Qt::endl << Qt::endl; + stream << "};" << Qt::endl + << Qt::endl; for (const Class& myClass : m_classes) { @@ -1795,10 +1799,12 @@ QString XFACodeGenerator::generateSource() const if (myClass.valueType) { - stream << QString(" const %1* getNodeValue() const { return m_nodeValue.getValue(); }").arg(myClass.valueType->typeName) << Qt::endl << Qt::endl; + stream << QString(" const %1* getNodeValue() const { return m_nodeValue.getValue(); }").arg(myClass.valueType->typeName) << Qt::endl + << Qt::endl; } - stream << QString(" virtual void accept(XFA_AbstractVisitor* visitor) const override { visitor->visit(this); }") << Qt::endl << Qt::endl; + stream << QString(" virtual void accept(XFA_AbstractVisitor* visitor) const override { visitor->visit(this); }") << Qt::endl + << Qt::endl; stream << QString(" static std::optional parse(const QDomElement& element);").arg(myClass.className) << Qt::endl; @@ -1827,13 +1833,19 @@ QString XFACodeGenerator::generateSource() const stream << QString(" XFA_Value<%1> m_nodeValue;").arg(myClass.valueType->typeName) << Qt::endl; } - stream << "};" << Qt::endl << Qt::endl; + stream << "};" << Qt::endl + << Qt::endl; // Class loader stream << QString("std::optional XFA_%1::parse(const QDomElement& element)").arg(myClass.className) << Qt::endl; stream << "{" << Qt::endl; - stream << " if (element.isNull())" << Qt::endl << " {" << Qt::endl << " return std::nullopt;" << Qt::endl << " }" << Qt::endl << Qt::endl; - stream << QString(" XFA_%1 myClass;").arg(myClass.className) << Qt::endl << Qt::endl; + stream << " if (element.isNull())" << Qt::endl + << " {" << Qt::endl + << " return std::nullopt;" << Qt::endl + << " }" << Qt::endl + << Qt::endl; + stream << QString(" XFA_%1 myClass;").arg(myClass.className) << Qt::endl + << Qt::endl; // Load attributes stream << " // load attributes" << Qt::endl; @@ -1860,18 +1872,21 @@ QString XFACodeGenerator::generateSource() const { stream << Qt::endl; stream << " // load node value" << Qt::endl; - stream << QString(" parseValue(element, myClass.m_nodeValue);") << Qt::endl << Qt::endl; + stream << QString(" parseValue(element, myClass.m_nodeValue);") << Qt::endl + << Qt::endl; } stream << " myClass.setOrderFromElement(element);" << Qt::endl; stream << " return myClass;" << Qt::endl; stream << "}" << Qt::endl; - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; } stream << "} // namespace xfa" << Qt::endl; - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; } return QString::fromUtf8(ba); @@ -1906,13 +1921,15 @@ QString XFACodeGenerator::generateHeader() const stream.setRealNumberPrecision(3); stream.setRealNumberNotation(QTextStream::FixedNotation); - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; stream << "namespace xfa" << Qt::endl; stream << "{" << Qt::endl; stream << "} // namespace xfa" << Qt::endl; - stream << Qt::endl << Qt::endl; + stream << Qt::endl + << Qt::endl; } return QString::fromUtf8(ba); diff --git a/CodeGenerator/codegenerator.h b/tools/CodeGenerator/codegenerator.h similarity index 99% rename from CodeGenerator/codegenerator.h rename to tools/CodeGenerator/codegenerator.h index 4793ee2b1..39a15db4f 100644 --- a/CodeGenerator/codegenerator.h +++ b/tools/CodeGenerator/codegenerator.h @@ -52,7 +52,7 @@ class Serializer static void store(QObject* object, QDomElement& element); static QObject* clone(QObject* object, QObject* parent); - template + template static inline QString convertEnumToString(T enumValue) { QMetaEnum metaEnum = QMetaEnum::fromType(); @@ -60,7 +60,7 @@ class Serializer return metaEnum.valueToKey(enumValue); } - template + template static inline void convertStringToEnum(const QString enumString, T& value) { QMetaEnum metaEnum = QMetaEnum::fromType(); @@ -74,7 +74,7 @@ class Serializer } } - template + template static inline void fillComboBox(QComboBox* comboBox, T value) { QMetaEnum metaEnum = QMetaEnum::fromType(); @@ -266,7 +266,6 @@ class GeneratedPDFObject : public GeneratedBase using BaseClass = GeneratedBase; public: - enum ObjectType { Object, @@ -317,7 +316,6 @@ class GeneratedAction : public GeneratedBase using BaseClass = GeneratedBase; public: - enum ActionType { Parameters, @@ -370,7 +368,6 @@ class GeneratedFunction : public GeneratedBase using BaseClass = GeneratedBase; public: - enum FunctionType { Structure, @@ -461,7 +458,6 @@ class XFACodeGenerator void generateCode(const QDomDocument& document, QString headerName, QString sourceName); private: - struct Type { QString id; @@ -508,4 +504,4 @@ class XFACodeGenerator Q_DECLARE_METATYPE(codegen::GeneratedCodeStorage*) Q_DECLARE_METATYPE(codegen::GeneratedFunction*) -#endif // CODEGENERATOR_H +#endif // CODEGENERATOR_H diff --git a/CodeGenerator/generatormainwindow.cpp b/tools/CodeGenerator/generatormainwindow.cpp similarity index 99% rename from CodeGenerator/generatormainwindow.cpp rename to tools/CodeGenerator/generatormainwindow.cpp index 4f2858adf..c47c3f189 100644 --- a/CodeGenerator/generatormainwindow.cpp +++ b/tools/CodeGenerator/generatormainwindow.cpp @@ -32,7 +32,7 @@ #include #include -GeneratorMainWindow::GeneratorMainWindow(QWidget *parent) : +GeneratorMainWindow::GeneratorMainWindow(QWidget* parent) : QMainWindow(parent), ui(new Ui::GeneratorMainWindow), m_generator(new codegen::CodeGenerator(this)), diff --git a/CodeGenerator/generatormainwindow.h b/tools/CodeGenerator/generatormainwindow.h similarity index 99% rename from CodeGenerator/generatormainwindow.h rename to tools/CodeGenerator/generatormainwindow.h index a13461a03..0d062ad88 100644 --- a/CodeGenerator/generatormainwindow.h +++ b/tools/CodeGenerator/generatormainwindow.h @@ -119,4 +119,4 @@ private slots: QString m_XFAsourceFileName; }; -#endif // GENERATORMAINWINDOW_H +#endif // GENERATORMAINWINDOW_H diff --git a/CodeGenerator/generatormainwindow.ui b/tools/CodeGenerator/generatormainwindow.ui similarity index 100% rename from CodeGenerator/generatormainwindow.ui rename to tools/CodeGenerator/generatormainwindow.ui diff --git a/CodeGenerator/main.cpp b/tools/CodeGenerator/main.cpp similarity index 97% rename from CodeGenerator/main.cpp rename to tools/CodeGenerator/main.cpp index 156c55572..87e1cc1c3 100644 --- a/CodeGenerator/main.cpp +++ b/tools/CodeGenerator/main.cpp @@ -27,7 +27,7 @@ #include #include -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { QHashSeed::globalSeed().setDeterministicGlobalSeed(); diff --git a/JBIG2_Viewer/CMakeLists.txt b/tools/JBIG2_Viewer/CMakeLists.txt similarity index 100% rename from JBIG2_Viewer/CMakeLists.txt rename to tools/JBIG2_Viewer/CMakeLists.txt diff --git a/JBIG2_Viewer/main.cpp b/tools/JBIG2_Viewer/main.cpp similarity index 97% rename from JBIG2_Viewer/main.cpp rename to tools/JBIG2_Viewer/main.cpp index 475ec89b0..522f3dcef 100644 --- a/JBIG2_Viewer/main.cpp +++ b/tools/JBIG2_Viewer/main.cpp @@ -26,7 +26,7 @@ #include -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { QApplication a(argc, argv); diff --git a/JBIG2_Viewer/mainwindow.cpp b/tools/JBIG2_Viewer/mainwindow.cpp similarity index 97% rename from JBIG2_Viewer/mainwindow.cpp rename to tools/JBIG2_Viewer/mainwindow.cpp index d437053f5..1274f2c8b 100644 --- a/JBIG2_Viewer/mainwindow.cpp +++ b/tools/JBIG2_Viewer/mainwindow.cpp @@ -124,7 +124,8 @@ void MainWindow::on_actionAdd_JBIG2_image_triggered() QImage image(imageData.getWidth(), imageData.getHeight(), QImage::Format_Mono); const uchar* sourceData = reinterpret_cast(imageData.getData().constData()); Q_ASSERT(imageData.getData().size() == image.sizeInBytes()); - std::transform(sourceData, sourceData + imageData.getData().size(), image.bits(), [](const uchar value) { return value; }); + std::transform(sourceData, sourceData + imageData.getData().size(), image.bits(), [](const uchar value) + { return value; }); addImage(file.fileName() + QString(", Decoded in %1 [msec]").arg(time), qMove(image)); } } diff --git a/JBIG2_Viewer/mainwindow.h b/tools/JBIG2_Viewer/mainwindow.h similarity index 92% rename from JBIG2_Viewer/mainwindow.h rename to tools/JBIG2_Viewer/mainwindow.h index 1e44e2367..21d4cc1f0 100644 --- a/JBIG2_Viewer/mainwindow.h +++ b/tools/JBIG2_Viewer/mainwindow.h @@ -6,7 +6,10 @@ #include "pdfexception.h" QT_BEGIN_NAMESPACE -namespace Ui { class MainWindow; } +namespace Ui +{ +class MainWindow; +} QT_END_NAMESPACE class MainWindow : public QMainWindow, public pdf::PDFRenderErrorReporter @@ -31,4 +34,4 @@ private slots: Ui::MainWindow* ui; QString m_directory; }; -#endif // MAINWINDOW_H +#endif // MAINWINDOW_H diff --git a/JBIG2_Viewer/mainwindow.ui b/tools/JBIG2_Viewer/mainwindow.ui similarity index 100% rename from JBIG2_Viewer/mainwindow.ui rename to tools/JBIG2_Viewer/mainwindow.ui diff --git a/PdfExampleGenerator/CMakeLists.txt b/tools/PdfExampleGenerator/CMakeLists.txt similarity index 100% rename from PdfExampleGenerator/CMakeLists.txt rename to tools/PdfExampleGenerator/CMakeLists.txt diff --git a/PdfExampleGenerator/main.cpp b/tools/PdfExampleGenerator/main.cpp similarity index 98% rename from PdfExampleGenerator/main.cpp rename to tools/PdfExampleGenerator/main.cpp index cc2c23c86..d401e5e77 100644 --- a/PdfExampleGenerator/main.cpp +++ b/tools/PdfExampleGenerator/main.cpp @@ -26,7 +26,7 @@ #include "pdfsettings.h" #include "pdfexamplesgenerator.h" -int main(int argc, char *argv[]) +int main(int argc, char* argv[]) { QApplication a(argc, argv); pdf::initializeApplicationIdentity(pdf::PDFApplicationSurface::PdfExampleGenerator); diff --git a/PdfExampleGenerator/pdfexamplesgenerator.cpp b/tools/PdfExampleGenerator/pdfexamplesgenerator.cpp similarity index 90% rename from PdfExampleGenerator/pdfexamplesgenerator.cpp rename to tools/PdfExampleGenerator/pdfexamplesgenerator.cpp index 03a7f9587..878a11f13 100644 --- a/PdfExampleGenerator/pdfexamplesgenerator.cpp +++ b/tools/PdfExampleGenerator/pdfexamplesgenerator.cpp @@ -37,14 +37,14 @@ void PDFExamplesGenerator::generateAnnotationsExample() builder.setLanguage(QLocale::system()); pdf::PDFObjectReference page1 = builder.appendPage(QRectF(0, 0, 400, 400)); - builder.createAnnotationText(page1, QRectF(50, 50, 24, 24), pdf::TextAnnotationIcon::Comment, "Title1", "Subject1", "Comment", false); + builder.createAnnotationText(page1, QRectF(50, 50, 24, 24), pdf::TextAnnotationIcon::Comment, "Title1", "Subject1", "Comment", false); builder.createAnnotationText(page1, QRectF(50, 100, 24, 24), pdf::TextAnnotationIcon::Help, "Title1", "Subject1", "Help", false); builder.createAnnotationText(page1, QRectF(50, 150, 24, 24), pdf::TextAnnotationIcon::Insert, "Title1", "Subject1", "Insert", false); builder.createAnnotationText(page1, QRectF(50, 200, 24, 24), pdf::TextAnnotationIcon::Key, "Title1", "Subject1", "Key", false); builder.createAnnotationText(page1, QRectF(50, 250, 24, 24), pdf::TextAnnotationIcon::NewParagraph, "Title1", "Subject1", "NewParagraph", false); builder.createAnnotationText(page1, QRectF(50, 300, 24, 24), pdf::TextAnnotationIcon::Note, "Title1", "Subject1", "Note", false); builder.createAnnotationText(page1, QRectF(50, 350, 24, 24), pdf::TextAnnotationIcon::Paragraph, "Title1", "Subject1", "Paragraph", false); - builder.createAnnotationText(page1, QRectF(250, 50, 24, 24), pdf::TextAnnotationIcon::Comment, "Title1", "Subject1", "Comment", true); + builder.createAnnotationText(page1, QRectF(250, 50, 24, 24), pdf::TextAnnotationIcon::Comment, "Title1", "Subject1", "Comment", true); builder.createAnnotationText(page1, QRectF(250, 100, 24, 24), pdf::TextAnnotationIcon::Help, "Title1", "Subject1", "Help", true); builder.createAnnotationText(page1, QRectF(250, 150, 24, 24), pdf::TextAnnotationIcon::Insert, "Title1", "Subject1", "Insert", true); builder.createAnnotationText(page1, QRectF(250, 200, 24, 24), pdf::TextAnnotationIcon::Key, "Title1", "Subject1", "Key", true); @@ -53,18 +53,18 @@ void PDFExamplesGenerator::generateAnnotationsExample() builder.createAnnotationText(page1, QRectF(250, 350, 24, 24), pdf::TextAnnotationIcon::Paragraph, "Title1", "Subject1", "Paragraph", true); pdf::PDFObjectReference page2 = builder.appendPage(QRectF(0, 0, 400, 400)); - builder.createAnnotationLink(page2, QRectF(50, 50, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Invert); - builder.createAnnotationLink(page2, QRectF(50, 150, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::None); - builder.createAnnotationLink(page2, QRectF(50, 250, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Outline); - builder.createAnnotationLink(page2, QRectF(50, 350, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Push); + builder.createAnnotationLink(page2, QRectF(50, 50, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Invert); + builder.createAnnotationLink(page2, QRectF(50, 150, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::None); + builder.createAnnotationLink(page2, QRectF(50, 250, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Outline); + builder.createAnnotationLink(page2, QRectF(50, 350, 200, 50), "www.seznam.cz", pdf::LinkHighlightMode::Push); pdf::PDFObjectReference page3 = builder.appendPage(QRectF(0, 0, 400, 400)); - builder.createAnnotationFreeText(page3, QRectF(50, 50, 100, 50), "Title", "Subject", "Toto je dolni text, желтая лошадь", Qt::AlignLeft); - builder.createAnnotationFreeText(page3, QRectF(50, 150, 100, 50), "Title", "Subject", "Toto je stredni text, желтая лошадь", Qt::AlignCenter); - builder.createAnnotationFreeText(page3, QRectF(50, 250, 100, 50), "Title", "Subject", "Toto je horni text, желтая лошадь", Qt::AlignRight); - builder.createAnnotationFreeText(page3, QRectF(250, 50, 100, 50), QRectF(300, 50, 50, 50), "Title", "Subject", "Toto je dolni text, желтая лошадь", Qt::AlignLeft, QPointF(250, 50), QPointF(300, 100), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); - builder.createAnnotationFreeText(page3, QRectF(250, 150, 100, 50), QRectF(300, 150, 50, 50), "Title", "Subject", "Toto je stredni text, желтая лошадь", Qt::AlignCenter, QPointF(250, 150), QPointF(300, 200), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); - pdf::PDFObjectReference ref = builder.createAnnotationFreeText(page3, QRectF(250, 250, 100, 50), QRectF(300, 250, 50, 50), "Title", "Subject", "Toto je horni text, желтая лошадь", Qt::AlignRight, QPointF(260, 250), QPointF(260, 290), QPointF(300, 290), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); + builder.createAnnotationFreeText(page3, QRectF(50, 50, 100, 50), "Title", "Subject", "Toto je dolni text, желтая лошадь", Qt::AlignLeft); + builder.createAnnotationFreeText(page3, QRectF(50, 150, 100, 50), "Title", "Subject", "Toto je stredni text, желтая лошадь", Qt::AlignCenter); + builder.createAnnotationFreeText(page3, QRectF(50, 250, 100, 50), "Title", "Subject", "Toto je horni text, желтая лошадь", Qt::AlignRight); + builder.createAnnotationFreeText(page3, QRectF(250, 50, 100, 50), QRectF(300, 50, 50, 50), "Title", "Subject", "Toto je dolni text, желтая лошадь", Qt::AlignLeft, QPointF(250, 50), QPointF(300, 100), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); + builder.createAnnotationFreeText(page3, QRectF(250, 150, 100, 50), QRectF(300, 150, 50, 50), "Title", "Subject", "Toto je stredni text, желтая лошадь", Qt::AlignCenter, QPointF(250, 150), QPointF(300, 200), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); + pdf::PDFObjectReference ref = builder.createAnnotationFreeText(page3, QRectF(250, 250, 100, 50), QRectF(300, 250, 50, 50), "Title", "Subject", "Toto je horni text, желтая лошадь", Qt::AlignRight, QPointF(260, 250), QPointF(260, 290), QPointF(300, 290), pdf::AnnotationLineEnding::OpenArrow, pdf::AnnotationLineEnding::ClosedArrow); builder.setAnnotationContents(ref, "UPDATED: Horni text"); builder.setAnnotationTitle(ref, "Updated title"); builder.setAnnotationSubject(ref, "Updated subject"); @@ -77,8 +77,7 @@ void PDFExamplesGenerator::generateAnnotationsExample() int lineRows = 400 / (baseRect.height() * spaceCoef); int lineCols = 400 / (baseRect.width() * spaceCoef); int lineNumber = 0; - constexpr pdf::AnnotationLineEnding lineEndings[] = - { + constexpr pdf::AnnotationLineEnding lineEndings[] = { pdf::AnnotationLineEnding::None, pdf::AnnotationLineEnding::Square, pdf::AnnotationLineEnding::Circle, @@ -123,8 +122,7 @@ void PDFExamplesGenerator::generateAnnotationsExample() int lineRows = 400 / (baseRect.height() * spaceCoef); int lineCols = 400 / (baseRect.width() * spaceCoef); int lineNumber = 0; - constexpr pdf::AnnotationLineEnding lineEndings[] = - { + constexpr pdf::AnnotationLineEnding lineEndings[] = { pdf::AnnotationLineEnding::None, pdf::AnnotationLineEnding::Square, pdf::AnnotationLineEnding::Circle, diff --git a/PdfExampleGenerator/pdfexamplesgenerator.h b/tools/PdfExampleGenerator/pdfexamplesgenerator.h similarity index 97% rename from PdfExampleGenerator/pdfexamplesgenerator.h rename to tools/PdfExampleGenerator/pdfexamplesgenerator.h index ef56e7e48..a8a1dbefa 100644 --- a/PdfExampleGenerator/pdfexamplesgenerator.h +++ b/tools/PdfExampleGenerator/pdfexamplesgenerator.h @@ -34,4 +34,4 @@ class PDFExamplesGenerator static void generatePageDrawExample(); }; -#endif // PDFEXAMPLESGENERATOR_H +#endif // PDFEXAMPLESGENERATOR_H diff --git a/ProductQuickAccessibilitySmoke/CMakeLists.txt b/tools/ProductQuickAccessibilitySmoke/CMakeLists.txt similarity index 85% rename from ProductQuickAccessibilitySmoke/CMakeLists.txt rename to tools/ProductQuickAccessibilitySmoke/CMakeLists.txt index 019e2fddb..56110702b 100644 --- a/ProductQuickAccessibilitySmoke/CMakeLists.txt +++ b/tools/ProductQuickAccessibilitySmoke/CMakeLists.txt @@ -2,14 +2,14 @@ qt_policy(SET QTP0001 NEW) qt_add_executable(ProductQuickAccessibilitySmoke main.cpp - ../LoopEditor/editorhost.cpp - ../LoopEditor/editorhost.h - ../LoopEditor/focusrestoration.cpp - ../LoopEditor/focusrestoration.h - ../LoopEditor/quickdocumentmodel.cpp - ../LoopEditor/quickdocumentmodel.h - ../LoopEditor/documentviewsession.cpp - ../LoopEditor/documentviewsession.h + ../../LoopEditor/editorhost.cpp + ../../LoopEditor/editorhost.h + ../../LoopEditor/focusrestoration.cpp + ../../LoopEditor/focusrestoration.h + ../../LoopEditor/quickdocumentmodel.cpp + ../../LoopEditor/quickdocumentmodel.h + ../../LoopEditor/documentviewsession.cpp + ../../LoopEditor/documentviewsession.h ) # The QML sources below intentionally mirror LoopEditor/qml/ file-for-file. diff --git a/ProductQuickAccessibilitySmoke/main.cpp b/tools/ProductQuickAccessibilitySmoke/main.cpp similarity index 100% rename from ProductQuickAccessibilitySmoke/main.cpp rename to tools/ProductQuickAccessibilitySmoke/main.cpp diff --git a/ProductQuickAccessibilitySmoke/qml/ActionListPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/ActionListPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/ActionListPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/ActionListPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/CanvasPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/CanvasPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/CanvasPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/CanvasPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/DocumentPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/DocumentPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/DocumentPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/DocumentPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/InspectPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/InspectPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/InspectPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/InspectPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/InspectorPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/InspectorPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/InspectorPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/InspectorPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/Main.qml b/tools/ProductQuickAccessibilitySmoke/qml/Main.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/Main.qml rename to tools/ProductQuickAccessibilitySmoke/qml/Main.qml diff --git a/ProductQuickAccessibilitySmoke/qml/MenuModel.qml b/tools/ProductQuickAccessibilitySmoke/qml/MenuModel.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/MenuModel.qml rename to tools/ProductQuickAccessibilitySmoke/qml/MenuModel.qml diff --git a/ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/PagesProductionPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/PreflightPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/PreflightPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/ProductionPreviewPane.qml diff --git a/ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml b/tools/ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml rename to tools/ProductQuickAccessibilitySmoke/qml/ShellMenuBar.qml diff --git a/ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml b/tools/ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml rename to tools/ProductQuickAccessibilitySmoke/qml/ShellToolBar.qml diff --git a/ProductQuickAccessibilitySmoke/qml/StateBadge.qml b/tools/ProductQuickAccessibilitySmoke/qml/StateBadge.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/StateBadge.qml rename to tools/ProductQuickAccessibilitySmoke/qml/StateBadge.qml diff --git a/ProductQuickAccessibilitySmoke/qml/Workspace.qml b/tools/ProductQuickAccessibilitySmoke/qml/Workspace.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/Workspace.qml rename to tools/ProductQuickAccessibilitySmoke/qml/Workspace.qml diff --git a/ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml b/tools/ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml similarity index 100% rename from ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml rename to tools/ProductQuickAccessibilitySmoke/qml/WorkspacePlaceholderPane.qml diff --git a/QuickShellSmoke/CMakeLists.txt b/tools/QuickShellSmoke/CMakeLists.txt similarity index 100% rename from QuickShellSmoke/CMakeLists.txt rename to tools/QuickShellSmoke/CMakeLists.txt diff --git a/QuickShellSmoke/QuickShellSmoke.qml b/tools/QuickShellSmoke/QuickShellSmoke.qml similarity index 100% rename from QuickShellSmoke/QuickShellSmoke.qml rename to tools/QuickShellSmoke/QuickShellSmoke.qml diff --git a/QuickShellSmoke/main.cpp b/tools/QuickShellSmoke/main.cpp similarity index 100% rename from QuickShellSmoke/main.cpp rename to tools/QuickShellSmoke/main.cpp diff --git a/translations/LOOP_cs.ts b/translations/LOOP_cs.ts index bde396720..9f47c8d6d 100644 --- a/translations/LOOP_cs.ts +++ b/translations/LOOP_cs.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator Generátor kódu - + Remove Odebrat - + Clone Klonovat - + New Nový - + Parameters Parametry - + Data type Datový typ - + Value Hodnota - + Item type Typ položky - + Name Název - + Text description / C++ code Textový popis / kód C++ - + Delete Smazat - + Up Nahoru - + Down Dolů - + New Child Nový potomek - + New Sibling Nový sourozenec - + File Soubor - + Code Kód - + XFA XFA - + Load Načíst - + Ctrl+O Ctrl+O - + Save Uložit - + Ctrl+S Ctrl+S - + Save As... Uložit jako… - + Set code header (*.h) Nastavit hlavičkový soubor (*.h) - + Set code source (*.cpp) Nastavit zdroj kódu (*.cpp) - + Generate code Generovat kód - + Ctrl+G Ctrl+G - + Set code header XFA Nastavit hlavičku XFA - + Set code source XFA Nastavit zdroj XFA - + Generate XFA code Generovat kód XFA - + Set XFA description Nastavit popis XFA - - + + Select XML definition file Vybrat definiční soubor XML - + Create function Vytvořit funkci - + Enter function name Zadejte název funkce - - + + Select cpp header Vybrat hlavičkový soubor C++ - - + + Select cpp source Vybrat zdrojový soubor C++ - + Select xml definition Vybrat definici XML @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer Prohlížeč obrázků JBIG2 - + Images Obrázky - + File Soubor - + Add image Přidat obrázek - + Ctrl+O Ctrl+O - + Clear Vyčistit - + Ctrl+W Ctrl+W - + Add JBIG2 image Přidat obrázek JBIG2 - + Ctrl+J Ctrl+J - - - + + + Error Chyba - - + + Open image Otevřít obrázek @@ -6617,32 +6617,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object Objekt - + Array (simple) Pole (jednoduché) - + Array (complex) Pole (komplexní) - + Dictionary Slovník - + Item (simple), name = '%1' Položka (jednoduchá), název = '%1' - + Item (complex), name = '%1' Položka (komplexní), název = '%1' diff --git a/translations/LOOP_de.ts b/translations/LOOP_de.ts index 09ca224ac..e76993f1e 100644 --- a/translations/LOOP_de.ts +++ b/translations/LOOP_de.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator Codegenerator - + Remove Entfernen - + Clone Klonen - + New Neu - + Parameters Parameter - + Data type Datentyp - + Value Wert - + Item type Elementtyp - + Name Name - + Text description / C++ code Textbeschreibung / C++-Code - + Delete Löschen - + Up Hoch - + Down Ausgefallen - + New Child Neues untergeordnetes Element - + New Sibling Neues Geschwister - + File Datei - + Code Code - + XFA XFA - + Load Laden - + Ctrl+O Ctrl+O - + Save Speichern - + Ctrl+S Ctrl+S - + Save As... Speichern unter... - + Set code header (*.h) Code-Header festlegen (*.h) - + Set code source (*.cpp) Codequelle festlegen (*.cpp) - + Generate code Code generieren - + Ctrl+G Ctrl+G - + Set code header XFA Code-Header XFA festlegen - + Set code source XFA Codequelle XFA festlegen - + Generate XFA code XFA-Code generieren - + Set XFA description XFA-Beschreibung festlegen - - + + Select XML definition file XML-Definitionsdatei auswählen - + Create function Funktion erstellen - + Enter function name Geben Sie den Funktionsnamen ein - - + + Select cpp header CPP-Header auswählen - - + + Select cpp source CPP-Quelle auswählen - + Select xml definition XML-Definition auswählen @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer JBIG2-Bildbetrachter - + Images Bilder - + File Datei - + Add image Bild hinzufügen - + Ctrl+O Ctrl+O - + Clear Klar - + Ctrl+W Ctrl+W - + Add JBIG2 image JBIG2-Image hinzufügen - + Ctrl+J Ctrl+J - - - + + + Error Fehler - - + + Open image Bild öffnen @@ -6617,32 +6617,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object Objekt - + Array (simple) Array (einfach) - + Array (complex) Array (komplex) - + Dictionary Wörterbuch - + Item (simple), name = '%1' Element (einfach), Name = „%1“ - + Item (complex), name = '%1' Element (komplex), Name = „%1“ diff --git a/translations/LOOP_en.ts b/translations/LOOP_en.ts index b8def02f7..2b0b8f8ca 100644 --- a/translations/LOOP_en.ts +++ b/translations/LOOP_en.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator - + Remove - + Clone - + New - + Parameters - + Data type - + Value - + Item type - + Name - + Text description / C++ code - + Delete - + Up - + Down - + New Child - + New Sibling - + File - + Code - + XFA - + Load - + Ctrl+O - + Save - + Ctrl+S - + Save As... - + Set code header (*.h) - + Set code source (*.cpp) - + Generate code - + Ctrl+G - + Set code header XFA - + Set code source XFA - + Generate XFA code - + Set XFA description - - + + Select XML definition file - + Create function - + Enter function name - - + + Select cpp header - - + + Select cpp source - + Select xml definition @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer - + Images - + File - + Add image - + Ctrl+O - + Clear - + Ctrl+W - + Add JBIG2 image - + Ctrl+J - - - + + + Error - - + + Open image @@ -6527,32 +6527,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object - + Array (simple) - + Array (complex) - + Dictionary - + Item (simple), name = '%1' - + Item (complex), name = '%1' diff --git a/translations/LOOP_es.ts b/translations/LOOP_es.ts index 91bd5ec9a..5833093e2 100644 --- a/translations/LOOP_es.ts +++ b/translations/LOOP_es.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator Generador de código - + Remove Quitar - + Clone Clonar - + New Nuevo - + Parameters Parámetros - + Data type Tipo de datos - + Value Valor - + Item type Tipo de elemento - + Name Nombre - + Text description / C++ code Descripción de texto/código C++ - + Delete Eliminar - + Up Arriba - + Down Abajo - + New Child Nuevo elemento secundario - + New Sibling Nuevo hermano - + File Archivo - + Code Código - + XFA XFA - + Load Cargar - + Ctrl+O Ctrl+O - + Save Guardar - + Ctrl+S Ctrl+S - + Save As... Guardar como... - + Set code header (*.h) Establecer encabezado de código (*.h) - + Set code source (*.cpp) Establecer código fuente (*.cpp) - + Generate code Generar código - + Ctrl+G Ctrl+G - + Set code header XFA Establecer encabezado de código XFA - + Set code source XFA Establecer fuente de código XFA - + Generate XFA code Generar código XFA - + Set XFA description Establecer descripción XFA - - + + Select XML definition file Seleccione el archivo de definición XML - + Create function Crear función - + Enter function name Ingrese el nombre de la función - - + + Select cpp header Seleccione el encabezado cpp - - + + Select cpp source Seleccione la fuente de cpp - + Select xml definition Seleccione la definición xml @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer Visor de imágenes JBIG2 - + Images Imágenes - + File Archivo - + Add image Agregar imagen - + Ctrl+O Ctrl+O - + Clear Borrar - + Ctrl+W Ctrl+W - + Add JBIG2 image Agregar imagen JBIG2 - + Ctrl+J Ctrl+J - - - + + + Error Error - - + + Open image Abrir imagen @@ -6617,32 +6617,32 @@ li.checked::marker { contenido: "\2612"; } codegen::GeneratedPDFObject - + Object Objeto - + Array (simple) Matriz (simple) - + Array (complex) Matriz (compleja) - + Dictionary Diccionario - + Item (simple), name = '%1' elemento (simple), nombre = '%1' - + Item (complex), name = '%1' elemento (complejo), nombre = '%1' diff --git a/translations/LOOP_fr.ts b/translations/LOOP_fr.ts index 02669724c..339ddabc9 100644 --- a/translations/LOOP_fr.ts +++ b/translations/LOOP_fr.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator Générateur de code - + Remove Supprimer - + Clone Clone - + New Nouveau - + Parameters Parameters - + Data type Type de données - + Value Value - + Item type Type d'élément - + Name Nom - + Text description / C++ code Description textuelle / code C++ - + Delete Supprimer - + Up Haut - + Down Bas - + New Child Nouvel enfant - + New Sibling Nouveau frère ou sœur - + File Fichier - + Code Code - + XFA XFA - + Load Load - + Ctrl+O Ctrl+O - + Save Enregistrer - + Ctrl+S Ctrl+S - + Save As... Enregistrer sous... - + Set code header (*.h) Définir l'en-tête de code (*.h) - + Set code source (*.cpp) Définir la source du code (*.cpp) - + Generate code Générer du code - + Ctrl+G Ctrl+G - + Set code header XFA Définir l'en-tête de code XFA - + Set code source XFA Définir la source du code XFA - + Generate XFA code Générer du code XFA - + Set XFA description Définir la description XFA - - + + Select XML definition file Sélectionnez le fichier de définition XML - + Create function Créer une fonction - + Enter function name Saisissez le nom de la fonction - - + + Select cpp header Sélectionnez l'en-tête cpp - - + + Select cpp source Sélectionnez la source cpp - + Select xml definition Sélectionnez la définition XML @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer Visionneuse d'images JBIG2 - + Images Images - + File Fichier - + Add image Ajouter une image - + Ctrl+O Ctrl+O - + Clear Effacer - + Ctrl+W Ctrl+W - + Add JBIG2 image Ajouter une image JBIG2 - + Ctrl+J Ctrl+J - - - + + + Error Error - - + + Open image Image ouverte @@ -6617,32 +6617,32 @@ li.checked::marker { contenu : "\2612" ; } codegen::GeneratedPDFObject - + Object Object - + Array (simple) Array (simple) - + Array (complex) Tableau (complexe) - + Dictionary Dictionary - + Item (simple), name = '%1' élément (simple), nom = '%1' - + Item (complex), name = '%1' élément (complexe), nom = '%1' diff --git a/translations/LOOP_ko.ts b/translations/LOOP_ko.ts index 324cc16ce..6f24d7273 100644 --- a/translations/LOOP_ko.ts +++ b/translations/LOOP_ko.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator 코드 생성기 - + Remove 제거 - + Clone 복제 - + New 신규 - + Parameters 매개변수 - + Data type 데이터 유형 - + Value 값 - + Item type 항목 유형 - + Name 이름 - + Text description / C++ code 텍스트 설명/C++ 코드 - + Delete 삭제 - + Up 위로 - + Down 아래에 - + New Child 새 하위 - + New Sibling 새로운 형제 - + File 파일 - + Code 코드 - + XFA XFA - + Load 로드 - + Ctrl+O Ctrl+O - + Save 저장 - + Ctrl+S Ctrl+S - + Save As... 다른 이름으로 저장... - + Set code header (*.h) 코드 헤더 설정(*.h) - + Set code source (*.cpp) 코드 소스 설정(*.cpp) - + Generate code 코드 생성 - + Ctrl+G Ctrl+G - + Set code header XFA 코드 헤더 XFA 설정 - + Set code source XFA 코드 소스 XFA 설정 - + Generate XFA code XFA 코드 생성 - + Set XFA description XFA 설명 설정 - - + + Select XML definition file XML 정의 파일 선택 - + Create function 기능 생성 - + Enter function name 기능 이름 입력 - - + + Select cpp header Cpp 헤더 선택 - - + + Select cpp source Cpp 소스 선택 - + Select xml definition Xml 정의 선택 @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer JBIG2 이미지 뷰어 - + Images 이미지 - + File 파일 - + Add image 이미지 추가 - + Ctrl+O Ctrl+O - + Clear 지우기 - + Ctrl+W Ctrl+W - + Add JBIG2 image JBIG2 이미지 추가 - + Ctrl+J Ctrl+J - - - + + + Error 오류 - - + + Open image 이미지 열기 @@ -6617,32 +6617,32 @@ li.checked::marker { 내용: "\2612"; } codegen::GeneratedPDFObject - + Object 개체 - + Array (simple) 배열(단순) - + Array (complex) 어레이(복잡함) - + Dictionary 사전 - + Item (simple), name = '%1' 항목(단순), 이름 = '%1' - + Item (complex), name = '%1' 항목(복합), 이름 = '%1' diff --git a/translations/LOOP_ru.ts b/translations/LOOP_ru.ts index 297e93066..30138a8c8 100644 --- a/translations/LOOP_ru.ts +++ b/translations/LOOP_ru.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator генератор кода - + Remove удалить - + Clone Клонировать - + New Новое - + Parameters Параметры - + Data type тип данных - + Value значение - + Item type тип элемента - + Name Имя - + Text description / C++ code текстовое описание/код C++. - + Delete Удалить - + Up вверх - + Down вниз - + New Child Новый ребенок - + New Sibling новый одноуровневый вариант - + File Файл - + Code Код - + XFA XFA - + Load Загрузить - + Ctrl+O Ctrl+O - + Save Сохранить - + Save As... Сохранить как... - + Ctrl+S Ctrl+S - + Set code header (*.h) Установите заголовок кода (*.h) - + Set code source (*.cpp) Установите источник кода (*.cpp). - + Generate code создание кода - + Ctrl+G Ctrl+G - + Set code header XFA установите заголовок кода XFA - + Set code source XFA установка источника кода XFA - + Generate XFA code создание кода XFA - + Set XFA description установите описание XFA - - + + Select XML definition file выберите файл определения XML. - + Create function Создать функцию - + Enter function name введите имя функции. - - + + Select cpp header Выбрать cpp-заголовок - - + + Select cpp source Выбрать cpp-код - + Select xml definition выберите определение XML. @@ -508,49 +508,49 @@ MainWindow - + JBIG2 Image Viewer Просмотр изображений JBIG2 - + Images Изображения - + Ctrl+O Ctrl+O - + File Файл - + Add image Добавить изображение - + Clear Очистить - + Ctrl+W Ctrl+W - + Add JBIG2 image добавьте изображение JBIG2. - + Ctrl+J Ctrl+J @@ -1103,15 +1103,15 @@ маркеры &отображения - - - + + + Error Ошибка - - + + Open image Открыть изображение @@ -6617,32 +6617,32 @@ __ТК7____ТК8____ТК9__ codegen::GeneratedPDFObject - + Object Объект - + Array (simple) массив (простой) - + Array (complex) массив (сложный) - + Dictionary словарь - + Item (simple), name = '%1' элемент (простой), имя = '%1' - + Item (complex), name = '%1' элемент (сложный), имя = '%1' diff --git a/translations/LOOP_tr.ts b/translations/LOOP_tr.ts index 040419e17..52721c49a 100644 --- a/translations/LOOP_tr.ts +++ b/translations/LOOP_tr.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator Kod üreticisi - + Remove Kaldır - + Clone Klon - + New Yeni - + Parameters Parametreler - + Data type Veri türü - + Value Değer - + Item type Öğe türü - + Name Ad - + Text description / C++ code Metin Açıklaması / C ++ Kodu - + Delete Sil - + Up Yukarı - + Down Aşağı - + New Child Yeni Çocuk - + New Sibling Yeni Kardeş - + File Dosya - + Code Kod - + XFA XFA - + Load Yükle - + Ctrl+O - + Save Kaydet - + Ctrl+S - + Save As... Farklı Kaydet... - + Set code header (*.h) Kod başlığını ayarlayın (*.h) - + Set code source (*.cpp) Kod kaynağını ayarlayın (*.cpp) - + Generate code Kod Üretin - + Ctrl+G - + Set code header XFA Kod başlığını ayarlayın XFA - + Set code source XFA Kod kaynağını ayarlayın XFA - + Generate XFA code XFA Kodu Oluşturun - + Set XFA description XFA açıklamasını ayarlayın - - + + Select XML definition file XML tanım dosyasını seçin - + Create function Fonksiyon oluşturun - + Enter function name Fonksiyon adını yazın - - + + Select cpp header CPP başlığını seçin - - + + Select cpp source CPP kaynağını seçin - + Select xml definition XML Tanımını seçin @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer JBIG2 Resim Görüntüleyici - + Images Resimler - + File Dosya - + Add image Resim ekle - + Ctrl+O - + Clear Temizle - + Ctrl+W - + Add JBIG2 image JBIG2 resmi ekle - + Ctrl+J - - - + + + Error Hata - - + + Open image Resmi Aç @@ -6618,32 +6618,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object Nesne - + Array (simple) Dizi (basit) - + Array (complex) Dizi (karmaşık) - + Dictionary Sözlük - + Item (simple), name = '%1' Öğe (basit), ad = '%1' - + Item (complex), name = '%1' Öğe (karmaşık), ad = '%1' diff --git a/translations/LOOP_zh_CN.ts b/translations/LOOP_zh_CN.ts index 579adf0e8..d0119d33b 100644 --- a/translations/LOOP_zh_CN.ts +++ b/translations/LOOP_zh_CN.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator 代码生成器 - + Remove 移除 - + Clone 克隆 - + New 新建 - + Parameters 参数 - + Data type 数据类型 - + Value 值 - + Item type 条目类型 - + Name 名称 - + Text description / C++ code 文本描述 / C++代码 - + Delete 删除 - + Up - + Down - + New Child 新子结点 - + New Sibling 新兄弟结点 - + File 文件 - + Code 代码 - + XFA - + Load 载入 - + Ctrl+O - + Save 保存 - + Ctrl+S - + Save As... 另存为... - + Set code header (*.h) 设置代码头文件（*.h） - + Set code source (*.cpp) 设置代码源 - + Generate code 生成代码 - + Ctrl+G - + Set code header XFA 设置代码头文件 XFA - + Set code source XFA 设置代码源 XFA - + Generate XFA code 生成XFA代码 - + Set XFA description 设置XFA描述 - - + + Select XML definition file 选取XML定义文件 - + Create function 创建函数 - + Enter function name 键入函数名 - - + + Select cpp header 选取cpp头文件 - - + + Select cpp source 选取cpp源 - + Select xml definition 选取xml定义 @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer JBIG2图像浏览器 - + Images 图像 - + File 文件 - + Add image 增加图像 - + Ctrl+O - + Clear 清除 - + Ctrl+W - + Add JBIG2 image 增加JBIG2图像 - + Ctrl+J - - - + + + Error 错误 - - + + Open image 打开图像 @@ -6628,32 +6628,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object 对象 - + Array (simple) 数组（简单） - + Array (complex) 数组（复杂） - + Dictionary 词典 - + Item (simple), name = '%1' 项目（简单），名称 = '%1' - + Item (complex), name = '%1' 项目（复杂），名称 = '%1' diff --git a/translations/LOOP_zh_TW.ts b/translations/LOOP_zh_TW.ts index 03497ddb7..97a6bc695 100644 --- a/translations/LOOP_zh_TW.ts +++ b/translations/LOOP_zh_TW.ts @@ -228,190 +228,190 @@ GeneratorMainWindow - + Code Generator 代碼生成器 - + Remove 移除 - + Clone 克隆 - + New 新建 - + Parameters 參數 - + Data type 數據類型 - + Value 值 - + Item type 條目類型 - + Name 名稱 - + Text description / C++ code 文本描述 / C++代碼 - + Delete 刪除 - + Up Up - + Down Down - + New Child 新子結點 - + New Sibling 新兄弟結點 - + File 文檔 - + Code 代碼 - + XFA XFA - + Load 載入 - + Ctrl+O Ctrl+O - + Save 保存 - + Ctrl+S Ctrl+S - + Save As... 另存為... - + Set code header (*.h) 設置代碼頭文件（*.h） - + Set code source (*.cpp) 設置代碼源 - + Generate code 生成代碼 - + Ctrl+G Ctrl+G - + Set code header XFA 設置代碼頭文件 XFA - + Set code source XFA 設置代碼源 XFA - + Generate XFA code 生成XFA代碼 - + Set XFA description 設置XFA描述 - - + + Select XML definition file 選取XML定義文件 - + Create function 創建函數 - + Enter function name 鍵入函數名 - - + + Select cpp header 選取cpp頭文件 - - + + Select cpp source 選取cpp源 - + Select xml definition 選取xml定義 @@ -508,62 +508,62 @@ MainWindow - + JBIG2 Image Viewer JBIG2圖像檢視器 - + Images 圖像 - + File 文件 - + Add image 增加圖像 - + Ctrl+O Ctrl+O - + Clear 清除 - + Ctrl+W Ctrl+W - + Add JBIG2 image 增加JBIG2圖像 - + Ctrl+J Ctrl+J - - - + + + Error 錯誤 - - + + Open image 打開圖像 @@ -6622,32 +6622,32 @@ li.checked::marker { content: "\2612"; } codegen::GeneratedPDFObject - + Object 物件 - + Array (simple) 數組（簡單） - + Array (complex) 數組（複雜） - + Dictionary 詞典 - + Item (simple), name = '%1' 項目（簡單），名稱 = '%1' - + Item (complex), name = '%1' 項目（複雜），名稱 = '%1'