From 1140dad0ca6c8960e8426c99de88ce41124878c5 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 07:50:49 +0200 Subject: [PATCH 01/22] Fix ARM-compatible controller and Pluto setup for 1.0.3 --- .github/workflows/ci.yml | 4 +-- .github/workflows/native-vsix.yml | 34 +++++++++++++++++-------- CHANGELOG.md | 9 +++++++ README.md | 30 ++++++++++++---------- package-lock.json | 4 +-- package.json | 2 +- src/plutoNotebook.ts | 13 +++++----- src/workspaceSetup.ts | 12 ++++----- test/native-core-contract.mjs | 6 ++--- test/native-core-contract.test.mjs | 8 +++--- test/native-platform-contract.mjs | 26 +++++++++++++++++++ test/native-platform-contract.test.mjs | 35 ++++++++++++++++++++++++++ test/pluto.test.mjs | 35 ++++++++++++++++++++++++-- test/public-vsix-host.cjs | 5 +++- test/run-public-vsix-host.mjs | 15 +++++++---- test/workspace-setup.test.mjs | 18 +++++++++---- 16 files changed, 195 insertions(+), 61 deletions(-) create mode 100644 test/native-platform-contract.mjs create mode 100644 test/native-platform-contract.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acc2d96..d5f4c17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v7 with: repository: Mirage-Interactive-Fr/PerfChecker.jl - ref: 00e94c62a2080dba02782e451422ed98ea0b358a + ref: d079efd5c24c3dcac5b7550781822038e0ff780f path: validation-core persist-credentials: false - uses: actions/setup-node@v7 @@ -53,7 +53,7 @@ jobs: - run: npm run compile - name: Verify the pinned Core candidate tree shell: bash - run: test "$(git -C validation-core rev-parse HEAD^{tree})" = "2563a09e7904f19592e8f658289566e76bf93c6c" + run: test "$(git -C validation-core rev-parse HEAD^{tree})" = "749f8eac351ed2a1f0bc110456f2eb37a3bc08ee" - name: Prepare exact Core candidate controller source shell: bash run: julia --startup-file=no -e 'using Pkg; Pkg.activate(ENV["PERFCHECKER_TEST_CONTROLLER"]); Pkg.develop(path=ENV["PERFCHECKER_TEST_CORE"]); Pkg.instantiate()' diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 4c88787..96271f5 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -26,7 +26,7 @@ on: scope: description: All supported hosts, or an explicit focused platform type: choice - options: [all, linux, linux-stable, windows, windows-stable, macos-stable] + options: [all, linux, linux-stable, windows, windows-stable, macos-stable, arm-bootstrap] default: all core: description: Published General minimum, explicit legacy compatibility, or a pinned candidate @@ -150,6 +150,10 @@ jobs: NATIVE_EVENT: ${{ github.event_name }} NATIVE_VSIX_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.artifact || 'candidate' }} NATIVE_NEW_ORIGIN: ${{ inputs.new_candidate_origin || 'false' }} + NATIVE_SCOPE: ${{ inputs.scope }} + NATIVE_STAGE: ${{ inputs.stage }} + NATIVE_CASE_GROUP: ${{ inputs.case_group }} + NATIVE_CORE: ${{ inputs.core }} REUSE_RUN: ${{ inputs.candidate_run }} REUSE_NAME: ${{ inputs.candidate_name }} REUSE_SHA: ${{ inputs.candidate_sha }} @@ -157,9 +161,14 @@ jobs: - uses: actions/checkout@v7 - name: Select the declared archive origin for qualification run: | - node <<'NODE' - const fs=require('fs'),e=process.env; + node --input-type=module <<'NODE' + import fs from 'node:fs'; + import {assertNativeCampaignSelection} from './test/native-platform-contract.mjs'; + const e=process.env; + assertNativeCampaignSelection({scope:e.NATIVE_SCOPE,event:e.NATIVE_EVENT,artifact:e.NATIVE_VSIX_MODE, + stage:e.NATIVE_STAGE,group:e.NATIVE_CASE_GROUP,core:e.NATIVE_CORE}); const requested=[e.REUSE_RUN,e.REUSE_NAME,e.REUSE_SHA].some(Boolean),newOrigin=e.NATIVE_NEW_ORIGIN==='true'; + if(e.NATIVE_SCOPE==='arm-bootstrap'&&(!requested||newOrigin))throw Error('ARM bootstrap must reuse the original candidate archive; it never packages a second origin.'); if(e.NATIVE_VSIX_MODE==='public'){ if(requested||newOrigin)throw Error('Public reproduction uses its published release, not candidate-origin inputs'); process.exit(0); @@ -229,11 +238,11 @@ jobs: native-workflows: if: github.event_name != 'workflow_dispatch' || inputs.stage != 'connector-workers' needs: package - name: Native VSIX (${{ matrix.os }}, VS Code ${{ matrix.vscode }}) + name: Native VSIX (${{ matrix.os }}, VS Code ${{ matrix.vscode }}${{ matrix.julia && format(', Julia {0}', matrix.julia) || '' }}) strategy: fail-fast: false matrix: - include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && '[{"os":"macos-15-intel","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]') }} + include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && '[{"os":"macos-15-intel","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]') }} runs-on: ${{ matrix.os }} timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }} env: @@ -242,9 +251,11 @@ jobs: PERFCHECKER_NATIVE_CANDIDATE_SHA: ${{ needs.package.outputs.sha }} PERFCHECKER_NATIVE_STAGE: ${{ github.event_name == 'workflow_dispatch' && inputs.stage || 'full' }} PERFCHECKER_NATIVE_CASE_GROUP: ${{ inputs.case_group }} + PERFCHECKER_NATIVE_SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'all' }} + PERFCHECKER_NATIVE_EXPECTED_ARCH: ${{ matrix.arch || 'x64' }} PERFCHECKER_NATIVE_CORE: ${{ github.event_name == 'workflow_dispatch' && inputs.core || 'candidate' }} - PERFCHECKER_NATIVE_CORE_COMMIT: ${{ (github.event_name != 'workflow_dispatch' || inputs.core == 'candidate') && (inputs.core_commit || '00e94c62a2080dba02782e451422ed98ea0b358a') || '' }} - PERFCHECKER_NATIVE_CORE_TREE: ${{ (github.event_name != 'workflow_dispatch' || inputs.core == 'candidate') && (inputs.core_tree || '2563a09e7904f19592e8f658289566e76bf93c6c') || '' }} + PERFCHECKER_NATIVE_CORE_COMMIT: ${{ (github.event_name != 'workflow_dispatch' || inputs.core == 'candidate') && (inputs.core_commit || 'd079efd5c24c3dcac5b7550781822038e0ff780f') || '' }} + PERFCHECKER_NATIVE_CORE_TREE: ${{ (github.event_name != 'workflow_dispatch' || inputs.core == 'candidate') && (inputs.core_tree || '749f8eac351ed2a1f0bc110456f2eb37a3bc08ee') || '' }} PERFCHECKER_NATIVE_VIDEO: ${{ matrix.os == 'ubuntu-latest' && matrix.vscode == 'stable' && '1' || '0' }} PERFCHECKER_TEST_JULIA: julia JULIA_NUM_THREADS: '1' @@ -258,17 +269,18 @@ jobs: - uses: actions/setup-node@v7 with: node-version: 22 + architecture: ${{ matrix.arch || 'x64' }} cache: npm - uses: julia-actions/setup-julia@v3 id: runner-julia with: - version: ${{ github.event_name == 'workflow_dispatch' && inputs.backend_julia || '1' }} - arch: x64 + version: ${{ matrix.julia || (github.event_name == 'workflow_dispatch' && inputs.backend_julia || '1') }} + arch: ${{ matrix.arch == 'arm64' && 'aarch64' || 'x64' }} - uses: julia-actions/setup-julia@v3 id: official-julia with: version: '1.12' - arch: x64 + arch: ${{ matrix.arch == 'arm64' && 'aarch64' || 'x64' }} - name: Preserve separate runtimes for PerfChecker and official Julia integration env: RUNNER_JULIA_BINDIR: ${{ steps.runner-julia.outputs.julia-bindir }} @@ -315,5 +327,5 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: native-workflows-${{ matrix.os }}-${{ matrix.vscode }} + name: native-workflows-${{ matrix.os }}-${{ matrix.vscode }}${{ matrix.julia && format('-julia-{0}', matrix.julia) || '' }} path: native-qualification-results/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b03a2ed..8ab215c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 1.0.3 — ARM-compatible setup + +- Install PerfChecker.jl 1.1.1 or newer in the 1.x series when creating a controller + or installing/upgrading the separate Pluto environment. This excludes Core + 1.1.0's ARM import failure while allowing compatible updates. +- Resolve Core and the requested packages together before precompilation, then + verify the installed Core version. Existing prepared environments still accept + Core 1.1.0 in 1.x; installation and upgrades require an explicit choice. + ## 1.0.2 — Narrow chat and process lifecycle fixes - Keep MCP chat controls usable in narrow VS Code editor panels with the host's diff --git a/README.md b/README.md index fe90ee0..4bef8a9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # PerfChecker for VS Code -PerfChecker for VS Code 1.0.2 provides Studio, interactive plots, Julia tools, MCP +PerfChecker for VS Code 1.0.3 provides Studio, interactive plots, Julia tools, MCP conversation, and reviewed agent implementation with Git recovery. Chat adapts to -narrow VS Code webviews, and explicit controller setup installs PerfChecker.jl -1.1.0. Guided setup and Pluto notebooks require Core 1.1.0; existing CLI -controllers with Core 1.0.1 remain compatible. Core 1.1.0 is recommended for its -read-only worker-environment fixes. Opening a workspace never installs or +narrow VS Code webviews. New controller and Pluto installations require +PerfChecker.jl 1.1.1 or newer in 1.x, which fixes loading on ARM. +Existing prepared environments still accept Core 1.1.0, and CLI controllers +with Core 1.0.1 remain compatible. Opening a workspace never installs or upgrades Julia packages. The user guides are maintained with PerfChecker.jl on its existing documentation host: @@ -24,8 +24,9 @@ modify and asks before downloading packages. An existing package `Project.toml` alone does not make a measurement controller ready. Use **Create controller environment**, **Use an existing controller**, or **Read the setup guide** -as appropriate. Guided setup requires registered PerfChecker.jl 1.1.0 or later. -It asks before upgrading an existing controller; other CLI actions remain +as appropriate. New installations require registered PerfChecker.jl 1.1.1 or +newer in the 1.x series. Prepared environments still accept Core 1.1.0. +Setup asks before installing or upgrading packages; other CLI actions remain compatible with a controller containing PerfChecker.jl 1.0.1. The starter installs measurement dependencies; add your package and its test dependencies explicitly to the selected controller before measuring test items. @@ -43,7 +44,8 @@ For notebooks, select **New Pluto notebook** and choose **Feature suite** or **Investigation**. The extension uses real reactive Pluto `.jl` notebooks in an editor tab. Stable Pluto 1.0.4 and PerfCheckerPluto 1.0.1 use a separate `perf/pluto` Julia environment. The companion stays on its independent public `v1.0.1` tag; -the Core requirement is 1.1.0. Installing or upgrading it requires an explicit choice and +prepared environments require Core 1.1.0, while installation/upgrades request +Core 1.1.1 or newer in 1.x. Installing or upgrading it requires an explicit choice and does not modify the MCP controller. Opening the notebook or changing selectors does not launch measurements. Use its Launch/Cancel/Refresh controls, and **Save completed reports** in a suite @@ -163,8 +165,8 @@ JSONL and Markdown outputs. See [CONTRACT.md](CONTRACT.md) for the exact boundar Set-Location C:\path\to\PerfCheckerVSCode npm ci npm test -npm run package -- --out perfchecker-vscode-1.0.2.vsix -code --install-extension .\perfchecker-vscode-1.0.2.vsix --force +npm run package -- --out perfchecker-vscode-1.0.3.vsix +code --install-extension .\perfchecker-vscode-1.0.3.vsix --force ``` Reload VS Code, open a Julia package workspace, then select the PerfChecker icon @@ -363,7 +365,7 @@ against an existing project or your normal editor profile. ```powershell npm ci npm test -npm run package -- --out perfchecker-vscode-1.0.2.vsix +npm run package -- --out perfchecker-vscode-1.0.3.vsix ``` `npm test` compiles TypeScript and runs the model/contract tests. The generated @@ -371,8 +373,10 @@ npm run package -- --out perfchecker-vscode-1.0.2.vsix ## Marketplace publication -Publish the stable extension only after PerfChecker.jl 1.1.0 is available in +Publish version 1.0.3 only after PerfChecker.jl 1.1.1 is available in Julia's General registry. Build and test the exact VSIX before publishing it. +Declare a new candidate origin for this version and reuse its qualified bytes +for publication. The immutable Marketplace identity is `mirage-interactive-fr.perfchecker-vscode`. Repository extraction therefore does @@ -382,7 +386,7 @@ For a manual stable release: ```powershell npx vsce login mirage-interactive-fr -npx vsce publish --packagePath .\perfchecker-vscode-1.0.2.vsix +npx vsce publish --packagePath .\perfchecker-vscode-1.0.3.vsix ``` Publishing requires a free Microsoft identity and Marketplace publisher, not a diff --git a/package-lock.json b/package-lock.json index e479a44..8272e6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "perfchecker-vscode", - "version": "1.0.2", + "version": "1.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "perfchecker-vscode", - "version": "1.0.2", + "version": "1.0.3", "license": "MIT", "devDependencies": { "@types/node": "^22.10.0", diff --git a/package.json b/package.json index 55c3f2c..d2b7e0b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "perfchecker-vscode", "displayName": "PerfChecker", "description": "Measure Julia performance, explore interactive results and improve code with a connected agent.", - "version": "1.0.2", + "version": "1.0.3", "publisher": "mirage-interactive-fr", "author": "Mirage Interactive", "repository": { diff --git a/src/plutoNotebook.ts b/src/plutoNotebook.ts index da76d05..c2110d2 100644 --- a/src/plutoNotebook.ts +++ b/src/plutoNotebook.ts @@ -16,12 +16,13 @@ export async function shutdownPlutoSessions(): Promise { const installCode = ` using Pkg -Pkg.add(PackageSpec(name="PerfChecker", version="1.1.0")) -Pkg.add([PackageSpec(name="Pluto", version="1.0.4"), PackageSpec(name="PlutoUI"), - PackageSpec(name="BenchmarkTools"), PackageSpec(name="Chairmarks")]) -Pkg.add(PackageSpec(url="https://github.com/Mirage-Interactive-Fr/PerfChecker.jl", - subdir="packages/PerfCheckerPluto", rev="v1.0.1")) +Pkg.add([PackageSpec(name="PerfChecker", version="1.1.1 - 1"), + PackageSpec(name="Pluto", version="1.0.4"), PackageSpec(name="PlutoUI"), + PackageSpec(name="BenchmarkTools"), PackageSpec(name="Chairmarks"), + PackageSpec(url="https://github.com/Mirage-Interactive-Fr/PerfChecker.jl", + subdir="packages/PerfCheckerPluto", rev="v1.0.1")]) using PerfChecker, PerfCheckerPluto, Pluto, PlutoUI +v"1.1.1" <= Base.pkgversion(PerfChecker) < v"2.0.0" || error("Pluto installation requires registered PerfChecker 1.1.1 or newer in the 1.x series.") `; const generateCode = ` using PerfChecker, PerfCheckerPluto, Base64 @@ -397,7 +398,7 @@ export class PlutoNotebooks implements vscode.Disposable { } const action = exists ? 'Upgrade Pluto environment' : 'Install Pluto environment'; const choice = await vscode.window.showWarningMessage( - `${exists ? 'This existing separate Pluto environment needs an explicit upgrade' : 'Pluto needs its own Julia environment'} at ${project}. Install registered PerfChecker 1.1.0, PerfCheckerPluto 1.0.1, Pluto 1.0.4 and PlutoUI there? This downloads packages and updates that environment. Your MCP controller stays separate.`, + `${exists ? 'This existing separate Pluto environment needs an explicit upgrade' : 'Pluto needs its own Julia environment'} at ${project}. Install registered PerfChecker 1.1.1 or newer in the 1.x series, PerfCheckerPluto 1.0.1, Pluto 1.0.4 and PlutoUI there? This downloads packages and updates that environment. Your MCP controller stays separate.`, {modal: true}, action, 'Open setup guide'); if (choice === 'Open setup guide') { await vscode.env.openExternal(vscode.Uri.parse('https://perfchecker.mirageinteractive.fr/interfaces/repl-pluto.html')); return; diff --git a/src/workspaceSetup.ts b/src/workspaceSetup.ts index 4e2d14c..07b21e5 100644 --- a/src/workspaceSetup.ts +++ b/src/workspaceSetup.ts @@ -25,7 +25,7 @@ export async function prepareWorkspaceController(folder: vscode.WorkspaceFolder, if(JSON.stringify(snapshot())!==JSON.stringify(expected))throw new Error('The PerfChecker controller settings changed during setup. Your new settings were preserved; start setup again.'); }; const verifyCode='using PerfChecker; v"1.1.0" <= Base.pkgversion(PerfChecker) < v"2.0.0" || error("PerfChecker 1.1.0 or newer in the 1.x series is required. Explicitly upgrade this controller environment from General before continuing.")'; - const run=async(project:string,code:string,title:string)=>{ + const run=async(project:string,code:string,title:string,minimumVersion='1.1.0')=>{ await vscode.window.withProgress({location:vscode.ProgressLocation.Notification,title,cancellable:true},async(_progress,token)=>{ current(); if(token.isCancellationRequested)throw new vscode.CancellationError(); @@ -45,7 +45,7 @@ export async function prepareWorkspaceController(folder: vscode.WorkspaceFolder, child.once('close',status=>{cleanup();try{current();}catch(error){reject(error);return;} if(token.isCancellationRequested&&status!==0&&status!==130){reject(new ControllerCancellationFailure('Controller cancellation failed. Inspect PerfChecker output for cleanup errors; no installation wizard or settings change was performed.'));return;} if(token.isCancellationRequested||status===130){reject(new vscode.CancellationError());return;} - status===0?resolve():reject(new Error('Controller setup did not finish. This integration requires registered PerfChecker 1.1.0 or newer in the 1.x series. Install or upgrade it explicitly in this controller environment; inspect PerfChecker output for the Julia dependency error. No controller setting was changed.'));}); + status===0?resolve():reject(new Error(`Controller setup did not finish. This integration requires registered PerfChecker ${minimumVersion} or newer in the 1.x series. Install or upgrade it explicitly in this controller environment; inspect PerfChecker output for the Julia dependency error. No controller setting was changed.`));}); }); }); }; @@ -53,7 +53,7 @@ export async function prepareWorkspaceController(folder: vscode.WorkspaceFolder, try{const selected=resolveControllerProject(folder.uri.fsPath,settings);await run(selected.project,verifyCode,'PerfChecker · Verify controller');return true;}catch(error){if(error instanceof vscode.CancellationError)return false;if(error instanceof ControllerCancellationFailure)throw error;output.appendLine(`Controller prerequisite: ${error}`);current();} const choice=await vscode.window.showQuickPick([ - {label:'Create controller environment',description:'Install registered PerfChecker 1.1.0 and collectors in perf/controller.',action:'create'}, + {label:'Create controller environment',description:'Install registered PerfChecker 1.1.1 or newer in the 1.x series and collectors in perf/controller.',action:'create'}, {label:'Use an existing controller',description:'Choose a Julia project containing PerfChecker.',action:'existing'}, {label:'Read the setup guide',description:'Manual setup, Julia prerequisites and environment configuration.',action:'guide'}, ],{title:'PerfChecker · Set up this workspace'}); @@ -66,7 +66,7 @@ export async function prepareWorkspaceController(folder: vscode.WorkspaceFolder, project=selected[0].fsPath; if(!await fs.stat(path.join(project,'Project.toml')).then(stat=>stat.isFile()).catch(()=>false))throw new Error('Choose a Julia environment containing Project.toml and PerfChecker.'); }else{ - const confirmed=await vscode.window.showWarningMessage(`Install PerfChecker 1.1.0, BenchmarkTools, Chairmarks and TestItemRunner in ${project}? This downloads Julia packages from General and creates or updates only that controller environment. Existing package tests also need the package under test and its test dependencies in this environment; they are not installed automatically.`,{modal:true},'Install controller'); + const confirmed=await vscode.window.showWarningMessage(`Install PerfChecker 1.1.1 or newer in the 1.x series, BenchmarkTools, Chairmarks and TestItemRunner in ${project}? This downloads Julia packages from General and creates or updates only that controller environment. Existing package tests also need the package under test and its test dependencies in this environment; they are not installed automatically.`,{modal:true},'Install controller'); if(confirmed!=='Install controller')return false; current(); // Do not follow a perf symlink out of this workspace during automatic setup. @@ -78,8 +78,8 @@ export async function prepareWorkspaceController(folder: vscode.WorkspaceFolder, } const existing=choice.action==='existing'; const code=existing ? verifyCode : - 'using Pkg; Pkg.add(PackageSpec(name="PerfChecker",version="1.1.0")); Pkg.add(["BenchmarkTools","Chairmarks","TestItemRunner"]); using PerfChecker; @assert Base.pkgversion(PerfChecker)==v"1.1.0"'; - try{await run(project,code,existing?'PerfChecker · Verify controller':'PerfChecker · Install controller');} + 'using Pkg; Pkg.add([PackageSpec(name="PerfChecker",version="1.1.1 - 1"),PackageSpec(name="BenchmarkTools"),PackageSpec(name="Chairmarks"),PackageSpec(name="TestItemRunner")]); using PerfChecker; v"1.1.1" <= Base.pkgversion(PerfChecker) < v"2.0.0" || error("Controller installation requires registered PerfChecker 1.1.1 or newer in the 1.x series.")'; + try{await run(project,code,existing?'PerfChecker · Verify controller':'PerfChecker · Install controller',existing?'1.1.0':'1.1.1');} catch(error){if(error instanceof vscode.CancellationError)return false;throw error;} current(); const value=path.relative(folder.uri.fsPath,project)||'.'; diff --git a/test/native-core-contract.mjs b/test/native-core-contract.mjs index b16bfd4..1fecf6c 100644 --- a/test/native-core-contract.mjs +++ b/test/native-core-contract.mjs @@ -1,7 +1,7 @@ // Qualification contracts are explicit: legacy compatibility never lowers the // guided-setup version exercised by the current candidate campaign. // Expected source identity; this constant does not assert General registration. -export const CORE110_CANDIDATE={version:'1.1.0',commit:'4e81603c57b47bc08372992492fdf2cbbd718b3b',tree:'753e41cad1c984640a2337b314b70ad6932626ad'}; +export const CORE111_CANDIDATE={version:'1.1.1',commit:'d079efd5c24c3dcac5b7550781822038e0ff780f',tree:'749f8eac351ed2a1f0bc110456f2eb37a3bc08ee'}; export const GENERAL100_TREE='7af0cc74194b953c5e998efd7523f0c5f455e395'; export function nativeCoreContract({core='general',artifact='public',stage='smoke',group='narrative',commit='',tree=''}) { if(!['general','candidate','general100'].includes(core))throw new Error('Choose a registered minimum, pinned candidate, or explicit General 1.0.0 compatibility campaign.'); @@ -14,6 +14,6 @@ export function nativeCoreContract({core='general',artifact='public',stage='smok if(group==='general100')throw new Error('The General 1.0.0 group requires its explicit registry contract.'); if(core==='candidate'&&![commit,tree].every(value=>/^[a-f0-9]{40}$/.test(value))) throw new Error('Core candidate mode requires an immutable commit and expected Git tree.'); - return {mode:core,version:core==='candidate'||artifact==='candidate'?'1.1.0':'1.0.0', - ...(core==='candidate'?{commit,tree}:{registry:'General',...(artifact==='candidate'?{tree:CORE110_CANDIDATE.tree}:{})})}; + return {mode:core,version:core==='candidate'||artifact==='candidate'?'1.1.1':'1.0.0', + ...(core==='candidate'?{commit,tree}:{registry:'General',...(artifact==='candidate'?{tree:CORE111_CANDIDATE.tree}:{})})}; } diff --git a/test/native-core-contract.test.mjs b/test/native-core-contract.test.mjs index 29b55ab..f1efb95 100644 --- a/test/native-core-contract.test.mjs +++ b/test/native-core-contract.test.mjs @@ -1,11 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import {nativeCoreContract,GENERAL100_TREE,CORE110_CANDIDATE} from './native-core-contract.mjs'; +import {nativeCoreContract,GENERAL100_TREE,CORE111_CANDIDATE} from './native-core-contract.mjs'; const legacy={core:'general100',artifact:'candidate',stage:'focused',group:'general100'}; -test('the 1.0.2 campaign qualifies guided setup with Core 1.1.0 without claiming registration',()=>{ - assert.deepEqual(nativeCoreContract({artifact:'candidate'}),{mode:'general',version:'1.1.0',registry:'General',tree:CORE110_CANDIDATE.tree}); +test('the 1.0.3 campaign qualifies corrected setup with Core 1.1.1 without claiming registration',()=>{ + assert.deepEqual(nativeCoreContract({artifact:'candidate'}),{mode:'general',version:'1.1.1',registry:'General',tree:CORE111_CANDIDATE.tree}); assert.equal(nativeCoreContract({artifact:'public'}).version,'1.0.0'); - for(const override of [{commit:'a'.repeat(40)},{tree:CORE110_CANDIDATE.tree}]) + for(const override of [{commit:'a'.repeat(40)},{tree:CORE111_CANDIDATE.tree}]) assert.throws(()=>nativeCoreContract({artifact:'candidate',...override}),/without Git source overrides/); }); test('legacy declares an exact registered tree without Git fallback',()=>{ diff --git a/test/native-platform-contract.mjs b/test/native-platform-contract.mjs new file mode 100644 index 0000000..f7420f8 --- /dev/null +++ b/test/native-platform-contract.mjs @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; + +export function assertNativeCampaignSelection({scope='all',event,artifact,stage,group,core}) { + if(scope!=='arm-bootstrap')return; + assert.equal(event,'workflow_dispatch','ARM bootstrap is an explicit manual campaign'); + assert.equal(artifact,'candidate','ARM bootstrap requires the immutable candidate VSIX'); + assert.equal(stage,'focused','ARM bootstrap is restricted to focused native controls'); + assert.equal(group,'bootstrap','ARM bootstrap exercises the production installers'); + assert.equal(core,'general','ARM bootstrap requires registered Core, without Git overrides'); +} + +export function nativePlatformContract({scope='all',event,artifact,stage,group,core, + platform=process.platform,arch=process.arch,runnerArch='',expectedArch='x64'}) { + assertNativeCampaignSelection({scope,event,artifact,stage,group,core}); + if(scope==='arm-bootstrap') { + assert(['linux','darwin'].includes(platform),'ARM bootstrap supports Linux and Apple Silicon'); + assert.equal(expectedArch,'arm64','ARM bootstrap explicitly requests native ARM64'); + }else assert.equal(expectedArch,'x64','Other native campaigns retain their x64 contract'); + assert.equal(arch,expectedArch,'The qualification driver must use the native Node architecture'); + if(runnerArch)assert.equal(runnerArch.toLowerCase(),expectedArch,'Node must match the actual runner architecture'); + return {node:expectedArch,julia:expectedArch==='arm64'?'aarch64':'x86_64'}; +} + +export function assertNativeJuliaArchitecture(runtime,contract,label) { + assert.equal(runtime.arch,contract.julia,`${label} must use the native Julia architecture`); +} diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs new file mode 100644 index 0000000..349a18a --- /dev/null +++ b/test/native-platform-contract.test.mjs @@ -0,0 +1,35 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture} from './native-platform-contract.mjs'; + +const arm={scope:'arm-bootstrap',event:'workflow_dispatch',artifact:'candidate',stage:'focused', + group:'bootstrap',core:'general',arch:'arm64',runnerArch:'ARM64',expectedArch:'arm64'}; + +test('native ARM bootstrap requires the explicit production-General campaign',()=>{ + for(const platform of ['linux','darwin']) + assert.deepEqual(nativePlatformContract({...arm,platform}),{node:'arm64',julia:'aarch64'}); + for(const change of [{event:'pull_request'},{artifact:'public'},{stage:'full'}, + {group:'pluto'},{core:'candidate'},{core:'general100'}]) + assert.throws(()=>assertNativeCampaignSelection({...arm,...change})); + assert.throws(()=>nativePlatformContract({...arm,platform:'win32'}),/Linux and Apple Silicon/); +}); + +test('Rosetta and mismatched Node or runner architecture cannot qualify ARM',()=>{ + for(const change of [{arch:'x64'},{runnerArch:'X64'},{expectedArch:'x64'}]) + assert.throws(()=>nativePlatformContract({...arm,platform:'darwin',...change})); + const contract=nativePlatformContract({...arm,platform:'linux'}); + for(const label of ['PerfChecker runtime','Official Julia runtime']) { + assertNativeJuliaArchitecture({arch:'aarch64'},contract,label); + assert.throws(()=>assertNativeJuliaArchitecture({arch:'x86_64'},contract,label),/native Julia architecture/); + assert.throws(()=>assertNativeJuliaArchitecture({},contract,label),/native Julia architecture/); + } +}); + +test('existing campaigns retain x64 and reject incidental ARM execution',()=>{ + for(const scope of ['all','linux','macos-stable']) { + assert.deepEqual(nativePlatformContract({scope,platform:'linux',arch:'x64',runnerArch:'X64'}), + {node:'x64',julia:'x86_64'}); + assert.throws(()=>nativePlatformContract({scope,platform:'linux',arch:'arm64'}),/native Node architecture/); + assert.throws(()=>nativePlatformContract({scope,platform:'darwin',arch:'arm64',expectedArch:'arm64'}),/x64 contract/); + } +}); diff --git a/test/pluto.test.mjs b/test/pluto.test.mjs index ac1d535..d08ac7a 100644 --- a/test/pluto.test.mjs +++ b/test/pluto.test.mjs @@ -361,12 +361,13 @@ test('Pluto requires explicit installation, a trusted workspace and a native not const root=await mkdtemp(path.join(os.tmpdir(),'perfchecker-pluto-boundaries-')); const uri=file=>({scheme:'file',fsPath:file,toString:()=>`file://${file}`}); const folder={name:'fixture',uri:uri(root)},messages=[],choices=[],scopes=[],values=new Map(); + let selectedAction; const disposable=()=>({dispose(){}}); const vscode={env:{openExternal:async()=>false},workspace:{isTrusted:true,workspaceFolders:[folder],getWorkspaceFolder:source=>source.fsPath.startsWith(root+path.sep)?folder:undefined, getConfiguration:(_name,scope)=>{scopes.push(scope);return{get:(key,fallback)=>values.has(key)?values.get(key):fallback};},onDidChangeWorkspaceFolders:disposable}, Uri:{file:uri,joinPath:(base,...pieces)=>uri(path.join(base.fsPath,...pieces)),parse:value=>({toString:()=>value})},ViewColumn:{One:1}, window:{createOutputChannel:()=>({append(){},appendLine(){},dispose(){}}), - showWarningMessage:async(message,_options,...actions)=>{messages.push(message);choices.push(actions);return undefined;}}, + showWarningMessage:async(message,_options,...actions)=>{messages.push(message);choices.push(actions);return selectedAction;}}, }; const original=Module._load; Module._load=function(name,...args){return name==='vscode'?vscode:original.call(this,name,...args);}; @@ -377,7 +378,7 @@ test('Pluto requires explicit installation, a trusted workspace and a native not assert.equal(await pluto.create(notebook,{kind:'suite'}),undefined); assert.equal(messages.length,1,'No package manager runs before the installation choice'); assert.match(messages[0],/own Julia environment.*perf.*pluto/); - assert.match(messages[0],/Install registered PerfChecker 1\.1\.0, PerfCheckerPluto 1\.0\.1, Pluto 1\.0\.4/); + assert.match(messages[0],/Install registered PerfChecker 1\.1\.1 or newer in the 1\.x series, PerfCheckerPluto 1\.0\.1, Pluto 1\.0\.4/); assert.deepEqual(choices[0],['Install Pluto environment','Open setup guide']); assert.deepEqual(await readdir(root),[],'Declining setup creates neither an environment nor a notebook'); vscode.workspace.isTrusted=false; @@ -412,6 +413,36 @@ test('Pluto requires explicit installation, a trusted workspace and a native not assert.equal((await readdir(outdated)).length,2,'Declining does not install or rewrite this environment'); }finally{pluto.command=originalCheck;} + // Capture both production installer routes after explicit consent. This + // checks their Julia recipes and isolation; it does not install packages. + try{ + for(const [relative,action,expectedCalls] of [ + ['perf/fresh-pluto','Install Pluto environment',1], + ['perf/pluto','Upgrade Pluto environment',2], + ]){ + values.set('plutoProject',relative);selectedAction=action; + const installs=[]; + pluto.command=async(...args)=>{ + installs.push(args); + if(args[3]==='PerfChecker · Check Pluto environment')throw Error('Existing Pluto needs an explicit upgrade'); + return ''; + }; + assert.equal(await pluto.ensureEnvironment(folder),path.join(root,relative)); + assert.equal(installs.length,expectedCalls); + const recipe=installs.at(-1)[2]; + assert.match(recipe,/Pkg\.add\(\[PackageSpec\(name="PerfChecker", version="1\.1\.1 - 1"\)/); + assert.match(recipe,/v"1\.1\.1" <= Base\.pkgversion\(PerfChecker\) < v"2\.0\.0"/); + assert.match(recipe,/subdir="packages\/PerfCheckerPluto", rev="v1\.0\.1"/); + assert.equal(recipe.match(/Pkg\.add\(/g).length,1,'Core, Pluto and its companion resolve together before precompilation'); + assert(!recipe.includes('PRESERVE_ALL')); + assert(!recipe.includes('Pkg.pin'),'Future compatible Core patches remain updateable'); + assert.equal(installs.at(-1)[1],path.join(root,relative)); + assert.match(messages.at(-1),/Install registered PerfChecker 1\.1\.1 or newer in the 1\.x series/); + } + assert.equal(await readFile(path.join(outdated,'Project.toml'),'utf8'),projectBytes); + assert.equal(await readFile(path.join(outdated,'Manifest.toml'),'utf8'),manifestBytes); + }finally{pluto.command=originalCheck;selectedAction=undefined;values.delete('plutoProject');} + const server=createServer((request,response)=>{ const query=new URL(request.url,'http://localhost').searchParams; response.statusCode=query.get('secret')==='isolated-test-secret' ? 200 : 403; diff --git a/test/public-vsix-host.cjs b/test/public-vsix-host.cjs index 71cbf93..ab148a1 100644 --- a/test/public-vsix-host.cjs +++ b/test/public-vsix-host.cjs @@ -455,6 +455,9 @@ async function measureNativeTestItem(context,expectedPassed,options={}){ exports.run = async () => { assert.equal(process.env.CI, 'true', 'Never run this host against a human VS Code installation'); + const expectedArch=process.env.PERFCHECKER_NATIVE_EXPECTED_ARCH||'x64'; + assert(['x64','arm64'].includes(expectedArch),'The runner declares the expected native architecture'); + assert.equal(process.arch,expectedArch,'The actual Electron extension host must use the native architecture'); const phase = process.env.PERFCHECKER_NATIVE_PHASE; const minimumVersion=process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_VERSION; assert(/^\d+\.\d+\.\d+$/.test(minimumVersion),'The runner declares the production registry minimum explicitly'); @@ -472,7 +475,7 @@ exports.run = async () => { const timing=()=>({observedAt:new Date().toISOString(),...(process.env.PERFCHECKER_NATIVE_VIDEO_STARTED_AT? {videoOffsetSeconds:(Date.now()-Date.parse(process.env.PERFCHECKER_NATIVE_VIDEO_STARTED_AT))/1000}:{})}); const persist=(status='running')=>{ - const report=JSON.stringify({status,phase,platform:process.platform,vscode:vscode.version,extension:process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION,invocation:process.env.PERFCHECKER_NATIVE_INVOCATION, + const report=JSON.stringify({status,phase,platform:process.platform,processArch:process.arch,vscode:vscode.version,extension:process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION,invocation:process.env.PERFCHECKER_NATIVE_INVOCATION, coverage:process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?'first-install-studio-investigations-mcp-pluto':'first-install-and-first-run-smoke', core:phase==='fresh'?{mode:'production-first-install',registry:'General',version:minimumVersion,available:process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE==='true'}:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE), commands:commandCoverage(commands,checks),configuration:configurationCoverage(configurationProperties,checks),buttons:buttonCoverage(checks),activeCase,checks,failures},null,2); diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index b2d3fc9..d8540f4 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -9,7 +9,8 @@ import {pipeline} from 'node:stream/promises'; import {fileURLToPath} from 'node:url'; import {createRequire} from 'node:module'; import {downloadAndUnzipVSCode, resolveCliArgsFromVSCodeExecutablePath} from '@vscode/test-electron'; -import {nativeCoreContract,CORE110_CANDIDATE} from './native-core-contract.mjs'; +import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; +import {nativePlatformContract,assertNativeJuliaArchitecture} from './native-platform-contract.mjs'; import {nativeVSCodeApplication} from './native-vscode-application.mjs'; import {createCoalescingObserver,beforeDeadline} from './native-landscape-controls.cjs'; import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable} from './native-controller-preflight.mjs'; @@ -30,7 +31,7 @@ const coreTree=coreMode==='candidate'?(process.env.PERFCHECKER_NATIVE_CORE_TREE const coreProvenance=nativeCoreContract({core:coreMode,artifact:mode,stage:process.env.PERFCHECKER_NATIVE_STAGE||'smoke', group:process.env.PERFCHECKER_NATIVE_CASE_GROUP||'narrative',commit:process.env.PERFCHECKER_NATIVE_CORE_COMMIT||'',tree:process.env.PERFCHECKER_NATIVE_CORE_TREE||''}); const expectedCoreVersion=coreProvenance.version; -const minimumVersion=mode==='public'?'1.0.0':'1.1.0'; +const minimumVersion=mode==='public'?'1.0.0':'1.1.1'; const installCore=coreMode==='candidate'?`${candidateGitSetupScript};Pkg.add(Pkg.PackageSpec(url="https://github.com/Mirage-Interactive-Fr/PerfChecker.jl",rev=ARGS[2]));${candidateCheckoutPermissionsScript}`:`Pkg.add(Pkg.PackageSpec(name="PerfChecker",version="${expectedCoreVersion}"))${coreMode==='general100'?'; Pkg.pin(Pkg.PackageSpec(name="PerfChecker",version="1.0.0"))':''}`; // Read the actual dependency and Manifest route, not the declared fixture pin. const companionProvenance=String.raw`function native_provider_provenance(m) @@ -499,7 +500,7 @@ println("LANDSCAPE_SDK_PREPARATION_COMPLETE_V1 ",(time_ns()-started)/1e9) try { await fs.mkdir(output, {recursive: true}); - assertCI(); + const architecture=assertCI(); if(landscapeOnly){ assert.equal(process.platform,'linux');await fs.mkdir(runnerEnvironment.XDG_RUNTIME_DIR,{mode:0o700}); process.on('SIGINT',abortLandscape);process.on('SIGTERM',abortLandscape); @@ -524,6 +525,7 @@ try { }else{ const runtime = JSON.parse((await execute(julia, ['--startup-file=no', '-e', "print(\"{\\\"executable\\\":\", repr(joinpath(Sys.BINDIR, Base.julia_exename())), \",\\\"version\\\":\", repr(string(VERSION)), \",\\\"arch\\\":\", repr(string(Sys.ARCH)), \",\\\"cpuName\\\":\", repr(Sys.CPU_NAME), \",\\\"threads\\\":\", Threads.nthreads(), \",\\\"hardwareThreads\\\":\", length(Sys.cpu_info()), \"}\")"])).trim()); + assertNativeJuliaArchitecture(runtime,architecture,'PerfChecker runtime'); julia = runtime.executable; let nativeNode,nodeProvenance; if(stage==='focused'&&caseGroup==='mcp-stdio'){ @@ -535,6 +537,7 @@ try { } const officialRuntime=JSON.parse((await execute(process.env.PERFCHECKER_NATIVE_OFFICIAL_JULIA||julia,['--startup-file=no','-e', "print(\"{\\\"executable\\\":\", repr(joinpath(Sys.BINDIR, Base.julia_exename())), \",\\\"version\\\":\", repr(string(VERSION)), \",\\\"arch\\\":\", repr(string(Sys.ARCH)), \",\\\"cpuName\\\":\", repr(Sys.CPU_NAME), \",\\\"threads\\\":\", Threads.nthreads(), \",\\\"hardwareThreads\\\":\", length(Sys.cpu_info()), \"}\")"])).trim()); + assertNativeJuliaArchitecture(officialRuntime,architecture,'Official Julia runtime'); const expectedVersion = mode === 'public' ? '1.0.0' : JSON.parse(await fs.readFile(path.join(repository, 'package.json'), 'utf8')).version; const vscode = await downloadAndUnzipVSCode({version, cachePath: path.join(session, 'vscode')}); const [cli, ...cliArgs] = resolveCliArgsFromVSCodeExecutablePath(vscode, {reuseMachineInstall: true}); @@ -600,7 +603,7 @@ try { const versionsText=await minimumResponse.text(),minimumSection=versionsText.split('["'+minimumVersion+'"]')[1]?.split(/^\[/m)[0]; const minimumTree=/^git-tree-sha1\s*=\s*"([a-f0-9]{40})"/m.exec(minimumSection||'')?.[1]; const minimumAvailable=Boolean(minimumTree); - if(minimumAvailable&&mode==='candidate')assert.equal(minimumTree,CORE110_CANDIDATE.tree,'Published General 1.1.0 must match the independently pinned Core source tree'); + if(minimumAvailable&&mode==='candidate')assert.equal(minimumTree,CORE111_CANDIDATE.tree,'Published General 1.1.1 must match the independently pinned Core source tree'); const plutoTagResponse=await fetch('https://raw.githubusercontent.com/Mirage-Interactive-Fr/PerfChecker.jl/v1.0.1/packages/PerfCheckerPluto/Project.toml'); if(!plutoTagResponse.ok&&plutoTagResponse.status!==404)throw new Error(`Cannot verify published Pluto companion tag: ${plutoTagResponse.status}`); const plutoTagAvailable=plutoTagResponse.ok&&/^version\s*=\s*"1\.0\.1"\s*$/m.test(await plutoTagResponse.text()); @@ -1039,5 +1042,7 @@ finally { function assertCI() { if (process.env.CI !== 'true') throw new Error('Native VS Code installation is restricted to disposable remote CI.'); - if (process.arch !== 'x64') throw new Error(`The registered core hardware dependency requires the explicitly tested x64 host, got ${process.arch}`); + return nativePlatformContract({scope:process.env.PERFCHECKER_NATIVE_SCOPE||'all',event:process.env.GITHUB_EVENT_NAME, + artifact:mode,stage,group:caseGroup,core:coreMode,runnerArch:process.env.RUNNER_ARCH||'', + expectedArch:process.env.PERFCHECKER_NATIVE_EXPECTED_ARCH||'x64'}); } diff --git a/test/workspace-setup.test.mjs b/test/workspace-setup.test.mjs index d4a2c5c..7426ecb 100644 --- a/test/workspace-setup.test.mjs +++ b/test/workspace-setup.test.mjs @@ -6,7 +6,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import os from 'node:os'; -test('explicit controller setup targets Core 1.1.0 without rewriting the selected older environment',async()=>{ +test('explicit controller setup requires corrected Core 1.x without rewriting the selected older environment',async()=>{ const root=await fs.mkdtemp(path.join(os.tmpdir(),'perfchecker-controller-upgrade-')); const oldProject=path.join(root,'perf');await fs.mkdir(oldProject); const projectBytes='[deps]\n',manifestBytes='# Existing controller is preserved\n'; @@ -24,9 +24,10 @@ test('explicit controller setup targets Core 1.1.0 without rewriting the selecte showWarningMessage:async message=>{warnings.push(message);return 'Install controller';}}}; // Exercise the real UI/setup orchestration while capturing Julia's invocation; // this fixture does not claim a package installation or run Julia. + let failInstall=false; const launch=(executable,args,options)=>{ calls.push({executable,args,options}); - const child=spawn(process.execPath,['-e',`process.exit(${calls.length===1?1:0})`], + const child=spawn(process.execPath,['-e',`process.exit(${calls.length===1||failInstall?1:0})`], {stdio:['pipe','pipe','pipe'],detached:process.platform!=='win32'}); children.push(child);return child; }; @@ -37,15 +38,22 @@ test('explicit controller setup targets Core 1.1.0 without rewriting the selecte assert.equal(await prepare(folder,{append(){},appendLine(){}}),true); assert.equal(calls.length,2,'A rejected existing controller requires an explicit install choice'); assert.match(calls[0].args.at(-1),/v"1\.1\.0" <= Base\.pkgversion\(PerfChecker\) < v"2\.0\.0"/); - assert.match(calls[1].args.at(-1),/Pkg\.add\(PackageSpec\(name="PerfChecker",version="1\.1\.0"\)\)/); - assert.match(calls[1].args.at(-1),/Base\.pkgversion\(PerfChecker\)==v"1\.1\.0"/); + assert.match(calls[1].args.at(-1),/Pkg\.add\(\[PackageSpec\(name="PerfChecker",version="1\.1\.1 - 1"\)/); + assert.match(calls[1].args.at(-1),/v"1\.1\.1" <= Base\.pkgversion\(PerfChecker\) < v"2\.0\.0"/); + assert.equal(calls[1].args.at(-1).match(/Pkg\.add\(/g).length,1,'Core and collectors resolve together before precompilation'); + assert(!calls[1].args.at(-1).includes('PRESERVE_ALL')); + assert(!calls[1].args.at(-1).includes('Pkg.pin'),'Future compatible Core patches remain updateable'); assert(!calls[1].args.at(-1).includes('Pkg.develop'),'Stable installation uses the registered package'); assert(calls[1].args.includes(`--project=${path.join(root,'perf','controller')}`)); - assert.match(warnings[0],/Install PerfChecker 1\.1\.0/); + assert.match(warnings[0],/Install PerfChecker 1\.1\.1 or newer in the 1\.x series/); assert.equal(values.get('runnerProject'),path.join('perf','controller')); assert.equal(values.get('scenarioProject'),path.join('perf','controller')); assert.equal(await fs.readFile(path.join(oldProject,'Project.toml'),'utf8'),projectBytes); assert.equal(await fs.readFile(path.join(oldProject,'Manifest.toml'),'utf8'),manifestBytes); + failInstall=true; + const savedSettings=new Map(values); + await assert.rejects(prepare(folder,{append(){},appendLine(){}}),/requires registered PerfChecker 1\.1\.1 or newer in the 1\.x series/); + assert.deepEqual(values,savedSettings,'A failed installation preserves the selected controller settings'); assert(children.every(child=>child.exitCode!==null),'Owned setup invocations finish before return'); }finally{ for(const child of children)if(child.exitCode===null){child.kill('SIGKILL');await new Promise(resolve=>child.once('close',resolve));} From ad271c389ca09ccba30276fc95d8f0f6e32e1a33 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 08:01:32 +0200 Subject: [PATCH 02/22] Freeze the immutable 1.0.3 qualification origin --- .github/workflows/native-vsix.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 96271f5..fe8dd1a 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -175,12 +175,17 @@ jobs: } if(newOrigin&&(e.NATIVE_EVENT!=='workflow_dispatch'||requested))throw Error('A new candidate origin requires an explicit manual run without archive-reuse inputs'); const version=JSON.parse(fs.readFileSync('package.json')).version; - if(version!=='1.0.2'&&!requested&&!newOrigin)throw Error('A new extension version requires an explicit candidate origin before archive reuse can be frozen.'); + if(!['1.0.2','1.0.3'].includes(version)&&!requested&&!newOrigin)throw Error('A new extension version requires an explicit candidate origin before archive reuse can be frozen.'); if(version==='1.0.2'&&!requested&&!newOrigin){ // Product commit df3085c9ccd92014586cd2b718ced606c0b912b7, tree cb36c394f91834a61fd58b6fe6a54cdc54f15fc2. fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38109028537\nREUSE_NAME=native-candidate-df3085c9ccd92014586cd2b718ced606c0b912b7\nREUSE_SHA=5fe530420b0ef8c7b8c0bdf19bc6223a6bdfd10ea6db9396b9f3f6d10d537f13\n'); console.log('Release 1.0.2 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } + if(version==='1.0.3'&&!requested&&!newOrigin){ + // Product commit 1140dad0ca6c8960e8426c99de88ce41124878c5, tree bf83cafca5641036dc21498458af6fd5b5f3cc58. + fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38518975105\nREUSE_NAME=native-candidate-1140dad0ca6c8960e8426c99de88ce41124878c5\nREUSE_SHA=068a3ed3bd01d823224f504dcfe134e3707789856c281ca8afb4329af41a83d3\n'); + console.log('Release 1.0.3 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); + } NODE - name: Validate optional exact-archive reuse if: env.NATIVE_VSIX_MODE != 'public' From 1059d1753fd2fe7e61a422374f59df3f8097aeb3 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 09:05:10 +0200 Subject: [PATCH 03/22] Bind controller readiness to its process inventory snapshot --- test/native-controller-preflight.mjs | 16 ++++++++ test/native-controller-preflight.test.mjs | 49 ++++++++++++++++++++++- test/run-public-vsix-host.mjs | 7 +++- 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/test/native-controller-preflight.mjs b/test/native-controller-preflight.mjs index 8ce6b5a..4c02ed4 100644 --- a/test/native-controller-preflight.mjs +++ b/test/native-controller-preflight.mjs @@ -2,6 +2,22 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import {TextDecoder} from 'node:util'; +// A READY received during an asynchronous process inventory belongs to its next +// snapshot. The caller still verifies that leader, parent and executable. +export async function captureControllerInventory(readReadyPid,readRows){ + const readyPid=readReadyPid(); + return {readyPid,rows:await readRows()}; +} + +export function controllerReadinessObserved({readyPid,started,identities,live,parent,expected,canonical}){ + if(!readyPid||started)return false; + const leader=identities.get(readyPid); + assert(leader&&live.has(readyPid)); + assert.equal(leader.parent,parent); + assert.equal(canonical(leader.executable),canonical(expected)); + return true; +} + // Pkg's LibGit2 checkout reads its own configuration search path. The Git CLI's // GIT_CONFIG_GLOBAL alone does not select that file for the embedded library. export const candidateGitSetupScript=String.raw`using LibGit2 diff --git a/test/native-controller-preflight.test.mjs b/test/native-controller-preflight.test.mjs index f3b6b63..85bb6e8 100644 --- a/test/native-controller-preflight.test.mjs +++ b/test/native-controller-preflight.test.mjs @@ -1,7 +1,54 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; -import {controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable} from './native-controller-preflight.mjs'; +import {controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; + +const readyLeader={pid:4924,parent:6176,executable:'C:\\Julia\\bin\\julia.exe'}; +const readiness={parent:6176,expected:readyLeader.executable,canonical:value=>value.toLowerCase()}; + +test('READY received during an inventory waits for the next coherent snapshot',async()=>{ + let readyPid,finishInventory,started=false; + const handshakes=[]; + const observe=snapshot=>{ + if(controllerReadinessObserved({...readiness,readyPid:snapshot.readyPid,started, + identities:new Map(snapshot.rows.map(row=>[row.pid,row])),live:new Set(snapshot.rows.map(row=>row.pid))})){ + started=true;handshakes.push('OBSERVED\n'); + } + }; + const first=captureControllerInventory(()=>readyPid,()=>new Promise(resolve=>{finishInventory=resolve;})); + readyPid=4924; + const earlierRows=[{pid:6176,parent:6592}]; + finishInventory(earlierRows); + const initial=await first; + assert.deepEqual(initial,{readyPid:undefined,rows:earlierRows},'Later stdout cannot qualify an earlier inventory'); + observe(initial);assert.deepEqual(handshakes,[],'The previous inventory cannot send a handshake'); + const currentRows=[...earlierRows,readyLeader]; + const next=await captureControllerInventory(()=>readyPid,async()=>currentRows); + assert.deepEqual(next,{readyPid:4924,rows:currentRows},'The next inventory binds the received READY to its own rows'); + observe(next);assert.deepEqual(handshakes,['OBSERVED\n']); + observe(next);assert.equal(handshakes.length,1,'The observed leader starts only once'); +}); + +test('an inventory keeps its initial READY identity and propagates read failures',async()=>{ + let readyPid=4924,finishInventory; + const pending=captureControllerInventory(()=>readyPid,()=>new Promise(resolve=>{finishInventory=resolve;})); + readyPid=9000; + const rows=[{pid:4924,parent:6176}];finishInventory(rows); + assert.deepEqual(await pending,{readyPid:4924,rows}); + const failure=new Error('CIM inventory failed'); + await assert.rejects(captureControllerInventory(()=>readyPid,async()=>{throw failure;}),error=>error===failure); +}); + +test('known READY still requires the live PID, original parent and executable',()=>{ + const qualified={...readiness,readyPid:4924,started:false,identities:new Map([[4924,readyLeader]]),live:new Set([4924])}; + assert.equal(controllerReadinessObserved(qualified),true); + for(const change of [ + {identities:new Map()}, + {live:new Set()}, + ...[{parent:undefined},{parent:9000},{executable:undefined},{executable:'C:\\Other\\julia.exe'}] + .map(change=>({identities:new Map([[4924,{...readyLeader,...change}]])})), + ])assert.throws(()=>controllerReadinessObserved({...qualified,...change}),'Missing or changed ownership evidence is still a failure'); +}); const project=path.resolve('controller','Project.toml'),coreRoot=path.resolve('packages','PerfChecker','Δ'),httpRoot=path.resolve('packages','HTTP','abc'); const hex=value=>Buffer.from(value).toString('hex'),tree='a'.repeat(40),httpTree='b'.repeat(40); diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index d8540f4..97ad9f4 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -13,7 +13,7 @@ import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; import {nativePlatformContract,assertNativeJuliaArchitecture} from './native-platform-contract.mjs'; import {nativeVSCodeApplication} from './native-vscode-application.mjs'; import {createCoalescingObserver,beforeDeadline} from './native-landscape-controls.cjs'; -import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable} from './native-controller-preflight.mjs'; +import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const output = path.join(repository, 'native-qualification-results'); @@ -235,10 +235,13 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=180 return {...after,executable:mappings.includes(expected)?expected:mappings[0],executableMappings:mappings}; }; const survey=async()=>{ + const {readyPid:observedReadyPid,rows}=await captureControllerInventory(()=>readyPid,async()=>{ let rows; if(windows){const value=await inspectCommand(ownerExecutable,['-NoProfile','-Command',"$ErrorActionPreference='Stop'; @((Get-CimInstance Win32_Process -ErrorAction Stop) | ForEach-Object { @{pid=$_.ProcessId;parent=$_.ParentProcessId;started=$(if($_.CreationDate){$_.CreationDate.ToUniversalTime().ToString('o')}else{$null});executable=$_.ExecutablePath} }) | ConvertTo-Json -Compress"],{timeout:limit(10000),maxBuffer:4*1024*1024});rows=JSON.parse(value.stdout);assert(Array.isArray(rows));} else{const value=await inspectCommand('ps',['-eo','pid=,ppid=,pgid=,lstart=,stat='],{timeout:limit(3000),maxBuffer:4*1024*1024});rows=value.stdout.split('\n').flatMap(line=>{ const match=/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s+(\S+)\s*$/.exec(line);return match?[{pid:Number(match[1]),parent:Number(match[2]),group:Number(match[3]),started:match[4],state:match[5]}]:[];});} + return rows; + }); lastRows=rows;const inspected=new Map(); const read=async row=>{if(inspected.has(row.pid))return inspected.get(row.pid);try{const current=await inspect(row);inspected.set(row.pid,current);return current;}catch(error){errors.push(controllerInspectionFailure(error,{stage:'inspect',pid:row.pid,ownerPid:child.pid}));inspected.set(row.pid,undefined);return undefined;}}; const original=rows.find(row=>row.pid===child.pid),owner=original&&await read(original); @@ -280,7 +283,7 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=180 const observation={observedAt:new Date().toISOString(),alive:[...live],unknown:unknown.map(row=>row.pid)}; const signature=JSON.stringify({alive:observation.alive,unknown:observation.unknown}); if(receipt.ownership.observations.at(-1)?.signature!==signature)receipt.ownership.observations.push({...observation,signature}); - if(readyPid&&!started){const leader=inspected.get(readyPid);assert(leader&&live.has(readyPid));assert.equal(leader.parent,windows?child.pid:process.pid);assert.equal(canonical(leader.executable),canonical(expected));started=true;child.stdin.write('OBSERVED\n');} + if(controllerReadinessObserved({readyPid:observedReadyPid,started,identities:inspected,live,parent:windows?child.pid:process.pid,expected,canonical})){started=true;child.stdin.write('OBSERVED\n');} return [...live].map(pid=>records.get(pid)); }; let observation=Promise.resolve(),sampling=false; From e782a33d947d817da8a8b8c0ad2a82053b378fb4 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 12:17:09 +0200 Subject: [PATCH 04/22] Qualify native discovery before partitioned Intel workflows --- .github/workflows/native-vsix.yml | 5 +- test/native-controller-preflight.mjs | 43 +++++++++++++- test/native-controller-preflight.test.mjs | 18 +++++- test/native-platform-contract.mjs | 26 +++++++++ test/native-platform-contract.test.mjs | 70 ++++++++++++++++++++++- test/public-vsix-host.cjs | 46 +++++++++------ test/run-public-vsix-host.mjs | 23 ++++++-- 7 files changed, 204 insertions(+), 27 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index fe8dd1a..6032659 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -247,7 +247,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && '[{"os":"macos-15-intel","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]') }} + include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && (inputs.artifact == 'candidate' && inputs.stage == 'full' && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"macos-15-intel","vscode":"stable"}]') || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || ((github.event_name != 'workflow_dispatch' || inputs.artifact == 'candidate' && inputs.stage == 'full') && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]')) }} runs-on: ${{ matrix.os }} timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }} env: @@ -257,6 +257,7 @@ jobs: PERFCHECKER_NATIVE_STAGE: ${{ github.event_name == 'workflow_dispatch' && inputs.stage || 'full' }} PERFCHECKER_NATIVE_CASE_GROUP: ${{ inputs.case_group }} PERFCHECKER_NATIVE_SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'all' }} + PERFCHECKER_NATIVE_FULL_PARTITION: ${{ matrix.partition || 'all' }} PERFCHECKER_NATIVE_EXPECTED_ARCH: ${{ matrix.arch || 'x64' }} PERFCHECKER_NATIVE_CORE: ${{ github.event_name == 'workflow_dispatch' && inputs.core || 'candidate' }} PERFCHECKER_NATIVE_CORE_COMMIT: ${{ (github.event_name != 'workflow_dispatch' || inputs.core == 'candidate') && (inputs.core_commit || 'd079efd5c24c3dcac5b7550781822038e0ff780f') || '' }} @@ -332,5 +333,5 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: native-workflows-${{ matrix.os }}-${{ matrix.vscode }}${{ matrix.julia && format('-julia-{0}', matrix.julia) || '' }} + name: native-workflows-${{ matrix.os }}-${{ matrix.vscode }}${{ matrix.julia && format('-julia-{0}', matrix.julia) || '' }}${{ matrix.partition && format('-{0}', matrix.partition) || '' }} path: native-qualification-results/ diff --git a/test/native-controller-preflight.mjs b/test/native-controller-preflight.mjs index 4c02ed4..1383f8e 100644 --- a/test/native-controller-preflight.mjs +++ b/test/native-controller-preflight.mjs @@ -74,10 +74,49 @@ end`; // prepared import; the owner/descendant observer also covers this process. export const controllerCachePreparationScript=String.raw`println("CONTROLLER_PREFLIGHT_READY ",getpid());flush(stdout);readline(stdin) using Pkg;Pkg.activate(ARGS[1]);started=time() -println("CONTROLLER_CACHE_PREPARATION_BEGIN PerfChecker HTTP strict=true");flush(stdout) -Pkg.precompile(["PerfChecker","HTTP"];strict=true) +packages=["PerfChecker","HTTP"];discovery=length(ARGS)==3 +if discovery;push!(packages,"TestItemRunner");end +println("CONTROLLER_CACHE_PREPARATION_BEGIN ",join(packages," ")," strict=true");flush(stdout) +Pkg.precompile(packages;strict=true) +if discovery + using Dates,PerfChecker,TestItemRunner + discovery_started=Dates.format(now(UTC),Dates.DateFormat("yyyy-mm-ddTHH:MM:SS.sssZ"));discovery_time=time() + @assert all(info.name!="PerfCheckerNativeFixture" for info in values(Pkg.dependencies())) + code=PerfChecker.perfchecker_main(["testitems","--root="*ARGS[2],"--project="*ARGS[1],"--output="*ARGS[3],"--list","--tags=","--exclude-tags="]) + @assert code==0 + listing=PerfChecker.JSON.parsefile(ARGS[3]);@assert listing["schema_version"]=="perfchecker-testitems/1";@assert listing["executed"]==false + matches=filter(item->item["name"]=="Vector reduction"&&replace(item["file"],'\\'=>'/')=="test/performance.jl",listing["items"]) + @assert length(matches)==1 + receipt=Dict("schema_version"=>"perfchecker-native-testitem-discovery/1","root"=>realpath(ARGS[2]), + "project"=>realpath(Base.active_project()),"startedAt"=>discovery_started, + "finishedAt"=>Dates.format(now(UTC),Dates.DateFormat("yyyy-mm-ddTHH:MM:SS.sssZ")),"elapsedSeconds"=>time()-discovery_time, + "listingSchema"=>listing["schema_version"],"listingRoot"=>listing["root"],"executed"=>listing["executed"],"item"=>only(matches),"targetPrepared"=>false, + "TestItemRunner"=>Dict("version"=>string(Base.pkgversion(TestItemRunner)),"source"=>realpath(pathof(TestItemRunner)))) + print("CONTROLLER_TESTITEM_DISCOVERY_V1 ");PerfChecker.JSON.print(receipt);println();flush(stdout) +end println("CONTROLLER_CACHE_PREPARATION_COMPLETE_V1 ",time()-started);flush(stdout)`; +export function parseControllerTestItemDiscoveryReceipt(text,{project,root}) { + const prefix='CONTROLLER_TESTITEM_DISCOVERY_V1 '; + const lines=text.split(/\r?\n/).filter(line=>line.startsWith('CONTROLLER_TESTITEM_DISCOVERY')); + assert.equal(lines.length,1,'Exactly one completed discovery receipt is required'); + assert(lines[0].startsWith(prefix),'Unknown discovery receipt protocol'); + const receipt=JSON.parse(lines[0].slice(prefix.length)); + assert.equal(receipt.schema_version,'perfchecker-native-testitem-discovery/1'); + assert.equal(receipt.project,project);assert.equal(receipt.root,root); + for(const key of ['startedAt','finishedAt'])assert(Number.isFinite(Date.parse(receipt[key])),'Discovery timestamps are required'); + assert(Date.parse(receipt.finishedAt)>=Date.parse(receipt.startedAt),'Discovery timestamps preserve their order'); + assert(Number.isFinite(receipt.elapsedSeconds)&&receipt.elapsedSeconds>=0); + assert.equal(receipt.listingSchema,'perfchecker-testitems/1');assert.equal(receipt.targetPrepared,false, + 'Discovery preparation must not install the intentionally missing target'); + assert.equal(receipt.executed,false,'The listing must not execute the missing-target workload'); + assert(path.isAbsolute(receipt.listingRoot),'The actual listing root is retained'); + assert.equal(receipt.item?.name,'Vector reduction');assert.equal(receipt.item?.file?.replaceAll('\\','/'),'test/performance.jl'); + assert(/^\d+\.\d+\.\d+(?:[-+][\w.-]+)?$/.test(receipt.TestItemRunner?.version)); + assert(path.isAbsolute(receipt.TestItemRunner?.source),'The loaded discovery dependency records its real source'); + return receipt; +} + // Emit primitive fields rather than compiling a generic JSON writer after the // prepared HTTP import. All loading, identity checks and the 180 s budget remain real. export const controllerPreflightScript=String.raw`println("CONTROLLER_PREFLIGHT_READY ",getpid());flush(stdout);readline(stdin) diff --git a/test/native-controller-preflight.test.mjs b/test/native-controller-preflight.test.mjs index 85bb6e8..4d0bb15 100644 --- a/test/native-controller-preflight.test.mjs +++ b/test/native-controller-preflight.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; -import {controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; +import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; const readyLeader={pid:4924,parent:6176,executable:'C:\\Julia\\bin\\julia.exe'}; const readiness={parent:6176,expected:readyLeader.executable,canonical:value=>value.toLowerCase()}; @@ -56,6 +56,22 @@ const fields=['CONTROLLER_IMPORT_RECEIPT_V1','175.935475','HTTPAdvisorExt',hex(p 'PerfChecker','1.0.1',hex(coreRoot),hex(path.join(coreRoot,'src','PerfChecker.jl')),tree,tree, 'HTTP','2.9.0',hex(httpRoot),hex(path.join(httpRoot,'src','HTTP.jl')),httpTree,httpTree]; const parse=value=>parseControllerImportReceipt(value,{project,version:'1.0.1',tree}); +const discoveryRoot=path.resolve('fixture'),discovery={schema_version:'perfchecker-native-testitem-discovery/1', + project,root:discoveryRoot,startedAt:'2026-10-14T07:00:00.000Z',finishedAt:'2026-10-14T07:00:04.000Z', + elapsedSeconds:4,listingSchema:'perfchecker-testitems/1',listingRoot:discoveryRoot,executed:false,targetPrepared:false, + item:{name:'Vector reduction',file:'test/performance.jl'},TestItemRunner:{version:'1.3.2',source:path.resolve('packages','TestItemRunner','src','TestItemRunner.jl')}}; +const parseDiscovery=value=>parseControllerTestItemDiscoveryReceipt(value,{project,root:discoveryRoot}); +const discoveryLine=receipt=>'CONTROLLER_TESTITEM_DISCOVERY_V1 '+JSON.stringify(receipt); +test('actual discovery preparation keeps timestamps, dependency identity and the missing target',()=>{ + assert.deepEqual(parseDiscovery(discoveryLine(discovery)),discovery); + for(const change of [{project:path.resolve('other','Project.toml')},{root:path.resolve('other')}, + {startedAt:'unknown'},{finishedAt:'2026-10-14T06:59:59.000Z'},{elapsedSeconds:-1}, + {listingSchema:'other'},{listingRoot:'relative'},{executed:true},{targetPrepared:true},{item:{name:'Other',file:'test/performance.jl'}}, + {item:{name:'Vector reduction',file:'different.jl'}},{TestItemRunner:{version:'unknown',source:'relative'}}]) + assert.throws(()=>parseDiscovery(discoveryLine({...discovery,...change}))); + assert.throws(()=>parseDiscovery('')); + assert.throws(()=>parseDiscovery(discoveryLine(discovery)+'\n'+discoveryLine(discovery))); +}); test('primitive preflight receipt preserves real import time, Unicode paths and both pinned source trees',()=>{ const value=parse(`CONTROLLER_IMPORT_AFTER HTTP elapsed=175.935475\n${fields.join('\t')}\n`); assert.equal(value.elapsedSeconds,175.935475);assert.equal(value.packages[0].root,coreRoot); diff --git a/test/native-platform-contract.mjs b/test/native-platform-contract.mjs index f7420f8..07a0006 100644 --- a/test/native-platform-contract.mjs +++ b/test/native-platform-contract.mjs @@ -24,3 +24,29 @@ export function nativePlatformContract({scope='all',event,artifact,stage,group,c export function assertNativeJuliaArchitecture(runtime,contract,label) { assert.equal(runtime.arch,contract.julia,`${label} must use the native Julia architecture`); } + +// Every partition retains fresh/configured and the complete prepared suite +// prefix. Only this ordered tail is divided between isolated Intel Mac jobs. +const fullCasePartitions=Object.freeze({ + measure:Object.freeze(['native-all-studio-controls','native-workbench-controls']), + investigate:Object.freeze(['native-investigation-controls','native-investigation-limits']), + agentic:Object.freeze(['native-enabled-narrative-protocol','native-advisor-controls', + 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']) +}); + +export function nativeFullCasePartition({partition='all',platform=process.platform, + arch=process.arch,stage,artifact}) { + assert(partition==='all'||Object.hasOwn(fullCasePartitions,partition),'Unknown full native case partition'); + if(partition!=='all') { + assert.equal(platform,'darwin','Full partitions are restricted to Intel macOS'); + assert.equal(arch,'x64','Full partitions require native Intel macOS'); + assert.equal(stage,'full','Partitions retain the complete native campaign'); + assert.equal(artifact,'candidate','Historical public campaigns remain unchanged'); + } + const all=Object.values(fullCasePartitions).flat().filter(name=>artifact==='candidate'||name!=='native-pluto-controls'); + return {name:partition,cases:partition==='all'?all:[...fullCasePartitions[partition]], + sharedPrefix:['controller-visible-in-studio','native-julia-terminal', + artifact==='candidate'?'native-pluto-without-jupyter':'native-notebook-with-jupyter', + 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', + 'native-testitem-measurement','native-complete-suite-measurements']}; +} diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs index 349a18a..f0b6ad9 100644 --- a/test/native-platform-contract.test.mjs +++ b/test/native-platform-contract.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture} from './native-platform-contract.mjs'; +import {readFileSync} from 'node:fs'; +import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; const arm={scope:'arm-bootstrap',event:'workflow_dispatch',artifact:'candidate',stage:'focused', group:'bootstrap',core:'general',arch:'arm64',runnerArch:'ARM64',expectedArch:'arm64'}; @@ -33,3 +34,70 @@ test('existing campaigns retain x64 and reject incidental ARM execution',()=>{ assert.throws(()=>nativePlatformContract({scope,platform:'darwin',arch:'arm64',expectedArch:'arm64'}),/x64 contract/); } }); + +test('Intel partitions preserve the full ordered case union and every shared prefix',()=>{ + const campaign={platform:'darwin',arch:'x64',stage:'full',artifact:'candidate'}; + const all=nativeFullCasePartition(campaign); + const partitions=['measure','investigate','agentic'].map(partition=>nativeFullCasePartition({...campaign,partition})); + const historical=['native-all-studio-controls','native-workbench-controls','native-investigation-controls', + 'native-investigation-limits','native-enabled-narrative-protocol','native-advisor-controls', + 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']; + assert.deepEqual(all.cases,historical,'The independently enumerated historical tail must remain complete'); + assert.deepEqual(partitions.flatMap(partition=>partition.cases),historical); + assert.equal(new Set(all.cases).size,all.cases.length,'No full case is accidentally duplicated or lost'); + for(const partition of partitions)assert.deepEqual(partition.sharedPrefix,all.sharedPrefix); + assert.deepEqual(partitions[2].cases.slice(-3),['native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto'], + 'The MCP to Pluto to discovery/sync transition remains in one physical session'); +}); + +test('the real host dispatches every advertised shared-prefix case before its partition',async()=>{ + const source=readFileSync(new URL('./public-vsix-host.cjs',import.meta.url),'utf8'); + const start=source.indexOf("await runCase('controller-visible-in-studio'"); + const end=source.indexOf("if (phase === 'configured'",start); + assert(start>=0&&end>start,'The common prepared controller/terminal/Pluto path remains present'); + const calls=[],runCase=async name=>{calls.push(name);return true;}; + const AsyncFunction=Object.getPrototypeOf(async function(){}).constructor; + // Execute only the actual dispatch path. Native callbacks are deliberately + // not invoked here; their effects remain the responsibility of native CI. + await new AsyncFunction('runCase','process',source.slice(start,end))(runCase,{env:{PERFCHECKER_NATIVE_MODE:'candidate'}}); + assert.deepEqual(calls,['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter']); + const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); + assert(suiteStart>=0&&suiteEnd>suiteStart); + const suite=Function(`return (${source.slice(suiteStart,suiteEnd).trim()})`)(); + await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture'},runCase,{testItem:true,complete:true}); + const historicalPrefix=['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter', + 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', + 'native-testitem-measurement','native-complete-suite-measurements']; + assert.deepEqual(calls,historicalPrefix); + assert.deepEqual(nativeFullCasePartition({artifact:'candidate'}).sharedPrefix,historicalPrefix); + assert(source.includes("await runSuiteCases(context,runCase,{testItem:true,complete:process.env.PERFCHECKER_NATIVE_STAGE==='full'});"), + 'Every full prepared partition still dispatches both the native test item and the complete suite'); +}); + +test('partitions cannot silently replace focused, public, ARM or other platform campaigns',()=>{ + const campaign={partition:'measure',platform:'darwin',arch:'x64',stage:'full',artifact:'candidate'}; + for(const change of [{partition:'unknown'},{platform:'linux'},{platform:'win32'}, + {arch:'arm64'},{stage:'focused'},{stage:'smoke'},{artifact:'public'}]) + assert.throws(()=>nativeFullCasePartition({...campaign,...change})); + assert(!nativeFullCasePartition({artifact:'public'}).cases.includes('native-pluto-controls')); +}); + +test('the actual workflow partitions only complete candidate Intel jobs and keeps every other lane',()=>{ + const workflow=readFileSync(new URL('../.github/workflows/native-vsix.yml',import.meta.url),'utf8'); + const expression=workflow.split('\n').find(line=>line.includes('include: ${{ fromJSON(')).split('${{ ')[1].split(' }}')[0]; + const matrix=(event,inputs={})=>Function('github','inputs','fromJSON',`return ${expression}`)({event_name:event},inputs,JSON.parse); + const full=matrix('pull_request'); + assert.deepEqual(full.filter(lane=>lane.os!=='macos-15-intel'),[ + {os:'ubuntu-latest',vscode:'1.96.0'},{os:'ubuntu-latest',vscode:'stable'}, + {os:'windows-latest',vscode:'1.96.0'},{os:'windows-latest',vscode:'stable'}]); + assert.deepEqual(full.filter(lane=>lane.os==='macos-15-intel').map(lane=>lane.partition),['measure','investigate','agentic']); + const manual=(scope,stage='full',artifact='candidate')=>matrix('workflow_dispatch',{scope,stage,artifact}); + assert.equal(manual('macos-stable').length,3); + assert.deepEqual(manual('macos-stable','focused'),[{os:'macos-15-intel',vscode:'stable'}]); + assert.equal(manual('all','full','public').length,5); + assert(manual('all','full','public').every(lane=>!lane.partition)); + assert.equal(manual('all','focused').length,5); + assert.equal(manual('arm-bootstrap','focused').length,4); + assert(manual('arm-bootstrap','focused').every(lane=>lane.arch==='arm64'&&!lane.partition)); + assert.deepEqual(manual('windows-stable'),[{os:'windows-latest',vscode:'stable'}]); +}); diff --git a/test/public-vsix-host.cjs b/test/public-vsix-host.cjs index ab148a1..466b9b6 100644 --- a/test/public-vsix-host.cjs +++ b/test/public-vsix-host.cjs @@ -399,7 +399,13 @@ async function measureNativeTestItem(context,expectedPassed,options={}){ await controls.clickStudioAction(context,'testing'); const filename=options.file||'test/performance.jl',name=options.name||'Vector reduction'; const row=windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).first(); - await row.waitFor({state:'visible',timeout:120000});await row.hover(); + try{await row.waitFor({state:'visible',timeout:120000});} + catch(error){ + try{await context.observeWork?.('testitems-discovery-not-ready');} + catch(diagnosticError){context.log('native-work-observation-error',{case:'testitems-discovery-not-ready',error:redact(diagnosticError)});} + throw error; + } + await row.hover(); if(context.editorQualification)context.log('native-editor-visible-testing-item',{name,file:filename,matchingRows:await windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).count(), visible:await row.isVisible(),ariaLabel:await row.getAttribute('aria-label'),runButtons:await row.locator('.action-label[title="Run Test"],.action-label[aria-label="Run Test"],.action-label.codicon-testing-run-icon').count()}); await context.editorQualification?.surface('tagged-testing-before-run'); @@ -459,6 +465,9 @@ exports.run = async () => { assert(['x64','arm64'].includes(expectedArch),'The runner declares the expected native architecture'); assert.equal(process.arch,expectedArch,'The actual Electron extension host must use the native architecture'); const phase = process.env.PERFCHECKER_NATIVE_PHASE; + const {nativeFullCasePartition}=await import('./native-platform-contract.mjs'); + const fullCasePartition=nativeFullCasePartition({partition:process.env.PERFCHECKER_NATIVE_FULL_PARTITION||'all', + stage:process.env.PERFCHECKER_NATIVE_STAGE,artifact:process.env.PERFCHECKER_NATIVE_MODE}); const minimumVersion=process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_VERSION; assert(/^\d+\.\d+\.\d+$/.test(minimumVersion),'The runner declares the production registry minimum explicitly'); const workspace = process.env.PERFCHECKER_NATIVE_WORKSPACE; @@ -476,8 +485,9 @@ exports.run = async () => { {videoOffsetSeconds:(Date.now()-Date.parse(process.env.PERFCHECKER_NATIVE_VIDEO_STARTED_AT))/1000}:{})}); const persist=(status='running')=>{ const report=JSON.stringify({status,phase,platform:process.platform,processArch:process.arch,vscode:vscode.version,extension:process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION,invocation:process.env.PERFCHECKER_NATIVE_INVOCATION, - coverage:process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?'first-install-studio-investigations-mcp-pluto':'first-install-and-first-run-smoke', + coverage:process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?(fullCasePartition.name==='all'?'first-install-studio-investigations-mcp-pluto':`full-native-${fullCasePartition.name}`):'first-install-and-first-run-smoke', core:phase==='fresh'?{mode:'production-first-install',registry:'General',version:minimumVersion,available:process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE==='true'}:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE), + fullCasePartition:process.env.PERFCHECKER_NATIVE_STAGE==='full'?fullCasePartition:undefined, commands:commandCoverage(commands,checks),configuration:configurationCoverage(configurationProperties,checks),buttons:buttonCoverage(checks),activeCase,checks,failures},null,2); pendingReport=pendingReport.then(()=>fs.writeFile(path.join(output,`${phase}.json`),report)); return pendingReport; @@ -485,7 +495,7 @@ exports.run = async () => { const retainEvidence=async name=>{ const directory=path.join(output,'worker-evidence',phase,name); const origins=['perf/results','perf/notebooks'].map(relative=>({root:workspace,relative,prefix:''})); - if(['testitems-ready','suite','diagnosis','pluto'].includes(phase))origins.push({ + if(['testitems','testitems-ready','suite','diagnosis','pluto','configured','prepared'].includes(phase))origins.push({ root:path.join(process.env.PERFCHECKER_NATIVE_PROFILE,'User','globalStorage','mirage-interactive-fr.perfchecker-vscode'),relative:'native-testitems',prefix:'host-storage'}); if(phase==='landscape'){ const root=process.env.PERFCHECKER_NATIVE_LANDSCAPE_WORKSPACE; @@ -613,7 +623,8 @@ exports.run = async () => { core:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE),coreVersion:process.env.PERFCHECKER_NATIVE_CORE_VERSION,flushReport:()=>persist()}; nativeContext=context; if(process.platform==='darwin')context.processInventory=(stage,port,known)=>macProcessInventory(context,stage,port,known); - if(['testitems-ready','suite','diagnosis','pluto'].includes(phase))context.observeWork=stage=>observeNativeWork(context,stage); + if(['testitems','testitems-ready','suite','diagnosis','pluto'].includes(phase))context.observeWork=stage=>observeNativeWork(context,stage); + else if(['configured','prepared'].includes(phase))context.observeWork=stage=>/testitems?/.test(stage)?observeNativeWork(context,stage):undefined; if(phase==='diagnosis'&&process.platform==='darwin'&&process.env.PERFCHECKER_NATIVE_DIAGNOSTIC_TEMP)context.observeDiagnosisRequest=(stage,deadline)=>macProcessInventory(context,stage,undefined,[],deadline); context.measureTestItem=options=>measureNativeTestItem(context,true,options); log('core-installation-provenance',phase==='fresh'?{mode:'first-install',controllerInitiallyAbsent:true,productionInstaller:`General ${minimumVersion}`,minimumAvailable:process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE==='true'}:context.core); @@ -827,18 +838,21 @@ exports.run = async () => { }); await runSuiteCases(context,runCase,{testItem:true,complete:process.env.PERFCHECKER_NATIVE_STAGE==='full'}); if (process.env.PERFCHECKER_NATIVE_STAGE === 'full') { - await runCase('native-all-studio-controls', () => controls.run(context)); - await runCase('native-workbench-controls',()=>workbench.run(context)); - await runCase('native-investigation-controls', () => investigations.run(context)); - await runCase('native-investigation-limits',()=>require('./native-investigation-limits.cjs').run(context)); - await runCase('native-enabled-narrative-protocol',()=>require('./native-narrative-controls.cjs').run(context)); - await runCase('native-advisor-controls', () => advisor.run(context)); - await runCase('native-mcp-controls', () => mcp.run(context)); - if(process.env.PERFCHECKER_NATIVE_MODE==='candidate')await runCase('native-pluto-controls',()=>pluto.run(context)); - await runCase('native-discovery-and-sync-after-pluto',async()=>{ - await vscode.commands.executeCommand('perfchecker.discoverScenarios'); - await vscode.commands.executeCommand('perfchecker.syncScenarios'); - }); + const callbacks={ + 'native-all-studio-controls':()=>controls.run(context), + 'native-workbench-controls':()=>workbench.run(context), + 'native-investigation-controls':()=>investigations.run(context), + 'native-investigation-limits':()=>require('./native-investigation-limits.cjs').run(context), + 'native-enabled-narrative-protocol':()=>require('./native-narrative-controls.cjs').run(context), + 'native-advisor-controls':()=>advisor.run(context), + 'native-mcp-controls':()=>mcp.run(context), + 'native-pluto-controls':()=>pluto.run(context), + 'native-discovery-and-sync-after-pluto':async()=>{ + await vscode.commands.executeCommand('perfchecker.discoverScenarios'); + await vscode.commands.executeCommand('perfchecker.syncScenarios'); + } + }; + for(const name of fullCasePartition.cases){assert.equal(typeof callbacks[name],'function');await runCase(name,callbacks[name]);} } } } diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index 97ad9f4..f57a2e7 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -10,10 +10,10 @@ import {fileURLToPath} from 'node:url'; import {createRequire} from 'node:module'; import {downloadAndUnzipVSCode, resolveCliArgsFromVSCodeExecutablePath} from '@vscode/test-electron'; import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; -import {nativePlatformContract,assertNativeJuliaArchitecture} from './native-platform-contract.mjs'; +import {nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; import {nativeVSCodeApplication} from './native-vscode-application.mjs'; import {createCoalescingObserver,beforeDeadline} from './native-landscape-controls.cjs'; -import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; +import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const output = path.join(repository, 'native-qualification-results'); @@ -53,6 +53,7 @@ async function assertCompanionRoute(info,name,revision){ } const version = process.env.PERFCHECKER_VSCODE_VERSION || 'stable'; const stage=process.env.PERFCHECKER_NATIVE_STAGE||'smoke'; +const fullCasePartition=nativeFullCasePartition({partition:process.env.PERFCHECKER_NATIVE_FULL_PARTITION||'all',stage,artifact:mode}); if(!['smoke','full','targeted','focused','core-external'].includes(stage))throw new Error('Choose smoke, full, targeted lifecycle/protocol, focused native controls, or the explicit Core-only external-process regression.'); const caseGroup=process.env.PERFCHECKER_NATIVE_CASE_GROUP||'narrative'; if(stage==='focused'&&!['bootstrap','general100','narrative','mcp','mcp-stdio','mcp-documented-install','mcp-pluto','pluto-plots','pluto','pluto-start-stop','workbench','advisor','saved-report','investigation','investigation-limits','diagnosis','studio','suite','studio-ordering','editor','testitems','testitems-ready','restricted','landscape','studio-color'].includes(caseGroup))throw new Error('Choose one of the explicit native-control groups.'); @@ -597,6 +598,7 @@ try { const cliProfile = await privateProfile(profile); await execute(cli, [...cliArgs, ...cliProfile, '--install-extension', vsix, '--force'], {shell: process.platform === 'win32' && cli.endsWith('.cmd')}); const artifactRecord={mode,retainedVsix,sha256: sha,vscodeRequested: version,core:coreProvenance,packageProvenance, + fullCasePartition:stage==='full'?fullCasePartition:undefined, runner:{platform:process.platform,processArch:process.arch,runnerArch:process.env.RUNNER_ARCH||null},runtimes:{perfchecker:runtime,officialJulia:officialRuntime},hostPreferences:{scope:'disposable-application-profile',dialogStyle:'custom',dialogAPI:'real-VS-Code-no-interception',actualVersion:hostVersion, actualCommit:application.commit,applicationName:application.applicationName,applicationRelativePath:path.relative(path.dirname(vscode),application.application), cliRelativePath:path.relative(path.dirname(vscode),cli),privateDirectoryFlags}}; @@ -789,19 +791,30 @@ try { coreProvenance.diagnosticWorker={file:worker,sha256:createHash('sha256').update(await fs.readFile(worker)).digest('hex')}; } await fs.writeFile(path.join(output, 'artifact.json'), JSON.stringify(artifactRecord, null, 2)); - if(stage==='targeted'||stage==='full'||stage==='focused'&&(['general100','mcp','mcp-stdio','mcp-pluto','advisor','narrative'].includes(caseGroup)||caseGroup==='editor'&&process.platform==='linux')){ + const prepareTestItemDiscovery=mode==='candidate'&&(stage==='full'||stage==='focused'&&['testitems','testitems-ready'].includes(caseGroup)); + if(prepareTestItemDiscovery||stage==='targeted'||stage==='full'||stage==='focused'&&(['general100','mcp','mcp-stdio','mcp-pluto','advisor','narrative'].includes(caseGroup)||caseGroup==='editor'&&process.platform==='linux')){ // This is installation/cache preparation, not first-Send or a cold-start // measurement. Keep its explicit compilation cost and physical ownership // separate from the unchanged 180-second prepared-import qualification. const preparationBefore=Object.fromEntries(await Promise.all(['Project.toml','Manifest.toml'].map(async name=>[name,createHash('sha256').update(await fs.readFile(path.join(controller,name))).digest('hex')]))); const preparation={status:'running',phase:'cache preparation',startedAt:new Date().toISOString(),project:controller, - packages:['PerfChecker','HTTP'],strict:true,timeoutMilliseconds:900000,hashesBefore:preparationBefore, + packages:['PerfChecker','HTTP',...(prepareTestItemDiscovery?['TestItemRunner']:[])],strict:true,timeoutMilliseconds:900000,hashesBefore:preparationBefore, automaticPrecompile:runnerEnvironment.JULIA_PKG_PRECOMPILE_AUTO, scope:'Explicit selected-package cache preparation with owned compilation children; no cold-start or native first-Send claim'}; artifactRecord.controllerCachePreparation=preparation;await fs.writeFile(path.join(output,'artifact.json'),JSON.stringify(artifactRecord,null,2)); try{ - const text=await executeControllerPreflight(julia,['--startup-file=no',`--project=${controller}`,'-e',controllerCachePreparationScript,controller],preparation,{timeoutMs:preparation.timeoutMilliseconds}); + const discoveryFile=path.join(output,'controller-testitem-discovery.json'); + const text=await executeControllerPreflight(julia,['--startup-file=no',`--project=${controller}`,'-e',controllerCachePreparationScript,controller, + ...(prepareTestItemDiscovery?[workspace,discoveryFile]:[])],preparation,{timeoutMs:preparation.timeoutMilliseconds}); await fs.writeFile(path.join(output,'controller-cache-preparation.log'),text); + if(prepareTestItemDiscovery){ + preparation.testItemDiscovery=parseControllerTestItemDiscoveryReceipt(text,{project:await fs.realpath(path.join(controller,'Project.toml')),root:await fs.realpath(workspace)}); + const listing=JSON.parse(await fs.readFile(discoveryFile,'utf8')); + assert.equal(listing.schema_version,preparation.testItemDiscovery.listingSchema); + assert.equal(listing.executed,false);assert.equal(await fs.realpath(listing.root),preparation.testItemDiscovery.root); + assert.deepEqual(listing.items.filter(item=>item.name==='Vector reduction'&&item.file.replaceAll('\\','/')==='test/performance.jl'),[preparation.testItemDiscovery.item]); + preparation.testItemDiscovery.listingSha256=createHash('sha256').update(await fs.readFile(discoveryFile)).digest('hex'); + } const complete=text.split(/\r?\n/).filter(line=>line.startsWith('CONTROLLER_CACHE_PREPARATION_COMPLETE_V1 ')); assert.equal(complete.length,1,'Exactly one completed explicit cache-preparation receipt is required'); preparation.precompileSeconds=Number(complete[0].slice('CONTROLLER_CACHE_PREPARATION_COMPLETE_V1 '.length)); From adebd5e75c7fab9b1104cc58b23cfb4d44166cd7 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 12:31:52 +0200 Subject: [PATCH 05/22] Isolate HTTP Git authentication fixture from runner helpers --- test/git-references.test.mjs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/test/git-references.test.mjs b/test/git-references.test.mjs index ac8d5b7..d3b993d 100644 --- a/test/git-references.test.mjs +++ b/test/git-references.test.mjs @@ -84,13 +84,32 @@ test('actual remote Git timeout and cancellation close connections and allow a f test('unauthenticated HTTP Git fails visibly without asking for credentials', async () => { const root = await mkdtemp(path.join(tmpdir(), 'perfchecker-git-auth-')); + const keys=['GIT_CONFIG_GLOBAL','GIT_CONFIG_NOSYSTEM','GIT_CONFIG_COUNT','GIT_CONFIG_PARAMETERS']; + const previous=Object.fromEntries(keys.map(key=>[key,process.env[key]])); try { - await serverFixture((_request, response) => { + // Exercise Git's unauthenticated HTTP path, independently of credential + // helpers installed in a runner's user/system configuration. + const configuration=path.join(root,'empty.gitconfig');await writeFile(configuration,''); + process.env.GIT_CONFIG_GLOBAL=configuration;process.env.GIT_CONFIG_NOSYSTEM='1'; + process.env.GIT_CONFIG_COUNT='0';delete process.env.GIT_CONFIG_PARAMETERS; + assert(!/credential\.helper\s*=/.test(await git(root,'config','--list')),'The HTTP fixture has no inherited credential helper'); + let challenges=0,authorizationSent=false; + await serverFixture((request, response) => { + challenges++;authorizationSent||=Boolean(request.headers.authorization); response.writeHead(401, {'WWW-Authenticate': 'Basic realm="Fixture"'}); response.end(); }, async source => { const started = Date.now(); - await assert.rejects(discoverGitReferences(source, root), /Authenticate Git outside PerfChecker/); - assert.ok(Date.now() - started < 10000); + let rejection; + await assert.rejects(discoverGitReferences(source, root),error=>{ + rejection=error;return /Authenticate Git outside PerfChecker/.test(String(error)); + }); + assert(challenges>0,'Git reached the actual unauthenticated HTTP server'); + assert.equal(authorizationSent,false,'The unauthenticated fixture must not send credentials'); + const elapsed=Date.now()-started; + assert.ok(elapsed<10000,`Unauthenticated HTTP rejection took ${elapsed} ms: ${String(rejection)}`); }); - } finally {await rm(root, {recursive: true, force: true});} + } finally { + for(const key of keys){if(previous[key]===undefined)delete process.env[key];else process.env[key]=previous[key];} + await rm(root, {recursive: true, force: true}); + } }); From 09439a0e8f8752c7a39e566ecace76645e00aaf1 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 12:53:26 +0200 Subject: [PATCH 06/22] Wait for owned Windows MCP fixture readiness before cleanup RPCs --- test/mcp-stdio.test.mjs | 58 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/test/mcp-stdio.test.mjs b/test/mcp-stdio.test.mjs index 8a0851c..226db4e 100644 --- a/test/mcp-stdio.test.mjs +++ b/test/mcp-stdio.test.mjs @@ -452,8 +452,62 @@ test('real owned server retries cleanup after a transient identity observation f }finally{foreign.kill('SIGKILL');await new Promise(resolve=>foreign.once('close',resolve));} },'ignore-eof')); -test('Windows owned Job retries a failed owner stop while childClosed is false',{skip:process.platform!=='win32'},()=>fixture(async({root,create})=>{ - const connector=await create().start(),pid=Number(await readFile(path.join(root,'server.pid'),'utf8')); +test('Windows owned Job retries a failed owner stop while childClosed is false',{skip:process.platform!=='win32'},context=>fixture(async({root,create})=>{ + const connector=create(),rpc=connector.rpc,startedAt=new Date().toISOString(),requests=[]; + let pid,readyAt; + const safeError=error=>error?{name:/^[A-Za-z]+$/.test(error.name??'')?error.name:null, + code:/^[A-Z0-9_]+$/.test(error.code??'')?error.code:null, + rpcTimeout:error.message==='MCP request timed out. Reconnect the local server to continue.'}:null; + const state=async()=>({at:new Date().toISOString(),ownerPid:connector.child?.pid??null,serverPid:pid??null, + ownerAlive:connector.child?.pid?await alive(connector.child.pid).catch(()=>null):false, + serverAlive:pid?await alive(pid).catch(()=>null):false,childClosed:connector.childClosed, + closed:connector.closed,closing:!!connector.closing,storedFailure:safeError(connector.failure)}); + connector.rpc=async function(...args){ + const row={method:args[0],enteredAt:new Date().toISOString(),id:null,stage:'readiness'};requests.push(row); + try { + if(!readyAt){ + // This test concerns retrying an owned stop, not cold PowerShell launch. + // Use the existing Windows private-output handshake bound before the + // first real RPC arms its unchanged five-second timer. The real server + // writes this PID only after its owner authenticated the private pipe, + // assigned it to the Job and resumed it; no protocol reply is supplied. + const deadline=Date.now()+10000; + while(true){ + assert(Date.now(){if(error.code==='ENOENT')return '';throw error;}); + if(value){ + assert(/^[1-9][0-9]*$/.test(value)&&Number.isSafeInteger(Number(value)),'Fixture must report its real numeric PID'); + pid=Number(value);assert.notEqual(pid,this.child.pid); + row.stage='owned-processes-alive'; + assert(await alive(this.child.pid)&&await alive(pid),'Real Windows owner and fixture server must remain alive'); + row.stage='private-reader'; + assert(this.output&&this.child.protocolOutput===this.output&&!this.output.destroyed,'The private owned output reader must remain available'); + if(this.failure)throw this.failure; + assert(!this.closed&&!this.closing&&!this.childClosed,'Windows owner must remain open before the first RPC'); + break; + } + await pause(20); + } + row.readyState=await state(); + assert(row.readyState.ownerAlive&&row.readyState.serverAlive,'Both real owned processes must still be alive before the first RPC'); + if(this.failure)throw this.failure; + assert(Date.now()sequence?this.sequence:null; + const result=await pending;row.resolved=true;return result; + }catch(error){row.resolved=false;row.error=safeError(error);throw error;} + finally{row.settledAt=new Date().toISOString();row.settledState=await state();} + }; + try{await connector.start();} + finally{connector.rpc=rpc;context.diagnostic(JSON.stringify({windowsOwnedRetryStartup:{startedAt,readyAt:readyAt??null, + readinessLimitMs:10000,rpcTimeoutMs:connector.options.timeoutMs,requests,state:await state()}}));} const owner=connector.child,kill=owner.kill.bind(owner),foreign=spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'}); let first=true; owner.kill=(...args)=>{if(first){first=false;throw new Error('Transient fixture owner stop failure');}return kill(...args);}; From b96d58abd5295d21405df677144dcda15e7acb17 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 16:46:49 +0200 Subject: [PATCH 07/22] Retain bounded native operation progress without changing test oracles --- test/native-controller-preflight.mjs | 72 ++++++++++++++++++++ test/native-controller-preflight.test.mjs | 80 ++++++++++++++++++++++- test/native-investigation-controls.cjs | 9 +++ test/native-studio-controls.cjs | 4 ++ test/public-vsix-host.cjs | 61 ++++++++++++++--- 5 files changed, 215 insertions(+), 11 deletions(-) diff --git a/test/native-controller-preflight.mjs b/test/native-controller-preflight.mjs index 1383f8e..4c2182b 100644 --- a/test/native-controller-preflight.mjs +++ b/test/native-controller-preflight.mjs @@ -2,6 +2,78 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import {TextDecoder} from 'node:util'; +// Passive native-test evidence only. No argv, environment, source, request or +// arbitrary log payload is accepted; the process reader proves incarnations. +export function nativeOperationSnapshot(value){ + assert(Number.isSafeInteger(value.hostPid)&&value.hostPid>0); + assert(typeof value.expectedExecutable==='string'&&path.isAbsolute(value.expectedExecutable)); + assert(Array.isArray(value.rows)&&value.rows.length<=32); + assert(Array.isArray(value.files??[])&&(value.files??[]).length<=4); + const rows=new Map(value.rows.map(row=>[row.pid,row])); + assert.equal(rows.size,value.rows.length,'Observation refuses ambiguous process identities'); + const owned=pid=>{ + const seen=new Set(); + while(pid!==value.hostPid){ + assert(!seen.has(pid),'Observation refuses a cyclic or foreign process ancestry');seen.add(pid); + const row=rows.get(pid);assert(row&&!row.identityOnly,'Observation requires verified process ownership'); + assert(Number.isSafeInteger(row.pid)&&row.pid>0&&Number.isSafeInteger(row.parent)&&row.parent>0); + assert(typeof row.createdAt==='string'&&Number.isFinite(Date.parse(row.createdAt))); + pid=row.parent; + } + }; + for(const row of rows.values())owned(row.pid); + return {hostPid:value.hostPid,processes:[...rows.values()].filter(row=>row.canonicalExecutable===value.expectedExecutable).map(row=>({ + pid:row.pid,parent:row.parent,createdAt:row.createdAt,executable:row.canonicalExecutable, + ...(row.progress?{progress:Object.fromEntries(['state','elapsed','cpuTime','cpuPercent'].map(key=>{ + const field=row.progress[key],pattern=key==='state'?/^[A-Z+{ + assert(['suite','plan','controller-project','controller-manifest'].includes(file.name)); + assert(typeof file.present==='boolean'); + if(!file.present)return{name:file.name,present:false}; + assert(Number.isSafeInteger(file.bytes)&&file.bytes>=0&&Number.isFinite(Date.parse(file.modifiedAt))); + return{name:file.name,present:true,bytes:file.bytes,modifiedAt:file.modifiedAt}; + })}; +} + +export async function captureQualifiedNativeProgress(expected,beforeMappings,readProgress,readMappings){ + assert(beforeMappings.includes(expected),'Progress observation requires the canonical executable before reading states'); + const progress=await readProgress(); + assert((await readMappings()).includes(expected),'Progress observation refuses a changed or uncertain executable'); + return progress; +} + +export async function observeNativeOperation(operation,run,{sample,record,deadlineAt, + intervalMs=15000,sampleTimeoutMs=2000,maxSamples=40}={}){ + assert(['initialize','plan','diagnose'].includes(operation));assert.equal(typeof run,'function'); + assert.equal(typeof sample,'function');assert.equal(typeof record,'function'); + assert(Number.isFinite(deadlineAt)&&Number.isInteger(intervalMs)&&intervalMs>0&&Number.isInteger(sampleTimeoutMs)&&sampleTimeoutMs>0); + assert(Number.isInteger(maxSamples)&&maxSamples>0&&maxSamples<=64); + let stopped=false,active,timer,controller,count=0; + const emit=detail=>{try{record({operation,at:new Date().toISOString(),...detail});}catch{/* A diagnostic sink cannot change the real operation. */}}; + const collect=()=>{ + if(stopped||active||count>=maxSamples||Date.now()>=deadlineAt)return; + const abort=controller=new AbortController(),sampleDeadline=Math.min(deadlineAt,Date.now()+sampleTimeoutMs); + const expiry=setTimeout(()=>abort.abort(),Math.max(1,sampleDeadline-Date.now()));count++; + active=Promise.resolve().then(()=>sample({signal:abort.signal,deadlineAt:sampleDeadline})).then(value=>{ + if(!stopped&&!abort.signal.aborted&&Date.now(){if(!stopped)emit({stage:'inspection-error',sequence:count, + errorClass:/^[A-Za-z]+$/.test(error?.name??'')?error.name:null,code:/^[A-Z0-9_]+$/.test(error?.code??'')?error.code:null});}) + .finally(()=>{clearTimeout(expiry);active=undefined;}); + }; + try{ + // Invoke the actual action before scheduling any potentially blocking read. + const pending=run();emit({stage:'started',deadlineAt:new Date(deadlineAt).toISOString()}); + collect();timer=setInterval(collect,intervalMs); + return await pending; + }finally{ + stopped=true;clearInterval(timer);controller?.abort(); + let drained=!active,expiry; + try{if(active)drained=await Promise.race([active.then(()=>true),new Promise(resolve=>{expiry=setTimeout(()=>resolve(false),sampleTimeoutMs);})]);} + finally{clearTimeout(expiry);emit({stage:'observer-stopped',samples:count,drained});} + } +} + // A READY received during an asynchronous process inventory belongs to its next // snapshot. The caller still verifies that leader, parent and executable. export async function captureControllerInventory(readReadyPid,readRows){ diff --git a/test/native-controller-preflight.test.mjs b/test/native-controller-preflight.test.mjs index 4d0bb15..7a29df6 100644 --- a/test/native-controller-preflight.test.mjs +++ b/test/native-controller-preflight.test.mjs @@ -1,7 +1,85 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; -import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; +import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,nativeOperationSnapshot,observeNativeOperation,captureQualifiedNativeProgress} from './native-controller-preflight.mjs'; + +const observationDelay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +const operationSnapshot=()=>({hostPid:10,expectedExecutable:process.execPath, + rows:[{pid:11,parent:10,createdAt:'2026-10-14T10:00:00.000Z',canonicalExecutable:process.execPath, + progress:{state:'R',elapsed:'00:12',cpuTime:'0:03.10',cpuPercent:'25.0'},argv:'private source',token:'private token'}], + files:[{name:'suite',present:false,contents:'private source'}],environment:'private environment'}); + +test('native state observations revalidate the executable after reading progress',async()=>{ + const order=[],progress={state:'R'}; + assert.equal(await captureQualifiedNativeProgress(process.execPath,[process.execPath],async()=>{order.push('states');return progress;}, + async()=>{order.push('image-after');return[process.execPath];}),progress); + assert.deepEqual(order,['states','image-after']); + await assert.rejects(captureQualifiedNativeProgress(process.execPath,[process.execPath],async()=>progress,async()=>['/other/executable']),/changed or uncertain executable/); + const unknown=Object.assign(new Error('canonical executable unavailable'),{code:'ENOENT'}); + await assert.rejects(captureQualifiedNativeProgress(process.execPath,[process.execPath],async()=>progress,async()=>{throw unknown;}),error=>error===unknown); + let read=false; + await assert.rejects(captureQualifiedNativeProgress(process.execPath,[],async()=>{read=true;return progress;},async()=>[process.execPath]),/canonical executable before/); + assert.equal(read,false); +}); + +test('native operation evidence refuses foreign or ambiguous ownership and strips private payloads',()=>{ + const value=operationSnapshot(),safe=nativeOperationSnapshot(value); + assert(!JSON.stringify(safe).includes('private')); + assert.deepEqual(safe.files,[{name:'suite',present:false}]); + for(const rows of [[{...value.rows[0],parent:999}],[...value.rows,...value.rows], + [{...value.rows[0],identityOnly:true}],[{...value.rows[0],createdAt:'unverified'}]]) + assert.throws(()=>nativeOperationSnapshot({...value,rows})); + assert.throws(()=>nativeOperationSnapshot({...value,files:[{name:'user-source',present:true,bytes:1,modifiedAt:'2026-10-14T10:00:00Z'}]})); +}); + +test('native operation invokes the real action first and preserves its exact result',async()=>{ + const order=[],records=[],result={actual:'result'}; + const actual=await observeNativeOperation('plan',async()=>{order.push('action');await observationDelay(15);return result;},{ + deadlineAt:Date.now()+1000,intervalMs:2,sampleTimeoutMs:100,maxSamples:2, + sample:async()=>{order.push('sample');return operationSnapshot();},record:row=>records.push(row)}); + assert.equal(actual,result);assert.equal(order[0],'action');assert(order.includes('sample')); + assert(records.filter(row=>row.stage==='sample').length<=2);assert.equal(records.at(-1).stage,'observer-stopped'); + assert.equal(records.at(-1).drained,true); +}); + +test('observation failure neither replaces a real rejection nor records its private contents',async()=>{ + const primary=new Error('original operation rejection'),records=[]; + await assert.rejects(observeNativeOperation('initialize',async()=>{await observationDelay(10);throw primary;},{ + deadlineAt:Date.now()+1000,intervalMs:1,sampleTimeoutMs:100,maxSamples:1, + sample:async()=>{throw Object.assign(new Error('private token and argv'),{code:'EACCES'});},record:row=>records.push(row)}),error=>error===primary); + assert(records.some(row=>row.stage==='inspection-error'&&row.code==='EACCES')); + assert(!JSON.stringify(records).includes('private'));assert.equal(records.at(-1).drained,true); +}); + +test('native observer coalesces pending reads and aborts and drains them before returning',async()=>{ + let active=0,maximum=0,calls=0,aborted=0;const records=[]; + await observeNativeOperation('diagnose',async()=>{await observationDelay(20);return 'unchanged';},{ + deadlineAt:Date.now()+1000,intervalMs:1,sampleTimeoutMs:100,maxSamples:10, + sample:({signal})=>new Promise(resolve=>{calls++;maximum=Math.max(maximum,++active); + signal.addEventListener('abort',()=>{aborted++;active--;resolve(operationSnapshot());},{once:true});}),record:row=>records.push(row)}); + assert.equal(maximum,1);assert.equal(calls,1);assert.equal(active,0);assert.equal(aborted,1); + assert.equal(records.at(-1).drained,true);const count=records.length; + await observationDelay(10);assert.equal(calls,1);assert.equal(records.length,count); +}); + +test('late observations cannot qualify a snapshot or extend an expired operation oracle',async()=>{ + const records=[],primary=new Error('original deadline');let finishSample,rejectOperation,started; + const entered=new Promise(resolve=>{started=resolve;}); + const pending=observeNativeOperation('plan',()=>new Promise((resolve,reject)=>{rejectOperation=reject;}),{ + deadlineAt:Date.now()+1000,intervalMs:1,sampleTimeoutMs:2,maxSamples:1, + sample:()=>new Promise(resolve=>{finishSample=()=>resolve(operationSnapshot());started();}),record:row=>records.push(row)}); + await entered;rejectOperation(primary); + await assert.rejects(pending,error=>error===primary);assert.equal(records.at(-1).drained,false); + finishSample();await new Promise(resolve=>setImmediate(resolve)); + assert(!records.some(row=>row.stage==='sample')); +}); + +test('a diagnostic sink failure preserves synchronous operation refusal and schedules no observer',async()=>{ + const primary=new Error('original refusal');let sampled=false; + await assert.rejects(observeNativeOperation('initialize',()=>{throw primary;},{deadlineAt:Date.now()+100, + sample:async()=>{sampled=true;return operationSnapshot();},record:()=>{throw new Error('sink failed');}}),error=>error===primary); + assert.equal(sampled,false); +}); const readyLeader={pid:4924,parent:6176,executable:'C:\\Julia\\bin\\julia.exe'}; const readiness={parent:6176,expected:readyLeader.executable,canonical:value=>value.toLowerCase()}; diff --git a/test/native-investigation-controls.cjs b/test/native-investigation-controls.cjs index 4d71a03..8f53ada 100644 --- a/test/native-investigation-controls.cjs +++ b/test/native-investigation-controls.cjs @@ -87,6 +87,11 @@ async function reportAfter(context, before, action, timeout = 360000) { } async function action(context, label, name, timeout = 360000) { + if(name==='diagnose'&&context.observeOperation)return context.observeOperation('diagnose',()=>performAction(context,label,name,timeout),Date.now()+timeout); + return performAction(context,label,name,timeout); +} + +async function performAction(context, label, name, timeout) { const before = await directories(reportRoot(context)); const current = await view(context); await context.observeWork?.(`investigation-${name}-before-action`); @@ -254,6 +259,10 @@ async function diagnose(context) { // This observation allowance leaves every execution limit and assertion intact. const result = await action(context, 'Diagnose selected', 'diagnose', process.platform==='darwin'?900000:360000); assert.equal(result.report.schema_version, 'perfchecker-diagnosis/1'); + context.log('investigation-diagnosis-record-statuses',{records:result.report.records.slice(0,16).map(record=>({ + tool:tools.includes(record.tool)?record.tool:'unlisted', + status:['complete','unavailable','timeout','cancelled','error','failed','invalid_evidence'].includes(record.status)?record.status:'unlisted', + ...(Number.isFinite(record.elapsed_seconds??record.worker_elapsed_seconds)?{elapsedSeconds:record.elapsed_seconds??record.worker_elapsed_seconds}:{})}))}); for (const tool of tools) { const records = result.report.records.filter(record => record.tool === tool); assert.equal(records.length, 1, `${tool} has one actual worker result`); diff --git a/test/native-studio-controls.cjs b/test/native-studio-controls.cjs index 3f9017d..340137f 100644 --- a/test/native-studio-controls.cjs +++ b/test/native-studio-controls.cjs @@ -85,6 +85,10 @@ async function options(view, selector) { } async function designer(context) { + return context.observeOperation?context.observeOperation('plan',()=>openDesigner(context),Date.now()+600000):openDesigner(context); +} + +async function openDesigner(context) { await clickStudioAction(context, 'suite'); // An attached empty #cards has no height while the real controller is loading. // Its visible owner and generated check-type controls identify a ready plan. diff --git a/test/public-vsix-host.cjs b/test/public-vsix-host.cjs index 466b9b6..75cdadf 100644 --- a/test/public-vsix-host.cjs +++ b/test/public-vsix-host.cjs @@ -245,11 +245,12 @@ async function readDiagnosticRequest(context,argumentsText,deadline){ scope:'Whitelisted request fields and response existence only; no import/analysis phase or log payload inferred'}; } -async function macProcessInventory(context,stage,port,knownIdentities=[],deadline){ +async function macProcessInventory(context,stage,port,knownIdentities=[],deadline,progressOnly){ assert.equal(process.platform,'darwin'); const bounded=maximum=>{if(deadline===undefined)return maximum;const remaining=deadline-Date.now();assert(remaining>0,'Observation stays inside the existing diagnosis report deadline');return Math.min(maximum,remaining);}; const expectedExecutable=await fs.realpath(process.env.PERFCHECKER_NATIVE_JULIA); - const survey=execute('ps',['-eo','pid=,ppid=,args='],{timeout:bounded(10000),maxBuffer:4*1024*1024}); + const observationOptions=progressOnly?{signal:progressOnly.signal}:{}; + const survey=execute('ps',['-eo',progressOnly?'pid=,ppid=':'pid=,ppid=,args='],{timeout:bounded(10000),maxBuffer:4*1024*1024,...observationOptions}); const observerPid=survey.child.pid,{stdout,stderr}=await survey;assert(!stderr.trim(),'Process survey has no stderr inspection error'); const all=stdout.split('\n').flatMap(line=>{const match=/^\s*(\d+)\s+(\d+)(?:\s+(.*))?$/.exec(line);return match?[{pid:Number(match[1]),parent:Number(match[2]),arguments:match[3]||''}]:[];}); const observer=all.find(row=>row.pid===observerPid); @@ -261,24 +262,38 @@ async function macProcessInventory(context,stage,port,knownIdentities=[],deadlin }} const errors=[],rows=[]; const relevant=all.filter(row=>row.pid!==process.pid&&(owned.has(row.pid)||knownIdentities.some(prior=>prior.pid===row.pid))); + if(progressOnly)assert(relevant.length<=32,'Passive operation observations have at most 32 owned processes'); await Promise.all(relevant.map(async row=>{ let before; try{ const identity=async()=>{ - const observed=await execute('ps',['-p',String(row.pid),'-o','ppid=','-o','lstart='],{timeout:bounded(3000)}); + const observed=await execute('ps',['-p',String(row.pid),'-o','ppid=','-o','lstart='],{timeout:bounded(3000),...observationOptions}); assert(!observed.stderr.trim(),'Process identity has no stderr inspection error');const value=observed.stdout.trim(); const match=/^(\d+)\s+(.+)$/.exec(value);assert(match&&Number.isFinite(Date.parse(match[2])),'macOS supplies a real parent and lstart'); return {parent:Number(match[1]),started:match[2],createdAt:new Date(Date.parse(match[2])).toISOString()}; }; before=await identity();assert.equal(before.parent,row.parent); - const observedMappings=await execute('lsof',['-a','-p',String(row.pid),'-d','txt','-Fn'],{timeout:bounded(3000)}); + const observedMappings=await execute('lsof',['-a','-p',String(row.pid),'-d','txt','-Fn'],{timeout:bounded(3000),...observationOptions}); assert(!observedMappings.stderr.trim(),'Executable inspection has no stderr error'); const mappings=observedMappings.stdout.split('\n').filter(line=>line.startsWith('n')).map(line=>line.slice(1)); - const canonicalMappings=await Promise.all(mappings.map(value=>fs.realpath(value).catch(()=>value))); + const canonicalMappings=await Promise.all(mappings.map(value=>progressOnly?fs.realpath(value):fs.realpath(value).catch(()=>value))); + let progress; + if(progressOnly&&canonicalMappings.includes(expectedExecutable)){ + const {captureQualifiedNativeProgress}=await import('./native-controller-preflight.mjs'); + progress=await captureQualifiedNativeProgress(expectedExecutable,canonicalMappings,async()=>{ + const value=await execute('ps',['-p',String(row.pid),'-o','stat=','-o','etime=','-o','time=','-o','%cpu='],{timeout:bounded(3000),...observationOptions}); + assert(!value.stderr.trim());const fields=value.stdout.trim().split(/\s+/);assert.equal(fields.length,4); + return{state:fields[0],elapsed:fields[1],cpuTime:fields[2],cpuPercent:fields[3]}; + },async()=>{ + const after=await execute('lsof',['-a','-p',String(row.pid),'-d','txt','-Fn'],{timeout:bounded(3000),...observationOptions}); + assert(!after.stderr.trim(),'Post-state executable inspection has no stderr error'); + return Promise.all(after.stdout.split('\n').filter(line=>line.startsWith('n')).map(line=>fs.realpath(line.slice(1)))); + }); + } assert.deepEqual(await identity(),before,'The observed process retains its parent and start identity while its executable is inspected'); const projects=[...row.arguments.matchAll(/--project(?:=|\s+)([^\s]+)/g)].map(match=>match[1]).filter(value=>value.startsWith(process.env.PERFCHECKER_NATIVE_SESSION+path.sep)); let diagnosticRequest; - if(process.env.PERFCHECKER_NATIVE_PHASE==='diagnosis'&&canonicalMappings.includes(expectedExecutable)&&row.arguments.includes('diagnostic_worker.jl')){ + if(!progressOnly&&process.env.PERFCHECKER_NATIVE_PHASE==='diagnosis'&&canonicalMappings.includes(expectedExecutable)&&row.arguments.includes('diagnostic_worker.jl')){ try{diagnosticRequest=await readDiagnosticRequest(context,row.arguments,deadline); const observedAfter=await execute('lsof',['-a','-p',String(row.pid),'-d','txt','-Fn'],{timeout:bounded(3000)}); assert(!observedAfter.stderr.trim(),'Post-read executable inspection has no stderr error'); @@ -288,8 +303,8 @@ async function macProcessInventory(context,stage,port,knownIdentities=[],deadlin }catch(error){diagnosticRequest={status:'inspection-error',errorClass:error.name||'Error',code:typeof error.code==='string'?error.code:undefined};} } rows.push({pid:row.pid,...before,canonicalExecutable:canonicalMappings.includes(expectedExecutable)?expectedExecutable:undefined, - executableMappings:canonicalMappings.slice(0,8),projects,argumentsCharacters:row.arguments.length,...(diagnosticRequest?{diagnosticRequest}: {})}); - }catch(error){if(before)rows.push({pid:row.pid,...before,identityOnly:true});errors.push({pid:row.pid,parent:row.parent,error:redact(error)});} + executableMappings:canonicalMappings.slice(0,8),projects,argumentsCharacters:row.arguments.length,...(progress?{progress}:{}),...(diagnosticRequest?{diagnosticRequest}: {})}); + }catch(error){if(before)rows.push({pid:row.pid,...before,identityOnly:true});errors.push(progressOnly?{pid:row.pid,parent:row.parent,inspectionFailed:true}:{pid:row.pid,parent:row.parent,error:redact(error)});} })); let listeners=[]; if(port!==undefined){ @@ -305,10 +320,28 @@ async function macProcessInventory(context,stage,port,knownIdentities=[],deadlin const inventory={stage,observedAt:new Date().toISOString(),hostPid:process.pid,expectedExecutable,observerPid, observerParentVerified:!!observer,presentPids:relevant.map(row=>row.pid),rows,direct:rows.filter(row=>row.parent===process.pid&&row.canonicalExecutable===expectedExecutable),listeners,errors}; const signature=JSON.stringify({stage,rows,listeners,errors}); - if(processObservationSignatures.get(context)!==signature){context.log('native-macos-owned-processes',inventory);processObservationSignatures.set(context,signature);} + if(!progressOnly&&processObservationSignatures.get(context)!==signature){context.log('native-macos-owned-processes',inventory);processObservationSignatures.set(context,signature);} return inventory; } +async function observeOperationProgress(context,stage,{signal,deadlineAt}){ + const inventory=await macProcessInventory(context,stage,undefined,[],deadlineAt,{signal}); + assert(!inventory.errors.length,'An inspection error is not evidence of process absence'); + const profile=process.env.PERFCHECKER_NATIVE_PROFILE; + const controller=stage==='initialize'?path.join(context.workspace,'perf','controller'):context.controller; + const files=[]; + for(const [name,file]of [['suite',path.join(context.workspace,'perf','suite.jl')], + ['plan',path.join(profile,'User','globalStorage','mirage-interactive-fr.perfchecker-vscode','suite-plan.json')], + ['controller-project',path.join(controller,'Project.toml')],['controller-manifest',path.join(controller,'Manifest.toml')]]){ + signal.throwIfAborted();assert(Date.now(){if(error.code==='ENOENT')return undefined;throw error;}); + if(!stat){files.push({name,present:false});continue;} + assert(stat.isFile()&&!stat.isSymbolicLink(),'Metadata observations refuse links and non-files'); + files.push({name,present:true,bytes:stat.size,modifiedAt:stat.mtime.toISOString()}); + } + return{hostPid:process.pid,expectedExecutable:inventory.expectedExecutable,rows:inventory.rows,files}; +} + async function observeNativeWork(context,stage){ const settings=context.vscode.workspace.getConfiguration('perfchecker',context.vscode.Uri.file(context.workspace)); const diagnostic={stage,observedAt:new Date().toISOString(),hostPid:process.pid,processArch:process.arch, @@ -622,6 +655,10 @@ exports.run = async () => { results: path.join(workspace, 'perf', 'results', 'vscode'), log, proof, findFrame, core:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE),coreVersion:process.env.PERFCHECKER_NATIVE_CORE_VERSION,flushReport:()=>persist()}; nativeContext=context; + const {observeNativeOperation}=await import('./native-controller-preflight.mjs'); + context.observeOperation=(operation,run,deadlineAt)=>process.platform==='darwin'&&['fresh','configured','prepared'].includes(phase)? + observeNativeOperation(operation,run,{deadlineAt,sample:options=>observeOperationProgress(context,operation,options), + record:detail=>log('native-operation-progress',detail)}):run(); if(process.platform==='darwin')context.processInventory=(stage,port,known)=>macProcessInventory(context,stage,port,known); if(['testitems','testitems-ready','suite','diagnosis','pluto'].includes(phase))context.observeWork=stage=>observeNativeWork(context,stage); else if(['configured','prepared'].includes(phase))context.observeWork=stage=>/testitems?/.test(stage)?observeNativeWork(context,stage):undefined; @@ -733,6 +770,7 @@ exports.run = async () => { await assert.rejects(fs.access(suitePlanFile),{code:'ENOENT'},'Fresh setup must not reuse a stale plan from any workspace'); const initializationStartedAt=Date.now(),initializationDeadline=initializationStartedAt+600000; const initializationRemaining=()=>{assert(Date.now(){ await studio.locator('#setup-workspace').click(); await picker.waitFor({state:'visible',timeout:60000}); await picker.locator('.monaco-list-row').filter({hasText:'Create controller environment'}).click(); @@ -745,7 +783,7 @@ exports.run = async () => { assert.equal(vscode.workspace.getConfiguration('perfchecker',uri).get('runnerProject','perf'),'perf','Failed production setup preserves the selected controller'); assert(!await fs.stat(path.join(workspace,'perf','suite.jl')).then(()=>true).catch(()=>false)); proof('bootstrap-awaiting-registration',{status:'prerequisite',minimum:minimumVersion,registry:'General',positiveBootstrapQualified:false,explicitConfirmation:true,nativeStudioClick:true}); - return; + return false; } await eventually(()=>fs.stat(path.join(workspace,'perf','suite.jl')).then(()=>true).catch(()=>false),'Explicit first-use setup creates a real suite',initializationRemaining()); await eventually(()=>vscode.workspace.getConfiguration('perfchecker',uri).get('runnerProject')===path.join('perf','controller'),'Controller setting is saved only after successful setup',initializationRemaining()); @@ -758,6 +796,9 @@ exports.run = async () => { }catch(error){if(error.code==='ENOENT'||error instanceof SyntaxError)return false;throw error;} },'The real initialization plan finishes before opening Feature suite',initializationRemaining()); log('native-bootstrap-plan-ready',{...bootstrapPlan,elapsedMs:Date.now()-initializationStartedAt,maximumMs:600000,noAdditionalPlanRequested:true}); + return true; + },initializationDeadline); + if(!initialized)return; await controls.runSelection(context); proof('bootstrap-first-install-and-measurement',{existingProjectWithoutPerfChecker:true,explicitConfirmation:true,core:`General ${minimumVersion}`,nativeStudioClick:true}); }); From e485b5a8458fb08c47ae1eba8c1cfdab46deb71f Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 17:26:49 +0200 Subject: [PATCH 08/22] Observe sacrificial CLI entry without changing preflight deadlines --- test/codex-connector.test.mjs | 183 ++++++++++++++++++++++++++++++++-- 1 file changed, 173 insertions(+), 10 deletions(-) diff --git a/test/codex-connector.test.mjs b/test/codex-connector.test.mjs index 71a5e1f..69035f5 100644 --- a/test/codex-connector.test.mjs +++ b/test/codex-connector.test.mjs @@ -3,10 +3,94 @@ import assert from 'node:assert/strict'; import Module, {createRequire} from 'node:module'; import {spawn as nativeSpawn} from 'node:child_process'; import * as nativeChildProcess from 'node:child_process'; -import {mkdtemp, writeFile, readFile, mkdir, rm, realpath} from 'node:fs/promises'; +import * as entryFs from 'node:fs'; +import {randomBytes} from 'node:crypto'; +import {EventEmitter} from 'node:events'; +import {PassThrough} from 'node:stream'; +import {mkdtemp, writeFile, readFile, readdir, mkdir, rm, realpath} from 'node:fs/promises'; import {tmpdir} from 'node:os'; import path from 'node:path'; +const entryVariable='PERFCHECKER_SACRIFICIAL_ENTRY',entryLimit=2048; +const entryPhase=args=>args[0]==='--version'?'version':args[0]==='--help'?'global-help': + args[0]==='exec'&&args[1]==='--help'?'exec-help':args[0]==='login'&&args[1]==='status'?'login-status':'fixture-operation'; +const safeEntryError=error=>({name:['Error','TypeError','RangeError'].includes(error?.name)?error.name:'Error', + ...(typeof error?.code==='string'&&/^(?:E[A-Z0-9_]{1,30}|ERR_[A-Z0-9_]{1,40})$/.test(error.code)?{code:error.code}:{})}); +const sameEntryFile=(left,right)=>left.isFile()&&right.isFile()&&left.nlink===1n&&right.nlink===1n&& + left.ino!==0n&&left.birthtimeNs>0n&&left.dev===right.dev&&left.ino===right.ino&&left.birthtimeNs===right.birthtimeNs; +function createEntryProbe(script,args){ + const nonce=randomBytes(24).toString('hex'),file=path.join(path.dirname(script),`entry-${nonce}.json`); + const descriptor=entryFs.openSync(file,entryFs.constants.O_CREAT|entryFs.constants.O_EXCL|entryFs.constants.O_RDWR,0o600); + try{ + const identity=entryFs.fstatSync(descriptor,{bigint:true}); + if(!identity.isFile()||identity.nlink!==1n||identity.ino===0n||identity.birthtimeNs<=0n)throw new Error('Ambiguous private entry file'); + return {file,nonce,identity,phase:entryPhase(args),image:entryFs.realpathSync(process.execPath),started:Date.now()}; + }catch(error){entryFs.rmSync(file,{force:true});throw error;} + finally{entryFs.closeSync(descriptor);} +} +function inspectEntryProbe(probe,ownerPid,closed=Date.now()){ + let descriptor; + const unavailable=(status,reason)=>({status,reason}); + try{ + const pathBefore=entryFs.lstatSync(probe.file,{bigint:true}); + if(pathBefore.isSymbolicLink()||!sameEntryFile(probe.identity,pathBefore))return unavailable('refused','private-file-identity'); + if(pathBefore.size>BigInt(entryLimit))return unavailable('refused','size-limit'); + descriptor=entryFs.openSync(probe.file,entryFs.constants.O_RDONLY|(entryFs.constants.O_NOFOLLOW??0)); + const before=entryFs.fstatSync(descriptor,{bigint:true}); + if(!sameEntryFile(pathBefore,before)||before.size!==pathBefore.size)return unavailable('refused','private-file-identity'); + const bytes=Buffer.alloc(entryLimit+1),count=entryFs.readSync(descriptor,bytes,0,bytes.length,0); + const after=entryFs.fstatSync(descriptor,{bigint:true}),pathAfter=entryFs.lstatSync(probe.file,{bigint:true}); + if(!sameEntryFile(before,after)||!sameEntryFile(after,pathAfter)||before.size!==after.size||after.size!==BigInt(count)|| + before.mtimeNs!==after.mtimeNs||after.mtimeNs!==pathAfter.mtimeNs)return unavailable('refused','private-file-changed'); + if(count===0)return unavailable('unavailable','script-entry-not-observed'); + if(count>entryLimit)return unavailable('refused','size-limit'); + const value=JSON.parse(bytes.subarray(0,count).toString('utf8')); + if(!value||Array.isArray(value)||Object.keys(value).sort().join(',')!=='at,image,nonce,phase,pid,ppid,schema'|| + value.schema!==1||value.nonce!==probe.nonce||value.phase!==probe.phase||value.image!==probe.image|| + !Number.isSafeInteger(ownerPid)||ownerPid<=0||value.ppid!==ownerPid||!Number.isSafeInteger(value.pid)||value.pid<=0||value.pid===ownerPid|| + !Number.isSafeInteger(value.at)||value.atclosed)return unavailable('refused','entry-identity'); + return {status:'script-entry-observed',phase:value.phase,pid:value.pid,ppid:value.ppid,image:'expected-native-node',at:value.at}; + }catch(error){return {status:'unavailable',reason:'inspection-error',error:safeEntryError(error)};} + finally{if(descriptor!==undefined)try{entryFs.closeSync(descriptor);}catch{}} +} +function observeFixtureSpawn(delegate,args,options,script,record=value=>console.log(JSON.stringify(value))){ + let probe,probeError,child; + try{probe=createEntryProbe(script,args);}catch(error){probeError=safeEntryError(error);} + const started=Date.now();let stdoutBytes=0,stderrBytes=0,firstOutput,spawnError,finished=false; + const finish=(code,signal,terminalEvent='close')=>{ + if(finished)return;finished=true; + const closed=Date.now(),marker=probe?inspectEntryProbe(probe,child?.pid,closed):{status:'unavailable',reason:'probe-setup-error',error:probeError}; + let markerRetired=true; + try{if(probe)entryFs.rmSync(probe.file,{force:true});}catch{markerRetired=false;} + try{record({event:'sacrificial-windows-cli-timing',phase:entryPhase(args),ownerPid:child?.pid??null, + terminalEvent,startedAt:started,...(terminalEvent==='close'?{closedAt:closed}:{eventAt:closed}),milliseconds:closed-started,firstOutputMilliseconds:firstOutput, + stdoutBytes,stderrBytes,code,signal,spawnError,marker,markerRetired});}catch{/* Diagnostic output never replaces the operation result. */} + }; + try{ + child=delegate(probe?{...options,env:{...options.env,[entryVariable]:JSON.stringify({file:probe.file,nonce:probe.nonce, + dev:String(probe.identity.dev),ino:String(probe.identity.ino),birthtimeNs:String(probe.identity.birthtimeNs),phase:probe.phase})}}:options); + }catch(error){spawnError=safeEntryError(error);finish(null,null,'synchronous-spawn-error');throw error;} + const output=(kind,chunk)=>{if(firstOutput===undefined)firstOutput=Date.now()-started;kind==='stdout'?stdoutBytes+=Buffer.byteLength(chunk):stderrBytes+=Buffer.byteLength(chunk);}; + child.stdout.on('data',chunk=>output('stdout',chunk));child.stderr.on('data',chunk=>output('stderr',chunk)); + child.once('error',error=>{spawnError=safeEntryError(error);finish(null,null,'spawn-error');}); + child.once('close',finish); + return child; +} +function fixtureChildClose(child,cleanupMs=2000){ + return new Promise((resolve,reject)=>{ + let failure,timer; + const retire=()=>{clearTimeout(timer);child.removeListener('error',onError);child.removeListener('close',onClose);}; + const onError=error=>{ + failure??=error; + timer??=setTimeout(()=>{ + retire();reject(Object.assign(new AggregateError([failure], 'Fixture child close was not observed after its original error.'),{cleanupQualified:false})); + },cleanupMs); + }; + const onClose=(code,signal)=>{retire();failure?reject(failure):resolve({code,signal});}; + child.on('error',onError);child.once('close',onClose); + }); +} + // Exercise HTTP and real process lifetime with a sacrificial CLI, not a model response claim. let fixture; const require = createRequire(import.meta.url), original = Module._load; @@ -14,15 +98,9 @@ Module._load = function(name, ...args) { if (name === './windowsOwnedProcess') { const owner = original.call(this, name, ...args); return {...owner, spawnWindowsOwnedProcess: (cli, argv, options) => { - const child=cli === 'sacrificial-codex' ? owner.spawnWindowsOwnedProcess(process.execPath,[fixture,...argv],options) : owner.spawnWindowsOwnedProcess(cli,argv,options); - if(cli==='sacrificial-codex'){ - const started=Date.now();let stdoutBytes=0,stderrBytes=0,firstOutput; - const output=(kind,chunk)=>{if(firstOutput===undefined)firstOutput=Date.now()-started;kind==='stdout'?stdoutBytes+=Buffer.byteLength(chunk):stderrBytes+=Buffer.byteLength(chunk);}; - child.stdout.on('data',chunk=>output('stdout',chunk));child.stderr.on('data',chunk=>output('stderr',chunk)); - child.once('close',(code,signal)=>console.log(JSON.stringify({event:'sacrificial-windows-cli-timing',phase:argv[0]==='exec'?'exec-help':argv[0], - milliseconds:Date.now()-started,firstOutputMilliseconds:firstOutput,stdoutBytes,stderrBytes,code,signal}))); - } - return child; + const script=fixture; + return cli === 'sacrificial-codex' ? observeFixtureSpawn(observedOptions=> + owner.spawnWindowsOwnedProcess(process.execPath,[script,...argv],observedOptions),argv,options,script) : owner.spawnWindowsOwnedProcess(cli,argv,options); }}; } if (name === 'node:child_process') return {...nativeChildProcess, spawn: (cli, argv, options) => @@ -37,6 +115,17 @@ async function environment(run) { fixture = path.join(root, 'cli.cjs'); await writeFile(fixture, `const fs=require('node:fs'),path=require('node:path'),cp=require('node:child_process'); const args=process.argv.slice(2); +const entry=process.env.PERFCHECKER_SACRIFICIAL_ENTRY;delete process.env.PERFCHECKER_SACRIFICIAL_ENTRY; +if(entry){let fd;try{ +const probe=JSON.parse(entry);if(fs.lstatSync(probe.file).isSymbolicLink())throw Error('Private marker is a link'); +fd=fs.openSync(probe.file,fs.constants.O_WRONLY|(fs.constants.O_NOFOLLOW||0)); +const identity=fs.fstatSync(fd,{bigint:true}); +if(!identity.isFile()||identity.nlink!==1n||identity.size!==0n||String(identity.dev)!==probe.dev||String(identity.ino)!==probe.ino||String(identity.birthtimeNs)!==probe.birthtimeNs)throw Error('Private marker identity changed'); +const current=fs.lstatSync(probe.file,{bigint:true});if(current.isSymbolicLink()||current.dev!==identity.dev||current.ino!==identity.ino||current.birthtimeNs!==identity.birthtimeNs)throw Error('Private marker path changed'); +const phase=args[0]==='--version'?'version':args[0]==='--help'?'global-help':args[0]==='exec'&&args[1]==='--help'?'exec-help':args[0]==='login'&&args[1]==='status'?'login-status':'fixture-operation'; +const value=Buffer.from(JSON.stringify({schema:1,nonce:probe.nonce,phase,pid:process.pid,ppid:process.ppid,image:fs.realpathSync(process.execPath),at:Date.now()})); +if(value.length<=2048)fs.writeSync(fd,value); +}catch{}finally{if(fd!==undefined)try{fs.closeSync(fd)}catch{}}} function execImage(){const child=cp.spawn('/bin/sh',['-c','while [ ! -e "$1" ]; do sleep 0.02; done; exec /bin/sleep 1000','--',path.join(process.cwd(),'release-exec')],{stdio:'ignore'});fs.writeFileSync('exec-image.json',JSON.stringify({leader:process.pid,child:child.pid}));setInterval(()=>{if(fs.existsSync('release-parent'))process.exit(0)},10)} if(args[0]==='--exec-owner'){execImage();return} if(args[0]==='--cohort-detached'){const leaf=cp.spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});fs.writeFileSync('cohort.json',JSON.stringify({middle:process.ppid,detached:process.pid,leaf:leaf.pid}));setInterval(()=>{},1000);return} @@ -76,6 +165,80 @@ async function call(connector, method, params, signal) { const tool = (connector, name, args, signal) => call(connector, 'tools/call', {name, arguments: args}, signal); async function until(read) {for (let i=0;i<(process.platform==='win32'?1000:100);i++) {try {return await read();} catch {await new Promise(r=>setTimeout(r,20));}} throw new Error('Fixture process did not start.');} +test('CLI entry diagnostics preserve the delegated child and original spawn errors',async()=>environment(async root=>{ + const child=Object.assign(new EventEmitter(),{pid:12345,stdout:new PassThrough(),stderr:new PassThrough()}); + const options={cwd:root,env:{UNCHANGED:'yes'}},events=[];let calls=0,privateEntry; + assert.equal(observeFixtureSpawn(received=>{ + calls++;assert.equal(received.cwd,options.cwd);assert.equal(received.env.UNCHANGED,'yes'); + privateEntry=JSON.parse(received.env[entryVariable]);return child; + },['--version'],options,fixture,value=>events.push(value)),child); + assert.equal(calls,1);assert.equal(options.env[entryVariable],undefined); + await writeFile(privateEntry.file,JSON.stringify({schema:1,nonce:privateEntry.nonce,phase:'version',pid:12346,ppid:12345, + image:entryFs.realpathSync(process.execPath),at:Date.now()})); + child.stdout.write('private response payload');child.stderr.write('private error payload');child.emit('close',7,'SIGKILL'); + assert.equal(events.length,1);assert.equal(events[0].code,7);assert.equal(events[0].signal,'SIGKILL'); + assert.equal(events[0].marker.status,'script-entry-observed');assert.equal(events[0].marker.image,'expected-native-node'); + assert.equal(events[0].stdoutBytes,24);assert.equal(events[0].stderrBytes,21);assert.equal(events[0].markerRetired,true); + assert(!entryFs.existsSync(privateEntry.file));assert.doesNotMatch(JSON.stringify(events),/private response|private error|entry-.*\.json/); + const originalError=Object.assign(new Error('private spawn details'),{code:'ENOENT'}); + assert.throws(()=>observeFixtureSpawn(()=>{throw originalError;},['unlisted-sensitive-argument'],options,fixture,value=>events.push(value)),error=>error===originalError); + assert.equal(events.at(-1).phase,'fixture-operation');assert.equal(events.at(-1).terminalEvent,'synchronous-spawn-error'); + assert.deepEqual(events.at(-1).spawnError,{name:'Error',code:'ENOENT'});assert.equal(events.at(-1).markerRetired,true); + assert.doesNotMatch(JSON.stringify(events),/private spawn details|unlisted-sensitive-argument/); + const asyncChild=Object.assign(new EventEmitter(),{stdout:new PassThrough(),stderr:new PassThrough()}); + observeFixtureSpawn(()=>asyncChild,['--help'],options,fixture,value=>events.push(value)); + asyncChild.emit('error',originalError);asyncChild.emit('close',null,null); + assert.equal(events.length,3);assert.equal(events.at(-1).terminalEvent,'spawn-error');assert.equal(events.at(-1).ownerPid,null); + assert.equal(events.at(-1).markerRetired,true); + assert.deepEqual(await readdir(root),['cli.cjs']); + const closing=new EventEmitter(),closed=fixtureChildClose(closing,100);let settled=false; + const handled=closed.catch(error=>{settled=true;return error;});closing.emit('error',originalError); + await Promise.resolve();assert.equal(settled,false,'An error must not finish the fixture before physical close'); + closing.emit('close',null,'SIGTERM');assert.equal(await handled,originalError); + assert.equal(closing.listenerCount('error'),0);assert.equal(closing.listenerCount('close'),0); + const missingClose=new EventEmitter(),incomplete=fixtureChildClose(missingClose,10);missingClose.emit('error',originalError); + await assert.rejects(incomplete,error=>error instanceof AggregateError&&error.errors[0]===originalError&&error.cleanupQualified===false); + assert.equal(missingClose.listenerCount('error'),0);assert.equal(missingClose.listenerCount('close'),0); +})); + +test('CLI entry diagnostics refuse stale, foreign or arbitrary marker records',async()=>environment(async root=>{ + const probe=createEntryProbe(fixture,['--version']),owner=12345; + try{ + assert.deepEqual(inspectEntryProbe(probe,owner),{status:'unavailable',reason:'script-entry-not-observed'}); + const valid={schema:1,nonce:probe.nonce,phase:'version',pid:12346,ppid:owner,image:probe.image,at:Date.now()}; + for(const changed of [{ppid:owner+1},{pid:owner},{nonce:'foreign'},{phase:'login-status'},{image:'unexpected'}, + {at:probe.started-1},{at:Date.now()+60000},{token:'private secret'}]){ + await writeFile(probe.file,JSON.stringify({...valid,...changed})); + const result=inspectEntryProbe(probe,owner);assert.deepEqual(result,{status:'refused',reason:'entry-identity'}); + assert.doesNotMatch(JSON.stringify(result),/private secret|unexpected|foreign/); + } + await writeFile(probe.file,'{private invalid JSON'); + assert.deepEqual(inspectEntryProbe(probe,owner),{status:'unavailable',reason:'inspection-error',error:{name:'Error'}}); + await writeFile(probe.file,'x'.repeat(entryLimit+1));assert.deepEqual(inspectEntryProbe(probe,owner),{status:'refused',reason:'size-limit'}); + }finally{await rm(probe.file,{force:true});} +})); + +test('CLI entry diagnostics observe actual Node entry and refuse replaced files and links',async()=>environment(async root=>{ + const probe=createEntryProbe(fixture,['--version']),retired=probe.file+'.retired'; + try{ + const child=nativeSpawn(process.execPath,[fixture,'--version'],{cwd:root,env:{...process.env,[entryVariable]:JSON.stringify({file:probe.file, + nonce:probe.nonce,dev:String(probe.identity.dev),ino:String(probe.identity.ino),birthtimeNs:String(probe.identity.birthtimeNs),phase:probe.phase})}, + signal:AbortSignal.timeout(5000),stdio:['ignore','pipe','pipe']}); + let stdout='',stderr='';child.stdout.on('data',chunk=>{stdout+=chunk;});child.stderr.on('data',chunk=>{stderr+=chunk;}); + const result=await fixtureChildClose(child); + assert.deepEqual(result,{code:0,signal:null});assert.equal(stdout,'codex-cli 0.159.2\n');assert.equal(stderr,''); + const observed=inspectEntryProbe(probe,process.pid);assert.equal(observed.status,'script-entry-observed'); + assert.equal(observed.pid,child.pid);assert.equal(observed.ppid,process.pid);assert.equal(observed.image,'expected-native-node'); + entryFs.renameSync(probe.file,retired);await writeFile(probe.file,'private replacement'); + assert.deepEqual(inspectEntryProbe(probe,12345),{status:'refused',reason:'private-file-identity'}); + if(process.platform!=='win32'){ + await rm(probe.file);entryFs.symlinkSync(retired,probe.file); + assert.deepEqual(inspectEntryProbe(probe,12345),{status:'refused',reason:'private-file-identity'}); + assert.equal(JSON.parse(await readFile(retired,'utf8')).pid,child.pid); + } + }finally{await rm(probe.file,{force:true});await rm(retired,{force:true});} +})); + test('modern discovery is authenticated, stateless and never starts a CLI; legacy 2025 remains distinct',async()=>environment(async root=>{ const connector=await new CodexConnector({cli:path.join(root,'must-not-run'),root}).start(); try{ From 7937002c262b8d9b12014e1e13a9e30af295ed0b Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Wed, 14 Oct 2026 23:53:50 +0200 Subject: [PATCH 09/22] Coalesce native test item discovery and reject stale results --- CHANGELOG.md | 2 + src/testitems.ts | 73 +++++++++++----- test/native-items-host.test.mjs | 146 ++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab215c..7222ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - Resolve Core and the requested packages together before precompilation, then verify the installed Core version. Existing prepared environments still accept Core 1.1.0 in 1.x; installation and upgrades require an explicit choice. +- Share simultaneous test-item discovery requests and prevent outdated results + from replacing the current workspace selection. ## 1.0.2 — Narrow chat and process lifecycle fixes diff --git a/src/testitems.ts b/src/testitems.ts index 4eaeeec..1770707 100644 --- a/src/testitems.ts +++ b/src/testitems.ts @@ -8,6 +8,10 @@ import {resolveControllerProject, resolveWorkspaceFolder} from './workspace-root import {cancellableJulia, controllerCancellation} from './controllerCancellation'; interface NativeItem {id: string; name: string; file: string; tags: string[]; source_sha256: string} +interface NativeInvocation { + readonly folder: string; readonly root: string; readonly project: string; readonly reason: string; + readonly executable: string; readonly tags: readonly string[]; readonly excludeTags: readonly string[]; +} /** Native TestItemRunner declarations, with one controller per workspace folder. */ export function registerNativeTestItems(context: vscode.ExtensionContext): void { @@ -21,28 +25,40 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void context.subscriptions.push(tests); let declarations: NativeItem[] = []; let busy = false; + const discoveries = new Map}>(); + let latestDiscovery = 0; const setting = (key: string, fallback: T) => vscode.workspace.getConfiguration('perfchecker', folder.uri).get(key, fallback); const assertWorkspace = () => { if (!vscode.workspace.isTrusted || !vscode.workspace.workspaceFolders?.some(f => f.uri.toString() === folder.uri.toString())) throw new Error('Open the original trusted workspace before executing test items.'); }; - const invoke = async (args: string[], token?: vscode.CancellationToken): Promise<{code: number; payload: any}> => { + const captureInvocation = (): NativeInvocation => { assertWorkspace(); - const controller = resolveControllerProject(folder.uri.fsPath, - vscode.workspace.getConfiguration('perfchecker', folder.uri)); - output.appendLine(`Controller project: ${controller.project} (${controller.reason})`); + const configuration = vscode.workspace.getConfiguration('perfchecker', folder.uri); + const controller = resolveControllerProject(folder.uri.fsPath, configuration); + return Object.freeze({folder:folder.uri.toString(),root:folder.uri.fsPath,...controller, + executable:configuration.get('juliaExecutable','julia'), + tags:Object.freeze([...configuration.get('testItemTags',[])]), + excludeTags:Object.freeze([...configuration.get('testItemExcludeTags',[])])}); + }; + const invocationKey = (invocation: NativeInvocation) => JSON.stringify([ + invocation.folder,invocation.root,invocation.project,invocation.executable,invocation.tags,invocation.excludeTags]); + const invoke = async (args: string[], token?: vscode.CancellationToken, + invocation: NativeInvocation = captureInvocation()): Promise<{code: number; payload: any}> => { + output.appendLine(`Controller project: ${invocation.project} (${invocation.reason})`); const directory = path.join(context.globalStorageUri.fsPath, 'native-testitems', randomUUID()); await fs.mkdir(directory, {recursive:true}); const destination = path.join(directory, 'result.json'); - const project = controller.project; + const project = invocation.project; const env: NodeJS.ProcessEnv = {...process.env, JULIA_LOAD_PATH:['@','@stdlib'].join(path.delimiter)}; delete env.JULIA_PROJECT; const code = await new Promise((resolve,reject) => { if (token?.isCancellationRequested) {resolve(130); return;} - const child = spawn(setting('juliaExecutable','julia'), ['--startup-file=no',`--project=${project}`, + assertWorkspace(); + const child = spawn(invocation.executable, ['--startup-file=no',`--project=${project}`, '-e',cancellableJulia('using PerfChecker, TestItemRunner; exit(perfchecker_main(ARGS))'),'--','testitems', - `--root=${folder.uri.fsPath}`,`--project=${project}`,`--output=${destination}`, ...args], - {cwd:folder.uri.fsPath,windowsHide:true,detached:process.platform!=='win32',env}); + `--root=${invocation.root}`,`--project=${project}`,`--output=${destination}`, ...args], + {cwd:invocation.root,windowsHide:true,detached:process.platform!=='win32',env}); const stop = controllerCancellation(child, (message, forced) => { output.appendLine(message); if (forced) void vscode.window.showWarningMessage(message); }); @@ -63,18 +79,30 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void output.appendLine(`Evidence: ${destination}`); return {code,payload}; }; - const selectionArgs = () => [`--tags=${setting('testItemTags',[]).join(',')}`, - `--exclude-tags=${setting('testItemExcludeTags',[]).join(',')}`]; - const refresh = async () => { - const {code,payload} = await invoke(['--list',...selectionArgs()]); - if (code || payload.schema_version !== 'perfchecker-testitems/1' || !Array.isArray(payload.items)) throw new Error('Unsupported test item listing.'); - declarations = payload.items; - tests.items.replace(declarations.map(item=>{ - const test = tests.createTestItem(item.id,item.name,vscode.Uri.file(path.resolve(folder.uri.fsPath,item.file))); - test.description = `${item.file} · ${item.tags.join(', ') || 'shared'}`; - test.tags = item.tags.map(tag=>new vscode.TestTag(tag)); - return test; - })); + const selectionArgs = (invocation: NativeInvocation) => [`--tags=${invocation.tags.join(',')}`, + `--exclude-tags=${invocation.excludeTags.join(',')}`]; + const refresh = (): Promise => { + const generation = {value:++latestDiscovery}; + let invocation: NativeInvocation; + try {invocation = captureInvocation();} catch (error) {return Promise.reject(error);} + const key = invocationKey(invocation); + const existing = discoveries.get(key); + if (existing) {existing.generation.value = generation.value;return existing.promise;} + const pending = (async () => { + const {code,payload} = await invoke(['--list',...selectionArgs(invocation)],undefined,invocation); + if (code || payload.schema_version !== 'perfchecker-testitems/1' || !Array.isArray(payload.items)) throw new Error('Unsupported test item listing.'); + if (latestDiscovery !== generation.value) return; + try {if (invocationKey(captureInvocation()) !== key) return;} catch {return;} + declarations = payload.items; + tests.items.replace(declarations.map(item=>{ + const test = tests.createTestItem(item.id,item.name,vscode.Uri.file(path.resolve(invocation.root,item.file))); + test.description = `${item.file} · ${item.tags.join(', ') || 'shared'}`; + test.tags = item.tags.map(tag=>new vscode.TestTag(tag)); + return test; + })); + })().finally(() => {if (discoveries.get(key)?.promise === pending) discoveries.delete(key);}); + discoveries.set(key,{generation,promise:pending}); + return pending; }; const run = async (request: vscode.TestRunRequest, token: vscode.CancellationToken) => { if (busy) throw new Error('A native test item run is already active in this folder.'); @@ -99,8 +127,9 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void if (token.isCancellationRequested) {execution.skipped(item);continue;} execution.started(item); const samples = setting('testItemSamples',1); - const {code,payload} = await invoke([`--item-id=${item.id}`,...selectionArgs(), - `--samples=${samples}`,`--timeout=${setting('analysisTimeout',120)}`],token); + const invocation = captureInvocation(); + const {code,payload} = await invoke([`--item-id=${item.id}`,...selectionArgs(invocation), + `--samples=${samples}`,`--timeout=${setting('analysisTimeout',120)}`],token,invocation); if (token.isCancellationRequested) {execution.skipped(item);continue;} const rows = payload?.schema_version==='perfchecker-testitem-run/1' ? payload.runs?.filter((r:any)=>r.item?.id===item.id) : undefined; const details=rows?.[0]?.samples?.map((sample:any)=>sample.message).filter((message:any)=>typeof message==='string').join('\n')??''; diff --git a/test/native-items-host.test.mjs b/test/native-items-host.test.mjs index ffe2d96..d0889e0 100644 --- a/test/native-items-host.test.mjs +++ b/test/native-items-host.test.mjs @@ -83,3 +83,149 @@ test('native item controller preserves selection, exclusions, cancellation and t assert.equal(calls.length,afterFailure); } finally {await fs.rm(root,{recursive:true,force:true});} }); + +test('native discovery shares equivalent callbacks and only publishes the latest requested selection', {timeout:10000}, async()=>{ + const root=await fs.mkdtemp(path.join(os.tmpdir(),'perfchecker-discovery-')); + const uri=p=>({scheme:'file',fsPath:p,toString:()=>`file://${p}`}); + const folders=['one','two'].map(name=>({name,uri:uri(path.join(root,name))})); + for(const folder of folders)for(const name of ['controller-a','controller-b']){ + await fs.mkdir(path.join(folder.uri.fsPath,name),{recursive:true}); + await fs.writeFile(path.join(folder.uri.fsPath,name,'Project.toml'),'name = "Controller"\n'); + } + const calls=[],controllers=[],commands=new Map(),requests=[],spawned=new EventEmitter(); + const disposable=()=>({dispose(){}}),base=()=>({runnerProject:'controller-a',juliaExecutable:'julia-a',testItemTags:['a'],testItemExcludeTags:['excluded-a']}); + let settings=base(),finishing=false,holdMkdir=false; + const heldDirectories=[]; + const mkdir=async(directory,options)=>{ + await fs.mkdir(directory,options); + if(holdMkdir&&directory.startsWith(path.join(root,'storage','native-testitems')+path.sep)) + await new Promise(resolve=>{heldDirectories.push(resolve);spawned.emit('mkdir');}); + }; + class Items {values=[];replace(values){this.values=values;}forEach(fn){this.values.forEach(fn);}} + const mock={Uri:{file:uri},TestTag:class {constructor(id){this.id=id;}},TestRunProfileKind:{Run:1}, + workspace:{isTrusted:true,workspaceFolders:folders,textDocuments:[],getConfiguration:()=>({get:(key,fallback)=>settings[key]??fallback})}, + window:{createOutputChannel:()=>({...disposable(),append(){},appendLine(){},show(){}})}, + commands:{registerCommand:(name,fn)=>{commands.set(name,fn);return disposable();}}, + tests:{createTestController:(id,label)=>{ + const c={id,label,items:new Items(),...disposable(),createTestItem:(id,label,uri)=>({id,label,uri}),createRunProfile(){}}; + controllers.push(c);return c; + }}}; + const close=(call,code)=>{call.child.exitCode=code;call.child.stdout.end();call.child.stderr.end();call.child.emit('close',code);}; + const spawn=(executable,args,options)=>{ + const child=new EventEmitter();child.stdout=new PassThrough();child.stderr=new PassThrough();child.exitCode=null;child.signalCode=null; + const call={executable,args,options,child};calls.push(call);spawned.emit('spawn'); + if(finishing)queueMicrotask(()=>close(call,2)); + return child; + }; + const track=promise=>{requests.push(promise);void promise.catch(()=>{});return promise;}; + const waitForCalls=count=>calls.length>=count?Promise.resolve():new Promise(resolve=>{ + const next=()=>{if(calls.length>=count){spawned.off('spawn',next);resolve();}};spawned.on('spawn',next); + }); + const complete=async(call,id)=>{ + await fs.writeFile(call.args.find(a=>a.startsWith('--output=')).slice(9),JSON.stringify({schema_version:'perfchecker-testitems/1',items:[{id,name:id,file:'items.jl',tags:[],source_sha256:'fixture'}]})); + close(call,0); + }; + const original=Module._load,require=createRequire(import.meta.url),modulePath=require.resolve('../dist/testitems.js'); + delete require.cache[modulePath]; + Module._load=function(name,...args){return name==='vscode'?mock:name==='node:child_process'?{spawn}: + name==='node:fs'?{...original.call(this,name,...args),promises:{...fs,mkdir}}:original.call(this,name,...args);}; + let register;try {({registerNativeTestItems:register}=require(modulePath));} finally {Module._load=original;} + try { + register({subscriptions:[],globalStorageUri:uri(path.join(root,'storage'))}); + const command=commands.get('perfchecker.discoverTestItems'); + const tags=settings.testItemTags,excluded=settings.testItemExcludeTags; + const first=track(command(folders[0].uri)),controller=controllers[0]; + const resolved=track(controller.resolveHandler()),refreshed=track(controller.refreshHandler()); + assert.equal(resolved,refreshed,'Resolve and Refresh share the actual in-flight promise'); + tags.push('changed-after-request');excluded.push('changed-after-request'); + settings={runnerProject:'controller-b',juliaExecutable:'julia-b',testItemTags:['b'],testItemExcludeTags:['excluded-b']}; + const middle=track(command(folders[0].uri)); + settings=base(); + const latest=track(controller.resolveHandler()); + assert.equal(latest,resolved,'A → B → A joins the existing A invocation'); + await waitForCalls(2); + const a=calls.find(c=>c.executable==='julia-a'),b=calls.find(c=>c.executable==='julia-b'); + assert.equal(a.executable,'julia-a');assert.equal(b.executable,'julia-b'); + assert(a.args.includes(`--project=${path.join(folders[0].uri.fsPath,'controller-a')}`)); + assert(b.args.includes(`--project=${path.join(folders[0].uri.fsPath,'controller-b')}`)); + assert(a.args.includes('--tags=a'));assert(a.args.includes('--exclude-tags=excluded-a')); + assert(b.args.includes('--tags=b'));assert(b.args.includes('--exclude-tags=excluded-b')); + assert.equal(a.options.cwd,folders[0].uri.fsPath);assert(a.args.includes(`--root=${folders[0].uri.fsPath}`)); + assert(a.args.includes('--list'));assert.equal(a.options.env.JULIA_PROJECT,undefined); + await complete(a,'latest-a');await Promise.all([first,resolved,refreshed,latest]); + await complete(b,'obsolete-b');await middle; + assert.equal(calls.length,2);assert.deepEqual(controller.items.values.map(i=>i.id),['latest-a']); + + for(const [change,label] of [[{runnerProject:'controller-b'},'project'],[{juliaExecutable:'julia-b'},'executable'], + [{testItemTags:['b']},'tags'],[{testItemExcludeTags:['excluded-b']},'exclusions']]){ + const before=calls.length;settings=base();const older=track(controller.resolveHandler()); + settings={...base(),...change};const newer=track(controller.refreshHandler()); + assert.notEqual(older,newer,`${label} changes require a distinct invocation`); + await waitForCalls(before+2); + const pair=calls.slice(before),newCall=pair.find(c=>label==='project'?c.args.includes(`--project=${path.join(folders[0].uri.fsPath,'controller-b')}`): + label==='executable'?c.executable==='julia-b':c.args.includes(label==='tags'?'--tags=b':'--exclude-tags=excluded-b')); + await complete(newCall,label);await newer; + await complete(pair.find(c=>c!==newCall),`obsolete-${label}`);await older; + assert.equal(calls.length,before+2);assert.deepEqual(controller.items.values.map(i=>i.id),[label]); + } + + settings=base();let before=calls.length; + const failedCommand=track(command(folders[0].uri)),failedResolve=track(controller.resolveHandler()),failedRefresh=track(controller.refreshHandler()); + assert.equal(failedResolve,failedRefresh); + const originalError=new Error('owned discovery spawn failed'); + const failures=[failedCommand,failedResolve,failedRefresh].map(p=>assert.rejects(p,error=>error===originalError)); + await waitForCalls(before+1);calls[before].child.emit('error',originalError);close(calls[before],2); + await Promise.all(failures);assert.equal(calls.length,before+1); + before=calls.length;const recovery=track(controller.resolveHandler()); + await waitForCalls(before+1);await complete(calls[before],'recovered');await recovery; + assert.deepEqual(controller.items.values.map(i=>i.id),['recovered']); + + before=calls.length;const superseded=track(controller.resolveHandler()); + settings={...base(),runnerProject:'missing-controller'}; + await assert.rejects(controller.refreshHandler(),/controller Project.toml not found/); + settings=base();await waitForCalls(before+1);await complete(calls[before],'obsolete-after-refusal');await superseded; + assert.equal(calls.length,before+1);assert.deepEqual(controller.items.values.map(i=>i.id),['recovered']); + + for(const change of ['settings','trust','folder']){ + before=calls.length;const obsolete=track(controller.resolveHandler());await waitForCalls(before+1); + if(change==='settings')settings={...base(),testItemTags:['changed-without-refresh']}; + else if(change==='trust')mock.workspace.isTrusted=false; + else mock.workspace.workspaceFolders=[folders[1]]; + await complete(calls[before],`obsolete-after-${change}`);await obsolete; + assert.deepEqual(controller.items.values.map(i=>i.id),['recovered'],`${change} changes without Refresh prevent stale publication`); + settings=base();mock.workspace.isTrusted=true;mock.workspace.workspaceFolders=folders; + } + + for(const change of ['trust','folder']){ + before=calls.length;holdMkdir=true; + const entered=new Promise(resolve=>spawned.once('mkdir',resolve)); + const refused=track(controller.resolveHandler());await entered; + if(change==='trust')mock.workspace.isTrusted=false; + else mock.workspace.workspaceFolders=[folders[1]]; + const rejected=assert.rejects(refused,/trusted workspace/); + holdMkdir=false;heldDirectories.shift()();await rejected; + assert.equal(calls.length,before,`${change} revoked during directory preparation prevents spawn`); + mock.workspace.isTrusted=true;mock.workspace.workspaceFolders=folders; + } + + before=calls.length; + const own=track(controller.resolveHandler()),other=track(command(folders[1].uri)); + mock.workspace.isTrusted=false; + await assert.rejects(command(folders[0].uri),/trusted workspace/); + mock.workspace.isTrusted=true; + await waitForCalls(before+2); + assert.notEqual(calls[before].options.cwd,calls[before+1].options.cwd); + for(const call of calls.slice(before))await complete(call,call.options.cwd===folders[0].uri.fsPath?'one':'two'); + await Promise.all([own,other]);assert.equal(controllers.length,2); + assert.deepEqual(controllers.map(c=>c.items.values.map(i=>i.id)),[['recovered'],['two']], + 'The refused newer request invalidates folder one only'); + mock.workspace.isTrusted=false; + await assert.rejects(command(folders[0].uri),/trusted workspace/); + assert.equal(calls.length,before+2,'Untrusted discovery never starts or joins an invocation'); + } finally { + finishing=true;holdMkdir=false;for(const release of heldDirectories.splice(0))release(); + for(const call of calls)if(call.child.exitCode===null)close(call,2); + await Promise.allSettled(requests);spawned.removeAllListeners();delete require.cache[modulePath]; + await fs.rm(root,{recursive:true,force:true}); + } +}); From a6e9a40ea506248f92a2e4c8c9431dc223550e5a Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 00:04:31 +0200 Subject: [PATCH 10/22] Freeze discovery candidate origin for native qualification --- .github/workflows/native-vsix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 6032659..2fb0673 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -182,8 +182,8 @@ jobs: console.log('Release 1.0.2 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } if(version==='1.0.3'&&!requested&&!newOrigin){ - // Product commit 1140dad0ca6c8960e8426c99de88ce41124878c5, tree bf83cafca5641036dc21498458af6fd5b5f3cc58. - fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38518975105\nREUSE_NAME=native-candidate-1140dad0ca6c8960e8426c99de88ce41124878c5\nREUSE_SHA=068a3ed3bd01d823224f504dcfe134e3707789856c281ca8afb4329af41a83d3\n'); + // Product commit 7937002c262b8d9b12014e1e13a9e30af295ed0b, tree 40853c0cce3c6416b63bd68dfbf64878037a73ee. + fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38643783540\nREUSE_NAME=native-candidate-7937002c262b8d9b12014e1e13a9e30af295ed0b\nREUSE_SHA=b4cc9352bb408441a6550cbc35ef79139f8f70eff3f727ca6cd30b1f6f4ba74e\n'); console.log('Release 1.0.3 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } NODE From 73b99deb87efaf45bd5e96036f755e42c748bdab Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 01:19:37 +0200 Subject: [PATCH 11/22] Clarify ARM-compatible Core requirements for 1.0.3 --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4bef8a9..cc89926 100644 --- a/README.md +++ b/README.md @@ -153,8 +153,11 @@ Sorting breaks equivalent numeric ties by target label, with a release before `perfchecker.juliaExecutable`. - A controller environment in the opened package workspace, normally `perf`, containing PerfChecker.jl 1.0.1 or later and a `suite.jl` with `build_suite()`. - Guided setup and Pluto notebooks require 1.1.0; use that version for the - read-only worker-environment fixes. + New controller setup and Pluto installations or upgrades require Core 1.1.1 + or newer in 1.x for ARM loading and the read-only worker-environment fixes. Existing + prepared controller and Pluto environments still accept Core 1.1.0; other CLI + actions remain compatible with Core 1.0.1. Installation and upgrades require + an explicit choice. The extension invokes the public PerfChecker CLI and reads only versioned JSON, JSONL and Markdown outputs. See [CONTRACT.md](CONTRACT.md) for the exact boundary. @@ -220,8 +223,9 @@ the quality contract and `perf/live_provider.jl` before launch. The provider mus write `perfchecker-provider-result/1` JSON to `PERFCHECKER_OUTPUT`, using the explicit `--quality=` argument. It runs in the game project; PerfChecker runs in the folder's configured controller project. The controller must already contain -PerfChecker.jl 1.0.1 or later. Core 1.1.0 is recommended for its read-only -worker-environment fixes. No package installation or suite factory runs automatically. +PerfChecker.jl 1.0.1 or later. Core 1.1.1 or newer in 1.x is recommended for ARM +loading and the read-only worker-environment fixes. No package installation or +suite factory runs automatically. The provider's `suite` is `etendu-beautiful-landscape-live`. Its `environment` contains `quality_profile`, effective `width` and `height`, `resolution_source`, @@ -412,12 +416,12 @@ Implementation requires a separate explicit tool configuration in the chat's use a trusted agent that respects the path and can access the local filesystem. Preparation creates a checkpoint; application requires a separate reviewed action. The controller's public `chat` and `implement` commands remain compatible with -PerfChecker.jl 1.0.1 or later. Core 1.1.0 is recommended for its read-only -worker-environment fixes; upgrading requires an explicit choice. The extension does not -install an agent backend. An MCP server may supply ordinary tools without an LLM -or an implementation tool. Choose tools whose documented inputs and capabilities -match the requested operation. Codex CLI is an optional connector, not a prerequisite -for MCP. +PerfChecker.jl 1.0.1 or later. Core 1.1.1 or newer in 1.x is recommended for ARM +loading and the read-only worker-environment fixes; upgrading requires an explicit +choice. The extension does not install an agent backend. An MCP server may supply +ordinary tools without an LLM or an implementation tool. Choose tools whose +documented inputs and capabilities match the requested operation. Codex CLI is +an optional connector, not a prerequisite for MCP. See the canonical [MCP guide](https://perfchecker.mirageinteractive.fr/mcp-advisor.html) for provider configuration, credentials, supported revisions, transmission, From d2823ebfa423ae8e1a807d2d9065807e924d013e Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 01:28:46 +0200 Subject: [PATCH 12/22] Freeze the corrected 1.0.3 README candidate origin --- .github/workflows/native-vsix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 2fb0673..1e8ff9f 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -182,8 +182,8 @@ jobs: console.log('Release 1.0.2 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } if(version==='1.0.3'&&!requested&&!newOrigin){ - // Product commit 7937002c262b8d9b12014e1e13a9e30af295ed0b, tree 40853c0cce3c6416b63bd68dfbf64878037a73ee. - fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38643783540\nREUSE_NAME=native-candidate-7937002c262b8d9b12014e1e13a9e30af295ed0b\nREUSE_SHA=b4cc9352bb408441a6550cbc35ef79139f8f70eff3f727ca6cd30b1f6f4ba74e\n'); + // Product commit 73b99deb87efaf45bd5e96036f755e42c748bdab, tree 3722727a870402497078e63da78be90d606f5ed5. + fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38653410110\nREUSE_NAME=native-candidate-73b99deb87efaf45bd5e96036f755e42c748bdab\nREUSE_SHA=bc61f9fa172069c61722e0691900d3ddf8a39ce143fd9d553b252304dbc55850\n'); console.log('Release 1.0.3 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } NODE From fd2e61d67fb3c1575a198e4f0d3ab07eed02408b Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 03:42:35 +0200 Subject: [PATCH 13/22] Sustain target workload inside fresh profiling fixture --- test/run-public-vsix-host.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index f57a2e7..8f1b5fe 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -568,7 +568,7 @@ try { await fs.writeFile(path.join(workspace, 'Project.toml'), 'name = "PerfCheckerNativeFixture"\nuuid = "6af56806-e0b1-4f34-88bf-fde69d8a8679"\nversion = "0.1.0"\n'); // Keep actual workload frames in the target package, rather than inlining them // into an external perf script. Both workloads allocate inside their own source. - await fs.writeFile(path.join(workspace, 'src', 'PerfCheckerNativeFixture.jl'), 'module PerfCheckerNativeFixture\nBase.@noinline sum_squares(xs) = sum(xs .^ 2)\nBase.@noinline wait_task(x) = (values=fill(x,1_000_000); sleep(0.005); values[1])\nend\n'); + await fs.writeFile(path.join(workspace, 'src', 'PerfCheckerNativeFixture.jl'), 'module PerfCheckerNativeFixture\nBase.@noinline sum_squares(xs) = sum(xs .^ 2)\nBase.@noinline function profile_sum_squares(xs)\n total=0.0\n for _ in 1:64\n total+=sum_squares(xs)\n end\n total\nend\nBase.@noinline wait_task(x) = (values=fill(x,1_000_000); sleep(0.005); values[1])\nend\n'); await fs.writeFile(path.join(workspace, 'test', 'performance.jl'), 'using TestItems\n@testitem "Vector reduction" tags=[:performance] begin\n data=collect(1:10_000)\n @test sum(data)==50_005_000\nend\n'); const vsix = path.join(session, 'perfchecker.vsix'); if (mode === 'public') { @@ -852,7 +852,9 @@ try { await fs.rm(path.join(workspace,'perf','results'),{recursive:true,force:true}); await fs.rm(path.join(workspace,'perf','perfchecker-ui.json'),{force:true}); await fs.writeFile(path.join(workspace, 'perf', 'sum.jl'), 'using PerfCheckerNativeFixture\nperf_setup() = collect(1.0:1000.0)\nperf_workload(xs) = PerfCheckerNativeFixture.sum_squares(xs)\nperf_oracle(xs) = perf_workload(xs) == 333833500.0\n'); - await fs.writeFile(path.join(workspace,'perf','profile-sum.jl'),'using PerfCheckerNativeFixture\nperf_setup() = collect(1.0:1_000_000.0)\nperf_workload(xs) = PerfCheckerNativeFixture.sum_squares(xs)\nperf_oracle(xs,result) = isapprox(result,1_000_000.0*1_000_001.0*2_000_001.0/6;rtol=1e-12)\n'); + // Keep a sustained, checked batch inside the target module while each fresh + // profile operation owns its own sampling window; ordinary collectors stay small. + await fs.writeFile(path.join(workspace,'perf','profile-sum.jl'),'using PerfCheckerNativeFixture\nperf_setup() = collect(1.0:1_000_000.0)\nperf_workload(xs) = PerfCheckerNativeFixture.profile_sum_squares(xs)\nperf_oracle(xs,result) = isapprox(result,64*(1_000_000.0*1_000_001.0*2_000_001.0/6);rtol=1e-12)\n'); await fs.writeFile(path.join(workspace, 'perf', 'wait.jl'), 'using PerfCheckerNativeFixture\nperf_setup() = 42\nperf_workload(x) = PerfCheckerNativeFixture.wait_task(x)\nperf_oracle(x) = perf_workload(x) == 42\n'); await fs.writeFile(path.join(workspace, 'perf', 'cases.jl'), 'make_sum_case(p) = (prepare=()->collect(1.0:1000.0), operation=xs->sum(xs.^2), verify=(xs,result)->result==333833500.0)\nmake_wait_case(p) = (prepare=()->42, operation=x->(sleep(0.005);x), verify=(x,result)->result==42)\nmake_cancel_case(p) = (prepare=()->42, operation=x->(write(p["marker"],"running");sleep(30);x), verify=(x,result)->result==42)\n'); // A distinct bounded workload gives CPU sampling actual operation time; From 5e8d6ad69fb9dbb3bf3776aa2bd2ea936ff44cab Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 04:50:22 +0200 Subject: [PATCH 14/22] Wait for exact saved measurement before native narrative actions --- test/native-narrative-controls.cjs | 61 +++++++++++++--- test/native-narrative-controls.test.mjs | 95 +++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 test/native-narrative-controls.test.mjs diff --git a/test/native-narrative-controls.cjs b/test/native-narrative-controls.cjs index a49ee71..caadd99 100644 --- a/test/native-narrative-controls.cjs +++ b/test/native-narrative-controls.cjs @@ -4,6 +4,7 @@ const fs = require('node:fs/promises'); const path = require('node:path'); const http = require('node:http'); const {createHash, randomUUID} = require('node:crypto'); +const {isDeepStrictEqual} = require('node:util'); const marker = 'Controlled protocol fixture: collect more comparable samples before drawing a performance conclusion.'; const customInstructions = 'Native qualification instruction: distinguish measured allocation evidence from hypotheses.'; @@ -36,6 +37,47 @@ async function tab(context, name) { return frame; } +async function reloadMeasuredEvidence(context, measured, stage) { + assert.equal(measured.report.schema_version, 'perfchecker-scenario-run/1'); + assert(measured.report.runs.length > 0); + assert(measured.report.runs.every(run => run.qualification.availability === 'complete' && run.qualification.correctness === 'passed')); + const expectedRuns = JSON.parse(JSON.stringify(measured.report.runs)); + context.log('native-narrative-history-reload-start', {stage, reportId: measured.id}); + const frame = await tab(context, 'Saved evidence'); + const card = frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^run ·/})}).first(); + const heading = await card.getByRole('heading', {name: /^run ·/}).innerText(); + assert.match(heading, /^run · \d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/); + const created = heading.slice('run · '.length); + context.log('native-narrative-history-open-click', {stage, created, reportId: measured.id}); + await card.getByRole('button', {name: 'Open evidence', exact: true}).click(); + // The click changes tabs before loadHistory has read and published its report. + // Match both the selected history timestamp and every rendered measured run. + await eventually(async () => { + const observed = await frame.locator('#app').evaluate(app => { + const status = app.querySelector('.status'); + const buttons = [...app.querySelectorAll('button')]; + return { + status: status?.textContent?.trim(), busy: status?.classList.contains('busy'), + view: app.querySelector('nav[aria-label="Investigation views"] button[aria-current="page"]')?.textContent, + cancelDisabled: buttons.find(button => button.textContent === 'Cancel')?.disabled, + explainDisabled: buttons.find(button => button.textContent === 'Explain with configured model')?.disabled, + narrative: [...app.querySelectorAll('h2')].some(heading => heading.textContent === 'Optional model explanation'), + imported: Boolean(app.querySelector('.imported-diagnostic')), + runs: [...app.querySelectorAll('details')] + .filter(detail => detail.querySelector(':scope > summary')?.textContent === 'Qualification and measurement evidence') + .map(detail => JSON.parse(detail.querySelector(':scope > pre').textContent)), + }; + }, undefined, {timeout: 1000}); + return observed.status === `Saved evidence · ${created}` && observed.busy === false && + observed.view === 'Findings & advice' && observed.cancelDisabled === true && observed.explainDisabled === false && + observed.narrative === false && observed.imported === false && isDeepStrictEqual(observed.runs, expectedRuns); + }, 'The exact saved measurement is loaded and idle before Explain'); + context.proof('native-narrative-history-measured-ack', {stage, created, reportId: measured.id, runs: expectedRuns.length}); + return frame; +} + +exports.reloadMeasuredEvidence = reloadMeasuredEvidence; + async function directories(root) { return new Set(await fs.readdir(root).catch(error => {if (error.code === 'ENOENT') return []; throw error;})); } @@ -276,19 +318,19 @@ exports.run = async context => { samples: 2, status: narrative.report.status, authority: narrative.report.authority, visible: true, jsonAndMarkdown: true}); // Reload the real saved measurement so no QuickPick or synthetic history is needed for another explanation. - frame = await tab(context, 'Saved evidence'); - await frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^run ·/})}).first() - .getByRole('button', {name: 'Open evidence', exact: true}).click(); - frame = await tab(context, 'Findings & advice'); + frame = await reloadMeasuredEvidence(context, measured, 'before-cancel'); fixture.mode = 'hold'; const before = await directories(root); await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click(); await eventually(() => fixture.blocked, 'second actual HTTP generation request arrives', 240000); + context.proof('native-narrative-http-held', {stage: 'before-cancel', requests: fixture.requests.length}); assert.equal(fixture.requests.length, 2); assert.equal(fixture.blocked.closed, false, 'Generation is actually in flight when Cancel is clicked'); assert.equal(await frame.getByRole('button', {name: 'Cancel', exact: true}).isEnabled(), true); + context.log('native-narrative-cancel-click', {stage: 'active-http'}); await frame.getByRole('button', {name: 'Cancel', exact: true}).click(); await eventually(() => fixture.blocked.closed, 'cancellation closes the active model HTTP socket before fixture teardown', 180000); + context.proof('native-narrative-http-closed', {stage: 'cancelled'}); frame = await idle(context); assert.match(await frame.locator('#app .status').innerText(), /Cancelled after controller cleanup/); assert.deepEqual(fixture.errors, []); @@ -306,15 +348,14 @@ exports.run = async context => { context.proof('native-narrative-cancel-active-http', {activeRequestReceived: true, socketClosedBeforeFixtureCleanup: true, controllerCleanupComplete: true, configurationPreserved: true, sourceAndMeasurementUnchanged: true, retainedLog: true}); + context.log('native-narrative-deadline-configuration-start', {configuredSeconds: 45}); await configuration(context).update('advisorTimeout', 45, context.vscode.ConfigurationTarget.WorkspaceFolder); fixture.blocked = undefined; const timeoutBefore = await directories(root); - frame = await tab(context, 'Saved evidence'); - await frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^run ·/})}).first() - .getByRole('button', {name: 'Open evidence', exact: true}).click(); - frame = await tab(context, 'Findings & advice'); + frame = await reloadMeasuredEvidence(context, measured, 'before-deadline'); await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click(); await eventually(() => fixture.blocked, 'The real third HTTP request reaches the deadline fixture', 180000); + context.proof('native-narrative-http-held', {stage: 'before-deadline', requests: fixture.requests.length}); const held = fixture.blocked; assert.equal(held.closed, false); const expired = await reportAfter(context, root, timeoutBefore, 'narrate'); assert.equal(expired.report.status, 'timeout', 'The Core request deadline expires without a Cancel click'); @@ -327,6 +368,7 @@ exports.run = async context => { noCancelClick: true, coreStatus: expired.report.status, socketClosedBeforeFixtureCleanup: true, nativeOutcomeVisible: true, measuredEvidencePreserved: true}); } finally { + context.log('native-narrative-finally-entered'); // Only the sacrificial workspace-folder settings are restored; global/user configuration is never written. try { const frame = await view(context); @@ -335,10 +377,13 @@ exports.run = async context => { await idle(context); } } finally { + context.log('native-narrative-fixture-close-start'); await fixture.close(); + context.proof('native-narrative-fixture-close-complete'); if (previousToken === undefined) delete process.env[keyEnvironment]; else process.env[keyEnvironment] = previousToken; for (const [key, value] of previous) await cfg.update(key, value, context.vscode.ConfigurationTarget.WorkspaceFolder); for (const [key, value] of previous) assert.deepEqual(configuration(context).inspect(key)?.workspaceFolderValue, value); + context.proof('native-narrative-settings-restored'); } } }; diff --git a/test/native-narrative-controls.test.mjs b/test/native-narrative-controls.test.mjs new file mode 100644 index 0000000..d242e83 --- /dev/null +++ b/test/native-narrative-controls.test.mjs @@ -0,0 +1,95 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import narrative from './native-narrative-controls.cjs'; + +const created = '2026-10-15T00:34:00.123Z'; +const runs = [{scenario: {id: 'sum_squares', implementation: 'allocating'}, collector: 'benchmark', + qualification: {availability: 'complete', correctness: 'passed'}, summaries: [{metric: 'julia.wall.time', median: 0.25, samples: 2}]}]; +const measured = {id: 'measured-fixture', report: {schema_version: 'perfchecker-scenario-run/1', runs}}; +const ready = () => ({status: `Saved evidence · ${created}`, busy: false, narrative: false, imported: false, + cancelDisabled: true, explainDisabled: false, runs: structuredClone(runs)}); + +function fixture(initial) { + let firstRead; + const seen = new Promise(resolve => {firstRead = resolve;}); + const state = {snapshot: initial, view: 'Saved evidence', opens: 0, explains: 0, proofs: []}; + const app = () => ({ + querySelector(selector) { + if (selector === '.status') return {textContent: state.snapshot.status, classList: {contains: () => state.snapshot.busy}}; + if (selector === '.imported-diagnostic') return state.snapshot.imported ? {} : null; + if (selector.startsWith('nav[')) return {textContent: state.view}; + throw new Error(`Unexpected DOM selector ${selector}`); + }, + querySelectorAll(selector) { + if (selector === 'button') return [{textContent: 'Cancel', disabled: state.snapshot.cancelDisabled}, + {textContent: 'Explain with configured model', disabled: state.snapshot.explainDisabled}]; + if (selector === 'h2') return state.snapshot.narrative ? [{textContent: 'Optional model explanation'}] : []; + if (selector === 'details') return state.snapshot.runs.map(run => ({querySelector: selector => + ({textContent: selector === ':scope > summary' ? 'Qualification and measurement evidence' : JSON.stringify(run)})})); + throw new Error(`Unexpected DOM collection ${selector}`); + }, + }); + const card = {getByRole(role, options) { + if (role === 'heading') return {innerText: async () => `run · ${created}`}; + assert.equal(options.name, 'Open evidence'); + return {click: async () => {state.opens++; state.view = 'Findings & advice';}}; + }}; + const frame = { + getByRole(role, options) { + if (role === 'heading') return {}; + return {click: async () => { + if (options.name === 'Explain with configured model') state.explains++; + else {assert.equal(options.name, 'Saved evidence'); state.view = options.name;} + }, getAttribute: async () => 'page'}; + }, + locator(selector) { + if (selector === 'article.card') return {filter: () => ({first: () => card})}; + assert.equal(selector, '#app'); + return {evaluate: async callback => {const observed = callback(app()); firstRead(); return observed;}}; + }, + }; + const context = {vscode: {commands: {executeCommand: async command => assert.equal(command, 'perfchecker.openInvestigations')}}, + findFrame: async () => frame, log: () => {}, proof: (name, detail) => state.proofs.push({name, ...detail})}; + return {state, context, seen}; +} + +async function waitForRealAck(initial) { + const f = fixture(initial); + let resolved = false; + const pending = narrative.reloadMeasuredEvidence(f.context, measured, 'before-cancel').then(async frame => { + resolved = true; + await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click(); + }); + try { + await f.seen; + await new Promise(resolve => setImmediate(resolve)); + assert.equal(resolved, false, 'Neither tab selection nor stale evidence acknowledges the reload'); + assert.equal(f.state.opens, 1, 'The actual Open evidence click is not retried'); + assert.equal(f.state.explains, 0, 'Explain waits for the measured report'); + assert(!f.state.proofs.some(check => check.name === 'native-narrative-history-measured-ack')); + f.state.snapshot = ready(); + await pending; + assert.equal(f.state.opens, 1); + assert.equal(f.state.explains, 1); + assert.equal(f.state.proofs.filter(check => check.name === 'native-narrative-history-measured-ack').length, 1); + } finally { + f.state.snapshot = ready(); + await pending; + } +} + +test('native narrative reload waits for the delayed DOM acknowledgement before Explain', async () => { + await waitForRealAck({...ready(), status: 'Previous narrative', narrative: true, runs: []}); +}); + +test('native narrative reload rejects a same-date report with different or incomplete deep measurement evidence', async () => { + const changed = structuredClone(runs); changed[0].summaries[0].median = 999; + for (const wrong of [changed, [], [{...runs[0], qualification: {availability: 'complete', correctness: 'failed'}}]]) + await waitForRealAck({...ready(), runs: wrong}); +}); + +test('native narrative reload rejects stale identity, narrative, imported and busy UI states', async () => { + for (const stale of [{status: 'Saved evidence · 2026-10-15T00:34:00.124Z'}, {narrative: true}, + {imported: true}, {busy: true}, {cancelDisabled: false}, {explainDisabled: true}]) + await waitForRealAck({...ready(), ...stale}); +}); From 78976f5e95bbbc5c4b133245277c6e9ff98f8d9c Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 07:58:08 +0200 Subject: [PATCH 15/22] Observe native test item discovery without changing its oracle --- .github/workflows/native-vsix.yml | 20 +- test/native-controller-preflight.mjs | 282 ++++++++++++++++++++++ test/native-controller-preflight.test.mjs | 199 ++++++++++++++- test/native-platform-contract.test.mjs | 26 +- test/public-vsix-host.cjs | 50 +++- test/run-public-vsix-host.mjs | 23 +- 6 files changed, 593 insertions(+), 7 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 1e8ff9f..34e51c6 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -48,6 +48,10 @@ on: candidate_sha: description: Full SHA256 of the original candidate (never repackage when supplied) type: string + discovery_diagnostic: + description: Nonqualifying Intel discovery observation with the exact full prelude and unchanged 120-second UI oracle + type: boolean + default: false new_candidate_origin: description: Explicitly declare a new candidate origin after product changes; otherwise reuse the frozen release archive type: boolean @@ -148,6 +152,10 @@ jobs: sha: ${{ steps.digest.outputs.sha }} env: NATIVE_EVENT: ${{ github.event_name }} + NATIVE_DISCOVERY_DIAGNOSTIC: ${{ inputs.discovery_diagnostic || 'false' }} + NATIVE_BACKEND_JULIA: ${{ inputs.backend_julia }} + NATIVE_CORE_COMMIT: ${{ inputs.core_commit }} + NATIVE_CORE_TREE: ${{ inputs.core_tree }} NATIVE_VSIX_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.artifact || 'candidate' }} NATIVE_NEW_ORIGIN: ${{ inputs.new_candidate_origin || 'false' }} NATIVE_SCOPE: ${{ inputs.scope }} @@ -164,10 +172,15 @@ jobs: node --input-type=module <<'NODE' import fs from 'node:fs'; import {assertNativeCampaignSelection} from './test/native-platform-contract.mjs'; + import {assertNativeDiscoveryDiagnostic} from './test/native-controller-preflight.mjs'; const e=process.env; assertNativeCampaignSelection({scope:e.NATIVE_SCOPE,event:e.NATIVE_EVENT,artifact:e.NATIVE_VSIX_MODE, stage:e.NATIVE_STAGE,group:e.NATIVE_CASE_GROUP,core:e.NATIVE_CORE}); const requested=[e.REUSE_RUN,e.REUSE_NAME,e.REUSE_SHA].some(Boolean),newOrigin=e.NATIVE_NEW_ORIGIN==='true'; + assertNativeDiscoveryDiagnostic({enabled:e.NATIVE_DISCOVERY_DIAGNOSTIC==='true',event:e.NATIVE_EVENT, + artifact:e.NATIVE_VSIX_MODE,stage:e.NATIVE_STAGE,scope:e.NATIVE_SCOPE,core:e.NATIVE_CORE,partition:'measure', + backendJulia:e.NATIVE_BACKEND_JULIA,originRun:e.REUSE_RUN,originName:e.REUSE_NAME,originSha:e.REUSE_SHA, + coreCommit:e.NATIVE_CORE_COMMIT,coreTree:e.NATIVE_CORE_TREE,newOrigin}); if(e.NATIVE_SCOPE==='arm-bootstrap'&&(!requested||newOrigin))throw Error('ARM bootstrap must reuse the original candidate archive; it never packages a second origin.'); if(e.NATIVE_VSIX_MODE==='public'){ if(requested||newOrigin)throw Error('Public reproduction uses its published release, not candidate-origin inputs'); @@ -243,14 +256,17 @@ jobs: native-workflows: if: github.event_name != 'workflow_dispatch' || inputs.stage != 'connector-workers' needs: package - name: Native VSIX (${{ matrix.os }}, VS Code ${{ matrix.vscode }}${{ matrix.julia && format(', Julia {0}', matrix.julia) || '' }}) + name: Native VSIX (${{ matrix.os }}, VS Code ${{ matrix.vscode }}${{ matrix.julia && format(', Julia {0}', matrix.julia) || '' }}${{ matrix.partition && format(', {0}', matrix.partition) || '' }}${{ inputs.discovery_diagnostic && ', discovery diagnostic' || '' }}) strategy: fail-fast: false matrix: - include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && (inputs.artifact == 'candidate' && inputs.stage == 'full' && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"macos-15-intel","vscode":"stable"}]') || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || ((github.event_name != 'workflow_dispatch' || inputs.artifact == 'candidate' && inputs.stage == 'full') && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]')) }} + include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.discovery_diagnostic && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && (inputs.artifact == 'candidate' && inputs.stage == 'full' && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"macos-15-intel","vscode":"stable"}]') || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || ((github.event_name != 'workflow_dispatch' || inputs.artifact == 'candidate' && inputs.stage == 'full') && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]')) }} runs-on: ${{ matrix.os }} timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }} env: + PERFCHECKER_NATIVE_DISCOVERY_DIAGNOSTIC: ${{ inputs.discovery_diagnostic && '1' || '0' }} + PERFCHECKER_NATIVE_EVENT: ${{ github.event_name }} + PERFCHECKER_NATIVE_BACKEND_JULIA: ${{ inputs.backend_julia }} PERFCHECKER_VSCODE_VERSION: ${{ matrix.vscode }} PERFCHECKER_VSIX_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.artifact || 'candidate' }} PERFCHECKER_NATIVE_CANDIDATE_SHA: ${{ needs.package.outputs.sha }} diff --git a/test/native-controller-preflight.mjs b/test/native-controller-preflight.mjs index 4c2182b..ec19bc7 100644 --- a/test/native-controller-preflight.mjs +++ b/test/native-controller-preflight.mjs @@ -1,6 +1,288 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import {TextDecoder} from 'node:util'; +import fs from 'node:fs/promises'; +import {constants as fileFlags} from 'node:fs'; +import {createHash} from 'node:crypto'; +import os from 'node:os'; +import {gunzipSync} from 'node:zlib'; + +const discoveryHash=value=>createHash('sha256').update(value).digest('hex'); + +// No stdout/stderr from an external inspection command reaches Actions logs. +export async function discoveryOwnedCommand(spawn,file,args,{signal,deadlineAt,states,stdio,allowStderr=false,record,stage,targetPid,sequence}={}){ + assert(deadlineAt>Date.now(),'An expired observation cannot start another process'); + let child,exited=false,closed=false,stopping=false,timer,escalation,drain,firstError,exitCode=null,exitSignal=null; + const startedAt=Date.now();let size=0,stderrBytes=0; + const stop=()=>{if(stopping)return;stopping=true;if(!exited&&!closed){try{child.kill('SIGTERM');}catch(error){firstError??=error;} + escalation=setTimeout(()=>{if(!exited&&!closed)try{child.kill('SIGKILL');}catch(error){firstError??=error;}},1000);} + drain=setTimeout(()=>{if(!closed){const cleanup=new Error('Owned diagnostic process close is unknown'); + finish(firstError?new AggregateError([firstError,cleanup],'Diagnostic command and physical drain failed'):cleanup);}},3000);}; + let finish; + const pending=new Promise((resolve,reject)=>{ + const chunks=[];let settled=false; + finish=error=>{if(settled)return;settled=true;clearTimeout(timer);clearTimeout(escalation);clearTimeout(drain); + signal?.removeEventListener('abort',stop); + try{record?.({stage:'tool-result',step:stage,sequence,targetPid,tool:path.basename(file),pid:child?.pid, + elapsedMs:Date.now()-startedAt,exitCode,signal:exitSignal,stdoutBytes:size,stderrBytes,physicalClose:closed, + status:error?'inconclusive':'completed'});}catch{/* An observation sink never throws from a child listener. */} + error?reject(error):resolve(Buffer.concat(chunks));}; + try{child=spawn(file,args,{stdio:stdio||['ignore','pipe','pipe']});}catch(error){finish(error);return;} + const state={child,get exited(){return exited;},get closed(){return closed;}};states?.push(state); + child.once('error',error=>{firstError??=error;});child.once('exit',(code,signal)=>{exited=true;exitCode=Number.isInteger(code)?code:null; + exitSignal=/^SIG[A-Z0-9]+$/.test(signal??'')?signal:null;}); + child.stdout?.on('data',bytes=>{size+=bytes.length;if(size>131072){firstError??=new Error('Diagnostic output exceeds its bound');stop();}else chunks.push(bytes);}); + child.stderr?.on('data',bytes=>{stderrBytes+=bytes.length;if(stderrBytes>16384){firstError??=new Error('Diagnostic stderr exceeds its bound');stop();}}); + child.once('close',(code)=>{closed=true;finish(firstError||(code!==0||stderrBytes&&!allowStderr?new Error('Owned diagnostic command failed'):undefined));}); + if(signal?.aborted)stop();else signal?.addEventListener('abort',stop,{once:true}); + timer=setTimeout(()=>{firstError??=new Error('Diagnostic command deadline');stop();},Math.max(0,deadlineAt-Date.now())); + }); + return await pending; +} + +export async function readBoundedNativeFile(handle,size,maximum){ + assert(Number.isSafeInteger(size)&&size>=0&&size<=maximum); + const bytes=Buffer.alloc(size+1);let count=0; + while(count=0&&result.bytesRead<=bytes.length-count); + if(!result.bytesRead)break;count+=result.bytesRead;} + assert.equal(count,size,'A bounded diagnostic read refuses growth or a short read'); + return bytes.subarray(0,count); +} + +export function nativeSampleProjection(bytes){ + assert(Buffer.isBuffer(bytes)&&bytes.length<=2000000); + const categories={jit:0,loading:0,io:0,waiting:0},symbols=new Map(),unknown=createHash('sha256'); + const known={jl_compile_method_internal:'jit',jl_generate_fptr_impl:'jit',jl_emit_code:'jit',jl_add_to_ee:'jit', + jl_load:'loading',jl_parse_eval_all:'loading',jl_toplevel_eval_flex:'loading',jl_require:'loading', + read:'io',pread:'io',open:'io',stat:'io',fstat:'io',kevent:'io',uv_fs_work:'io',__read:'io',__open:'io', + pthread_cond_wait:'waiting',pthread_mutex_lock:'waiting',semaphore_wait_trap:'waiting',mach_msg_trap:'waiting', + __psynch_cvwait:'waiting',__psynch_mutexwait:'waiting'}; + let unknownLines=0,unknownWeight=0,rejectedLines=0; + for(const line of bytes.toString('utf8').split(/\r?\n/)){ + if(line.length>2048){rejectedLines++;continue;} + const match=/^\s*(?:[+!|:]\s*)*(\d{1,9})\s+([A-Za-z_][A-Za-z0-9_:<>.]{0,159})(?:\s|$)/.exec(line);if(!match)continue; + const name=match[2],weight=Number(match[1]);assert(Number.isSafeInteger(weight)&&weight<=999999999); + const category=Object.hasOwn(known,name)?known[name]:undefined; + if(category){categories[category]+=weight;symbols.set(name,(symbols.get(name)||0)+weight);} + else{unknownLines++;unknownWeight+=weight;unknown.update(name+'\n');} + } + assert([...Object.values(categories),unknownWeight].every(Number.isSafeInteger)); + return{bytes:bytes.length,sha256:discoveryHash(bytes),categories, + symbols:[...symbols].map(([name,count])=>({name,count})), + unclassified:{lines:unknownLines,weight:unknownWeight,namesSha256:unknown.digest('hex')},rejectedLines, + scope:'Observed call-graph weights, not exclusive time or a historical cause'}; +} + +export function nativeSampleManualContract(raw){ + assert(Buffer.isBuffer(raw)&&raw.length<=1048576); + const text=raw.toString('utf8').replace(/.\x08/g,''); + const supported=/\bsample\b[\s\S]{0,600}\b(?:pid|process)\b/i.test(text)&& + /(?:-mayDie\b|\.Fl[ \t]+mayDie\b)/.test(text)&&/(?:-file\b|\.Fl[ \t]+file\b)/.test(text)&& + /\bduration\b/i.test(text)&&/\binterval\b/i.test(text); + return{supported,manualSha256:discoveryHash(raw),manualBytes:raw.length, + contract:'sample PID duration interval -mayDie -file output',descriptorOutputSupport:'unverified-until-tool-return-and-file-read',rawManualRetained:false}; +} + +export async function verifyNativeSampleSyntax(){ + // Read the installed manual directly: man can start a pager or formatter. + let raw; + for(const name of ['/usr/share/man/man1/sample.1','/usr/share/man/man1/sample.1.gz']){ + let handle;try{const before=await fs.lstat(name);assert(before.isFile()&&!before.isSymbolicLink()&&before.size<=1048576); + handle=await fs.open(name,fileFlags.O_RDONLY|fileFlags.O_NOFOLLOW);const opened=await handle.stat(); + assert.equal(opened.dev,before.dev);assert.equal(opened.ino,before.ino); + raw=await readBoundedNativeFile(handle,before.size,1048576); + const after=await handle.stat(),named=await fs.lstat(name); + for(const key of ['dev','ino','size','mtimeMs']){assert.equal(after[key],before[key]);assert.equal(named[key],before[key]);} + if(name.endsWith('.gz'))raw=gunzipSync(raw,{maxOutputLength:1048576});break; + }catch(error){if(error.code!=='ENOENT')throw error;}finally{await handle?.close();} + } + if(!raw)return{supported:false,reason:'installed-manual-unavailable'}; + return nativeSampleManualContract(raw); +} + +export function createNativeDiscoveryReaders(expected,{sampleSyntax,record}={}){ + const states=[],temporary=[];let incarnation,sampled=false; + const check=(capture,options)=>{assert(!options.signal.aborted&&Date.now()['dev','ino','size','mtimeMs','nlink'].every(key=>a[key]===b[key]); + const readListing=async(capture,options)=>{ + assert(!options.signal.aborted&&Date.now(){const after=await fs.lstat(directory);assert(after.isDirectory()&&!after.isSymbolicLink()); + assert.equal(after.dev,parent.dev);assert.equal(after.ino,parent.ino);assert.equal(await fs.realpath(directory),canonicalDirectory);}; + let before;try{before=await fs.lstat(capture.output);}catch(error){if(error.code==='ENOENT'){await recheckParent();return{status:'not-observed'};}throw error;} + assert(before.isFile()&&!before.isSymbolicLink()&&before.nlink===1&&before.size<=262144); + const handle=await fs.open(capture.output,fileFlags.O_RDONLY|fileFlags.O_NOFOLLOW); + try{assert(sameFile(before,await handle.stat()));const bytes=await readBoundedNativeFile(handle,before.size,262144); + assert.equal(bytes.length,before.size);assert(sameFile(before,await handle.stat()));assert(sameFile(before,await fs.lstat(capture.output))); + await recheckParent(); + const value=JSON.parse(bytes);assert.equal(value.schema_version,'perfchecker-testitems/1');assert.equal(value.root,expected.root); + assert.equal(value.executed,false);assert(Array.isArray(value.items)&&value.items.length<=1024); + const selected=value.items.filter(item=>item.name==='Vector reduction'&&typeof item.file==='string'&&item.file.replaceAll('\\','/')==='test/performance.jl'); + assert(selected.length<=4&&selected.every(item=>typeof item.id==='string'&&item.id.length<=128)); + assert(!options.signal.aborted&&Date.now()discoveryHash(String(item.id)))}; + }finally{await handle.close();} + }; + const observe=async(capture,options)=>{ + let stage='identity-before'; + const step=async(name,run)=>{stage=name;try{return await run();}catch(error){throw Object.assign(new Error('Discovery observation is inconclusive'), + {name:error?.name==='AssertionError'?'AssertionError':'Error',code:/^[A-Z0-9_]+$/.test(error?.code??'')?error.code:undefined,diagnosticStage:name});}}; + const command=(file,args)=>discoveryOwnedCommand(options.originalSpawn,file,args,{...options,states,record,stage,targetPid:capture.pid}); + const identity=async()=>{check(capture,options);const text=(await command('/bin/ps',['-p',String(capture.pid),'-o','ppid=','-o','lstart='])).toString('utf8').trim(); + const match=/^(\d+)\s+(.+)$/.exec(text);assert(match&&Number.isFinite(Date.parse(match[2]))); + const value={pid:capture.pid,parent:Number(match[1]),started:match[2],createdAt:Date.parse(match[2])}; + assert.equal(value.parent,capture.parent);assert(value.createdAt>=capture.startedAt-1000&&value.createdAt<=capture.spawnedAt+1000); + if(incarnation)assert.deepEqual(value,incarnation);else incarnation=value;return value;}; + const mappings=async()=>{check(capture,options);const raw=(await command('/usr/sbin/lsof',['-a','-p',String(capture.pid),'-d','txt','-Fn'])).toString('utf8'); + const images=await Promise.all(raw.split('\n').filter(line=>line.startsWith('n')).map(line=>fs.realpath(line.slice(1)))); + assert(images.includes(expected.canonicalExecutable),'The original canonical Julia image must remain mapped');return images;}; + const before=await step('identity-before',identity);await step('image-before',mappings); + const fields=await step('cpu',async()=>{const fields=(await command('/bin/ps',['-p',String(capture.pid),'-o','stat=','-o','etime=','-o','time=','-o','%cpu='])).toString('utf8').trim().split(/\s+/); + assert.equal(fields.length,4);assert(/^[A-Z+/^[\d:.-]{1,32}$/.test(v))); + assert(/^\d+(?:\.\d+)?$/.test(fields[3])&&fields[3].length<=32);return fields;}); + let stack; + if(options.sample&&!sampled&&sampleSyntax?.supported){ + sampled=true;check(capture,options); + const directory=await fs.mkdtemp(path.join(os.tmpdir(),'pc-discovery-sample-')),owned={directory};temporary.push(owned); + await fs.chmod(directory,0o700); + const file=path.join(directory,'sample.txt'),handle=await fs.open(file,'wx+',0o600);Object.assign(owned,{file,handle}); + const original=await handle.stat();assert(original.isFile()&&original.nlink===1); + // The output names the inherited descriptor, not a writer path that can be replaced. + await step('sample-tool',()=>discoveryOwnedCommand(options.originalSpawn,'/usr/bin/sample',[String(capture.pid),'1','10','-mayDie','-file','/dev/fd/3'], + {...options,states,record,stage,targetPid:capture.pid,allowStderr:true,stdio:['ignore','pipe','pipe',handle.fd]})); + stack=await step('sample-read',async()=>{const current=await handle.stat(),named=await fs.lstat(file); + assert(!named.isSymbolicLink()&&named.nlink===1&¤t.nlink===1&¤t.size<=2000000); + assert.equal(current.dev,original.dev);assert.equal(current.ino,original.ino);assert.equal(named.dev,current.dev);assert.equal(named.ino,current.ino); + const bytes=await readBoundedNativeFile(handle,current.size,2000000);assert(bytes.length>0,'Descriptor output support was not observed');const projection=nativeSampleProjection(bytes); + assert(sameFile(current,await handle.stat()));assert(sameFile(current,await fs.lstat(file)));return projection;}); + } + await step('image-after',mappings);await step('identity-after',async()=>{assert.deepEqual(await identity(),before);check(capture,options);}); + return{qualified:true,pid:before.pid,parent:before.parent,createdAt:new Date(before.createdAt).toISOString(), + executable:{basename:path.basename(expected.canonicalExecutable),pathSha256:discoveryHash(expected.canonicalExecutable)}, + state:fields[0],elapsed:fields[1],cpuTime:fields[2],cpuPercent:fields[3],stack, + sampleUnavailable:options.sample&&!sampleSyntax?.supported}; + }; + return{readListing,observe,async retire(){ + const closed=states.every(state=>state.closed),errors=[]; + try{record({stage:'owned-observers-retired',physicalClose:closed,observerPids:states.map(state=>state.child.pid), + sampled,diagnosticOnly:true,perturbing:true,qualifiesCanonicalProduct:false});}catch(error){errors.push(error);} + if(!closed)errors.push(new Error('Every owned diagnostic process must physically close before raw temporary output is removed')); + else for(const value of temporary){ + try{await value.handle?.close();}catch(error){errors.push(error);continue;} + try{await fs.rm(value.directory,{recursive:true});}catch(error){errors.push(error);} + } + if(errors.length===1)throw errors[0];if(errors.length)throw new AggregateError(errors,'Discovery reader retirement failed'); + }}; +} + +export function assertNativeDiscoveryDiagnostic(value){ + if(!value.enabled)return; + assert.equal(value.event,'workflow_dispatch');assert.equal(value.artifact,'candidate'); + assert.equal(value.stage,'full');assert.equal(value.scope,'macos-stable'); + assert.equal(value.core,'candidate');assert.equal(value.partition,'measure'); + if(value.platform!==undefined)assert.equal(value.platform,'darwin');if(value.arch!==undefined)assert.equal(value.arch,'x64'); + assert.equal(value.backendJulia,'1'); + assert.equal(value.originRun,'38653410110'); + assert.equal(value.originName,'native-candidate-73b99deb87efaf45bd5e96036f755e42c748bdab'); + assert.equal(value.originSha,'bc61f9fa172069c61722e0691900d3ddf8a39ce143fd9d553b252304dbc55850'); + assert.equal(value.coreCommit,'d079efd5c24c3dcac5b7550781822038e0ff780f'); + assert.equal(value.coreTree,'749f8eac351ed2a1f0bc110456f2eb37a3bc08ee'); + assert.equal(value.newOrigin,false); +} + +// Match private spawn inputs without publishing them or inspecting process argv. +export function nativeDiscoverySpawnOutput(file,args,options,expected){ + if(file!==expected.executable||!Array.isArray(args)||args.length!==12||options?.cwd!==expected.root)return; + const prefix=['--startup-file=no',`--project=${expected.project}`,'-e',expected.expression,'--','testitems', + `--root=${expected.root}`,`--project=${expected.project}`]; + if(!prefix.every((arg,index)=>args[index]===arg)||args[9]!=='--list'||args[10]!=='--tags='||args[11]!=='--exclude-tags=')return; + if(!args[8].startsWith('--output='))return; + const output=args[8].slice('--output='.length),directory=path.dirname(output); + if(path.basename(output)!=='result.json'||path.dirname(directory)!==expected.outputRoot|| + !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(path.basename(directory)))return; + return output; +} + +export function installNativeDiscoveryObserver({childProcess,expected,observe,readListing,record, + delays=[0,60000,85000],now=Date.now}){ + const originalSpawn=childProcess.spawn,captures=[],timers=new Set(),errors=[]; + let stopped=false,active,abortController; + const emit=value=>{try{record({diagnosticOnly:true,perturbing:true,qualifiesCanonicalProduct:false, + observedAt:new Date(now()).toISOString(),...value});}catch{errors.push(new Error('Discovery diagnostic sink failed'));}}; + const fail=error=>{try{return{errorClass:/^[A-Za-z]+$/.test(error?.name??'')?error.name:null, + code:/^[A-Z0-9_]+$/.test(error?.code??'')?error.code:null, + step:['identity-before','image-before','cpu','sample-tool','sample-read','image-after','identity-after','listing-open-read','retire'].includes(error?.diagnosticStage)?error.diagnosticStage:null};}catch{return{errorClass:null,code:null,step:null};}}; + const collect=(capture,sequence,onlyListing=false)=>{ + if(stopped||active||(!onlyListing&&(capture.exited||now()>=capture.deadlineAt)))return; + const abort=abortController=new AbortController(),deadlineAt=onlyListing?now()+2000:Math.min(capture.deadlineAt,now()+5000); + if(deadlineAt<=now())return; + const timeout=setTimeout(()=>abort.abort(),deadlineAt-now()); + active=Promise.resolve().then(async()=>{ + let listing;try{listing=await readListing(capture,{signal:abort.signal,deadlineAt});} + catch(error){throw Object.assign(new Error('Listing observation is inconclusive'),{diagnosticStage:'listing-open-read', + code:/^[A-Z0-9_]+$/.test(error?.code??'')?error.code:undefined});} + if(abort.signal.aborted||stopped||now()>=deadlineAt){ + emit({stage:'listing-unknown',sequence,pid:capture.pid,reason:'observation-deadline-or-stop'});return; + } + emit({stage:'listing-observed',sequence,pid:capture.pid,listing,historical:onlyListing}); + const process=onlyListing?undefined:await observe(capture,{signal:abort.signal,deadlineAt, + sequence,sample:sequence===3&&listing.status==='not-observed',originalSpawn}); + if(!abort.signal.aborted&&!stopped&&now()=capture.startedAt+120000:undefined,listing,process}); + else emit({stage:'snapshot-unknown',sequence,pid:capture.pid,reason:'observation-deadline-or-stop'}); + }).catch(error=>emit({stage:'snapshot-unknown',sequence,pid:capture.pid,...fail(error)})) + .finally(()=>{clearTimeout(timeout);active=undefined;}); + }; + function wrapper(file,args,options){ + const startedAt=now(); + // The actual call, its synchronous error and ChildProcess identity are untouched. + const child=Reflect.apply(originalSpawn,this,arguments),spawnedAt=now(); + try{ + const output=nativeDiscoverySpawnOutput(file,args,options,expected); + if(!output||stopped)return child; + const capture={pid:child.pid,parent:process.pid,startedAt,spawnedAt,deadlineAt:startedAt+118000, + output,child,exited:child.exitCode!=null||child.signalCode!=null,closed:false};captures.push(capture); + const outcome=(code,signal)=>({code:Number.isInteger(code)?code:null,signal:/^SIG[A-Z0-9]+$/.test(signal??'')?signal:null}); + const exit=(code,signal)=>{capture.exited=true;queueMicrotask(()=>emit({stage:'product-exit',pid:capture.pid, + ...outcome(code,signal),elapsedMs:now()-startedAt}));}; + const close=(code,signal)=>{capture.closed=true;queueMicrotask(()=>{emit({stage:'product-close',pid:capture.pid, + ...outcome(code,signal),elapsedMs:now()-startedAt}); + const pending=active;void(pending?pending.then(()=>collect(capture,0,true)):Promise.resolve(collect(capture,0,true)));});}; + const error=value=>queueMicrotask(()=>emit({stage:'product-error',pid:capture.pid,...fail(value)})); + capture.remove=()=>{child.removeListener('exit',exit);child.removeListener('close',close);child.removeListener('error',error);}; + child.once('exit',exit);child.once('close',close);child.once('error',error); + queueMicrotask(()=>{ + if(stopped)return; + emit({stage:'product-spawn-returned',pid:capture.pid,parent:capture.parent, + startedAt:new Date(startedAt).toISOString(),spawnedAt:new Date(spawnedAt).toISOString(),invocation:captures.length}); + if(captures.length!==1){errors.push(new Error('Multiple diagnostic listing invocations'));return;} + for(const [index,delay]of delays.entries()){ + const timer=setTimeout(()=>{timers.delete(timer);collect(capture,index+1);},delay);timers.add(timer); + } + }); + }catch(error){errors.push(new Error('Discovery observer setup failed'));queueMicrotask(()=>emit({stage:'observer-setup-unknown',...fail(error)}));} + return child; + } + childProcess.spawn=wrapper; + return{originalSpawn,captures,async retire(){ + stopped=true;for(const timer of timers)clearTimeout(timer);timers.clear();abortController?.abort(); + if(childProcess.spawn===wrapper)childProcess.spawn=originalSpawn; + else errors.push(new Error('Another spawn hook replaced the diagnostic observer')); + let drained=!active,expiry; + try{if(active)drained=await Promise.race([active.then(()=>true),new Promise(resolve=>{expiry=setTimeout(()=>resolve(false),5000);})]);} + finally{clearTimeout(expiry);for(const capture of captures)capture.remove?.();} + emit({stage:'observer-retired',drained,invocations:captures.length, + productCloseObserved:captures.length===1&&captures[0].closed}); + if(!drained)errors.push(new Error('Discovery observer physical drain was not established')); + if(errors.length)throw new AggregateError(errors,'Discovery diagnostic retirement failed'); + }}; +} // Passive native-test evidence only. No argv, environment, source, request or // arbitrary log payload is accepted; the process reader proves incarnations. diff --git a/test/native-controller-preflight.test.mjs b/test/native-controller-preflight.test.mjs index 7a29df6..84e3d4b 100644 --- a/test/native-controller-preflight.test.mjs +++ b/test/native-controller-preflight.test.mjs @@ -1,7 +1,204 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import path from 'node:path'; -import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,nativeOperationSnapshot,observeNativeOperation,captureQualifiedNativeProgress} from './native-controller-preflight.mjs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import {EventEmitter} from 'node:events'; +import {writeSync} from 'node:fs'; +import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,nativeOperationSnapshot,observeNativeOperation,captureQualifiedNativeProgress,assertNativeDiscoveryDiagnostic,nativeDiscoverySpawnOutput,installNativeDiscoveryObserver,readBoundedNativeFile,nativeSampleProjection,nativeSampleManualContract,createNativeDiscoveryReaders,discoveryOwnedCommand} from './native-controller-preflight.mjs'; + +const nextObservationTurn=()=>new Promise(resolve=>setImmediate(resolve)); +const discoveryExpected={executable:'/private/Julia/bin/julia',project:'/private/controller',root:'/private/workspace', + expression:'immutable installed cancellation wrapper',outputRoot:'/private/profile/native-testitems'}; +const discoveryArgs=()=>['--startup-file=no','--project=/private/controller','-e',discoveryExpected.expression,'--','testitems', + '--root=/private/workspace','--project=/private/controller','--output=/private/profile/native-testitems/12345678-1234-1234-1234-123456789abc/result.json', + '--list','--tags=','--exclude-tags=']; +function observationChild(pid=123){const child=new EventEmitter();child.pid=pid;child.exitCode=null;child.signalCode=null; + child.stdout=new EventEmitter();child.stderr=new EventEmitter();child.kill=()=>true;return child;} +const diagnosticSelection=()=>({enabled:true,event:'workflow_dispatch',artifact:'candidate',stage:'full',scope:'macos-stable', + core:'candidate',partition:'measure',platform:'darwin',arch:'x64',backendJulia:'1',originRun:'38653410110', + originName:'native-candidate-73b99deb87efaf45bd5e96036f755e42c748bdab', + originSha:'bc61f9fa172069c61722e0691900d3ddf8a39ce143fd9d553b252304dbc55850', + coreCommit:'d079efd5c24c3dcac5b7550781822038e0ff780f',coreTree:'749f8eac351ed2a1f0bc110456f2eb37a3bc08ee',newOrigin:false}); + +test('discovery diagnostic refuses another host, phase, product, Core or origin and is inert when off',()=>{ + assert.doesNotThrow(()=>assertNativeDiscoveryDiagnostic({enabled:false,stage:'irrelevant'})); + assert.doesNotThrow(()=>assertNativeDiscoveryDiagnostic(diagnosticSelection())); + for(const [key,value]of Object.entries({event:'pull_request',artifact:'public',stage:'focused',scope:'all',core:'general', + partition:'agentic',platform:'linux',arch:'arm64',backendJulia:'1.10',originRun:'other',originName:'other',originSha:'other', + coreCommit:'other',coreTree:'other',newOrigin:true}))assert.throws(()=>assertNativeDiscoveryDiagnostic({...diagnosticSelection(),[key]:value}),key); +}); + +test('discovery matcher uses the exact known private invocation and never inspects a running argv',()=>{ + const args=discoveryArgs(),options={cwd:discoveryExpected.root,env:{token:'PRIVATE'}}; + assert.equal(nativeDiscoverySpawnOutput(discoveryExpected.executable,args,options,discoveryExpected),args[8].slice(9)); + for(const index of [0,1,3,6,7,8,9,10,11]){const other=[...args];other[index]+='changed'; + assert.equal(nativeDiscoverySpawnOutput(discoveryExpected.executable,other,options,discoveryExpected),undefined);} + assert.equal(nativeDiscoverySpawnOutput('/other/julia',args,options,discoveryExpected),undefined); +}); + +test('discovery hook delegates once with identical arguments, this and ChildProcess before observers run',async()=>{ + const records=[],calls=[],child=observationChild();let reads=0; + const cp={spawn:function(){calls.push({self:this,args:[...arguments]});return child;}},original=cp.spawn; + const observer=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[0],record:row=>records.push(row), + readListing:async()=>{reads++;return{status:'not-observed'};},observe:async()=>({qualified:true})}); + const args=discoveryArgs(),options={cwd:discoveryExpected.root}; + try{assert.equal(cp.spawn(discoveryExpected.executable,args,options),child);assert.equal(reads,0);assert.equal(calls.length,1); + assert.equal(calls[0].self,cp);assert.equal(calls[0].args[1],args);assert.equal(calls[0].args[2],options); + await new Promise(resolve=>setTimeout(resolve,5));assert.equal(reads,1); + child.exitCode=0;child.emit('exit',0,null);child.emit('close',0,null);await nextObservationTurn(); + }finally{await observer.retire();} + assert.equal(cp.spawn,original);assert(records.some(row=>row.stage==='product-close')); + assert(!JSON.stringify(records).includes('/private/'));assert.equal(records.at(-1).drained,true); +}); + +test('diagnostic matcher and sink failures cannot lose an already spawned product or replace its synchronous error',async()=>{ + const child=observationChild(),calls=[],cp={spawn:function(){calls.push([...arguments]);return child;}}; + const observer=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[], + record:()=>{},observe:async()=>({}),readListing:async()=>({})}); + const args=discoveryArgs();Object.defineProperty(args,8,{get(){throw new Error('PRIVATE matcher failure');}}); + assert.equal(cp.spawn(discoveryExpected.executable,args,{cwd:discoveryExpected.root}),child);assert.equal(calls.length,1); + await assert.rejects(observer.retire(),error=>error instanceof AggregateError&&!String(error).includes('PRIVATE')); + const primary=new Error('original spawn refusal'),failing={spawn(){throw primary;}}; + const other=installNativeDiscoveryObserver({childProcess:failing,expected:discoveryExpected,record:()=>{throw new Error('PRIVATE sink');}, + observe:async()=>{assert.fail('No observer after refused spawn');},readListing:async()=>({})}); + assert.throws(()=>failing.spawn('anything',[]),error=>error===primary); + await assert.rejects(other.retire()); + const short={spawn:function(){assert.equal(arguments.length,2);return child;}}; + const exact=installNativeDiscoveryObserver({childProcess:short,expected:discoveryExpected,record:()=>{},observe:async()=>({}),readListing:async()=>({})}); + assert.equal(short.spawn('other',[]),child);await exact.retire(); +}); + +test('discovery observation keeps a valid listing when process inspection fails without leaking tool messages',async()=>{ + const records=[],child=observationChild(),cp={spawn:()=>child};let started; + const entered=new Promise(resolve=>{started=resolve;}); + const observer=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[0],record:row=>records.push(row), + readListing:async()=>({status:'valid-listing',sha256:'safe'}),observe:async()=>{started();throw Object.assign(new Error('PRIVATE argv'),{diagnosticStage:'image-after',code:'EACCES'});}}); + try{cp.spawn(discoveryExpected.executable,discoveryArgs(),{cwd:discoveryExpected.root});await entered;await nextObservationTurn();} + finally{await observer.retire();} + assert(records.some(row=>row.stage==='listing-observed'&&row.listing.status==='valid-listing')); + assert(records.some(row=>row.stage==='snapshot-unknown'&&row.step==='image-after'&&row.code==='EACCES')); + assert(!JSON.stringify(records).includes('PRIVATE')); +}); + +test('discovery observer coalesces readers, aborts and physically drains them, and refuses a replaced hook',async()=>{ + const child=observationChild(),cp={spawn:()=>child},records=[];let calls=0,aborted=false,entered; + const started=new Promise(resolve=>{entered=resolve;}); + const observer=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[0,0,0],record:row=>records.push(row), + readListing:async()=>({status:'not-observed'}),observe:(capture,{signal})=>new Promise(resolve=>{calls++;entered(); + signal.addEventListener('abort',()=>{aborted=true;resolve({qualified:true});},{once:true});})}); + cp.spawn(discoveryExpected.executable,discoveryArgs(),{cwd:discoveryExpected.root});await started;await nextObservationTurn(); + assert.equal(calls,1);await observer.retire();assert(aborted);assert.equal(records.at(-1).drained,true); + const other=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[],record:()=>{},observe:async()=>({}),readListing:async()=>({})}); + const foreign=()=>child;cp.spawn=foreign;await assert.rejects(other.retire(),/retirement failed/);assert.equal(cp.spawn,foreign); +}); + +test('discovery listing delayed beyond abort and retirement is unknown and never published as active',async()=>{ + const child=observationChild(),cp={spawn:()=>child},records=[];let resolveRead,entered; + const started=new Promise(resolve=>{entered=resolve;}); + const observer=installNativeDiscoveryObserver({childProcess:cp,expected:discoveryExpected,delays:[0],record:row=>records.push(row), + readListing:()=>new Promise(resolve=>{resolveRead=resolve;entered();}),observe:async()=>assert.fail('Late listing cannot inspect a process')}); + cp.spawn(discoveryExpected.executable,discoveryArgs(),{cwd:discoveryExpected.root});await started; + const retired=observer.retire();resolveRead({status:'valid-listing',sha256:'late'});await retired; + assert(!records.some(row=>row.stage==='listing-observed')); + assert(records.some(row=>row.stage==='listing-unknown'));assert.equal(records.at(-1).drained,true); +}); + +test('bounded reads refuse growth and short data and accumulate partial physical reads exactly',async()=>{ + const fake=bytes=>({async read(out,offset,length,position){const n=Math.min(2,length,Math.max(0,bytes.length-position));bytes.copy(out,offset,position,position+n);return{bytesRead:n};}}); + assert.equal(String(await readBoundedNativeFile(fake(Buffer.from('abc')),3,4)),'abc'); + await assert.rejects(readBoundedNativeFile(fake(Buffer.from('ab')),3,4),/short read/); + await assert.rejects(readBoundedNativeFile(fake(Buffer.from('abcd')),3,4),/growth/); + await assert.rejects(readBoundedNativeFile(fake(Buffer.from('a')),5,4)); +}); + +test('listing reader records current schema and hashed target and refuses links and stale live identity',async()=>{ + const root=await fs.mkdtemp(path.join(os.tmpdir(),'pc-discovery-test-')); + const output=path.join(root,'result.json'),records=[]; + const readers=createNativeDiscoveryReaders({...discoveryExpected,root,canonicalExecutable:process.execPath},{record:row=>records.push(row)}); + const options={signal:new AbortController().signal,deadlineAt:Date.now()+1000}; + const capture={output,pid:123,parent:process.pid,exited:false,closed:false}; + try{assert.deepEqual(await readers.readListing(capture,options),{status:'not-observed'}); + await fs.writeFile(output,JSON.stringify({schema_version:'perfchecker-testitems/1',root,executed:false, + items:[{id:'PRIVATE target',name:'Vector reduction',file:'test/performance.jl'}]})); + const value=await readers.readListing(capture,options);assert.equal(value.targetMatches,1);assert.equal(value.targetIds[0].length,64); + assert(!JSON.stringify(value).includes('PRIVATE'));await fs.unlink(output);await fs.symlink('missing',output); + await assert.rejects(readers.readListing(capture,options)); + await assert.rejects(readers.observe({...capture,exited:true},{...options,originalSpawn:()=>assert.fail('Exited product cannot start an observer')})); + }finally{await readers.retire();await fs.rm(root,{recursive:true,force:true});} +}); + +test('sample projection accepts only finite known symbols and hashes unknown or overlong names',()=>{ + const value=nativeSampleProjection(Buffer.from(' 12 jl_compile_method_internal (in Julia)\n 3 read\n 9 llvm::PRIVATE_TOKEN\n 8 jl_compilePRIVATE_TOKEN\n'+ + ' 999999999999999999999999 jl_load\n 1 '+ 'jl_compile'+ 'X'.repeat(3000))); + assert.equal(value.categories.jit,12);assert.equal(value.categories.io,3);assert.equal(value.unclassified.weight,17); + assert.equal(value.unclassified.lines,2);assert.equal(value.rejectedLines,1); + assert(!JSON.stringify(value).includes('PRIVATE'));assert(Object.values(value.categories).every(Number.isSafeInteger)); + assert.throws(()=>nativeSampleProjection(Buffer.alloc(2000001))); +}); + +test('sample temporary directories retire after chmod or open fails without launching a sample writer',async()=>{ + const realChmod=fs.chmod,realOpen=fs.open,realMkdtemp=fs.mkdtemp; + for(const failingMethod of ['chmod','open']){ + const records=[],directoryNames=[],toolNames=[],created=Date.now(),image=await fs.realpath(process.execPath); + const expected={...discoveryExpected,canonicalExecutable:image}; + const readers=createNativeDiscoveryReaders(expected,{sampleSyntax:{supported:true},record:row=>records.push(row)}); + const fakeSpawn=(file,args)=>{toolNames.push(path.basename(file));const child=observationChild(456); + queueMicrotask(()=>{const text=file.endsWith('lsof')?'p123\nn'+image+'\n':args.includes('lstart=')? + process.pid+' '+new Date(created).toString()+'\n':'R 00:01 0:00.01 1.0\n'; + child.stdout.emit('data',Buffer.from(text));child.emit('exit',0,null);child.emit('close',0,null);});return child;}; + const primary=new Error('controlled setup failure'); + fs.mkdtemp=async(...args)=>{const directory=await realMkdtemp(...args);directoryNames.push(directory);return directory;}; + fs.chmod=failingMethod==='chmod'?async()=>{throw primary;}:realChmod; + fs.open=failingMethod==='open'?async()=>{throw primary;}:realOpen; + try{await assert.rejects(readers.observe({pid:123,parent:process.pid,startedAt:created,spawnedAt:created,exited:false,closed:false}, + {signal:new AbortController().signal,deadlineAt:Date.now()+1000,originalSpawn:fakeSpawn,sample:true}),error=>error===primary); + }finally{fs.chmod=realChmod;fs.open=realOpen;fs.mkdtemp=realMkdtemp;await readers.retire();} + assert.equal(directoryNames.length,1);assert(!toolNames.includes('sample')); + await assert.rejects(fs.lstat(directoryNames[0]),error=>error.code==='ENOENT');assert.equal(records.at(-1).physicalClose,true); + } +}); + +test('reader retirement preserves a sink error after closing and removing safe sample output',async()=>{ + const primary=new Error('original diagnostic sink failure'),created=Date.now(),image=await fs.realpath(process.execPath); + const realMkdtemp=fs.mkdtemp,directories=[]; + const readers=createNativeDiscoveryReaders({...discoveryExpected,canonicalExecutable:image},{sampleSyntax:{supported:true}, + record:row=>{if(row.stage==='owned-observers-retired')throw primary;}}); + const fakeSpawn=(file,args,options)=>{const child=observationChild(456); + queueMicrotask(()=>{ + if(file.endsWith('sample')){assert.equal(args.at(-1),'/dev/fd/3');writeSync(options.stdio[3],' 4 jl_compile_method_internal\n');} + else{const text=file.endsWith('lsof')?'p123\nn'+image+'\n':args.includes('lstart=')? + process.pid+' '+new Date(created).toString()+'\n':'R 00:01 0:00.01 1.0\n';child.stdout.emit('data',Buffer.from(text));} + child.emit('exit',0,null);child.emit('close',0,null); + });return child;}; + fs.mkdtemp=async(...args)=>{const directory=await realMkdtemp(...args);directories.push(directory);return directory;}; + try{ + const result=await readers.observe({pid:123,parent:process.pid,startedAt:created,spawnedAt:created,exited:false,closed:false}, + {signal:new AbortController().signal,deadlineAt:Date.now()+1000,originalSpawn:fakeSpawn,sample:true}); + assert.equal(result.stack.categories.jit,4); + await assert.rejects(readers.retire(),error=>error===primary); + assert.equal(directories.length,1);await assert.rejects(fs.lstat(directories[0]),error=>error.code==='ENOENT'); + }finally{fs.mkdtemp=realMkdtemp;for(const directory of directories)await fs.rm(directory,{recursive:true,force:true});} +}); + +test('sample command syntax is supported only by explicit installed man or mdoc options',()=>{ + assert(nativeSampleManualContract(Buffer.from('sample pid duration interval [-mayDie] [-file output]')).supported); + assert(nativeSampleManualContract(Buffer.from('.Dt SAMPLE 1\n.Nm sample\n.Ar pid\n.Ar duration\n.Ar interval\n.Fl mayDie\n.Fl file')).supported); + for(const text of ['sample pid duration interval mayDie file','sample pid duration interval -mayDied -filename', + 'sample pid -file output -mayDie'])assert.equal(nativeSampleManualContract(Buffer.from(text)).supported,false); +}); + +test('owned inspection stops once and preserves the first error until real close with safe tool metadata',async()=>{ + const child=observationChild(),controller=new AbortController(),records=[],primary=new Error('PRIVATE original command error');let kills=0; + child.kill=()=>{kills++;return true;}; + const pending=discoveryOwnedCommand(()=>child,'/bin/ps',[],{signal:controller.signal,deadlineAt:Date.now()+1000,states:[], + record:row=>records.push(row),stage:'cpu',targetPid:456,sequence:1}); + child.emit('error',primary);child.stdout.emit('data',Buffer.alloc(131073));controller.abort();child.stderr.emit('data',Buffer.alloc(16385)); + assert.equal(kills,1);let finished=false;void pending.catch(()=>{finished=true;});await nextObservationTurn();assert.equal(finished,false); + child.emit('exit',null,'SIGTERM');child.emit('close',null,'SIGTERM');await assert.rejects(pending,error=>error===primary); + assert.equal(kills,1);assert.equal(records.length,1);assert.equal(records[0].physicalClose,true); + assert.equal(records[0].tool,'ps');assert.equal(records[0].stdoutBytes,131073);assert(!JSON.stringify(records).includes('PRIVATE')); +}); const observationDelay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); const operationSnapshot=()=>({hostPid:10,expectedExecutable:process.execPath, diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs index f0b6ad9..5d9e4e2 100644 --- a/test/native-platform-contract.test.mjs +++ b/test/native-platform-contract.test.mjs @@ -63,7 +63,7 @@ test('the real host dispatches every advertised shared-prefix case before its pa assert.deepEqual(calls,['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter']); const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); assert(suiteStart>=0&&suiteEnd>suiteStart); - const suite=Function(`return (${source.slice(suiteStart,suiteEnd).trim()})`)(); + const suite=Function('discoveryDiagnostic',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false); await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture'},runCase,{testItem:true,complete:true}); const historicalPrefix=['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter', 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', @@ -74,6 +74,28 @@ test('the real host dispatches every advertised shared-prefix case before its pa 'Every full prepared partition still dispatches both the native test item and the complete suite'); }); +test('the real diagnostic host keeps the seventh prefix and explicitly excludes only the following cases',async()=>{ + const source=readFileSync(new URL('./public-vsix-host.cjs',import.meta.url),'utf8'); + const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); + assert(suiteStart>=0&&suiteEnd>suiteStart); + const suite=Function('discoveryDiagnostic','process',`return (${source.slice(suiteStart,suiteEnd).trim()})`) + (true,{env:{PERFCHECKER_NATIVE_PHASE:'prepared'}}); + const calls=[],logs=[]; + await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture',log:(name,detail)=>logs.push({name,...detail})}, + async name=>{calls.push(name);return true;},{testItem:true,complete:true}); + assert.deepEqual(calls,['native-suite-run-button','native-suite-save-palette','native-run-selection-command','native-testitem-measurement']); + assert.equal(logs.length,1);assert.equal(logs[0].name,'native-discovery-diagnostic-skipped-after-target'); + assert.deepEqual(logs[0].cases,['native-complete-suite-measurements','native-all-studio-controls','native-workbench-controls']); + assert.equal(logs[0].qualifiesCanonicalProduct,false); + const tailStart=source.indexOf("if (process.env.PERFCHECKER_NATIVE_STAGE === 'full'&&!discoveryDiagnostic) {"); + assert(tailStart>=0);let end=source.indexOf('{',tailStart),depth=1; + while(depth&&++end{ const campaign={partition:'measure',platform:'darwin',arch:'x64',stage:'full',artifact:'candidate'}; for(const change of [{partition:'unknown'},{platform:'linux'},{platform:'win32'}, @@ -100,4 +122,6 @@ test('the actual workflow partitions only complete candidate Intel jobs and keep assert.equal(manual('arm-bootstrap','focused').length,4); assert(manual('arm-bootstrap','focused').every(lane=>lane.arch==='arm64'&&!lane.partition)); assert.deepEqual(manual('windows-stable'),[{os:'windows-latest',vscode:'stable'}]); + assert.deepEqual(matrix('workflow_dispatch',{scope:'macos-stable',stage:'full',artifact:'candidate',discovery_diagnostic:true}), + [{os:'macos-15-intel',vscode:'stable',partition:'measure'}]); }); diff --git a/test/public-vsix-host.cjs b/test/public-vsix-host.cjs index 75cdadf..ae8b133 100644 --- a/test/public-vsix-host.cjs +++ b/test/public-vsix-host.cjs @@ -8,6 +8,7 @@ const {execFile}=require('node:child_process'); const {promisify}=require('node:util'); const execute=promisify(execFile); const processObservationSignatures=new WeakMap(); +const discoveryDiagnostic=process.env.PERFCHECKER_NATIVE_DISCOVERY_DIAGNOSTIC==='1'; const {chromium} = require('playwright'); const controls = require('./native-studio-controls.cjs'); const mcp = require('./native-mcp-controls.cjs'); @@ -410,6 +411,10 @@ async function runSuiteCases(context,runCase,options={}){ assert.equal(report.runs.length,1);assert.equal(report.runs[0].status,'pass'); }); if(options.testItem)await runCase('native-testitem-measurement',()=>measureNativeTestItem(context,true)); + if(discoveryDiagnostic&&process.env.PERFCHECKER_NATIVE_PHASE==='prepared'){ + context.log('native-discovery-diagnostic-skipped-after-target',{cases:['native-complete-suite-measurements', + 'native-all-studio-controls','native-workbench-controls'],diagnosticOnly:true,perturbing:true,qualifiesCanonicalProduct:false});return; + } if(options.complete){ await runCase('native-complete-suite-measurements',async()=>{ if(options.blockDependents&&!selected)throw Object.assign(new Error('Blocked by native-suite-run-button; no dependent action was executed'),{blockedBy:'native-suite-run-button'}); @@ -426,18 +431,46 @@ async function runSuiteCases(context,runCase,options={}){ } async function measureNativeTestItem(context,expectedPassed,options={}){ + if(!discoveryDiagnostic||process.env.PERFCHECKER_NATIVE_PHASE!=='prepared')return measureNativeTestItemBody(context,expectedPassed,options); + const {installNativeDiscoveryObserver,createNativeDiscoveryReaders}=await import('./native-controller-preflight.mjs'); + const childProcess=require('node:child_process'),settings=context.vscode.workspace.getConfiguration('perfchecker',context.vscode.Uri.file(context.workspace)); + const {cancellableJulia}=require(path.join(context.extension.extensionPath,'dist','controllerCancellation.js')); + const expected={root:context.workspace,project:context.controller,executable:settings.get('juliaExecutable','julia'), + canonicalExecutable:await fs.realpath(process.env.PERFCHECKER_NATIVE_JULIA), + expression:cancellableJulia('using PerfChecker, TestItemRunner; exit(perfchecker_main(ARGS))'), + outputRoot:path.join(process.env.PERFCHECKER_NATIVE_PROFILE,'User','globalStorage','mirage-interactive-fr.perfchecker-vscode','native-testitems')}; + assert.equal(settings.get('runnerProject'),context.controller);assert.deepEqual(settings.get('testItemTags',[]),[]); + assert.deepEqual(settings.get('testItemExcludeTags',[]),[]); + const record=detail=>context.log('native-discovery-observation',detail); + const readers=createNativeDiscoveryReaders(expected,{sampleSyntax:context.discoveryObservation.sampleSyntax,record}); + const observer=installNativeDiscoveryObserver({childProcess,expected,observe:readers.observe,readListing:readers.readListing,record}); + let result,original;const cleanup=[]; + try{result=await measureNativeTestItemBody(context,expectedPassed,options);}catch(error){original=error;} + finally{ + try{await observer.retire();}catch(error){cleanup.push(error);} + try{await readers.retire();}catch(error){cleanup.push(error);} + } + if(cleanup.length)throw new AggregateError(original?[original,...cleanup]:cleanup,'Discovery observation or its physical retirement failed'); + if(original)throw original;return result; +} + +async function measureNativeTestItemBody(context,expectedPassed,options={}){ const {vscode,windowPage,workspace}=context; await context.observeWork?.('testitems-before-discovery'); await controls.clickStudioAction(context,'items'); await controls.clickStudioAction(context,'testing'); const filename=options.file||'test/performance.jl',name=options.name||'Vector reduction'; const row=windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).first(); + if(discoveryDiagnostic&&process.env.PERFCHECKER_NATIVE_PHASE==='prepared')context.log('native-discovery-ui-oracle-start', + {uiOracleMs:120000,diagnosticOnly:true,qualifiesCanonicalProduct:false}); try{await row.waitFor({state:'visible',timeout:120000});} catch(error){ try{await context.observeWork?.('testitems-discovery-not-ready');} catch(diagnosticError){context.log('native-work-observation-error',{case:'testitems-discovery-not-ready',error:redact(diagnosticError)});} throw error; } + if(discoveryDiagnostic&&process.env.PERFCHECKER_NATIVE_PHASE==='prepared')context.log('native-discovery-ui-row-observed', + {surface:'visible-Testing-row',testControllerPublication:'unknown',diagnosticOnly:true,qualifiesCanonicalProduct:false}); await row.hover(); if(context.editorQualification)context.log('native-editor-visible-testing-item',{name,file:filename,matchingRows:await windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).count(), visible:await row.isVisible(),ariaLabel:await row.getAttribute('aria-label'),runButtons:await row.locator('.action-label[title="Run Test"],.action-label[aria-label="Run Test"],.action-label.codicon-testing-run-icon').count()}); @@ -501,6 +534,17 @@ exports.run = async () => { const {nativeFullCasePartition}=await import('./native-platform-contract.mjs'); const fullCasePartition=nativeFullCasePartition({partition:process.env.PERFCHECKER_NATIVE_FULL_PARTITION||'all', stage:process.env.PERFCHECKER_NATIVE_STAGE,artifact:process.env.PERFCHECKER_NATIVE_MODE}); + let discoveryObservation; + if(discoveryDiagnostic){ + discoveryObservation=JSON.parse(process.env.PERFCHECKER_NATIVE_DISCOVERY_OBSERVATION); + const {assertNativeDiscoveryDiagnostic}=await import('./native-controller-preflight.mjs'); + const core=JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE); + assertNativeDiscoveryDiagnostic({enabled:true,event:process.env.GITHUB_EVENT_NAME,artifact:process.env.PERFCHECKER_NATIVE_MODE, + stage:process.env.PERFCHECKER_NATIVE_STAGE,scope:process.env.PERFCHECKER_NATIVE_SCOPE,core:core.mode,partition:fullCasePartition.name, + platform:process.platform,arch:process.arch,backendJulia:process.env.PERFCHECKER_NATIVE_BACKEND_JULIA, + originRun:discoveryObservation.origin.run,originName:`native-candidate-${discoveryObservation.origin.commit}`, + originSha:process.env.PERFCHECKER_NATIVE_SHA,coreCommit:core.commit,coreTree:core.tree,newOrigin:false}); + } const minimumVersion=process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_VERSION; assert(/^\d+\.\d+\.\d+$/.test(minimumVersion),'The runner declares the production registry minimum explicitly'); const workspace = process.env.PERFCHECKER_NATIVE_WORKSPACE; @@ -518,7 +562,8 @@ exports.run = async () => { {videoOffsetSeconds:(Date.now()-Date.parse(process.env.PERFCHECKER_NATIVE_VIDEO_STARTED_AT))/1000}:{})}); const persist=(status='running')=>{ const report=JSON.stringify({status,phase,platform:process.platform,processArch:process.arch,vscode:vscode.version,extension:process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION,invocation:process.env.PERFCHECKER_NATIVE_INVOCATION, - coverage:process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?(fullCasePartition.name==='all'?'first-install-studio-investigations-mcp-pluto':`full-native-${fullCasePartition.name}`):'first-install-and-first-run-smoke', + coverage:discoveryDiagnostic?'nonqualifying-full-prelude-testitem-discovery':process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?(fullCasePartition.name==='all'?'first-install-studio-investigations-mcp-pluto':`full-native-${fullCasePartition.name}`):'first-install-and-first-run-smoke', + ...(discoveryDiagnostic?{discoveryObservation}:{}), core:phase==='fresh'?{mode:'production-first-install',registry:'General',version:minimumVersion,available:process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE==='true'}:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE), fullCasePartition:process.env.PERFCHECKER_NATIVE_STAGE==='full'?fullCasePartition:undefined, commands:commandCoverage(commands,checks),configuration:configurationCoverage(configurationProperties,checks),buttons:buttonCoverage(checks),activeCase,checks,failures},null,2); @@ -651,6 +696,7 @@ exports.run = async () => { }else{ const uri = vscode.Uri.file(workspace); const context = {vscode, browser, windowPage, workspace, + ...(discoveryDiagnostic?{extension,discoveryObservation}:{}), controller: process.env.PERFCHECKER_NATIVE_CONTROLLER, target: process.env.PERFCHECKER_NATIVE_TARGET, results: path.join(workspace, 'perf', 'results', 'vscode'), log, proof, findFrame, core:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE),coreVersion:process.env.PERFCHECKER_NATIVE_CORE_VERSION,flushReport:()=>persist()}; @@ -878,7 +924,7 @@ exports.run = async () => { await vscode.commands.executeCommand('workbench.action.revertAndCloseActiveEditor'); }); await runSuiteCases(context,runCase,{testItem:true,complete:process.env.PERFCHECKER_NATIVE_STAGE==='full'}); - if (process.env.PERFCHECKER_NATIVE_STAGE === 'full') { + if (process.env.PERFCHECKER_NATIVE_STAGE === 'full'&&!discoveryDiagnostic) { const callbacks={ 'native-all-studio-controls':()=>controls.run(context), 'native-workbench-controls':()=>workbench.run(context), diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index 8f1b5fe..83ae24a 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -13,7 +13,7 @@ import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; import {nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; import {nativeVSCodeApplication} from './native-vscode-application.mjs'; import {createCoalescingObserver,beforeDeadline} from './native-landscape-controls.cjs'; -import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved} from './native-controller-preflight.mjs'; +import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,assertNativeDiscoveryDiagnostic,verifyNativeSampleSyntax} from './native-controller-preflight.mjs'; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const output = path.join(repository, 'native-qualification-results'); @@ -53,6 +53,7 @@ async function assertCompanionRoute(info,name,revision){ } const version = process.env.PERFCHECKER_VSCODE_VERSION || 'stable'; const stage=process.env.PERFCHECKER_NATIVE_STAGE||'smoke'; +const discoveryDiagnostic=process.env.PERFCHECKER_NATIVE_DISCOVERY_DIAGNOSTIC==='1'; const fullCasePartition=nativeFullCasePartition({partition:process.env.PERFCHECKER_NATIVE_FULL_PARTITION||'all',stage,artifact:mode}); if(!['smoke','full','targeted','focused','core-external'].includes(stage))throw new Error('Choose smoke, full, targeted lifecycle/protocol, focused native controls, or the explicit Core-only external-process regression.'); const caseGroup=process.env.PERFCHECKER_NATIVE_CASE_GROUP||'narrative'; @@ -590,6 +591,23 @@ try { if(packageProvenance.sha256!==sha||packageProvenance.version!==expectedVersion)throw new Error('The candidate package provenance differs from the installed archive.'); await fs.writeFile(path.join(output,'candidate.provenance.json'),JSON.stringify(packageProvenance,null,2)); } + let discoveryObservation; + if(discoveryDiagnostic){ + assertNativeDiscoveryDiagnostic({enabled:true,event:process.env.GITHUB_EVENT_NAME,artifact:mode,stage, + scope:process.env.PERFCHECKER_NATIVE_SCOPE,core:coreMode,partition:fullCasePartition.name,platform:process.platform,arch:process.arch, + backendJulia:process.env.PERFCHECKER_NATIVE_BACKEND_JULIA,originRun:packageProvenance?.run, + originName:`native-candidate-${packageProvenance?.commit}`,originSha:sha,coreCommit,coreTree,newOrigin:false}); + assert.equal(packageProvenance.commit,'73b99deb87efaf45bd5e96036f755e42c748bdab'); + assert.equal(packageProvenance.tree,'3722727a870402497078e63da78be90d606f5ed5'); + assert.equal(packageProvenance.packagedOnceOn,'Linux');assert.equal(packageProvenance.version,'1.0.3'); + let sampleSyntax;try{sampleSyntax=await verifyNativeSampleSyntax();} + catch(error){sampleSyntax={supported:false,reason:'installed-manual-inspection-unknown', + errorClass:/^[A-Za-z]+$/.test(error.name)?error.name:null,code:/^[A-Z0-9_]+$/.test(error.code||'')?error.code:null};} + discoveryObservation={diagnosticOnly:true,perturbing:true,qualifiesCanonicalProduct:false,sampleSyntax, + prelude:'Exact full fresh/configured preparation, then prepared prefix through native-testitem-measurement', + skippedAfter:['native-complete-suite-measurements','native-all-studio-controls','native-workbench-controls'], + origin:packageProvenance,uiOracleMs:120000,workerOracleMs:180000}; + } const privateProfile=async root=>{ const directories=privateDirectoryFlags.map(flag=>[flag,path.join(root,flag)]); await Promise.all(directories.map(([,directory])=>fs.mkdir(directory,{recursive:true}))); @@ -598,6 +616,7 @@ try { const cliProfile = await privateProfile(profile); await execute(cli, [...cliArgs, ...cliProfile, '--install-extension', vsix, '--force'], {shell: process.platform === 'win32' && cli.endsWith('.cmd')}); const artifactRecord={mode,retainedVsix,sha256: sha,vscodeRequested: version,core:coreProvenance,packageProvenance, + ...(discoveryDiagnostic?{discoveryObservation}:{}), fullCasePartition:stage==='full'?fullCasePartition:undefined, runner:{platform:process.platform,processArch:process.arch,runnerArch:process.env.RUNNER_ARCH||null},runtimes:{perfchecker:runtime,officialJulia:officialRuntime},hostPreferences:{scope:'disposable-application-profile',dialogStyle:'custom',dialogAPI:'real-VS-Code-no-interception',actualVersion:hostVersion, actualCommit:application.commit,applicationName:application.applicationName,applicationRelativePath:path.relative(path.dirname(vscode),application.application), @@ -667,6 +686,8 @@ try { assert(coreProvenance.diagnosticWorker,'The qualified Core preparation supplies its exact diagnostic worker'); } const environment={...runnerEnvironment,...externalBrowser?.environment,...(phase==='landscape'?landscapeFixture.environment:{}), + ...(discoveryDiagnostic?{PERFCHECKER_NATIVE_DISCOVERY_DIAGNOSTIC:'1', + PERFCHECKER_NATIVE_DISCOVERY_OBSERVATION:JSON.stringify(discoveryObservation)}:{}), ...(diagnosticTemporaryRoot?{TMPDIR:diagnosticTemporaryRoot,PERFCHECKER_NATIVE_DIAGNOSTIC_TEMP:diagnosticTemporaryRoot}:{}), PERFCHECKER_NATIVE_PHASE: phase, PERFCHECKER_NATIVE_INVOCATION:randomUUID(),PERFCHECKER_NATIVE_OUTPUT: output, PERFCHECKER_NATIVE_PROFILE: phaseProfile, PERFCHECKER_NATIVE_SESSION:session,PERFCHECKER_NATIVE_WORKSPACE: workspace, PERFCHECKER_NATIVE_CONTROLLER: controller, From aa362965136496e8bcba28056ece322b3a6d8f6d Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 08:10:11 +0200 Subject: [PATCH 16/22] Split full native qualification by named functional group --- .github/workflows/native-vsix.yml | 19 +++- test/native-platform-contract.mjs | 66 ++++++++++-- test/native-platform-contract.test.mjs | 142 ++++++++++++++++++++----- 3 files changed, 190 insertions(+), 37 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 34e51c6..d38c3bd 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -150,6 +150,7 @@ jobs: runs-on: ubuntu-latest outputs: sha: ${{ steps.digest.outputs.sha }} + matrix: ${{ steps.matrix.outputs.matrix }} env: NATIVE_EVENT: ${{ github.event_name }} NATIVE_DISCOVERY_DIAGNOSTIC: ${{ inputs.discovery_diagnostic || 'false' }} @@ -167,6 +168,18 @@ jobs: REUSE_SHA: ${{ inputs.candidate_sha }} steps: - uses: actions/checkout@v7 + - name: Select native functions and supported host configurations + id: matrix + run: | + node --input-type=module <<'NODE' + import fs from 'node:fs'; + import {nativeWorkflowMatrix} from './test/native-platform-contract.mjs'; + const e=process.env; + const matrix=nativeWorkflowMatrix({event:e.NATIVE_EVENT,scope:e.NATIVE_SCOPE||'all', + artifact:e.NATIVE_VSIX_MODE,stage:e.NATIVE_STAGE||'full',group:e.NATIVE_CASE_GROUP||'narrative', + core:e.NATIVE_CORE||'candidate',discoveryDiagnostic:e.NATIVE_DISCOVERY_DIAGNOSTIC==='true'}); + fs.appendFileSync(process.env.GITHUB_OUTPUT,'matrix='+JSON.stringify(matrix)+'\n'); + NODE - name: Select the declared archive origin for qualification run: | node --input-type=module <<'NODE' @@ -256,11 +269,11 @@ jobs: native-workflows: if: github.event_name != 'workflow_dispatch' || inputs.stage != 'connector-workers' needs: package - name: Native VSIX (${{ matrix.os }}, VS Code ${{ matrix.vscode }}${{ matrix.julia && format(', Julia {0}', matrix.julia) || '' }}${{ matrix.partition && format(', {0}', matrix.partition) || '' }}${{ inputs.discovery_diagnostic && ', discovery diagnostic' || '' }}) + name: ${{ matrix.name }} strategy: fail-fast: false matrix: - include: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.discovery_diagnostic && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'arm-bootstrap' && '[{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"ubuntu-24.04-arm","vscode":"stable","arch":"arm64","julia":"1"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1.10"},{"os":"macos-15","vscode":"stable","arch":"arm64","julia":"1"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'macos-stable' && (inputs.artifact == 'candidate' && inputs.stage == 'full' && '[{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"macos-15-intel","vscode":"stable"}]') || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows-stable' && '[{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'windows' && '[{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux-stable' && '[{"os":"ubuntu-latest","vscode":"stable"}]' || github.event_name == 'workflow_dispatch' && inputs.scope == 'linux' && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"}]' || ((github.event_name != 'workflow_dispatch' || inputs.artifact == 'candidate' && inputs.stage == 'full') && '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable","partition":"measure"},{"os":"macos-15-intel","vscode":"stable","partition":"investigate"},{"os":"macos-15-intel","vscode":"stable","partition":"agentic"}]' || '[{"os":"ubuntu-latest","vscode":"1.96.0"},{"os":"ubuntu-latest","vscode":"stable"},{"os":"windows-latest","vscode":"1.96.0"},{"os":"windows-latest","vscode":"stable"},{"os":"macos-15-intel","vscode":"stable"}]')) }} + include: ${{ fromJSON(needs.package.outputs.matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }} env: @@ -349,5 +362,5 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: native-workflows-${{ matrix.os }}-${{ matrix.vscode }}${{ matrix.julia && format('-julia-{0}', matrix.julia) || '' }}${{ matrix.partition && format('-{0}', matrix.partition) || '' }} + name: ${{ matrix.artifactName }} path: native-qualification-results/ diff --git a/test/native-platform-contract.mjs b/test/native-platform-contract.mjs index 07a0006..08c66b2 100644 --- a/test/native-platform-contract.mjs +++ b/test/native-platform-contract.mjs @@ -25,26 +25,76 @@ export function assertNativeJuliaArchitecture(runtime,contract,label) { assert.equal(runtime.arch,contract.julia,`${label} must use the native Julia architecture`); } -// Every partition retains fresh/configured and the complete prepared suite -// prefix. Only this ordered tail is divided between isolated Intel Mac jobs. -const fullCasePartitions=Object.freeze({ +// Historical Intel partitions remain available for reproduction and the +// explicit discovery diagnostic; they are not part of the new group union. +const legacyFullCasePartitions=Object.freeze({ measure:Object.freeze(['native-all-studio-controls','native-workbench-controls']), investigate:Object.freeze(['native-investigation-controls','native-investigation-limits']), agentic:Object.freeze(['native-enabled-narrative-protocol','native-advisor-controls', 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']) }); +export const nativeFullFunctionalGroups=Object.freeze([ + {id:'studio',label:'Studio & suites',cases:['native-all-studio-controls']}, + {id:'workbench',label:'Julia workbench',cases:['native-workbench-controls']}, + {id:'investigation',label:'Investigations & reports',cases:['native-investigation-controls']}, + {id:'investigation-limits',label:'Budgets & deadlines',cases:['native-investigation-limits']}, + {id:'narrative',label:'Narrative & HTTP lifecycle',cases:['native-enabled-narrative-protocol']}, + {id:'advisor',label:'Providers & model setup',cases:['native-advisor-controls']}, + {id:'mcp-pluto',label:'MCP + Pluto + discovery',cases:['native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']} +].map(group=>Object.freeze({...group,cases:Object.freeze(group.cases)}))); + +const nativeHostConfigurations=Object.freeze([ + {os:'ubuntu-latest',vscode:'1.96.0'},{os:'ubuntu-latest',vscode:'stable'}, + {os:'windows-latest',vscode:'1.96.0'},{os:'windows-latest',vscode:'stable'}, + {os:'macos-15-intel',vscode:'stable'} +].map(Object.freeze)); + +export function nativeWorkflowMatrix({event,scope='all',artifact='candidate',stage='full',group='narrative',core='general', + discoveryDiagnostic=false}={}){ + assert(['pull_request','workflow_dispatch'].includes(event),'Unknown native workflow event'); + assert(['all','linux','linux-stable','windows','windows-stable','macos-stable','arm-bootstrap'].includes(scope),'Unknown native scope'); + assert(['candidate','public'].includes(artifact),'Unknown native archive mode'); + assert(['smoke','full','targeted','focused','core-external','connector-workers'].includes(stage),'Unknown native stage'); + const named=(lane,label)=>{const platform=lane.os.startsWith('ubuntu')?`Linux ${lane.arch==='arm64'?'ARM64':'x64'}`: + lane.os.startsWith('windows')?'Windows x64':lane.os==='macos-15'?'Apple Silicon':'macOS Intel'; + return{...lane,name:`Native — ${platform} — VS Code ${lane.vscode}${lane.julia?` — Julia ${lane.julia==='1'?'stable':lane.julia}`:''} — ${label}`, + artifactName:`native-workflows-${lane.os}-${lane.vscode}${lane.julia?`-julia-${lane.julia}`:''}${lane.partition?`-${lane.partition}`:''}`};}; + if(discoveryDiagnostic){ + assert.equal(event,'workflow_dispatch');assert.equal(scope,'macos-stable');assert.equal(artifact,'candidate'); + assert.equal(stage,'full');assert.equal(core,'candidate'); + return[{...named({os:'macos-15-intel',vscode:'stable',partition:'measure'},'TestItem discovery'), + name:'Diagnostic — macOS Intel — VS Code stable — TestItem discovery — non-qualifying'}]; + } + assertNativeCampaignSelection({scope,event,artifact,stage,group,core}); + if(scope==='arm-bootstrap')return[ + {os:'ubuntu-24.04-arm',vscode:'stable',arch:'arm64',julia:'1.10'}, + {os:'ubuntu-24.04-arm',vscode:'stable',arch:'arm64',julia:'1'}, + {os:'macos-15',vscode:'stable',arch:'arm64',julia:'1.10'}, + {os:'macos-15',vscode:'stable',arch:'arm64',julia:'1'} + ].map(lane=>named(lane,'Fresh bootstrap')); + const selected=event==='pull_request'?nativeHostConfigurations:nativeHostConfigurations.filter(lane=> + scope==='all'||scope==='linux'&&lane.os.startsWith('ubuntu')||scope==='windows'&&lane.os.startsWith('windows')|| + scope==='linux-stable'&&lane.os==='ubuntu-latest'&&lane.vscode==='stable'|| + scope==='windows-stable'&&lane.os==='windows-latest'&&lane.vscode==='stable'||scope==='macos-stable'&&lane.os==='macos-15-intel'); + if(event==='pull_request'||artifact==='candidate'&&stage==='full') + return selected.flatMap(lane=>nativeFullFunctionalGroups.map(group=>named({...lane,partition:group.id},group.label))); + return selected.map(lane=>named(lane,stage==='focused'?`Focused ${group}`:stage)); +} + export function nativeFullCasePartition({partition='all',platform=process.platform, arch=process.arch,stage,artifact}) { - assert(partition==='all'||Object.hasOwn(fullCasePartitions,partition),'Unknown full native case partition'); + const functional=nativeFullFunctionalGroups.find(group=>group.id===partition),legacy=Object.hasOwn(legacyFullCasePartitions,partition); + assert(partition==='all'||functional||legacy,'Unknown full native case partition'); if(partition!=='all') { - assert.equal(platform,'darwin','Full partitions are restricted to Intel macOS'); - assert.equal(arch,'x64','Full partitions require native Intel macOS'); + if(legacy)assert.equal(platform,'darwin','Historical full partitions are restricted to Intel macOS'); + else assert(['linux','win32','darwin'].includes(platform),'Functional groups require a supported native host'); + assert.equal(arch,'x64','Full partitions require native x64'); assert.equal(stage,'full','Partitions retain the complete native campaign'); assert.equal(artifact,'candidate','Historical public campaigns remain unchanged'); } - const all=Object.values(fullCasePartitions).flat().filter(name=>artifact==='candidate'||name!=='native-pluto-controls'); - return {name:partition,cases:partition==='all'?all:[...fullCasePartitions[partition]], + const all=Object.values(legacyFullCasePartitions).flat().filter(name=>artifact==='candidate'||name!=='native-pluto-controls'); + return {name:partition,...(functional?{label:functional.label}:{}),cases:partition==='all'?all:[...(functional?.cases||legacyFullCasePartitions[partition])], sharedPrefix:['controller-visible-in-studio','native-julia-terminal', artifact==='candidate'?'native-pluto-without-jupyter':'native-notebook-with-jupyter', 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs index 5d9e4e2..144deb1 100644 --- a/test/native-platform-contract.test.mjs +++ b/test/native-platform-contract.test.mjs @@ -1,7 +1,14 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; -import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; +import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition,nativeFullFunctionalGroups,nativeWorkflowMatrix} from './native-platform-contract.mjs'; + +const historicalCases=['native-all-studio-controls','native-workbench-controls','native-investigation-controls', + 'native-investigation-limits','native-enabled-narrative-protocol','native-advisor-controls', + 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']; +const historicalPrefix=['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter', + 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', + 'native-testitem-measurement','native-complete-suite-measurements']; const arm={scope:'arm-bootstrap',event:'workflow_dispatch',artifact:'candidate',stage:'focused', group:'bootstrap',core:'general',arch:'arm64',runnerArch:'ARM64',expectedArch:'arm64'}; @@ -35,15 +42,25 @@ test('existing campaigns retain x64 and reject incidental ARM execution',()=>{ } }); -test('Intel partitions preserve the full ordered case union and every shared prefix',()=>{ +test('seven functional groups preserve every historical case and shared prefix without mixing legacy partitions',()=>{ + assert.deepEqual(nativeFullFunctionalGroups.map(group=>group.id), + ['studio','workbench','investigation','investigation-limits','narrative','advisor','mcp-pluto']); + assert.deepEqual(nativeFullFunctionalGroups.flatMap(group=>group.cases),historicalCases); + assert.equal(new Set(nativeFullFunctionalGroups.flatMap(group=>group.cases)).size,9); + for(const platform of ['linux','win32','darwin'])for(const group of nativeFullFunctionalGroups){ + const partition=nativeFullCasePartition({partition:group.id,platform,arch:'x64',stage:'full',artifact:'candidate'}); + assert.deepEqual(partition.sharedPrefix,historicalPrefix);assert.deepEqual(partition.cases,group.cases); + } + assert.deepEqual(nativeFullFunctionalGroups.at(-1).cases, + ['native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']); +}); + +test('historical Intel partitions remain separately reproducible with their original ordered union',()=>{ const campaign={platform:'darwin',arch:'x64',stage:'full',artifact:'candidate'}; const all=nativeFullCasePartition(campaign); const partitions=['measure','investigate','agentic'].map(partition=>nativeFullCasePartition({...campaign,partition})); - const historical=['native-all-studio-controls','native-workbench-controls','native-investigation-controls', - 'native-investigation-limits','native-enabled-narrative-protocol','native-advisor-controls', - 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']; - assert.deepEqual(all.cases,historical,'The independently enumerated historical tail must remain complete'); - assert.deepEqual(partitions.flatMap(partition=>partition.cases),historical); + assert.deepEqual(all.cases,historicalCases,'The independently enumerated historical tail must remain complete'); + assert.deepEqual(partitions.flatMap(partition=>partition.cases),historicalCases); assert.equal(new Set(all.cases).size,all.cases.length,'No full case is accidentally duplicated or lost'); for(const partition of partitions)assert.deepEqual(partition.sharedPrefix,all.sharedPrefix); assert.deepEqual(partitions[2].cases.slice(-3),['native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto'], @@ -65,15 +82,47 @@ test('the real host dispatches every advertised shared-prefix case before its pa assert(suiteStart>=0&&suiteEnd>suiteStart); const suite=Function('discoveryDiagnostic',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false); await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture'},runCase,{testItem:true,complete:true}); - const historicalPrefix=['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter', - 'native-suite-run-button','native-suite-save-palette','native-run-selection-command', - 'native-testitem-measurement','native-complete-suite-measurements']; assert.deepEqual(calls,historicalPrefix); assert.deepEqual(nativeFullCasePartition({artifact:'candidate'}).sharedPrefix,historicalPrefix); assert(source.includes("await runSuiteCases(context,runCase,{testItem:true,complete:process.env.PERFCHECKER_NATIVE_STAGE==='full'});"), 'Every full prepared partition still dispatches both the native test item and the complete suite'); }); +test('each real full dispatcher executes eight prefixes followed by only its functional tail in one context',async()=>{ + const source=readFileSync(new URL('./public-vsix-host.cjs',import.meta.url),'utf8'); + const prefixStart=source.indexOf("await runCase('controller-visible-in-studio'"),prefixEnd=source.indexOf("if (phase === 'configured'",prefixStart); + const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); + const tailStart=source.indexOf("if (process.env.PERFCHECKER_NATIVE_STAGE === 'full'&&!discoveryDiagnostic) {"); + assert(prefixStart>=0&&prefixEnd>prefixStart&&suiteStart>=0&&suiteEnd>suiteStart&&tailStart>=0); + let end=source.indexOf('{',tailStart),depth=1; + while(depth&&++endvalue}},workspace:'/disposable-fixture'}; + const noEffects=async name=>{calls.push(name);return true;}; + await prefix(noEffects,{env:{PERFCHECKER_NATIVE_MODE:'candidate'}});await suite(context,noEffects,{testItem:true,complete:true}); + const effect=name=>async actual=>{assert.equal(actual,context);effects.push(name);}; + const modules={ + './native-investigation-limits.cjs':{run:effect('native-investigation-limits')}, + './native-narrative-controls.cjs':{run:effect('native-enabled-narrative-protocol')} + }; + const vscode={commands:{executeCommand:async command=>{effects.push(command);}}}; + await tail({env:{PERFCHECKER_NATIVE_STAGE:'full'}},false,async(name,run)=>{calls.push(name);await run();},context, + nativeFullCasePartition({partition:group.id,platform,arch:'x64',stage:'full',artifact:'candidate'}),assert, + {run:effect('native-all-studio-controls')},{run:effect('native-workbench-controls')}, + {run:effect('native-investigation-controls')},name=>{assert(Object.hasOwn(modules,name));return modules[name];}, + {run:effect('native-advisor-controls')},{run:effect('native-mcp-controls')},{run:effect('native-pluto-controls')},vscode); + assert.deepEqual(calls,[...historicalPrefix,...group.cases]); + assert.deepEqual(effects,group.id==='mcp-pluto'? + ['native-mcp-controls','native-pluto-controls','perfchecker.discoverScenarios','perfchecker.syncScenarios']:group.cases); + } +}); + test('the real diagnostic host keeps the seventh prefix and explicitly excludes only the following cases',async()=>{ const source=readFileSync(new URL('./public-vsix-host.cjs',import.meta.url),'utf8'); const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); @@ -102,26 +151,67 @@ test('partitions cannot silently replace focused, public, ARM or other platform {arch:'arm64'},{stage:'focused'},{stage:'smoke'},{artifact:'public'}]) assert.throws(()=>nativeFullCasePartition({...campaign,...change})); assert(!nativeFullCasePartition({artifact:'public'}).cases.includes('native-pluto-controls')); + for(const partition of nativeFullFunctionalGroups.map(group=>group.id))for(const change of + [{partition:'unknown'},{platform:'freebsd'},{arch:'arm64'},{stage:'focused'},{stage:'smoke'},{artifact:'public'}]) + assert.throws(()=>nativeFullCasePartition({...campaign,partition,...change})); }); -test('the actual workflow partitions only complete candidate Intel jobs and keeps every other lane',()=>{ +test('the actual package output selects unique functional jobs for every complete candidate host',()=>{ const workflow=readFileSync(new URL('../.github/workflows/native-vsix.yml',import.meta.url),'utf8'); - const expression=workflow.split('\n').find(line=>line.includes('include: ${{ fromJSON(')).split('${{ ')[1].split(' }}')[0]; - const matrix=(event,inputs={})=>Function('github','inputs','fromJSON',`return ${expression}`)({event_name:event},inputs,JSON.parse); + const step=workflow.indexOf('id: matrix'),start=workflow.indexOf("node --input-type=module <<'NODE'",step)+"node --input-type=module <<'NODE'".length; + const end=workflow.indexOf('\n NODE',start);assert(step>=0&&end>start); + assert(step!line.trim().startsWith('import ')).join('\n'); + const matrix=(event,inputs={})=>{let output; + Function('fs','nativeWorkflowMatrix','process',script)({appendFileSync(file,text){assert.equal(file,'memory-output');assert.equal(output,undefined);output=text;}}, + nativeWorkflowMatrix,{env:{GITHUB_OUTPUT:'memory-output',NATIVE_EVENT:event,NATIVE_SCOPE:inputs.scope, + NATIVE_VSIX_MODE:inputs.artifact||'candidate',NATIVE_STAGE:inputs.stage,NATIVE_CASE_GROUP:inputs.group, + NATIVE_CORE:inputs.core||'candidate',NATIVE_DISCOVERY_DIAGNOSTIC:String(inputs.discovery_diagnostic||false)}}); + assert(output.startsWith('matrix=')&&output.endsWith('\n'));return JSON.parse(output.slice(7));}; + assert(workflow.includes('matrix: ${{ steps.matrix.outputs.matrix }}')); + assert(workflow.includes('include: ${{ fromJSON(needs.package.outputs.matrix) }}')); + assert(workflow.includes('name: ${{ matrix.name }}'));assert(workflow.includes('name: ${{ matrix.artifactName }}')); const full=matrix('pull_request'); - assert.deepEqual(full.filter(lane=>lane.os!=='macos-15-intel'),[ - {os:'ubuntu-latest',vscode:'1.96.0'},{os:'ubuntu-latest',vscode:'stable'}, - {os:'windows-latest',vscode:'1.96.0'},{os:'windows-latest',vscode:'stable'}]); - assert.deepEqual(full.filter(lane=>lane.os==='macos-15-intel').map(lane=>lane.partition),['measure','investigate','agentic']); - const manual=(scope,stage='full',artifact='candidate')=>matrix('workflow_dispatch',{scope,stage,artifact}); - assert.equal(manual('macos-stable').length,3); - assert.deepEqual(manual('macos-stable','focused'),[{os:'macos-15-intel',vscode:'stable'}]); + assert.equal(full.length,35);assert.equal(new Set(full.map(lane=>lane.name)).size,35); + assert.equal(new Set(full.map(lane=>lane.artifactName)).size,35); + for(const [os,vscode]of [['ubuntu-latest','1.96.0'],['ubuntu-latest','stable'],['windows-latest','1.96.0'],['windows-latest','stable'],['macos-15-intel','stable']]){ + const lanes=full.filter(lane=>lane.os===os&&lane.vscode===vscode);assert.equal(lanes.length,7); + assert.deepEqual(lanes.map(lane=>lane.partition),['studio','workbench','investigation','investigation-limits','narrative','advisor','mcp-pluto']); + assert(lanes.every(lane=>lane.name.includes(`VS Code ${vscode}`)&&lane.name.includes(os==='macos-15-intel'?'macOS Intel':'x64'))); + } + const manual=(scope,stage='full',artifact='candidate',extra={})=>matrix('workflow_dispatch',{scope,stage,artifact,...extra}); + assert.equal(manual('all').length,35);for(const scope of ['linux','windows'])assert.equal(manual(scope).length,14); + for(const scope of ['linux-stable','windows-stable','macos-stable'])assert.equal(manual(scope).length,7); + assert.deepEqual(manual('macos-stable','focused').map(({os,vscode})=>({os,vscode})),[{os:'macos-15-intel',vscode:'stable'}]); assert.equal(manual('all','full','public').length,5); assert(manual('all','full','public').every(lane=>!lane.partition)); - assert.equal(manual('all','focused').length,5); - assert.equal(manual('arm-bootstrap','focused').length,4); - assert(manual('arm-bootstrap','focused').every(lane=>lane.arch==='arm64'&&!lane.partition)); - assert.deepEqual(manual('windows-stable'),[{os:'windows-latest',vscode:'stable'}]); - assert.deepEqual(matrix('workflow_dispatch',{scope:'macos-stable',stage:'full',artifact:'candidate',discovery_diagnostic:true}), - [{os:'macos-15-intel',vscode:'stable',partition:'measure'}]); + assert.deepEqual(manual('all','full','public').map(({os,vscode})=>({os,vscode})),[ + {os:'ubuntu-latest',vscode:'1.96.0'},{os:'ubuntu-latest',vscode:'stable'}, + {os:'windows-latest',vscode:'1.96.0'},{os:'windows-latest',vscode:'stable'},{os:'macos-15-intel',vscode:'stable'}]); + for(const stage of ['focused','smoke','targeted','core-external','connector-workers']){ + assert.equal(manual('all',stage).length,5);assert(manual('all',stage).every(lane=>!lane.partition)); + } + const arm=manual('arm-bootstrap','focused','candidate',{core:'general',group:'bootstrap'});assert.equal(arm.length,4); + assert(arm.every(lane=>lane.arch==='arm64'&&!lane.partition&&lane.name.includes(lane.os==='macos-15'?'Apple Silicon':'Linux ARM64')&&lane.name.includes('Julia'))); + assert.equal(new Set(arm.map(lane=>lane.name)).size,4);assert.equal(new Set(arm.map(lane=>lane.artifactName)).size,4); + const diagnostic=manual('macos-stable','full','candidate',{discovery_diagnostic:true});assert.equal(diagnostic.length,1); + assert.equal(diagnostic[0].partition,'measure'); + assert.equal(diagnostic[0].name,'Diagnostic — macOS Intel — VS Code stable — TestItem discovery — non-qualifying'); + for(const change of [{scope:'unknown'},{artifact:'unknown'},{stage:'unknown'},{event:'unknown'}]) + assert.throws(()=>nativeWorkflowMatrix({event:'workflow_dispatch',...change})); + for(const change of [{scope:'all'},{stage:'focused'},{artifact:'public'},{core:'general'},{event:'pull_request'}]) + assert.throws(()=>nativeWorkflowMatrix({event:'workflow_dispatch',scope:'macos-stable',stage:'full',artifact:'candidate',core:'candidate',discoveryDiagnostic:true,...change})); +}); + +test('functional splitting preserves the real three-phase preparation and all existing time limits',()=>{ + const workflow=readFileSync(new URL('../.github/workflows/native-vsix.yml',import.meta.url),'utf8'); + const driver=readFileSync(new URL('./run-public-vsix-host.mjs',import.meta.url),'utf8'); + assert(workflow.includes("timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }}")); + assert(driver.includes("const completeCampaign=['smoke','full'].includes(stage);")); + assert(driver.includes("if(completeCampaign||bootstrapOnly)await launch('fresh');")); + assert(driver.includes("if(completeCampaign)await launch('configured');")); + assert(driver.includes("else await launch('prepared');")); + assert(driver.includes("phase==='prepared'&&environment.PERFCHECKER_NATIVE_STAGE==='full'?130")); + assert(workflow.includes("if: runner.os == 'Linux' && matrix.vscode == 'stable' && (env.PERFCHECKER_NATIVE_STAGE == 'full' || env.PERFCHECKER_NATIVE_CASE_GROUP == 'mcp-pluto')"), + 'Every complete group preserves the actual host prerequisite condition without a partition exception'); }); From 131430d4aa4226eac2c92eb820557b66f1ec04eb Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 13:46:44 +0200 Subject: [PATCH 17/22] Allow one hour for native qualification readiness --- .github/workflows/native-vsix.yml | 10 +- test/native-advisor-controls.cjs | 88 ++--- test/native-codex-controls.cjs | 11 +- test/native-editor-actions.cjs | 59 +-- test/native-external-browser.cjs | 13 +- test/native-general100-controls.cjs | 65 ++-- test/native-investigation-controls.cjs | 233 ++++++------ test/native-investigation-limits.cjs | 39 +- test/native-mcp-controls.cjs | 185 +++++----- test/native-mcp-identity.test.mjs | 22 +- test/native-narrative-controls.cjs | 87 ++--- test/native-platform-contract.test.mjs | 212 ++++++++++- test/native-pluto-controls.cjs | 278 ++++++++------- test/native-readiness.cjs | 28 ++ test/native-reload-host.cjs | 7 +- test/native-restricted-controls.cjs | 17 +- test/native-studio-controls.cjs | 475 +++++++++++++------------ test/native-suite-chat-evidence.cjs | 39 +- test/native-workbench-controls.cjs | 61 ++-- test/public-vsix-host.cjs | 106 +++--- test/run-public-vsix-host.mjs | 91 +++-- 21 files changed, 1212 insertions(+), 914 deletions(-) create mode 100644 test/native-readiness.cjs diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index d38c3bd..11cd72b 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -190,6 +190,7 @@ jobs: assertNativeCampaignSelection({scope:e.NATIVE_SCOPE,event:e.NATIVE_EVENT,artifact:e.NATIVE_VSIX_MODE, stage:e.NATIVE_STAGE,group:e.NATIVE_CASE_GROUP,core:e.NATIVE_CORE}); const requested=[e.REUSE_RUN,e.REUSE_NAME,e.REUSE_SHA].some(Boolean),newOrigin=e.NATIVE_NEW_ORIGIN==='true'; + if(e.NATIVE_DISCOVERY_DIAGNOSTIC==='true')throw Error('Historical discovery diagnostics must use their original branch and explicit oracle contract; this source qualifies the one-hour campaign'); assertNativeDiscoveryDiagnostic({enabled:e.NATIVE_DISCOVERY_DIAGNOSTIC==='true',event:e.NATIVE_EVENT, artifact:e.NATIVE_VSIX_MODE,stage:e.NATIVE_STAGE,scope:e.NATIVE_SCOPE,core:e.NATIVE_CORE,partition:'measure', backendJulia:e.NATIVE_BACKEND_JULIA,originRun:e.REUSE_RUN,originName:e.REUSE_NAME,originSha:e.REUSE_SHA, @@ -275,7 +276,7 @@ jobs: matrix: include: ${{ fromJSON(needs.package.outputs.matrix) }} runs-on: ${{ matrix.os }} - timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }} + timeout-minutes: 360 env: PERFCHECKER_NATIVE_DISCOVERY_DIAGNOSTIC: ${{ inputs.discovery_diagnostic && '1' || '0' }} PERFCHECKER_NATIVE_EVENT: ${{ github.event_name }} @@ -300,6 +301,9 @@ jobs: JULIA_PKG_PRECOMPILE_AUTO: '0' PERFCHECKER_UUID: 00000000-0000-4000-8000-000000000001 steps: + - name: Start the shared native activity budget (330 minutes plus 30 minutes technical reserve) + shell: bash + run: echo "PERFCHECKER_NATIVE_ACTIVITY_STARTED_AT=$(( $(date +%s) * 1000 ))" >> "$GITHUB_ENV" - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: @@ -334,7 +338,7 @@ jobs: run: npx playwright install chromium - name: Install screen capture tooling in the disposable Linux stable host if: runner.os == 'Linux' && env.PERFCHECKER_NATIVE_VIDEO == '1' - timeout-minutes: 15 + timeout-minutes: 60 run: | if ! command -v ffmpeg >/dev/null 2>&1; then sudo apt-get update @@ -343,7 +347,7 @@ jobs: ffmpeg -version - name: Install native colour-picker tools in the disposable Linux host if: runner.os == 'Linux' && env.PERFCHECKER_NATIVE_STAGE == 'focused' && env.PERFCHECKER_NATIVE_CASE_GROUP == 'studio-color' - timeout-minutes: 15 + timeout-minutes: 60 run: | if ! command -v xdotool >/dev/null 2>&1 || ! command -v xwd >/dev/null 2>&1 || ! command -v xwininfo >/dev/null 2>&1 || ! command -v convert >/dev/null 2>&1; then sudo apt-get update diff --git a/test/native-advisor-controls.cjs b/test/native-advisor-controls.cjs index d6090bf..3cedcc8 100644 --- a/test/native-advisor-controls.cjs +++ b/test/native-advisor-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Actual VS Code controls and Julia HTTP workers. This local protocol fixture // stores tiny test payloads; it is not an Ollama runtime or an inference model. const assert = require('node:assert/strict'); @@ -7,7 +8,7 @@ const os = require('node:os'); const path = require('node:path'); const {createHash, randomUUID} = require('node:crypto'); -async function eventually(read, label, timeout = 180000) { +async function eventually(read, label, timeout = NATIVE_READINESS_MS) { const deadline = Date.now() + timeout; let last; while (Date.now() < deadline) { @@ -110,34 +111,35 @@ async function panel(context) { return context.findFrame('#advisor-root'); } const row = (frame, name) => frame.locator('section[aria-label="Available models and tools"] article').filter({has: frame.getByRole('heading', {name, exact: true})}); -async function finished(frame, message) { +async function finished(frame, message,timeout=NATIVE_READINESS_MS) { + const readTimeout=timeout===NATIVE_READINESS_MS?timeout:Math.min(timeout,30000); try{ - await eventually(async () => await frame.locator('#advisor-root').getAttribute('aria-busy') === 'false' && - message.test(await frame.getByRole('status').innerText()), `Advisor action finishes: ${message}`); + await eventually(async () => await frame.locator('#advisor-root').getAttribute('aria-busy',{timeout:readTimeout}) === 'false' && + message.test(await frame.getByRole('status').innerText({timeout:readTimeout})), `Advisor action finishes: ${message}`,timeout); }catch(error){ - const actual={busy:await frame.locator('#advisor-root').getAttribute('aria-busy'),status:await frame.getByRole('status').innerText(), - mode:await frame.locator('#advisor-protocol').inputValue(),model:await frame.locator('#advisor-model').inputValue()}; + const actual={busy:await frame.locator('#advisor-root').getAttribute('aria-busy',{timeout:30000}),status:await frame.getByRole('status').innerText({timeout:30000}), + mode:await frame.locator('#advisor-protocol').inputValue({timeout:30000}),model:await frame.locator('#advisor-model').inputValue({timeout:30000})}; throw new Error(`${error.message}; actual declared fixture UI: ${JSON.stringify(actual)}`); } } -async function click(frame, label) {await frame.getByRole('button', {name: label, exact: true}).click();} +async function click(frame, label,timeout=NATIVE_READINESS_MS) {await frame.getByRole('button', {name: label, exact: true}).click({timeout});} async function confirm(frame, label) {await click(frame, label); await click(frame, 'Confirm');} async function ollama(context, fixture, configFile) { const frame = await panel(context), initialRequests = fixture.calls.length; - assert.equal(await frame.getByRole('button', {name: 'Cancel operation', exact: true}).isDisabled(), true); + assert.equal(await frame.getByRole('button', {name: 'Cancel operation', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), true); assert.equal(fixture.calls.length, initialRequests, 'Opening Advisor performs no provider request'); - await frame.locator('#advisor-protocol').selectOption('ollama'); - await frame.locator('#advisor-endpoint').fill(`${fixture.base}/api/chat`); - await frame.locator('#advisor-model').fill('missing:fixture'); - await frame.locator('#advisor-timeout').fill('120'); - await frame.locator('#advisor-instructions').fill('Protocol fixture only; preserve the public API.'); + await frame.locator('#advisor-protocol').selectOption('ollama',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-endpoint').fill(`${fixture.base}/api/chat`,{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-model').fill('missing:fixture',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-timeout').fill('120',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-instructions').fill('Protocol fixture only; preserve the public API.',{timeout:NATIVE_READINESS_MS}); await click(frame, 'Test connection / discover'); await finished(frame, /configured model \/ tool is missing/); assert(fixture.calls.some(call => call.method === 'GET' && call.url === '/api/tags')); assert.equal(await row(frame, 'seed:fixture').count(), 1); - await row(frame, 'seed:fixture').getByRole('button', {name: 'Use', exact: true}).click(); - assert.equal(await frame.locator('#advisor-model').inputValue(), 'seed:fixture'); + await row(frame, 'seed:fixture').getByRole('button', {name: 'Use', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await frame.locator('#advisor-model').inputValue({timeout:NATIVE_READINESS_MS}), 'seed:fixture'); const beforeSave = fixture.calls.length; await click(frame, 'Save configuration'); await finished(frame, /Configuration saved/); const saved = JSON.parse(await fs.readFile(configFile, 'utf8')); @@ -148,14 +150,14 @@ async function ollama(context, fixture, configFile) { await click(frame, 'Refresh models'); await finished(frame, /Ollama connected/); const original = await fs.readFile(fixture.modelFile('seed:fixture')); const beforeUnload = fixture.calls.length; - await row(frame, 'seed:fixture').getByRole('button', {name: 'Unload from memory…', exact: true}).click(); + await row(frame, 'seed:fixture').getByRole('button', {name: 'Unload from memory…', exact: true}).click({timeout:NATIVE_READINESS_MS}); assert.equal(fixture.calls.length, beforeUnload, 'Mutation waits for the explicit confirmation'); await click(frame, 'Back'); assert.equal(fixture.calls.length, beforeUnload); - await row(frame, 'seed:fixture').getByRole('button', {name: 'Unload from memory…', exact: true}).click(); + await row(frame, 'seed:fixture').getByRole('button', {name: 'Unload from memory…', exact: true}).click({timeout:NATIVE_READINESS_MS}); await click(frame, 'Confirm'); await finished(frame, /unloaded from memory/); assert.equal(fixture.models.get('seed:fixture').loaded, false); assert.deepEqual(await fs.readFile(fixture.modelFile('seed:fixture')), original); - await frame.locator('#advisor-download-model').fill('downloaded:fixture'); + await frame.locator('#advisor-download-model').fill('downloaded:fixture',{timeout:NATIVE_READINESS_MS}); const beforePull = fixture.calls.length; await click(frame, 'Download this model…'); assert.equal(fixture.calls.length, beforePull); assert.equal(fixture.models.has('downloaded:fixture'), false); @@ -164,7 +166,7 @@ async function ollama(context, fixture, configFile) { assert(fixture.calls.some(call => call.method === 'POST' && call.url === '/api/pull' && call.body.model === 'downloaded:fixture' && call.body.stream === false)); await click(frame, 'Refresh models'); await finished(frame, /Ollama connected/); assert.equal(await row(frame, 'downloaded:fixture').count(), 1); - await row(frame, 'downloaded:fixture').getByRole('button', {name: 'Delete from disk…', exact: true}).click(); + await row(frame, 'downloaded:fixture').getByRole('button', {name: 'Delete from disk…', exact: true}).click({timeout:NATIVE_READINESS_MS}); const beforeDelete = fixture.calls.length; await click(frame, 'Confirm'); await finished(frame, /Model removed/); assert.equal(fixture.models.has('downloaded:fixture'), false); @@ -172,20 +174,20 @@ async function ollama(context, fixture, configFile) { assert.equal(fixture.calls[beforeDelete].method, 'DELETE'); await click(frame, 'Refresh models'); await finished(frame, /Ollama connected/); assert.equal(await row(frame, 'downloaded:fixture').count(), 0); - await frame.locator('#advisor-download-model').fill('cancelled:fixture'); + await frame.locator('#advisor-download-model').fill('cancelled:fixture',{timeout:NATIVE_READINESS_MS}); await confirm(frame, 'Download this model…'); await eventually(() => fs.stat(fixture.partialFile('cancelled:fixture')).then(() => true).catch(() => false), 'The real fixture receives the active download'); - assert.equal(await frame.getByRole('button', {name: 'Cancel operation', exact: true}).isEnabled(), true); - await click(frame, 'Cancel operation'); await finished(frame, /Operation cancelled/); - await eventually(() => fixture.cancelled.has('cancelled:fixture'), 'Cancellation closes the real Julia HTTP connection'); + assert.equal(await frame.getByRole('button', {name: 'Cancel operation', exact: true}).isEnabled({timeout:NATIVE_READINESS_MS}), true); + await click(frame, 'Cancel operation',30000); await finished(frame, /Operation cancelled/,180000); + await eventually(() => fixture.cancelled.has('cancelled:fixture'), 'Cancellation closes the real Julia HTTP connection',180000); assert.equal(fixture.models.has('cancelled:fixture'), false); assert.equal(await fs.readFile(fixture.partialFile('cancelled:fixture'), 'utf8'), 'partial test payload\n'); assert.deepEqual(await fs.readFile(configFile), savedBytes, 'Cancelled download does not modify the saved provider'); await click(frame, 'Refresh models'); await finished(frame, /Ollama connected/); assert.equal(await row(frame, 'cancelled:fixture').count(), 0); - await frame.locator('#advisor-endpoint').fill(`${fixture.base}/unavailable/api/chat`); + await frame.locator('#advisor-endpoint').fill(`${fixture.base}/unavailable/api/chat`,{timeout:NATIVE_READINESS_MS}); await click(frame, 'Test connection / discover'); await finished(frame, /HTTP 503/); - assert.equal(await frame.getByRole('button', {name: 'Test connection / discover', exact: true}).isEnabled(), true); + assert.equal(await frame.getByRole('button', {name: 'Test connection / discover', exact: true}).isEnabled({timeout:NATIVE_READINESS_MS}), true); assert.deepEqual(await fs.readFile(configFile), savedBytes, 'Provider errors preserve the saved configuration'); context.proof('advisor-native-model-management', {protocolFixture: true, inferenceModelDownloaded: false, payloadBytes: fixture.payload.length, selectionSaved: true, confirmedPullDeleteUnload: true, @@ -194,27 +196,27 @@ async function ollama(context, fixture, configFile) { async function inventories(context, fixture, configFile) { const frame = await panel(context); - await frame.locator('#advisor-protocol').selectOption('chat_completions'); - await frame.locator('#advisor-endpoint').fill(`${fixture.base}/v1/chat/completions`); - await frame.locator('#advisor-model').fill('chat:fixture'); + await frame.locator('#advisor-protocol').selectOption('chat_completions',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-endpoint').fill(`${fixture.base}/v1/chat/completions`,{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-model').fill('chat:fixture',{timeout:NATIVE_READINESS_MS}); await click(frame, 'Test connection / discover'); await finished(frame, /Model inventory connected/); - await row(frame, 'chat:fixture').getByRole('button', {name: 'Use', exact: true}).click(); - assert.equal(await frame.locator('#advisor-model').inputValue(), 'chat:fixture'); - await frame.locator('#advisor-protocol').selectOption('mcp_http'); - await frame.locator('#advisor-endpoint').fill(`${fixture.base}/mcp`); - await frame.locator('#advisor-mcp_version').selectOption('2025-11-25'); - await frame.locator('#advisor-mcp_tool').fill(''); + await row(frame, 'chat:fixture').getByRole('button', {name: 'Use', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await frame.locator('#advisor-model').inputValue({timeout:NATIVE_READINESS_MS}), 'chat:fixture'); + await frame.locator('#advisor-protocol').selectOption('mcp_http',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-endpoint').fill(`${fixture.base}/mcp`,{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-mcp_version').selectOption('2025-11-25',{timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-mcp_tool').fill('',{timeout:NATIVE_READINESS_MS}); await click(frame, 'Test connection / discover'); await finished(frame, /MCP connected/); assert.equal(await row(frame, 'ask_fixture').count(), 1); assert.equal(await row(frame, 'simple_advice').count(), 1); assert.equal(await frame.locator('#advisor-root b').count(), 0, 'Tool descriptions remain literal text'); - await row(frame, 'ask_fixture').getByText('Required arguments and tool schema', {exact: true}).click(); - assert.match(await row(frame, 'ask_fixture').locator('pre').innerText(), /"context"/); - await row(frame,'ask_fixture').scrollIntoViewIfNeeded(); + await row(frame, 'ask_fixture').getByText('Required arguments and tool schema', {exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await row(frame, 'ask_fixture').locator('pre').innerText({timeout:NATIVE_READINESS_MS}), /"context"/); + await row(frame,'ask_fixture').scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); if(process.env.PERFCHECKER_NATIVE_VIDEO==='1')await new Promise(resolve=>setTimeout(resolve,2500)); - await row(frame, 'ask_fixture').getByRole('button', {name: 'Use', exact: true}).click(); - assert.equal(await frame.locator('#advisor-mcp_tool').inputValue(), 'ask_fixture'); - assert.equal(await frame.locator('#advisor-mcp_prompt_argument').inputValue(), 'question'); - await frame.locator('#advisor-mcp_arguments').fill('{"context":{"fixture":true}}'); + await row(frame, 'ask_fixture').getByRole('button', {name: 'Use', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await frame.locator('#advisor-mcp_tool').inputValue({timeout:NATIVE_READINESS_MS}), 'ask_fixture'); + assert.equal(await frame.locator('#advisor-mcp_prompt_argument').inputValue({timeout:NATIVE_READINESS_MS}), 'question'); + await frame.locator('#advisor-mcp_arguments').fill('{"context":{"fixture":true}}',{timeout:NATIVE_READINESS_MS}); await click(frame, 'Save configuration'); await finished(frame, /Configuration saved/); const saved = JSON.parse(await fs.readFile(configFile, 'utf8')); assert.equal(saved.mcp_tool, 'ask_fixture'); assert.equal(saved.mcp_prompt_argument, 'question'); @@ -227,7 +229,7 @@ async function inventories(context, fixture, configFile) { assert(calls.some(call => call.method === 'DELETE')); assert.equal(fixture.calls.filter(call => call.body.method === 'tools/call').length, 0); assert.equal(fixture.calls.filter(call => /chat\/completions|api\/chat/.test(call.url)).length, 0); - await frame.locator('#advisor-protocol').selectOption('none'); + await frame.locator('#advisor-protocol').selectOption('none',{timeout:NATIVE_READINESS_MS}); await click(frame, 'Save configuration'); await finished(frame, /Rule-based advice only/); assert.equal(context.vscode.workspace.getConfiguration('perfchecker', context.vscode.Uri.file(context.workspace)).get('advisorEnabled'), false); assert.deepEqual(await fs.readFile(fixture.modelFile('seed:fixture')), fixture.payload); @@ -267,7 +269,7 @@ exports.run = async context => { throw error; } finally { await context.vscode.commands.executeCommand('perfchecker.cancelAdvisorSetup').catch(() => {}); - const frame = await context.findFrame('#advisor-root').catch(() => undefined); + const frame = await context.findFrame('#advisor-root',{timeout:120000}).catch(() => undefined); if (frame) await context.vscode.commands.executeCommand('workbench.action.closeActiveEditor').catch(() => {}); for (const key of keys) await settings.update(key, previous[key], context.vscode.ConfigurationTarget.WorkspaceFolder); await fixture.close(); await fs.rm(path.dirname(configFile), {recursive: true, force: true}); diff --git a/test/native-codex-controls.cjs b/test/native-codex-controls.cjs index 95c8dad..39347d9 100644 --- a/test/native-codex-controls.cjs +++ b/test/native-codex-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Explicit local opt-in: existing authentication is read-only, no model is called. const assert=require('node:assert/strict'); const fs=require('node:fs/promises'); @@ -66,7 +67,7 @@ exports.run=async context=>{ assert.equal(process.env.PERFCHECKER_NATIVE_CODEX_NO_MODEL,'1'); const externalDeadline=Date.parse(process.env.PERFCHECKER_NATIVE_CODEX_DEADLINE); assert(Number.isFinite(externalDeadline)&&externalDeadline>Date.now(),'The external native-case deadline must be finite and future'); - const deadline=Math.min(Date.now()+120000,externalDeadline); + const deadline=Math.min(Date.now()+NATIVE_READINESS_MS,externalDeadline); assert.equal(await fs.realpath(workspace),path.join(await fs.realpath(session),'workspace')); const cli=await fs.realpath(process.env.PERFCHECKER_NATIVE_CODEX_EXECUTABLE); assert.equal(cli,'/usr/lib/chatgpt/resources/codex','Only the explicitly reviewed installed native CLI is selected'); @@ -130,13 +131,13 @@ exports.run=async context=>{ assert.equal(settings().inspect('codexExecutable')[level],cli); stage='open-chat'; await vscode.commands.executeCommand('perfchecker.openChat');const frame=await findFrame('#chat-root'); - await frame.locator('summary').filter({hasText:'Optional Codex CLI connector'}).click(); + await frame.locator('summary').filter({hasText:'Optional Codex CLI connector'}).click({timeout:NATIVE_READINESS_MS}); stage='native-connect-click'; - await frame.getByRole('button',{name:'Connect Codex CLI',exact:true}).click(); + await frame.getByRole('button',{name:'Connect Codex CLI',exact:true}).click({timeout:NATIVE_READINESS_MS}); stage='wait-connected-state'; await eventually(async()=>{ if(observerErrors.length)throw observerErrors[0];return (await state()).connected; - },'The selected authenticated native CLI connects without a model request',60000,deadline); + },'The selected authenticated native CLI connects without a model request',NATIVE_READINESS_MS,deadline); connected=true; const active=connection();assert.equal(active.kind,'codex'); assert.match(active.label,/^codex-cli 0\.162\.0-alpha\.17\.2/); @@ -156,7 +157,7 @@ exports.run=async context=>{ preflightKindsObserved:[...new Set([...records.values()].map(value=>value.kind))],preflightIdentities:[...records.values()], observedPreflightsAbsentBeforeTeardown:true,unobservedPreflightCountNotClaimed:true,savedProviderBytesPreserved:true,modelRequestsSent:0,implementationRequestsSent:0}); stage='native-disconnect-click'; - await frame.getByRole('button',{name:'Disconnect Codex',exact:true}).click(); + await frame.getByRole('button',{name:'Disconnect Codex',exact:true}).click({timeout:30000}); stage='wait-disconnected-state'; await eventually(async()=>!(await state()).connected,'Actual Disconnect clears the authenticated temporary connection',10000,deadline); connected=false;assert.equal(connection(),undefined); diff --git a/test/native-editor-actions.cjs b/test/native-editor-actions.cjs index 5da973a..66e040f 100644 --- a/test/native-editor-actions.cjs +++ b/test/native-editor-actions.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); // Actual CodeLens and quick-fix clicks in disposable native VS Code hosts. // Findings come from a real JET worker; no diagnostic or report is injected. const assert=require('node:assert/strict'); @@ -6,7 +7,7 @@ const path=require('node:path'); const {createHash}=require('node:crypto'); const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function eventually(read,description,timeout=180000){ +async function eventually(read,description,timeout=NATIVE_READINESS_MS){ const deadline=Date.now()+timeout;let last; while(Date.now()item.id==='native_editor_branch')); assert(discovery.payload.candidates.length,'The actual TestItem fixture exposes a proposal CodeLens'); @@ -201,7 +202,7 @@ async function resource(context,workspace,controller,threads,previousEditorActio const candidate=discovery.payload.candidates[0]; await vscode.commands.executeCommand('perfchecker.openInvestigationSource',candidate.origin.file,candidate.origin.line); const prepareLens=windowPage.locator('.codelens-decoration a').filter({hasText:'Prepare shared performance case'}).first(); - await prepareLens.waitFor({state:'visible',timeout:60000});await observeSurface(context,'proposal-codelens');await prepareLens.click(); + await prepareLens.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await observeSurface(context,'proposal-codelens');await prepareLens.click({timeout:NATIVE_READINESS_MS}); await eventually(()=>vscode.window.activeTextEditor?.document.isUntitled,'The real proposal CodeLens opens an untitled shared-case draft'); assert.match(vscode.window.activeTextEditor.document.getText(),/prepare/); assert.match(vscode.window.activeTextEditor.document.getText(),/verify/); @@ -213,7 +214,7 @@ async function resource(context,workspace,controller,threads,previousEditorActio const actualLenses=await vscode.commands.executeCommand('vscode.executeCodeLensProvider',sourceUri); const editorAction=actualLenses.find(item=>item.command?.title==='Diagnose native_editor_branch · conditional')?.command; assert.equal(editorAction?.arguments[1],uri.toString(),'The registered native CodeLens contains its owning workspace URI'); - await lens.waitFor({state:'visible',timeout:60000});await observeSurface(context,'diagnosis-codelens');before=await reports(root);await lens.click(); + await lens.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await observeSurface(context,'diagnosis-codelens');before=await reports(root);await lens.click({timeout:NATIVE_READINESS_MS}); const diagnosis=await reportAfter(root,before,'diagnosis'); const record=diagnosis.payload.records.find(item=>item.scenario==='native_editor_branch'); assert(record,'The clicked CodeLens diagnoses its own selected scenario'); @@ -230,7 +231,7 @@ async function resource(context,workspace,controller,threads,previousEditorActio {preview:false,selection:diagnostic.range}); await vscode.commands.executeCommand('editor.action.quickFix'); const action=windowPage.getByText('Read evidence and verification steps',{exact:true}); - await action.waitFor({state:'visible',timeout:30000}); + await action.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); if(context.editorQualification)context.log('native-editor-visible-quickfix',{entries:await action.count(),label:'Read evidence and verification steps',visible:await action.isVisible()}); const menu=windowPage.locator('.action-widget:visible');assert.equal(await menu.count(),1); const entries=await menu.getByRole('option').count();assert(entries>0); @@ -241,7 +242,7 @@ async function resource(context,workspace,controller,threads,previousEditorActio context.log('native-editor-quickfix-keyboard',{label:'Read evidence and verification steps',options:entries,arrowDown:next,exactRowFocused:true}); await windowPage.keyboard.press('Enter'); frame=await context.findFrame('#app'); - await eventually(async()=>(await frame.locator('body').innerText()).includes('native_editor_branch'), + await eventually(async()=>(await frame.locator('body').innerText({timeout:NATIVE_READINESS_MS})).includes('native_editor_branch'), 'The chosen native quick fix opens the evidence for the same workspace'); assert.equal(await fs.readFile(factory,'utf8'),source,'Quick fix opens evidence without editing source'); const currentSettings=vscode.workspace.getConfiguration('perfchecker',uri); @@ -257,9 +258,9 @@ async function resource(context,workspace,controller,threads,previousEditorActio }finally{ await vscode.commands.executeCommand('perfchecker.cancelInvestigation'); { - const current=await view(context); - await eventually(async()=>!((await current.locator('#app .status').getAttribute('class'))||'').includes('busy'), - 'Editor-action workers finish before restoring their fixture'); + const current=await view(context,120000); + await eventually(async()=>!((await current.locator('#app .status').getAttribute('class',{timeout:30000}))||'').includes('busy'), + 'Editor-action workers finish before restoring their fixture',180000); } for(const key of keys)await update(key,previous[key],'restore'); if(original)await fs.writeFile(catalog,original);else await fs.rm(catalog,{force:true}); @@ -289,7 +290,7 @@ exports.run=async context=>{ }); try{ added=vscode.workspace.updateWorkspaceFolders(vscode.workspace.workspaceFolders.length,0,{uri:secondUri,name:'Editor action second workspace'}); - assert(added);await eventually(()=>acknowledged,'The workbench acknowledges the second workspace and its configuration model',30000); + assert(added);await eventually(()=>acknowledged,'The workbench acknowledges the second workspace and its configuration model',NATIVE_READINESS_MS); }finally{changed.dispose();} assert.equal(vscode.workspace.getWorkspaceFolder(secondUri)?.uri.toString(),secondUri.toString()); context.log('native-editor-second-workspace-acknowledged',{resource:secondUri.toString(),actualWorkspaceEvent:true}); @@ -305,25 +306,25 @@ exports.run=async context=>{ exports.runSuiteLog=async context=>{ const {vscode,windowPage,workspace,controller}=context; await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',vscode.Uri.file(workspace)); - await vscode.commands.executeCommand('perfchecker.refresh'); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.refresh'),"perfchecker.refresh"); await vscode.commands.executeCommand('workbench.action.closePanel'); await vscode.commands.executeCommand('workbench.action.showCommands'); - const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible'}); - await picker.locator('input[type="text"]').fill('>PerfChecker: Show worker output'); - if(context.editorQualification){const action=picker.getByText('PerfChecker: Show worker output',{exact:true});await action.waitFor({state:'visible'}); + const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await picker.locator('input[type="text"]').fill('>PerfChecker: Show worker output',{timeout:NATIVE_READINESS_MS}); + if(context.editorQualification){const action=picker.getByText('PerfChecker: Show worker output',{exact:true});await action.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); context.log('native-editor-visible-palette',{command:'perfchecker.showLog',entries:await action.count(),visible:await action.isVisible()});} - await picker.getByText('PerfChecker: Show worker output',{exact:true}).click(); - await picker.waitFor({state:'hidden'}); + await picker.getByText('PerfChecker: Show worker output',{exact:true}).click({timeout:NATIVE_READINESS_MS}); + await picker.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS}); // OutputViewPane's official output-view class scopes the visible Monaco // editor; the native channel selector independently identifies the suite log. - const output=windowPage.locator('.output-view:visible');await output.waitFor({state:'visible'}); + const output=windowPage.locator('.output-view:visible');await output.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); const canonicalController=await fs.realpath(controller); // Monaco only renders the current viewport. Reveal the first physical log // line with a native editor gesture before asserting its visible contents. assert((await vscode.commands.getCommands(true)).includes('workbench.action.focusPanel')); await vscode.commands.executeCommand('workbench.action.focusPanel'); await eventually(()=>output.evaluate(element=>element.contains(document.activeElement)), - 'The native Focus into Panel command focuses the actual visible Output editor',30000); + 'The native Focus into Panel command focuses the actual visible Output editor',NATIVE_READINESS_MS); await windowPage.keyboard.press(process.platform==='darwin'?'Meta+Home':'Control+Home'); const file=`native-${process.platform}-vscode-${vscode.version}-suite-worker-output.png`; await windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,file)}); @@ -415,14 +416,14 @@ exports.runActiveSettings=async context=>{ await settings().update(key,value,vscode.ConfigurationTarget.WorkspaceFolder); await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',uri); let frame=await view(context);const root=path.join(workspace,'perf','results','active-settings'),before=await reports(root); - await frame.getByRole('button',{name:'Discover tests',exact:true}).click();await reportAfter(root,before,'discovery'); - await eventually(async()=>!((await frame.locator('#app .status').getAttribute('class'))||'').includes('busy'),'The actual declaration discovery finishes'); - await frame.getByRole('button',{name:'Scenarios',exact:true}).click();await frame.getByRole('button',{name:'Clear selection',exact:true}).click(); - await frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:'native_active_settings'})}).locator('.scenario-title input').check(); - await frame.getByRole('button',{name:'Measure selected',exact:true}).click(); + await frame.getByRole('button',{name:'Discover tests',exact:true}).click({timeout:NATIVE_READINESS_MS});await reportAfter(root,before,'discovery'); + await eventually(async()=>!((await frame.locator('#app .status').getAttribute('class',{timeout:NATIVE_READINESS_MS}))||'').includes('busy'),'The actual declaration discovery finishes'); + await frame.getByRole('button',{name:'Scenarios',exact:true}).click({timeout:NATIVE_READINESS_MS});await frame.getByRole('button',{name:'Clear selection',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:'native_active_settings'})}).locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS}); + await frame.getByRole('button',{name:'Measure selected',exact:true}).click({timeout:NATIVE_READINESS_MS}); const started=await eventually(async()=>{ const values=(await fs.readFile(marker,'utf8')).split('\n');return values.length===3?values:false; - },'The real owned workload records its process and original target environment',240000); + },'The real owned workload records its process and original target environment',NATIVE_READINESS_MS); ownedPid=Number(started[0]);assert(ownedPid>0&&alive(ownedPid)); assert.equal(vscode.Uri.file(started[1]).fsPath,vscode.Uri.file(controller).fsPath); assert.equal(started[2],process.env.PERFCHECKER_NATIVE_JULIA_VERSION); @@ -434,8 +435,8 @@ exports.runActiveSettings=async context=>{ const workerAliveAfterSettingsChanges=alive(ownedPid); context.log('native-active-settings-before-cancel',{pid:ownedPid,workerAlive:workerAliveAfterSettingsChanges, changedKeys:['runnerProject','scenarioProject','juliaExecutable']}); - frame=await view(context);await observeSurface(context,'active-settings-before-cancel',frame);await frame.getByRole('button',{name:'Cancel',exact:true}).click(); - await eventually(async()=>/Cancelled after controller cleanup/.test(await frame.locator('#app .status').innerText()),'Cancellation finishes despite changed controller/runtime settings',120000); + frame=await view(context);await observeSurface(context,'active-settings-before-cancel',frame);await frame.getByRole('button',{name:'Cancel',exact:true}).click({timeout:30000}); + await eventually(async()=>/Cancelled after controller cleanup/.test(await frame.locator('#app .status').innerText({timeout:30000})),'Cancellation finishes despite changed controller/runtime settings',120000); await eventually(()=>!alive(ownedPid),'The original owned worker is dead before any fixture cleanup',30000); assert.equal(settings().get('runnerProject'),nextController);assert.equal(settings().get('scenarioProject'),nextController);assert.equal(settings().get('juliaExecutable'),missingJulia); await assert.rejects(vscode.commands.executeCommand('perfchecker.discoverScenarios'),/ENOENT|spawn.*unavailable/i, diff --git a/test/native-external-browser.cjs b/test/native-external-browser.cjs index 3ca96e2..3722ac7 100644 --- a/test/native-external-browser.cjs +++ b/test/native-external-browser.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Disposable Linux default-browser fixture. env.openExternal is never mocked: // Electron launches xdg-open, which invokes this fixture's temporary handler. const fs=require('node:fs/promises'); @@ -32,12 +33,12 @@ exports.create=async(directory,{chromium,executablePath}={})=>{ assert(url.searchParams.get('secret')); if(origin){assert.equal(url.origin,origin);assert.equal(url.searchParams.get('secret'),secret);} else{origin=url.origin;secret=url.searchParams.get('secret');} - browser||=await chromium.launch({...(executablePath?{executablePath}:{}),headless:true,args:['--no-sandbox']}); + browser||=await chromium.launch({...(executablePath?{executablePath}:{}),headless:true,timeout:NATIVE_READINESS_MS,args:['--no-sandbox']}); const page=await browser.newPage(); const route=url.pathname;let proof; if(route==='/notebookexport'){ - const downloadPromise=page.waitForEvent('download'); - await page.goto(url.href).catch(error=>{if(!/ERR_ABORTED|Download is starting/i.test(error.message))throw error;}); + const downloadPromise=page.waitForEvent('download',{timeout:NATIVE_READINESS_MS}); + await page.goto(url.href,{timeout:NATIVE_READINESS_MS}).catch(error=>{if(!/ERR_ABORTED|Download is starting/i.test(error.message))throw error;}); const download=await downloadPromise,stream=await download.createReadStream(),chunks=[]; for await(const chunk of stream)chunks.push(chunk); const bytes=Buffer.concat(chunks),html=bytes.toString(); @@ -49,14 +50,14 @@ exports.create=async(directory,{chromium,executablePath}={})=>{ assert(!html.includes(secret)&&!source.includes(secret),'Exports never contain the session credential'); proof={route,kind:'html-download',bytes:bytes.length,sha256:createHash('sha256').update(bytes).digest('hex'),embeddedJulia:true,editedExpression:true,credentialAbsent:true}; }else if(route==='/notebookfile'){ - const result=await page.goto(url.href);assert.equal(result.status(),200); - const source=await page.locator('body').innerText(); + const result=await page.goto(url.href,{timeout:NATIVE_READINESS_MS});assert.equal(result.status(),200); + const source=await page.locator('body').innerText({timeout:NATIVE_READINESS_MS}); assert(source.startsWith('### A Pluto.jl notebook ###'));assert.match(source,/4 \+ 5/);assert(!source.includes(secret)); proof={route,kind:'julia-source-browser',bytes:Buffer.byteLength(source),sha256:createHash('sha256').update(source).digest('hex'),editedExpression:true,credentialAbsent:true}; }else{ let authenticatedWebSocket=false; page.on('websocket',socket=>{const ws=new URL(socket.url());if(ws.host===url.host&&ws.searchParams.get('secret')===secret)authenticatedWebSocket=true;}); - await page.goto(url.href);await page.locator('pluto-notebook').waitFor({timeout:90000}); + await page.goto(url.href,{timeout:NATIVE_READINESS_MS});await page.locator('pluto-notebook').waitFor({timeout:NATIVE_READINESS_MS}); const final=new URL(page.url());assert.equal(final.origin,origin);assert.equal(final.pathname,'/edit');assert.equal(final.searchParams.get('secret'),secret); assert(authenticatedWebSocket,'The actual new browser notebook connects its authenticated WebSocket'); proof={route,kind:'new-context-editor',authenticatedWebSocket:true,notebookId:final.searchParams.get('id')}; diff --git a/test/native-general100-controls.cjs b/test/native-general100-controls.cjs index 6722919..0eda90d 100644 --- a/test/native-general100-controls.cjs +++ b/test/native-general100-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); // Compatibility of the candidate VSIX with the registered General 1.0.0 tree. // APIs checked against v1.0.0: testitems CLI (src/testitems.jl), FeatureSpec / // plan_suite / run_suite / read_run_bundle (src/suites.jl, src/protocol.jl), @@ -7,10 +8,10 @@ const assert=require('node:assert/strict'),fs=require('node:fs/promises'),path=r const {createHash}=require('node:crypto'),{execFile}=require('node:child_process'),{promisify}=require('node:util'); const execute=promisify(execFile),hash=bytes=>createHash('sha256').update(bytes).digest('hex'); const documents=['manifest.json','measurement-definitions.json','observations.jsonl','diagnostics.jsonl','artifacts.json','integrity.json']; -async function eventually(read,label,timeout=180000){const until=Date.now()+timeout;do{const value=await read();if(value)return value;await new Promise(resolve=>setTimeout(resolve,100));}while(Date.now()setTimeout(resolve,100));}while(Date.now()({returned:true}),error=>({error})); - const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:180000}); - await picker.locator('.monaco-list-row').filter({hasText:'Use an existing controller'}).click(); + const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await picker.locator('.monaco-list-row').filter({hasText:'Use an existing controller'}).click({timeout:NATIVE_READINESS_MS}); const dialog=windowPage.locator('.quick-input-widget').filter({has:windowPage.locator('.quick-input-title').filter({hasText:/^PerfChecker · Choose controller project$/})}); - const input=dialog.locator('input[type="text"]');await input.waitFor({state:'visible',timeout:60000}); - await input.fill(context.controller+path.sep);await input.press('Enter');await input.waitFor({state:'hidden',timeout:60000}); + const input=dialog.locator('input[type="text"]');await input.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await input.fill(context.controller+path.sep,{timeout:NATIVE_READINESS_MS});await input.press('Enter',{timeout:NATIVE_READINESS_MS});await input.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS}); guidedResult=await guided; assert(guidedResult.error,'Choosing the real registered100 controller must fail its actual minimum-version check'); assert.match(String(guidedResult.error),/requires registered PerfChecker 1\.0\.1/); @@ -35,8 +36,8 @@ async function prerequisites(context){ actualVersionCheckRejected:true,error:String(guidedResult.error),upgradeNotAccepted:true,bytesPreserved:true,positiveSetupQualified:false}); const notebook=path.join(workspace,'perf','notebooks','general100-prerequisite.jl'); const requested=vscode.commands.executeCommand('perfchecker.newNotebook',vscode.Uri.file(notebook),{kind:'suite'}); - await windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}).waitFor({timeout:180000}); - assert.match(await windowPage.locator('.monaco-dialog-box').innerText(),/PerfChecker 1\.0\.1.*PerfCheckerPluto 1\.0\.1/s); + await windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}).waitFor({timeout:NATIVE_READINESS_MS}); + assert.match(await windowPage.locator('.monaco-dialog-box').innerText({timeout:NATIVE_READINESS_MS}),/PerfChecker 1\.0\.1.*PerfCheckerPluto 1\.0\.1/s); await windowPage.keyboard.press('Escape');assert.equal(await requested,undefined); assert.equal(await fs.stat(path.join(workspace,'perf','pluto','Project.toml')).then(()=>true).catch(error=>{if(error.code==='ENOENT')return false;throw error;}),false); assert.deepEqual(await hashes(),before); @@ -52,28 +53,28 @@ exports.run=async context=>{ await prerequisites(context); await context.measureTestItem({samples:1,proofName:'native-general100-testing-evidence'}); const {vscode,workspace,findFrame,proof}=context; - await vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',vscode.Uri.file(workspace)); - let designer=await findFrame('#cards');await designer.locator('#reset-filters').click();await designer.locator('#clear-all').click(); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',vscode.Uri.file(workspace)),"perfchecker.openDesignerForWorkspace"); + let designer=await findFrame('#cards');await designer.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS});await designer.locator('#clear-all').click({timeout:NATIVE_READINESS_MS}); const card=designer.locator('#cards .card').filter({has:designer.locator('.package',{hasText:/^PerfCheckerNativeFixture$/})}).filter({has:designer.locator('strong',{hasText:/^sum_squares$/})}).first(); - const check=card.locator('.check-option.ready').filter({hasText:'BenchmarkTools'});assert.equal(await check.count(),1);await check.locator('input').check(); - assert.match(await designer.locator('#count').innerText(),/^1 selected/);await designer.locator('#open-after-run').uncheck();await designer.locator('#run').click(); + const check=card.locator('.check-option.ready').filter({hasText:'BenchmarkTools'});assert.equal(await check.count(),1);await check.locator('input').check({timeout:NATIVE_READINESS_MS}); + assert.match(await designer.locator('#count').innerText({timeout:NATIVE_READINESS_MS}),/^1 selected/);await designer.locator('#open-after-run').uncheck({timeout:NATIVE_READINESS_MS});await designer.locator('#run').click({timeout:NATIVE_READINESS_MS}); const reports=path.join(workspace,'perf','results','vscode'); - await eventually(async()=>{const file=await fs.readFile(path.join(reports,'suite-result.json')).catch(error=>{if(error.code==='ENOENT')return undefined;throw error;});if(!file)return false;let value;try{value=JSON.parse(file);}catch(error){if(error instanceof SyntaxError)return false;throw error;}return value.runs.length===1&&value.runs[0].status==='pass'&&!(await designer.locator('#run').isDisabled());},'General100 completes the single real selected suite',360000); + await eventually(async()=>{const file=await fs.readFile(path.join(reports,'suite-result.json')).catch(error=>{if(error.code==='ENOENT')return undefined;throw error;});if(!file)return false;let value;try{value=JSON.parse(file);}catch(error){if(error instanceof SyntaxError)return false;throw error;}return value.runs.length===1&&value.runs[0].status==='pass'&&!(await designer.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS}));},'General100 completes the single real selected suite',NATIVE_READINESS_MS); await vscode.commands.executeCommand('perfchecker.openOutput');const output=await findFrame('button[data-report="suite-result.json"]'); assert(await output.locator('[data-result-item]').count()>0); const temporal=output.locator('.chart-card').filter({has:output.locator('header strong',{hasText:/^julia\.wall\.time distribution$/})}).first().locator('.distribution-view'); assert.equal(await temporal.count(),1,'Use the actual timing distribution, not constant allocation or GC measurements'); - const full={min:Number(await temporal.getAttribute('data-full-min')),max:Number(await temporal.getAttribute('data-full-max'))}; + const full={min:Number(await temporal.getAttribute('data-full-min',{timeout:NATIVE_READINESS_MS})),max:Number(await temporal.getAttribute('data-full-max',{timeout:NATIVE_READINESS_MS}))}; assert(full.max>full.min,'The actual temporal samples must have nonzero extent'); - const range=async()=>({min:Number(await temporal.locator('svg').getAttribute('data-current-min')),max:Number(await temporal.locator('svg').getAttribute('data-current-max'))}); + const range=async()=>({min:Number(await temporal.locator('svg').getAttribute('data-current-min',{timeout:NATIVE_READINESS_MS})),max:Number(await temporal.locator('svg').getAttribute('data-current-max',{timeout:NATIVE_READINESS_MS}))}); const fit=temporal.getByRole('button',{name:'Fit all samples',exact:true}),zoom=temporal.getByRole('button',{name:'Zoom in',exact:true}); - assert.deepEqual(await range(),full);assert(await fit.isDisabled());assert(await zoom.isEnabled()); - const points=temporal.locator('.sample'),pointCount=await points.count();assert(pointCount>1);await points.first().focus(); - assert((await temporal.locator('.plot-detail').innerText()).includes('sorted sample')); + assert.deepEqual(await range(),full);assert(await fit.isDisabled({timeout:NATIVE_READINESS_MS}));assert(await zoom.isEnabled({timeout:NATIVE_READINESS_MS})); + const points=temporal.locator('.sample'),pointCount=await points.count();assert(pointCount>1);await points.first().focus({timeout:NATIVE_READINESS_MS}); + assert((await temporal.locator('.plot-detail').innerText({timeout:NATIVE_READINESS_MS})).includes('sorted sample')); const before=await fs.readFile(path.join(reports,'version-series.json')); - await zoom.click();const zoomed=await eventually(async()=>{const current=await range();return current.max-current.min=full.min&&zoomed.max<=full.max);await fit.click(); - await eventually(async()=>{const current=await range();return current.min===full.min&¤t.max===full.max&&await fit.isDisabled();},'Native Fit restores the exact initial full temporal range'); + await zoom.click({timeout:NATIVE_READINESS_MS});const zoomed=await eventually(async()=>{const current=await range();return current.max-current.min=full.min&&zoomed.max<=full.max);await fit.click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>{const current=await range();return current.min===full.min&¤t.max===full.max&&await fit.isDisabled({timeout:NATIVE_READINESS_MS});},'Native Fit restores the exact initial full temporal range'); assert.equal(await points.count(),pointCount);assert.deepEqual(await fs.readFile(path.join(reports,'version-series.json')),before); proof('native-general100-suite-output',{selectedChecks:1,collector:'BenchmarkTools',realRunSelection:true,realSavedOutput:true, metric:'julia.wall.time',pointFocus:true,pointCount,fullRange:full,zoomedRange:zoomed,fitRestoresExactFullRange:true,reportBytesPreserved:true,noComparisonGainClaimed:true}); @@ -88,11 +89,11 @@ exports.measuredEvidence=async context=>{ const root=path.resolve(workspace,settings.get('investigationReports','perf/results/investigations')); const entries=()=>fs.readdir(root).catch(error=>{if(error.code==='ENOENT')return [];throw error;});const previous=new Set(await entries()); await vscode.commands.executeCommand('perfchecker.openInvestigations');const frame=await findFrame('#app nav[aria-label="Investigation views"]'); - await frame.getByRole('button',{name:'Discover tests',exact:true}).click(); - await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class')).includes('busy')&&await frame.locator('.scenario-title strong').filter({hasText:/^advisor_sum_squares$/}).count()===1,'General100 discovers the actual advisor scenario',240000); - await frame.getByRole('button',{name:'Scenarios',exact:true}).click();await frame.getByRole('button',{name:'Clear selection',exact:true}).click(); + await frame.getByRole('button',{name:'Discover tests',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class',{timeout:NATIVE_READINESS_MS})).includes('busy')&&await frame.locator('.scenario-title strong').filter({hasText:/^advisor_sum_squares$/}).count()===1,'General100 discovers the actual advisor scenario',NATIVE_READINESS_MS); + await frame.getByRole('button',{name:'Scenarios',exact:true}).click({timeout:NATIVE_READINESS_MS});await frame.getByRole('button',{name:'Clear selection',exact:true}).click({timeout:NATIVE_READINESS_MS}); const card=frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:/^advisor_sum_squares$/})}).filter({has:frame.locator('.implementation',{hasText:/^allocating$/})}); - assert.equal(await card.count(),1);await card.locator('.scenario-title input').check();await frame.getByRole('button',{name:'Measure selected',exact:true}).click(); + assert.equal(await card.count(),1);await card.locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS});await frame.getByRole('button',{name:'Measure selected',exact:true}).click({timeout:NATIVE_READINESS_MS}); let result; await eventually(async()=>{for(const id of await entries()){ if(previous.has(id))continue;const file=path.join(root,id,'run.json'),adviceFile=path.join(root,id,'advice','advice.json'); @@ -105,8 +106,8 @@ exports.measuredEvidence=async context=>{ const projection=await core(context,'using PerfChecker; advice=read_advice(ARGS[1]); config=AdvisorConfig(protocol=:mcp_http,mcp_tool="ask_perfchecker",mcp_response=:text); evidence=PerfChecker._advisor_evidence(advice,config); PerfChecker.JSON.print(Dict("evidence"=>evidence,"characters"=>length(PerfChecker.JSON.json(evidence)),"budget"=>config.max_evidence_chars))',[adviceFile]); assert(projection.evidence.length>0);assert(projection.characters<=projection.budget);assert(projection.evidence.every(row=>typeof row.rule==='string'&&!Object.hasOwn(row,'metric'))); result={id,file,adviceFile,evidence:projection.evidence,recommendations:advice.recommendations,evidenceCharacters:projection.characters,evidenceBudget:projection.budget,runSha256:hash(bytes),adviceSha256:hash(adviceBytes)}; - return !(await frame.locator('#app .status').getAttribute('class')).includes('busy'); - }return false;},'General100 saves its real measurements and legacy deterministic advice',360000); + return !(await frame.locator('#app .status').getAttribute('class',{timeout:NATIVE_READINESS_MS})).includes('busy'); + }return false;},'General100 saves its real measurements and legacy deterministic advice',NATIVE_READINESS_MS); return result; }; exports.suiteRefusal=async(context,{state,calls,receipts})=>{ @@ -117,25 +118,25 @@ exports.suiteRefusal=async(context,{state,calls,receipts})=>{ const hashes=async()=>Object.fromEntries(await Promise.all(files.map(async file=>[path.relative(root,file),hash(await fs.readFile(file))])));const before=await hashes(); const projection=await core(context,'using PerfChecker; bundle=read_run_bundle(ARGS[1];require_integrity=true); advice=advise(bundle); @assert !haskey(advice,"measurement_summaries"); PerfChecker.JSON.print(Dict("version"=>string(Base.pkgversion(PerfChecker)),"canonicalSummariesAvailable"=>false))',[directory]); assert.equal(projection.version,'1.0.0');const beforeCalls=calls.length,beforeReceipts=receipts.length; - await vscode.commands.executeCommand('perfchecker.openChat');const chat=await findFrame('#chat-root');await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click(); + await vscode.commands.executeCommand('perfchecker.openChat');const chat=await findFrame('#chat-root');await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click({timeout:NATIVE_READINESS_MS}); const entry=await eventually(async()=>(await state()).evidence.find(item=>item.id.startsWith('suite:')&&JSON.parse(item.id.slice(6))[1]===series.run_id&&!item.unavailable),'The real General100 bundle appears in Chat'); assert.equal(JSON.parse(entry.id.slice(6))[0],vscode.Uri.file(workspace).toString()); - await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(entry.id); + await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(entry.id,{timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return value.evidenceId===entry.id&&!value.busy&&value.messages.length===0;},'Suite selection is local and idle'); assert.equal(calls.length,beforeCalls);assert.equal(receipts.length,beforeReceipts); - await chat.locator('#chat-question').fill('Explain the saved suite without inventing canonical measurements.');await chat.getByRole('button',{name:'Send question',exact:true}).click(); + await chat.locator('#chat-question').fill('Explain the saved suite without inventing canonical measurements.',{timeout:NATIVE_READINESS_MS});await chat.getByRole('button',{name:'Send question',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return !value.busy&&/1\.0\.0 does not provide/.test(value.status)&&/No measurements were sent/.test(value.status);},'General100 explicitly refuses canonical Suite transmission'); assert.equal(calls.length,beforeCalls);assert.equal(receipts.length,beforeReceipts);assert.deepEqual((await state()).messages,[]);assert.deepEqual(await hashes(),before); proof('native-general100-suite-canonical-refusal',{runId:series.run_id,actualCoreProjection:projection,nativePicker:true,newProviderRequests:0,noInventedSummaries:true,uiIdle:true,noConversationRecorded:true,protocolAndIdentityHashes:before}); let original; try{original=await fs.readFile(path.join(directory,'observations.jsonl'));await fs.writeFile(path.join(directory,'observations.jsonl'),Buffer.concat([original,Buffer.from('\n')])); - await chat.locator('#chat-question').fill('Do not send the modified bundle.');await chat.getByRole('button',{name:'Send question',exact:true}).click(); + await chat.locator('#chat-question').fill('Do not send the modified bundle.',{timeout:NATIVE_READINESS_MS});await chat.getByRole('button',{name:'Send question',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return !value.busy&&/byte integrity check.*No measurements were sent/.test(value.status);},'General100 also refuses actual bundle corruption before provider'); assert.equal(calls.length,beforeCalls);assert.equal(receipts.length,beforeReceipts);assert.deepEqual((await state()).messages,[]); const changed={...before,[path.relative(root,path.join(directory,'observations.jsonl'))]:hash(Buffer.concat([original,Buffer.from('\n')]))};assert.deepEqual(await hashes(),changed); proof('native-general100-suite-corruption-refusal',{runId:series.run_id,actualByteChange:true,newProviderRequests:0,identityReportsPreserved:true}); }finally{if(original)await fs.writeFile(path.join(directory,'observations.jsonl'),original);} - assert.deepEqual(await hashes(),before);await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(''); + assert.deepEqual(await hashes(),before);await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption('',{timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return value.evidenceId===''&&!value.busy&&value.messages.length===0;},'Restore configuration-only Chat after General100 refusal'); proof('native-general100-suite-restored',{runId:series.run_id,allEightHashesPreserved:true,hashes:before});return chat; }; diff --git a/test/native-investigation-controls.cjs b/test/native-investigation-controls.cjs index 8f53ada..99c9248 100644 --- a/test/native-investigation-controls.cjs +++ b/test/native-investigation-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Real public VSIX controls, real General Julia workers, and disposable CI fixtures only. const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); @@ -9,7 +10,7 @@ const {tmpdir} = require('node:os'); const reportNames = {discover: 'discovery', run: 'run', diagnose: 'diagnosis', advise: 'advice', compare: 'comparison', tools: 'tools', sync: 'sync', investigate: 'investigation'}; -async function eventually(read, description, timeout = 120000) { +async function eventually(read, description, timeout = NATIVE_READINESS_MS) { const deadline = Date.now() + timeout; let last; while (Date.now() < deadline) { @@ -23,17 +24,17 @@ function configuration(context) { return context.vscode.workspace.getConfiguration('perfchecker', context.vscode.Uri.file(context.workspace)); } -async function view(context) { +async function view(context,timeout=NATIVE_READINESS_MS) { await context.vscode.commands.executeCommand('perfchecker.openInvestigations'); - const result = await context.findFrame('#app nav[aria-label="Investigation views"]'); - await result.locator('#app p.status').waitFor({state: 'visible'}); + const result = await context.findFrame('#app nav[aria-label="Investigation views"]',{timeout}); + await result.locator('#app p.status').waitFor({state: 'visible',timeout}); return result; } async function tab(context, name) { const result = await view(context); - await result.getByRole('button', {name, exact: true}).click(); - await eventually(async () => (await result.getByRole('button', {name, exact: true}).getAttribute('aria-current')) === 'page', `${name} investigation view`); + await result.getByRole('button', {name, exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => (await result.getByRole('button', {name, exact: true}).getAttribute('aria-current',{timeout:NATIVE_READINESS_MS})) === 'page', `${name} investigation view`); return result; } @@ -47,12 +48,12 @@ function reportRoot(context) { return path.resolve(context.workspace, configuration(context).get('investigationReports', 'perf/results/investigations')); } -async function reportAfter(context, before, action, timeout = 360000) { +async function reportAfter(context, before, action, timeout = NATIVE_READINESS_MS) { const root = reportRoot(context); const observationStarted=Date.now(),observationDeadline=observationStarted+timeout;let nextObservation=0; if(action==='diagnose')context.log('investigation-diagnosis-observation-budget',{ timeoutMilliseconds:timeout,startedAt:new Date(observationStarted).toISOString(),deadlineAt:new Date(observationDeadline).toISOString(), - scope:'Native test observation only; analyzer and investigation execution budgets are unchanged'}); + scope:'One-hour availability and positive analyzer fixture execution; intentional timeout/budget fixtures remain separate and unchanged'}); let reportObserved=false,cleanupConfirmed=false,reportObservedAt,cleanupConfirmedAt; try{ const found = await eventually(async () => { @@ -74,8 +75,8 @@ async function reportAfter(context, before, action, timeout = 360000) { reportObserved=true; reportObservedAt=new Date().toISOString(); const current = await view(context); - await eventually(async () => !(await current.locator('#app .status').getAttribute('class')).includes('busy'), `${action} controller completes cleanup`, Math.min(timeout,360000)); - assert.equal(await current.getByRole('button', {name: 'Cancel', exact: true}).isDisabled(), true); + await eventually(async () => !(await current.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'), `${action} controller completes cleanup`, Math.min(timeout,360000)); + assert.equal(await current.getByRole('button', {name: 'Cancel', exact: true}).isDisabled({timeout:30000}), true); cleanupConfirmed=true; cleanupConfirmedAt=new Date().toISOString(); return found; @@ -86,7 +87,7 @@ async function reportAfter(context, before, action, timeout = 360000) { } } -async function action(context, label, name, timeout = 360000) { +async function action(context, label, name, timeout = NATIVE_READINESS_MS) { if(name==='diagnose'&&context.observeOperation)return context.observeOperation('diagnose',()=>performAction(context,label,name,timeout),Date.now()+timeout); return performAction(context,label,name,timeout); } @@ -95,7 +96,7 @@ async function performAction(context, label, name, timeout) { const before = await directories(reportRoot(context)); const current = await view(context); await context.observeWork?.(`investigation-${name}-before-action`); - await current.getByRole('button', {name: label, exact: true}).click(); + await current.getByRole('button', {name: label, exact: true}).click({timeout:NATIVE_READINESS_MS}); const result=await reportAfter(context, before, name, timeout); await context.observeWork?.(`investigation-${name}-completed-before-teardown`); return result; @@ -103,26 +104,26 @@ async function performAction(context, label, name, timeout) { async function quickPick(context, title, choice = 0) { const widget = context.windowPage.locator('.quick-input-widget'); - await widget.waitFor({state: 'visible', timeout: 30000}); - await eventually(async () => (await widget.innerText()).includes(title), `native quick pick ${title}`, 30000); + await widget.waitFor({state: 'visible', timeout: NATIVE_READINESS_MS}); + await eventually(async () => (await widget.innerText({timeout:NATIVE_READINESS_MS})).includes(title), `native quick pick ${title}`, NATIVE_READINESS_MS); const choices = widget.locator('.monaco-list-row[role="option"]'); if (typeof choice === 'string') { const selected = choices.filter({hasText: choice}); - await eventually(async () => (await selected.count()) === 1, `saved evidence ${choice}`, 30000); - await selected.click(); + await eventually(async () => (await selected.count()) === 1, `saved evidence ${choice}`, NATIVE_READINESS_MS); + await selected.click({timeout:NATIVE_READINESS_MS}); } else { - await eventually(async () => (await choices.count()) > choice, `saved evidence choice ${choice}`, 30000); - await choices.nth(choice).click(); + await eventually(async () => (await choices.count()) > choice, `saved evidence choice ${choice}`, NATIVE_READINESS_MS); + await choices.nth(choice).click({timeout:NATIVE_READINESS_MS}); } } async function selectScenario(context, id, implementation) { const current = await tab(context, 'Scenarios'); - await current.getByRole('button', {name: 'Clear selection', exact: true}).click(); + await current.getByRole('button', {name: 'Clear selection', exact: true}).click({timeout:NATIVE_READINESS_MS}); const cards = current.locator('article.card').filter({has: current.locator('.scenario-title strong', {hasText: id})}); const card = cards.filter({has: current.locator('.implementation', {hasText: implementation})}).first(); - await card.locator('.scenario-title input').check(); - assert.match(await current.locator('#app').innerText(), /1 selected/); + await card.locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS}); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /1 selected/); return current; } @@ -133,7 +134,7 @@ async function selectTools(context, names) { for (const text of tools) { const tool = text.split(' · ')[0]; const input = fieldset.locator('label').filter({hasText: text}).locator('input'); - if (names.includes(tool)) await input.check(); else await input.uncheck(); + if (names.includes(tool)) await input.check({timeout:NATIVE_READINESS_MS}); else await input.uncheck({timeout:NATIVE_READINESS_MS}); } for (const name of names) assert(tools.some(text => text.startsWith(`${name} ·`)), `${name} analyzer is declared by General controller`); return current; @@ -148,29 +149,29 @@ async function discover(context) { assert(result.report.analyzers.some(item => item.tool === 'jet')); const current = await tab(context, 'Scenarios'); for (const name of ['Scenarios', 'Findings & advice', 'Before / after', 'Saved evidence']) { - await current.getByRole('button', {name, exact: true}).click(); - assert.equal(await current.getByRole('button', {name, exact: true}).getAttribute('aria-current'), 'page'); + await current.getByRole('button', {name, exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await current.getByRole('button', {name, exact: true}).getAttribute('aria-current',{timeout:NATIVE_READINESS_MS}), 'page'); } - await current.getByRole('button', {name: 'Scenarios', exact: true}).click(); - await current.getByRole('button', {name: 'Select all declared', exact: true}).click(); + await current.getByRole('button', {name: 'Scenarios', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await current.getByRole('button', {name: 'Select all declared', exact: true}).click({timeout:NATIVE_READINESS_MS}); const selected = result.report.declared.length; - assert.match(await current.locator('#app').innerText(), new RegExp(`${selected} selected`)); - await current.getByRole('searchbox', {name: 'Filter scenarios'}).fill('absent-native-fixture'); - await current.getByRole('searchbox', {name: 'Filter scenarios'}).press('Tab'); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), new RegExp(`${selected} selected`)); + await current.getByRole('searchbox', {name: 'Filter scenarios'}).fill('absent-native-fixture',{timeout:NATIVE_READINESS_MS}); + await current.getByRole('searchbox', {name: 'Filter scenarios'}).press('Tab',{timeout:NATIVE_READINESS_MS}); assert.equal(await current.locator('.scenario-title').count(), 0); - assert.match(await current.locator('#app').innerText(), new RegExp(`${selected} selected`)); - await current.getByRole('button', {name: 'Clear selection', exact: true}).click(); - assert.equal(await current.getByRole('button', {name: 'Measure selected', exact: true}).isDisabled(), true); - await current.getByRole('searchbox', {name: 'Filter scenarios'}).fill(''); - await current.getByRole('searchbox', {name: 'Filter scenarios'}).press('Tab'); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), new RegExp(`${selected} selected`)); + await current.getByRole('button', {name: 'Clear selection', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await current.getByRole('button', {name: 'Measure selected', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), true); + await current.getByRole('searchbox', {name: 'Filter scenarios'}).fill('',{timeout:NATIVE_READINESS_MS}); + await current.getByRole('searchbox', {name: 'Filter scenarios'}).press('Tab',{timeout:NATIVE_READINESS_MS}); await selectScenario(context, 'sum_squares', 'allocating'); await selectTools(context, []); - assert.equal(await current.getByRole('button', {name: 'Diagnose selected', exact: true}).isDisabled(), true); - assert.equal(await current.getByRole('button', {name: 'Investigate selected', exact: true}).isDisabled(), true); + assert.equal(await current.getByRole('button', {name: 'Diagnose selected', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), true); + assert.equal(await current.getByRole('button', {name: 'Investigate selected', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), true); await selectTools(context, ['jet', 'alloccheck', 'latency']); const candidate = result.report.candidates[0]; - await current.locator('article.proposal').first().getByRole('button', {name: 'Prepare shared case', exact: true}).click(); + await current.locator('article.proposal').first().getByRole('button', {name: 'Prepare shared case', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => context.vscode.window.activeTextEditor?.document.isUntitled, 'Prepare shared case opens a real untitled Julia draft'); assert(context.vscode.window.activeTextEditor.document.getText().includes('prepare')); const source = result.report.declared.find(item => item.id === 'sum_squares').source; @@ -185,17 +186,17 @@ async function discover(context) { async function adopt(context, id, factory, parameters = {}) { const current = await tab(context, 'Scenarios'); const form = current.locator('details.card').filter({has: current.locator('summary', {hasText: 'Adopt a shared factory into the catalogue'})}); - await form.locator('summary').click(); - await form.locator('input[name="id"]').fill(id); - await form.locator('input[name="source"]').fill('perf/cases.jl'); - await form.locator('input[name="factory"]').fill(factory); - await form.locator('input[name="implementation"]').fill('ui'); - await form.locator('textarea').nth(0).fill('{ invalid'); - await form.getByRole('button', {name: 'Add declaration', exact: true}).click(); - assert.match(await form.getByRole('alert').innerText(), /Invalid input/); - await form.locator('textarea').nth(0).fill(JSON.stringify(parameters)); + await form.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="id"]').fill(id,{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="source"]').fill('perf/cases.jl',{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="factory"]').fill(factory,{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="implementation"]').fill('ui',{timeout:NATIVE_READINESS_MS}); + await form.locator('textarea').nth(0).fill('{ invalid',{timeout:NATIVE_READINESS_MS}); + await form.getByRole('button', {name: 'Add declaration', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await form.getByRole('alert').innerText({timeout:NATIVE_READINESS_MS}), /Invalid input/); + await form.locator('textarea').nth(0).fill(JSON.stringify(parameters),{timeout:NATIVE_READINESS_MS}); const before = await directories(reportRoot(context)); - await form.getByRole('button', {name: 'Add declaration', exact: true}).click(); + await form.getByRole('button', {name: 'Add declaration', exact: true}).click({timeout:NATIVE_READINESS_MS}); const result = await reportAfter(context, before, 'discover'); assert(result.report.declared.some(item => item.id === id && item.factory === factory)); const catalog = path.resolve(context.workspace, configuration(context).get('scenarioCatalog', 'perf/scenarios.toml')); @@ -215,15 +216,15 @@ async function rejectAdoption(context) { for (const testCase of cases) { const current = await tab(context, 'Scenarios'); const form = current.locator('details.card').filter({has: current.locator('summary', {hasText: 'Adopt a shared factory into the catalogue'})}); - if ((await form.getAttribute('open')) === null) await form.locator('summary').click(); - await form.locator('input[name="id"]').fill(testCase.id); - await form.locator('input[name="source"]').fill('perf/cases.jl'); - await form.locator('input[name="factory"]').fill('make_sum_case'); - await form.locator('input[name="implementation"]').fill('ui'); - await form.locator('textarea').nth(0).fill(testCase.parameters); - await form.locator('textarea').nth(1).fill(testCase.fixtures); - await form.getByRole('button', {name: 'Add declaration', exact: true}).click(); - await eventually(async () => testCase.error.test(await current.locator('#app .status').innerText()), `Invalid or duplicate declaration ${testCase.id} is rejected`); + if ((await form.getAttribute('open',{timeout:NATIVE_READINESS_MS})) === null) await form.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="id"]').fill(testCase.id,{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="source"]').fill('perf/cases.jl',{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="factory"]').fill('make_sum_case',{timeout:NATIVE_READINESS_MS}); + await form.locator('input[name="implementation"]').fill('ui',{timeout:NATIVE_READINESS_MS}); + await form.locator('textarea').nth(0).fill(testCase.parameters,{timeout:NATIVE_READINESS_MS}); + await form.locator('textarea').nth(1).fill(testCase.fixtures,{timeout:NATIVE_READINESS_MS}); + await form.getByRole('button', {name: 'Add declaration', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => testCase.error.test(await current.locator('#app .status').innerText({timeout:NATIVE_READINESS_MS})), `Invalid or duplicate declaration ${testCase.id} is rejected`); assert.deepEqual(await fs.readFile(catalog), before, 'Invalid input does not alter the actual catalogue'); } context.proof('investigation-adoption-types-and-duplicate-rejected', {parametersArray: true, fixturesObject: true, duplicate: true, catalogUnchanged: true}); @@ -242,10 +243,10 @@ async function measure(context) { const current = await tab(context, 'Findings & advice'); const chart = current.locator('svg.sample-chart'); assert(await chart.count(), 'Actual measured observations produce a sample chart'); - await chart.locator('circle').first().focus(); - assert.match(await chart.locator('circle').first().getAttribute('aria-label'), /Displayed sample/); - await current.getByText('Qualification and measurement evidence', {exact: true}).click(); - assert.match(await current.locator('#app').innerText(), /passed/); + await chart.locator('circle').first().focus({timeout:NATIVE_READINESS_MS}); + assert.match(await chart.locator('circle').first().getAttribute('aria-label',{timeout:NATIVE_READINESS_MS}), /Displayed sample/); + await current.getByText('Qualification and measurement evidence', {exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /passed/); context.proof('investigation-real-measurement', {scenario: run.scenario.id, collector: run.collector, correctness: run.qualification.correctness, samples: run.summaries[0]?.samples, report: result.location}); context.investigationFirstRun ||= result; return result; @@ -255,9 +256,9 @@ async function diagnose(context) { await selectScenario(context, 'ui_adopted', 'ui'); const tools = ['jet', 'aqua', 'alloccheck', 'snoopcompile', 'latency', 'gc', 'memory', 'heap', 'locks']; await selectTools(context, tools); - // Nine real analyzers can outlive six minutes on a cold macOS CI host. - // This observation allowance leaves every execution limit and assertion intact. - const result = await action(context, 'Diagnose selected', 'diagnose', process.platform==='darwin'?900000:360000); + // Positive analyzer availability and execution use the requested one-hour + // fixture allowance; timeout/budget refusal fixtures remain separate. + const result = await action(context, 'Diagnose selected', 'diagnose', NATIVE_READINESS_MS); assert.equal(result.report.schema_version, 'perfchecker-diagnosis/1'); context.log('investigation-diagnosis-record-statuses',{records:result.report.records.slice(0,16).map(record=>({ tool:tools.includes(record.tool)?record.tool:'unlisted', @@ -291,7 +292,7 @@ async function diagnose(context) { assert.equal(createHash('sha256').update(bytes).digest('hex'), artifact.sha256); const cardFor = frame => frame.locator('article.card').filter({has: frame.getByRole('heading', {name: `${artifactRecord.scenario} · ${artifactRecord.implementation} · ${artifactRecord.tool}`, exact: true})}); const recordCard = cardFor(current); - await recordCard.getByRole('button', {name: `Open ${artifact.kind}`, exact: true}).first().click(); + await recordCard.getByRole('button', {name: `Open ${artifact.kind}`, exact: true}).first().click({timeout:NATIVE_READINESS_MS}); const artifactUri=context.vscode.Uri.file(artifact.path); await eventually(() => context.vscode.window.tabGroups.all.some(group=>group.tabs.some(tab=>tab.isActive&&tab.input?.uri?.fsPath===artifactUri.fsPath)), 'Evidence artifact opens in the actual native text or custom editor'); const opened=context.vscode.window.tabGroups.all.flatMap(group=>group.tabs).find(tab=>tab.isActive&&tab.input?.uri?.fsPath===artifactUri.fsPath); @@ -300,8 +301,8 @@ async function diagnose(context) { try { // Opening a native editor hides and unloads this non-retained webview. const reopened = await tab(context, 'Findings & advice'); - await cardFor(reopened).getByRole('button', {name: `Open ${artifact.kind}`, exact: true}).first().click(); - await eventually(async () => /Artifact changed/.test(await reopened.locator('#app .status').innerText()), 'Changed artifact is rejected by digest'); + await cardFor(reopened).getByRole('button', {name: `Open ${artifact.kind}`, exact: true}).first().click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => /Artifact changed/.test(await reopened.locator('#app .status').innerText({timeout:NATIVE_READINESS_MS})), 'Changed artifact is rejected by digest'); } finally {await fs.writeFile(artifact.path, bytes);} context.proof('investigation-native-artifact-and-integrity', {kind: artifact.kind, sha256: artifact.sha256}); } else context.log('investigation-artifact-prerequisite-gap', {reason: 'The real analyzers returned no evidence artifacts; artifact execution is not qualified by this run.'}); @@ -311,14 +312,14 @@ async function diagnose(context) { async function advise(context) { const current = await tab(context, 'Findings & advice'); const before = await directories(reportRoot(context)); - await current.getByRole('button', {name: 'Advise from saved evidence', exact: true}).click(); + await current.getByRole('button', {name: 'Advise from saved evidence', exact: true}).click({timeout:NATIVE_READINESS_MS}); await quickPick(context, 'Choose saved evidence', 0); const result = await reportAfter(context, before, 'advise'); assert.equal(result.report.schema_version, 'perfchecker-advice/1'); assert(Array.isArray(result.report.recommendations)); assert(await (await view(context)).getByRole('heading', {name: 'Recommendations', exact: true}).isVisible()); - await (await view(context)).getByRole('button', {name: 'Explain with configured model', exact: true}).click(); - await eventually(async () => /Optional advisor is disabled/.test(await current.locator('#app .status').innerText()), 'Unconfigured model explains the prerequisite without making a generation request'); + await (await view(context)).getByRole('button', {name: 'Explain with configured model', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => /Optional advisor is disabled/.test(await current.locator('#app .status').innerText({timeout:NATIVE_READINESS_MS})), 'Unconfigured model explains the prerequisite without making a generation request'); context.proof('investigation-saved-advice-and-disabled-model', {recommendations: result.report.recommendations.length, modelPrerequisiteExplained: true}); } @@ -331,8 +332,8 @@ async function compare(context) { const timestamp = directory => path.basename(directory).slice(0, 24).replace(/T(\d{2})-(\d{2})-(\d{2})/, 'T$1:$2:$3'); const pickMeasurement = async (title, directory) => { const widget = context.windowPage.locator('.quick-input-widget'); - await widget.waitFor({state:'visible',timeout:30000}); - await eventually(async()=>(await widget.innerText()).includes(title),`native quick pick ${title}`,30000); + await widget.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await eventually(async()=>(await widget.innerText({timeout:NATIVE_READINESS_MS})).includes(title),`native quick pick ${title}`,NATIVE_READINESS_MS); const target = Date.parse(timestamp(directory)); const rows = widget.locator('.monaco-list-row[role="option"]'); const labels = await rows.allTextContents(); @@ -340,17 +341,17 @@ async function compare(context) { .filter(item=>item.date&&Math.abs(Date.parse(item.date)-target)<10000); assert.equal(matches.length,1,`One actual saved measurement corresponds to ${timestamp(directory)}: ${JSON.stringify(labels)}`); context.log('investigation-native-measurement-picker',{title,directory:path.basename(directory),displayedTimestamp:matches[0].date}); - await rows.nth(matches[0].index).click(); + await rows.nth(matches[0].index).click({timeout:NATIVE_READINESS_MS}); }; const current = await tab(context, 'Before / after'); const before = await directories(reportRoot(context)); - await current.getByRole('button', {name: 'Choose baseline and candidate', exact: true}).click(); + await current.getByRole('button', {name: 'Choose baseline and candidate', exact: true}).click({timeout:NATIVE_READINESS_MS}); await pickMeasurement('Choose the baseline measurement', baseline.directory); await pickMeasurement('Choose the candidate measurement', candidate.directory); const result = await reportAfter(context, before, 'compare'); assert.equal(result.report.schema_version, 'perfchecker-scenario-comparison/1'); assert(result.report.configurations.some(item => item.scenario === 'ui_adopted' && item.implementation === 'ui')); - assert.match(await (await tab(context, 'Before / after')).locator('#app').innerText(), /ui_adopted/); + assert.match(await (await tab(context, 'Before / after')).locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /ui_adopted/); context.proof('investigation-real-baseline-candidate', {configurations: result.report.configurations.length, report: result.location}); } @@ -363,10 +364,10 @@ async function bounded(context) { assert(Array.isArray(result.report.unexecuted)); assert(Array.isArray(result.report.decisions)); const current = await tab(context, 'Findings & advice'); - assert.match(await current.locator('#app').innerText(), /Bounded investigation/); - await current.getByText('Limits', {exact: true}).click(); - await current.getByText('Experiments not executed', {exact: true}).click(); - await current.getByText('Optional model decisions', {exact: true}).click(); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /Bounded investigation/); + await current.getByText('Limits', {exact: true}).click({timeout:NATIVE_READINESS_MS}); + await current.getByText('Experiments not executed', {exact: true}).click({timeout:NATIVE_READINESS_MS}); + await current.getByText('Optional model decisions', {exact: true}).click({timeout:NATIVE_READINESS_MS}); context.proof('investigation-real-bounded-work', {status: result.report.status, experiments: result.report.experiments.length, limits: result.report.limits}); } @@ -380,22 +381,22 @@ async function syncAndHistory(context) { assert.equal(catalog.report.schema_version, 'perfchecker-tool-catalog/1'); const current = await tab(context, 'Findings & advice'); const search = current.getByRole('searchbox', {name: 'Search tool catalogue'}); - await search.fill('jet'); - assert.match(await current.locator('#app').innerText(), /JET/i); - await search.fill('no-native-tool-matches'); + await search.fill('jet',{timeout:NATIVE_READINESS_MS}); + assert.match(await current.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /JET/i); + await search.fill('no-native-tool-matches',{timeout:NATIVE_READINESS_MS}); assert.equal(await current.locator('article.card').count(), 0); - await search.fill(''); + await search.fill('',{timeout:NATIVE_READINESS_MS}); let history = await tab(context, 'Saved evidence'); const cardFor = frame => frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^run ·/})}).first(); for (const format of ['JSON', 'Markdown']) { - await cardFor(history).getByRole('button', {name: format, exact: true}).click(); + await cardFor(history).getByRole('button', {name: format, exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => context.vscode.window.activeTextEditor?.document.uri.fsPath.endsWith(format === 'JSON' ? 'run.json' : 'run.md'), `Native ${format} saved evidence export`); history = await tab(context, 'Saved evidence'); } - await cardFor(history).getByRole('button', {name: 'Open evidence', exact: true}).click(); - await eventually(async () => /Saved evidence/.test(await history.locator('#app .status').innerText()), 'History reloads actual evidence without starting a worker'); - await (await view(context)).getByRole('button', {name: 'Worker log', exact: true}).click(); - await eventually(async () => /PerfChecker investigations/.test(await context.windowPage.locator('body').innerText()), 'Investigation output channel is displayed'); + await cardFor(history).getByRole('button', {name: 'Open evidence', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => /Saved evidence/.test(await history.locator('#app .status').innerText({timeout:NATIVE_READINESS_MS})), 'History reloads actual evidence without starting a worker'); + await (await view(context)).getByRole('button', {name: 'Worker log', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => /PerfChecker investigations/.test(await context.windowPage.locator('body').innerText({timeout:NATIVE_READINESS_MS})), 'Investigation output channel is displayed'); context.proof('investigation-sync-tools-history-native-exports', {coverage: sync.report.coverage.length, tools: catalog.report.tools.length, formats: ['JSON', 'Markdown'], historyReload: true}); } @@ -419,12 +420,12 @@ async function profiles(context) { } for (let index = 0; index < await profiles.count(); index++) { const profile = profiles.nth(index); - await profile.locator('summary').first().click(); + await profile.locator('summary').first().click({timeout:NATIVE_READINESS_MS}); const filter = profile.getByRole('searchbox', {name: 'Filter profile stacks'}); - await filter.fill('no-stack-matches-native-test'); - assert.match(await profile.innerText(), /0 \/ 0 matching|No samples were collected/); - await filter.fill(''); - await profile.getByText('Raw profile', {exact: true}).click(); + await filter.fill('no-stack-matches-native-test',{timeout:NATIVE_READINESS_MS}); + assert.match(await profile.innerText({timeout:NATIVE_READINESS_MS}), /0 \/ 0 matching|No samples were collected/); + await filter.fill('',{timeout:NATIVE_READINESS_MS}); + await profile.getByText('Raw profile', {exact: true}).click({timeout:NATIVE_READINESS_MS}); } context.proof('investigation-real-collectors-and-profile-controls', {collectors: result.report.runs.map(item => item.collector), profileFilter: true, fullRawEvidence: true}); } @@ -434,14 +435,14 @@ async function cancel(context) { await fs.rm(marker, {force: true}); await adopt(context, 'ui_cancel', 'make_cancel_case', {marker}); const current = await selectScenario(context, 'ui_cancel', 'ui'); - await current.getByRole('button', {name: 'Measure selected', exact: true}).click(); - await eventually(async () => (await fs.readFile(marker, 'utf8')) === 'running', 'Actual isolated scenario operation starts before cancellation', 180000); - await (await view(context)).getByRole('button', {name: 'Cancel', exact: true}).click(); - await eventually(async () => /Cancelled after controller cleanup/.test(await current.locator('#app .status').innerText()), 'Cancel waits for actual controller and worker cleanup', 120000); + await current.getByRole('button', {name: 'Measure selected', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => (await fs.readFile(marker, 'utf8')) === 'running', 'Actual isolated scenario operation starts before cancellation', NATIVE_READINESS_MS); + await (await view(context)).getByRole('button', {name: 'Cancel', exact: true}).click({timeout:30000}); + await eventually(async () => /Cancelled after controller cleanup/.test(await current.locator('#app .status').innerText({timeout:30000})), 'Cancel waits for actual controller and worker cleanup', 120000); const history = await tab(context, 'Saved evidence'); const first = history.locator('article.card').first(); - assert.match(await first.innerText(), /cancelled/); - assert.equal(await first.getByRole('button', {name: 'Open evidence', exact: true}).isDisabled(), true); + assert.match(await first.innerText({timeout:NATIVE_READINESS_MS}), /cancelled/); + assert.equal(await first.getByRole('button', {name: 'Open evidence', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), true); await fs.rm(marker, {force: true}); context.proof('investigation-cancel-active-julia-worker', {workerStarted: true, cleanupCompleted: true, remainingConfigurationsQualified: false}); } @@ -463,32 +464,32 @@ async function savedImport(context) { const beforeDirectories=await directories(reportRoot(context)); const beforeHistory=await (await tab(context,'Saved evidence')).locator('article.card').allTextContents(); const choose=async file=>{ - const picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:30000}); - const input=picker.locator('input[type="text"]');await input.fill(file);await input.press('Enter'); - await picker.waitFor({state:'hidden',timeout:30000}); + const picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + const input=picker.locator('input[type="text"]');await input.fill(file,{timeout:NATIVE_READINESS_MS});await input.press('Enter',{timeout:NATIVE_READINESS_MS}); + await picker.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS}); }; try{ await files.update('simpleDialog.enable',true,context.vscode.ConfigurationTarget.Global); const pending=context.vscode.commands.executeCommand('perfchecker.openDiagnosticReport');await choose(filename);await pending; let current=await view(context); - const banner=current.locator('.imported-diagnostic');await banner.waitFor({state:'visible'}); - assert.match(await banner.innerText(),/not measured or independently verified in this editor session/); - assert((await banner.innerText()).includes(sha256));assert((await banner.innerText()).includes(await fs.realpath(filename))); + const banner=current.locator('.imported-diagnostic');await banner.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + assert.match(await banner.innerText({timeout:NATIVE_READINESS_MS}),/not measured or independently verified in this editor session/); + assert((await banner.innerText({timeout:NATIVE_READINESS_MS})).includes(sha256));assert((await banner.innerText({timeout:NATIVE_READINESS_MS})).includes(await fs.realpath(filename))); assert.deepEqual(await current.locator('.diagnostic-values dd').allTextContents(),['1.25 s','2.5 s','0.125 s']); assert.equal(await current.locator('.diagnostic-memory tbody tr').count(),5); - assert.match(await current.locator('article.card').last().innerText(),/Availability: unavailable/); - for(const name of ['Advise from saved evidence','Explain with configured model'])assert.equal(await current.getByRole('button',{name,exact:true}).isDisabled(),true); - await current.getByRole('button',{name:'Open imported JSON snapshot',exact:true}).click(); + assert.match(await current.locator('article.card').last().innerText({timeout:NATIVE_READINESS_MS}),/Availability: unavailable/); + for(const name of ['Advise from saved evidence','Explain with configured model'])assert.equal(await current.getByRole('button',{name,exact:true}).isDisabled({timeout:NATIVE_READINESS_MS}),true); + await current.getByRole('button',{name:'Open imported JSON snapshot',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(()=>context.vscode.window.activeTextEditor?.document.isUntitled,'Imported JSON opens an unsaved snapshot'); assert.equal(context.vscode.window.activeTextEditor.document.getText(),text); await context.vscode.commands.executeCommand('workbench.action.revertAndCloseActiveEditor'); current=await view(context); - await current.getByRole('button',{name:'Open saved diagnostic report',exact:true}).click(); - const picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible'});await picker.locator('input[type="text"]').press('Escape'); - await picker.waitFor({state:'hidden'});assert((await current.locator('.imported-diagnostic').innerText()).includes(sha256)); - await current.getByRole('button',{name:'Open saved diagnostic report',exact:true}).click();await choose(invalid); - await eventually(async()=>/Invalid diagnostic record/.test(await current.locator('#app > .status').innerText()),'Invalid saved JSON is refused without replacing the snapshot'); - assert((await current.locator('.imported-diagnostic').innerText()).includes(sha256)); + await current.getByRole('button',{name:'Open saved diagnostic report',exact:true}).click({timeout:NATIVE_READINESS_MS}); + const picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await picker.locator('input[type="text"]').press('Escape',{timeout:NATIVE_READINESS_MS}); + await picker.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS});assert((await current.locator('.imported-diagnostic').innerText({timeout:NATIVE_READINESS_MS})).includes(sha256)); + await current.getByRole('button',{name:'Open saved diagnostic report',exact:true}).click({timeout:NATIVE_READINESS_MS});await choose(invalid); + await eventually(async()=>/Invalid diagnostic record/.test(await current.locator('#app > .status').innerText({timeout:NATIVE_READINESS_MS})),'Invalid saved JSON is refused without replacing the snapshot'); + assert((await current.locator('.imported-diagnostic').innerText({timeout:NATIVE_READINESS_MS})).includes(sha256)); assert.deepEqual(await current.locator('.diagnostic-values dd').allTextContents(),['1.25 s','2.5 s','0.125 s']); assert.equal(await fs.readFile(filename,'utf8'),text); assert.deepEqual(await directories(reportRoot(context)),beforeDirectories); @@ -509,7 +510,7 @@ exports.run = async (context,options={}) => { const settings = configuration(context); const keys = ['scenarioSamples', 'analysisTimeout', 'investigationMaxExperiments', 'investigationBudgetSeconds', 'advisorEnabled', 'advisorInvestigates']; const previous = Object.fromEntries(keys.map(key => [key, settings.inspect(key)?.workspaceFolderValue])); - const values = {scenarioSamples: 2, analysisTimeout: 180, investigationMaxExperiments: 1, investigationBudgetSeconds: 120, advisorEnabled: false, advisorInvestigates: false}; + const values = {scenarioSamples: 2, analysisTimeout: NATIVE_READINESS_MS/1000, investigationMaxExperiments: 1, investigationBudgetSeconds: 120, advisorEnabled: false, advisorInvestigates: false}; const catalog = path.resolve(context.workspace, settings.get('scenarioCatalog', 'perf/scenarios.toml')); const originalCatalog = await fs.readFile(catalog); const failures = []; @@ -542,8 +543,8 @@ exports.run = async (context,options={}) => { } finally { await context.vscode.commands.executeCommand('perfchecker.cancelInvestigation').catch(() => {}); try { - const current = await view(context); - await eventually(async () => !(await current.locator('#app .status').getAttribute('class')).includes('busy'), 'Investigation teardown waits for worker cleanup', 120000); + const current = await view(context,120000); + await eventually(async () => !(await current.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'), 'Investigation teardown waits for worker cleanup', 120000); } catch (error) {failures.push(new Error(`investigation-teardown: ${error.message}`, {cause: error}));} await fs.writeFile(catalog, originalCatalog); await fs.rm(path.join(context.workspace, 'perf', 'native-investigation-running.marker'), {force: true}); diff --git a/test/native-investigation-limits.cjs b/test/native-investigation-limits.cjs index 40bc675..d69ceb7 100644 --- a/test/native-investigation-limits.cjs +++ b/test/native-investigation-limits.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Installed VSIX callbacks, actual Julia workers and an owned MCP protocol fixture. const assert=require('node:assert/strict'); const fs=require('node:fs/promises'); @@ -5,7 +6,7 @@ const path=require('node:path'); const http=require('node:http'); const {randomUUID}=require('node:crypto'); -async function eventually(read,label,timeout=180000){ +async function eventually(read,label,timeout=NATIVE_READINESS_MS){ const until=Date.now()+timeout;let last; while(Date.now()setTimeout(resolve,100));} @@ -19,10 +20,10 @@ async function alive(pid){ } return true; } -async function panel(context,tab){ +async function panel(context,tab,timeout=NATIVE_READINESS_MS){ await context.vscode.commands.executeCommand('perfchecker.openInvestigations'); - const frame=await context.findFrame('#app nav[aria-label="Investigation views"]'); - if(tab)await frame.getByRole('button',{name:tab,exact:true}).click(); + const frame=await context.findFrame('#app nav[aria-label="Investigation views"]',{timeout}); + if(tab)await frame.getByRole('button',{name:tab,exact:true}).click({timeout:NATIVE_READINESS_MS}); return frame; } async function entries(root){return new Set(await fs.readdir(root).catch(error=>{if(error.code==='ENOENT')return [];throw error;}));} @@ -35,8 +36,8 @@ async function completed(context,root,before,name){ } },`The real ${name} report is written`,240000); const frame=await panel(context); - await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class')).includes('busy'),'The actual controller completes cleanup'); - assert(await frame.getByRole('button',{name:'Cancel',exact:true}).isDisabled()); + await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'),'The actual controller completes cleanup',180000); + assert(await frame.getByRole('button',{name:'Cancel',exact:true}).isDisabled({timeout:30000})); return result; } async function decisionFixture(){ @@ -96,26 +97,26 @@ exports.run=async context=>{ await fs.writeFile(path.join(directory,'scenarios.toml'),`schema_version = "perfchecker-scenario-catalog/1"\nroot = "."\n[[scenarios]]\nid = "native_limits"\nimplementation = "sleeping"\nsource = "case.jl"\nfactory = "make_native_limit_case"\ncollectors = ["benchmark"]\nparameters = { marker = ${JSON.stringify(marker)} }\n`); try{ for(const [key,value]of Object.entries(values))await settings().update(key,value,context.vscode.ConfigurationTarget.WorkspaceFolder); - let frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:'Discover tests',exact:true}).click(); + let frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:'Discover tests',exact:true}).click({timeout:NATIVE_READINESS_MS}); await completed(context,root,new Set(),'discovery'); - frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:'Clear selection',exact:true}).click(); + frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:'Clear selection',exact:true}).click({timeout:NATIVE_READINESS_MS}); await frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:'native_limits'})}).first() - .locator('.scenario-title input').check(); + .locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS}); const fieldset=frame.locator('fieldset').filter({has:frame.getByText('Diagnostic tools',{exact:true})}); for(const label of await fieldset.locator('label span').allTextContents()){ const input=fieldset.locator('label').filter({hasText:label}).locator('input'); - if(label.split(' · ')[0]==='latency')await input.check();else await input.uncheck(); + if(label.split(' · ')[0]==='latency')await input.check({timeout:NATIVE_READINESS_MS});else await input.uncheck({timeout:NATIVE_READINESS_MS}); } - let before=await entries(root);await frame.getByRole('button',{name:'Investigate selected',exact:true}).click(); + let before=await entries(root);await frame.getByRole('button',{name:'Investigate selected',exact:true}).click({timeout:NATIVE_READINESS_MS}); const enabled=await completed(context,root,before,'investigation'); assert.equal(enabled.report.status,'advisor_stopped');assert.equal(enabled.report.experiments.length,0); assert.equal(enabled.report.decisions.length,1);assert.equal(enabled.report.decisions[0].status,'complete'); assert.equal(enabled.report.decisions[0].experiment_id,'stop');assert(enabled.report.unexecuted.length>=2); assert.equal(fixture.calls.length,1);assert.deepEqual(fixture.errors,[]); await assert.rejects(fs.stat(marker),{code:'ENOENT'},'A stop decision executes no sleeping scenario'); - frame=await panel(context,'Findings & advice');assert.match(await frame.locator('#app').innerText(),/advisor_stopped/); - await frame.getByText('Optional model decisions',{exact:true}).click(); - assert.match(await frame.locator('#app').innerText(),/"experiment_id": "stop"/); + frame=await panel(context,'Findings & advice');assert.match(await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS}),/advisor_stopped/); + await frame.getByText('Optional model decisions',{exact:true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS}),/"experiment_id": "stop"/); context.proof('native-investigation-enabled-model-decision',{nativeClick:true,core:context.core,customMcpFieldsAndInstructionsPreserved:true, structuredResponse:true,declaredMenu:fixture.calls[0].menu,status:enabled.report.status,noScenarioExecution:true,nativeDecisionVisible:true}); @@ -124,7 +125,7 @@ exports.run=async context=>{ for(const [name,label,budget]of [['run','Measure selected',180],['investigation','Investigate selected',45]]){ await fs.rm(marker,{force:true});await settings().update('investigationBudgetSeconds',budget,context.vscode.ConfigurationTarget.WorkspaceFolder); if(name==='investigation')await settings().update('analysisTimeout',180,context.vscode.ConfigurationTarget.WorkspaceFolder); - before=await entries(root);frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:label,exact:true}).click(); + before=await entries(root);frame=await panel(context,'Scenarios');await frame.getByRole('button',{name:label,exact:true}).click({timeout:NATIVE_READINESS_MS}); const pid=await eventually(async()=>{const value=Number(await fs.readFile(marker,'utf8'));return value>0?value:false;},'The actual sleeping scenario publishes its worker PID'); assert(await alive(pid),'The real operation is active before its configured limit expires'); context.log('native-limit-worker-active',{limit:name,pid,budgetSeconds:budget,analysisTimeout:settings().get('analysisTimeout')}); @@ -135,8 +136,8 @@ exports.run=async context=>{ if(name==='investigation'){ assert.equal(result.report.status,'budget_exhausted');assert.equal(result.report.limits.budget_seconds,45); assert.equal(result.report.experiments.length,1);assert(result.report.unexecuted.length>0); - assert.match(await frame.locator('#app').innerText(),/budget_exhausted/); - }else assert.match(await frame.locator('#app').innerText(),/timeout/); + assert.match(await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS}),/budget_exhausted/); + }else assert.match(await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS}),/timeout/); assert.equal(await fs.readFile(factory,'utf8'),source);assert.equal(fixture.calls.length,1); context.proof(name==='run'?'native-analysis-configured-deadline':'native-investigation-elapsed-budget',{ nativeClick:true,pid,operationActuallyStarted:true,configuredSeconds:45,noCancelClick:true, @@ -145,8 +146,8 @@ exports.run=async context=>{ } }finally{ await context.vscode.commands.executeCommand('perfchecker.cancelInvestigation'); - const frame=await panel(context); - await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class')).includes('busy'),'Limits teardown waits for owned controller cleanup'); + const frame=await panel(context,undefined,120000); + await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'),'Limits teardown waits for owned controller cleanup',180000); await fixture.close(); for(const [key,value]of Object.entries(previous))await settings().update(key,value,context.vscode.ConfigurationTarget.WorkspaceFolder); await fs.rm(directory,{recursive:true,force:true}); diff --git a/test/native-mcp-controls.cjs b/test/native-mcp-controls.cjs index 473cee7..bf27b72 100644 --- a/test/native-mcp-controls.cjs +++ b/test/native-mcp-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Real HTTP MCP transport and Julia workers, driven through the installed webview. const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); @@ -11,7 +12,7 @@ const {clickStudioAction}=require('./native-studio-controls.cjs'); const execute = promisify(execFile); const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); const hold = async()=>{if(process.env.PERFCHECKER_NATIVE_VIDEO==='1')await delay(3000);}; -async function eventually(read, label, timeout = 180000) { +async function eventually(read, label, timeout = NATIVE_READINESS_MS) { const deadline = Date.now() + timeout; while (Date.now() < deadline) {if (await read()) return; await delay(100);} throw new Error(label); @@ -307,22 +308,22 @@ async function verifyPersistedStdioSettings(context,fixture,stage){ assert.equal(state.workspace,vscode.workspace.getWorkspaceFolder(uri).name); assert.equal(state.connectionKind,undefined);assert.equal(state.busy,false); assert.deepEqual(state.implementation.arguments,fixture.values.advisorImplementationMcpArguments); - await chat.getByRole('tab',{name:'02 · Implementation',exact:true}).click(); - await chat.getByText('Configure the MCP implementation tool',{exact:true}).click(); - await eventually(async()=>JSON.stringify(JSON.parse(await chat.getByLabel('Other implementation tool arguments (JSON)',{exact:true}).inputValue()))===JSON.stringify(fixture.values.advisorImplementationMcpArguments), - 'The real Chat UI receives the selected folder implementation arguments',15000); - await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click(); - assert.equal(await chat.getByRole('tab',{name:'01 · Advice',exact:true}).getAttribute('aria-selected'),'true'); - await chat.getByRole('button',{name:'Connect local MCP server',exact:true}).click(); + await chat.getByRole('tab',{name:'02 · Implementation',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await chat.getByText('Configure the MCP implementation tool',{exact:true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>JSON.stringify(JSON.parse(await chat.getByLabel('Other implementation tool arguments (JSON)',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS})))===JSON.stringify(fixture.values.advisorImplementationMcpArguments), + 'The real Chat UI receives the selected folder implementation arguments',NATIVE_READINESS_MS); + await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await chat.getByRole('tab',{name:'01 · Advice',exact:true}).getAttribute('aria-selected',{timeout:NATIVE_READINESS_MS}),'true'); + await chat.getByRole('button',{name:'Connect local MCP server',exact:true}).click({timeout:NATIVE_READINESS_MS}); const setup=await findFrame('#advisor-root'); - assert.equal(await setup.getByLabel('Absolute MCP server executable',{exact:true}).inputValue(),fixture.values.advisorMcpStdioCommand); - assert.deepEqual(JSON.parse(await setup.getByLabel('Executable arguments (JSON array)',{exact:true}).inputValue()),fixture.values.advisorMcpStdioArguments); - assert.equal(await setup.getByLabel('Absolute server working directory',{exact:true}).inputValue(),fixture.values.advisorMcpStdioDirectory); + assert.equal(await setup.getByLabel('Absolute MCP server executable',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS}),fixture.values.advisorMcpStdioCommand); + assert.deepEqual(JSON.parse(await setup.getByLabel('Executable arguments (JSON array)',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS})),fixture.values.advisorMcpStdioArguments); + assert.equal(await setup.getByLabel('Absolute server working directory',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS}),fixture.values.advisorMcpStdioDirectory); // The untouched form has not called state(), so aria-busy is absent until // its first explicit operation. Assert the actual initial controls instead. - assert.equal(await setup.getByRole('button',{name:'Test connection / discover',exact:true}).isEnabled(),true); - assert.equal(await setup.getByRole('button',{name:'Cancel operation',exact:true}).isDisabled(),true); - assert.equal(await setup.getByRole('status').innerText(),'Ready.'); + assert.equal(await setup.getByRole('button',{name:'Test connection / discover',exact:true}).isEnabled({timeout:NATIVE_READINESS_MS}),true); + assert.equal(await setup.getByRole('button',{name:'Cancel operation',exact:true}).isDisabled({timeout:NATIVE_READINESS_MS}),true); + assert.equal(await setup.getByRole('status').innerText({timeout:NATIVE_READINESS_MS}),'Ready.'); // Inspect and close the actual form without discovering or connecting. await vscode.commands.executeCommand('workbench.action.closeActiveEditor'); await vscode.commands.executeCommand('perfchecker.openChat'); @@ -503,9 +504,9 @@ async function measuredEvidence(context) { let frame=await context.findFrame('#app nav[aria-label="Investigation views"]'); let declared,discoveryId; try{ - await frame.getByRole('button',{name:'Discover tests',exact:true}).click(); + await frame.getByRole('button',{name:'Discover tests',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{ - if((await frame.locator('#app .status').getAttribute('class')).includes('busy')|| + if((await frame.locator('#app .status').getAttribute('class',{timeout:NATIVE_READINESS_MS})).includes('busy')|| await frame.locator('.scenario-title strong').filter({hasText:/^advisor_sum_squares$/}).count()!==1)return false; const advisorCard=frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:/^advisor_sum_squares$/})}) .filter({has:frame.locator('.implementation',{hasText:/^allocating$/})}); @@ -527,13 +528,13 @@ async function measuredEvidence(context) { discoveryId=id; } return Boolean(declared); - },'Actual Core saves a new exact advisor catalogue and finishes its visible discovery',240000); + },'Actual Core saves a new exact advisor catalogue and finishes its visible discovery',NATIVE_READINESS_MS); assert(declared,'The saved discovery must identify the exact single-collector catalogue before launch'); }catch(primary){ const diagnostic={stage:'discovery',errorClass:primary.name,message:String(primary),recoveryAfterFailure:true}; context.log('native-mcp-discovery-primary-failure-before-recovery',diagnostic); try{ - diagnostic.status={text:await frame.locator('#app .status').innerText(),className:await frame.locator('#app .status').getAttribute('class')}; + diagnostic.status={text:await frame.locator('#app .status').innerText({timeout:30000}),className:await frame.locator('#app .status').getAttribute('class',{timeout:30000})}; diagnostic.newReportIds=(await entries()).filter(id=>!previousDiscovery.has(id)); diagnostic.screenshot=`native-${process.platform}-mcp-discovery-before-recovery.png`; await context.windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,diagnostic.screenshot)}); @@ -541,23 +542,23 @@ async function measuredEvidence(context) { context.log('native-mcp-discovery-state-before-recovery',diagnostic); try{ await context.vscode.commands.executeCommand('perfchecker.cancelInvestigation'); - await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class')).includes('busy'), + await eventually(async()=>!(await frame.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'), 'Explicit recovery waits for the failed discovery controller cleanup',120000); context.log('native-mcp-discovery-recovery-after-failure',{cancelCommand:'perfchecker.cancelInvestigation', - status:await frame.locator('#app .status').innerText(),naturalCompletionClaimed:false,originalFailureRetained:true}); + status:await frame.locator('#app .status').innerText({timeout:30000}),naturalCompletionClaimed:false,originalFailureRetained:true}); }catch(cleanup){throw new AggregateError([primary,cleanup],'Discovery failed and explicit recovery did not complete');} throw primary; } context.proof('native-mcp-benchmark-catalog-discovery',{id:declared.id,implementation:declared.implementation,catalog:declared.catalog, source:declared.source,collectors:declared.collectors,discoveryId,newSavedDiscovery:true,originalFourCollectorsPreserved:true,assertedBeforeMeasurement:true}); - await frame.getByRole('button',{name:'Scenarios',exact:true}).click(); - await frame.getByRole('button',{name:'Clear selection',exact:true}).click(); + await frame.getByRole('button',{name:'Scenarios',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await frame.getByRole('button',{name:'Clear selection',exact:true}).click({timeout:NATIVE_READINESS_MS}); const selectedCard=frame.locator('article.card').filter({has:frame.locator('.scenario-title strong',{hasText:/^advisor_sum_squares$/})}) .filter({has:frame.locator('.implementation',{hasText:/^allocating$/})}); assert.equal(await selectedCard.count(),1,'Only the exact advisor declaration is selected'); - await selectedCard.locator('.scenario-title input').check(); + await selectedCard.locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS}); const before=new Set(await entries()); - await frame.getByRole('button',{name:'Measure selected',exact:true}).click(); + await frame.getByRole('button',{name:'Measure selected',exact:true}).click({timeout:NATIVE_READINESS_MS}); let measured; await eventually(async()=>{ for(const id of await entries()){ @@ -618,10 +619,10 @@ async function measuredEvidence(context) { measurementEnvironments:environments.map(environment=>({path:environment.path,projectSha256:environment.project_sha256, manifestSha256:environment.manifest_sha256,collector:environment.resolved_packages.find(item=>item.name==='BenchmarkTools'), providerDependenciesAbsent:true}))}; - return !(await frame.locator('#app .status').getAttribute('class')).includes('busy'); + return !(await frame.locator('#app .status').getAttribute('class',{timeout:NATIVE_READINESS_MS})).includes('busy'); } return false; - },'Real Julia measurements and their saved deterministic advice complete',360000); + },'Real Julia measurements and their saved deterministic advice complete',NATIVE_READINESS_MS); return measured; } @@ -793,7 +794,7 @@ exports.run = async (context,options={}) => { const stdioRecords=async connection=>(await fs.readFile(path.join(connection.root,'stdio-requests.jsonl'),'utf8')).trim().split('\n').filter(Boolean).map(JSON.parse); const stdioSelect=async(panel,id,caption)=>{ const input=panel.locator('select#'+id);assert.equal(await input.count(),1); - assert.equal(await input.locator('..').locator(':scope > span').innerText(),caption);return input; + assert.equal(await input.locator('..').locator(':scope > span').innerText({timeout:NATIVE_READINESS_MS}),caption);return input; }; const connectStdio=async(version)=>{ assert.equal((await state()).connectionKind,undefined,'A new native connection starts from a disconnected chat'); @@ -802,31 +803,31 @@ exports.run = async (context,options={}) => { assert.equal(await fs.realpath(path.dirname(root)),await fs.realpath(process.env.PERFCHECKER_NATIVE_SESSION)); assert.equal((await fs.lstat(root)).isSymbolicLink(),false); await vscode.commands.executeCommand('perfchecker.openChat');let chat=await findFrame('#chat-root'); - await chat.getByRole('button',{name:'Connect local MCP server',exact:true}).click(); + await chat.getByRole('button',{name:'Connect local MCP server',exact:true}).click({timeout:NATIVE_READINESS_MS}); const panel=await findFrame('#advisor-root'); - await(await stdioSelect(panel,'advisor-protocol','Mode')).selectOption('mcp_stdio'); - await panel.getByLabel('Absolute MCP server executable',{exact:true}).fill(command); - await panel.getByLabel('Executable arguments (JSON array)',{exact:true}).fill(JSON.stringify(args)); - await panel.getByLabel('Absolute server working directory',{exact:true}).fill(root); - await(await stdioSelect(panel,'advisor-mcp_version','MCP version')).selectOption(version); - await panel.getByLabel('Connection / download timeout (seconds)',{exact:true}).fill(String(values.advisorTimeout)); - await panel.getByRole('button',{name:'Test connection / discover',exact:true}).click(); + await(await stdioSelect(panel,'advisor-protocol','Mode')).selectOption('mcp_stdio',{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Absolute MCP server executable',{exact:true}).fill(command,{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Executable arguments (JSON array)',{exact:true}).fill(JSON.stringify(args),{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Absolute server working directory',{exact:true}).fill(root,{timeout:NATIVE_READINESS_MS}); + await(await stdioSelect(panel,'advisor-mcp_version','MCP version')).selectOption(version,{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Connection / download timeout (seconds)',{exact:true}).fill(String(values.advisorTimeout),{timeout:NATIVE_READINESS_MS}); + await panel.getByRole('button',{name:'Test connection / discover',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await panel.getByRole('heading',{name:adviceTool,exact:true}).count()===1&&await panel.getByRole('heading',{name:implementationTool,exact:true}).count()===1&& - await panel.locator('#advisor-root').getAttribute('aria-busy')==='false','The real stdio transport displays both paginated tool schemas'); + await panel.locator('#advisor-root').getAttribute('aria-busy',{timeout:NATIVE_READINESS_MS})==='false','The real stdio transport displays both paginated tool schemas'); const row=name=>panel.locator('article.card').filter({has:panel.getByRole('heading',{name,exact:true})}); - await row(adviceTool).getByRole('button',{name:'Use',exact:true}).click(); - await row(implementationTool).getByRole('button',{name:'Use for implementation',exact:true}).click(); - await panel.getByLabel('Other tool arguments (JSON)',{exact:true}).fill(JSON.stringify(additional)); - await panel.getByLabel('Implementation prompt argument',{exact:true}).fill(implementationArgument); - await panel.getByLabel('Isolated checkout argument',{exact:true}).fill(workspaceArgument); - await panel.getByLabel('Other implementation arguments (JSON)',{exact:true}).fill(JSON.stringify(implementationAdditional)); - assert.equal(await panel.getByLabel('Selected MCP tool',{exact:true}).inputValue(),adviceTool); - assert.equal(await panel.getByLabel('Prompt argument',{exact:true}).inputValue(),adviceArgument); - assert.equal(await panel.getByLabel('Optional implementation tool',{exact:true}).inputValue(),implementationTool); + await row(adviceTool).getByRole('button',{name:'Use',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await row(implementationTool).getByRole('button',{name:'Use for implementation',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Other tool arguments (JSON)',{exact:true}).fill(JSON.stringify(additional),{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Implementation prompt argument',{exact:true}).fill(implementationArgument,{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Isolated checkout argument',{exact:true}).fill(workspaceArgument,{timeout:NATIVE_READINESS_MS}); + await panel.getByLabel('Other implementation arguments (JSON)',{exact:true}).fill(JSON.stringify(implementationAdditional),{timeout:NATIVE_READINESS_MS}); + assert.equal(await panel.getByLabel('Selected MCP tool',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS}),adviceTool); + assert.equal(await panel.getByLabel('Prompt argument',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS}),adviceArgument); + assert.equal(await panel.getByLabel('Optional implementation tool',{exact:true}).inputValue({timeout:NATIVE_READINESS_MS}),implementationTool); const schemas=[]; for(const name of [adviceTool,implementationTool]){ - await row(name).locator('summary').click(); - const schema=JSON.parse(await row(name).locator('pre').innerText());schemas.push({name,schema}); + await row(name).locator('summary').click({timeout:NATIVE_READINESS_MS}); + const schema=JSON.parse(await row(name).locator('pre').innerText({timeout:NATIVE_READINESS_MS}));schemas.push({name,schema}); assert.deepEqual(schema.required,name===adviceTool?[adviceArgument,'native_contract']:[implementationArgument,workspaceArgument,'native_contract']); } const discoveredPID=Number(await fs.readFile(path.join(root,'server.pid'),'utf8')); @@ -840,11 +841,11 @@ exports.run = async (context,options={}) => { proof('native-mcp-stdio-discovered-schema',{version,command,args,cwd:root,sourceSha256:hash(await fs.readFile(serverFile)),identity,windowsJobOwner, schemas,timeoutSeconds:values.advisorTimeout,timeoutScope:'Explicit qualification configuration, not the default',toolsCalled:0, provider:providerLabel,explicitNativeFieldGestures:true,independentImplementationArguments:implementationAdditional}); - await panel.getByLabel('Absolute MCP server executable',{exact:true}).scrollIntoViewIfNeeded(); + await panel.getByLabel('Absolute MCP server executable',{exact:true}).scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await context.windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,`native-${process.platform}-${vscode.version}-mcp-stdio-${version}-configuration.png`)}); - await row(implementationTool).scrollIntoViewIfNeeded(); + await row(implementationTool).scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await context.windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,`native-${process.platform}-${vscode.version}-mcp-stdio-${version}-schema.png`)}); - await panel.getByRole('button',{name:'Connect for this editor session',exact:true}).click(); + await panel.getByRole('button',{name:'Connect for this editor session',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>(await state()).connectionKind==='stdio','Native Save publishes the connection and does not cancel its server'); assert.deepEqual(await nativeIdentity(identity.pid,command),identity, 'Discovery and Connect retain exactly the same native server'); @@ -857,7 +858,7 @@ exports.run = async (context,options={}) => { adviceTool,implementationTool,adviceArgument,implementationArgument,workspaceArgument, independentImplementationArguments:implementationAdditional,sessionOnly:true,savedSettingsNotQualified:true,provider:providerLabel}); await vscode.commands.executeCommand('perfchecker.openChat');chat=await findFrame('#chat-root'); - await chat.getByRole('button',{name:'New conversation',exact:true}).click(); + await chat.getByRole('button',{name:'New conversation',exact:true}).click({timeout:NATIVE_READINESS_MS}); return chat; }; try { @@ -876,9 +877,9 @@ exports.run = async (context,options={}) => { let view = await findFrame('#chat-root'); await context.editorQualification?.surface('custom-mcp-before-connect',view); if(!stdio&&!documentedInstall){ - await view.locator('summary').filter({hasText: 'Optional Codex CLI connector'}).click(); - await view.getByRole('button', {name: 'Connect Codex CLI', exact: true}).click(); - await eventually(async () => /ENOENT|executable|could not|launch/i.test(await view.locator('[role="status"]').innerText()), 'Missing Codex explains its executable prerequisite'); + await view.locator('summary').filter({hasText: 'Optional Codex CLI connector'}).click({timeout:NATIVE_READINESS_MS}); + await view.getByRole('button', {name: 'Connect Codex CLI', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => /ENOENT|executable|could not|launch/i.test(await view.locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS})), 'Missing Codex explains its executable prerequisite'); assert.equal((await vscode.commands.executeCommand('perfchecker.codexConnectionState')).connected, false); proof('codex-missing-native-prerequisite',{command:'perfchecker.connectCodex',status:'prerequisite',reason:'Codex executable absent from disposable CI; no human credentials are transferred.'}); assert.deepEqual(await vscode.commands.executeCommand('perfchecker.disconnectCodex'),{connected:false}); @@ -887,7 +888,7 @@ exports.run = async (context,options={}) => { assert.deepEqual(await readSavedConfiguration(),savedConfig,'The failed temporary CLI connection and explicit disconnect preserve the saved provider bytes'); proof('codex-disconnected-command',{command:'perfchecker.disconnectCodex',returnValueVerified:true,alreadyDisconnected:true,savedConfigurationPreserved:true,optionalConfigurationPath:configuredPath,directoryRead:false,activeAuthenticatedDisconnection:false}); } - await view.getByRole('button', {name: 'New conversation', exact: true}).click(); + await view.getByRole('button', {name: 'New conversation', exact: true}).click({timeout:NATIVE_READINESS_MS}); const diagnoseSend=async(stage)=>{ const diagnostic={stage,extensionHost:process.pid,controllerProject,measurementProject, configuredTimeoutSeconds:values.advisorTimeout,externalTimeoutSeconds:values.advisorTimeout+60, @@ -939,9 +940,9 @@ exports.run = async (context,options={}) => { log('native-mcp-send-diagnostic',diagnostic); }; const send = async (question, count) => { - await view.locator('#chat-question').fill(question); + await view.locator('#chat-question').fill(question,{timeout:NATIVE_READINESS_MS}); log('native-ui-action',{surface:'MCP conversation',action:'Send question',turn:count/2}); - await view.getByRole('button', {name: 'Send question', exact: true}).click(); + await view.getByRole('button', {name: 'Send question', exact: true}).click({timeout:NATIVE_READINESS_MS}); let nextDiagnostic=Date.now()+60000; try{ await eventually(async () => {assert.deepEqual(providerErrors,[],'The real provider must accept the exact Core request');const value = await state(); @@ -960,14 +961,14 @@ exports.run = async (context,options={}) => { const connection=activeStdio; await legacyIdentity(connection,'after-completed-advice'); assert.equal((await state()).connectionKind,'stdio','The completed legacy advice retains its existing stdio connection'); - await view.getByRole('button',{name:'Configure MCP connection',exact:true}).click(); + await view.getByRole('button',{name:'Configure MCP connection',exact:true}).click({timeout:NATIVE_READINESS_MS}); const panel=await findFrame('#advisor-root'); - assert.equal(await(await stdioSelect(panel,'advisor-protocol','Mode')).inputValue(),'mcp_stdio'); - assert.equal(await(await stdioSelect(panel,'advisor-mcp_version','MCP version')).inputValue(),'2025-11-25', + assert.equal(await(await stdioSelect(panel,'advisor-protocol','Mode')).inputValue({timeout:NATIVE_READINESS_MS}),'mcp_stdio'); + assert.equal(await(await stdioSelect(panel,'advisor-mcp_version','MCP version')).inputValue({timeout:NATIVE_READINESS_MS}),'2025-11-25', 'Reopening configuration reads the connected legacy revision from memory, rather than the modern saved settings'); legacyMarker('probe-before',{server:connection.identity}); - await panel.getByRole('button',{name:'Test connection / discover',exact:true}).click(); - await eventually(async()=>await panel.locator('#advisor-root').getAttribute('aria-busy')==='false'&& + await panel.getByRole('button',{name:'Test connection / discover',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>await panel.locator('#advisor-root').getAttribute('aria-busy',{timeout:NATIVE_READINESS_MS})==='false'&& await panel.getByRole('heading',{name:adviceTool,exact:true}).count()===1,'Probe after a completed legacy conversation keeps the existing server usable'); legacyMarker('probe-after',{server:connection.identity}); await legacyIdentity(connection,'after-probe'); @@ -976,7 +977,7 @@ exports.run = async (context,options={}) => { assert(records.every(row=>row.version==='2025-11-25'));assert.equal(records.some(row=>row.method==='server/discover'),false); const disconnectUntil=Date.now()+60000; legacyMarker('disconnect-before-click',{server:connection.identity}); - await panel.getByRole('button',{name:'Disconnect local MCP server',exact:true}).click(); + await panel.getByRole('button',{name:'Disconnect local MCP server',exact:true}).click({timeout:30000}); legacyMarker('disconnect-after-click',{server:connection.identity}); await eventually(async()=>await sameNativeIdentityGone(connection.identity)&& (!connection.windowsJobOwner||await sameNativeIdentityGone(connection.windowsJobOwner))&&!(await state()).connectionKind&& @@ -1002,8 +1003,8 @@ exports.run = async (context,options={}) => { messages:completed.messages.length,assistantRole:completed.messages[1].role,actualProviderRequests:calls.length,configuredSeconds:90,externalSeconds:150}); const reply=view.locator('.transcript .message.assistant .message-text'); assert.equal(await reply.count(),1,'One actual assistant bubble is present in the conversation'); - assert.equal(await reply.innerText(),completed.messages[1].content,'The visible Advice bubble contains the full actual response'); - await reply.waitFor({state:'visible',timeout:15000}); + assert.equal(await reply.innerText({timeout:NATIVE_READINESS_MS}),completed.messages[1].content,'The visible Advice bubble contains the full actual response'); + await reply.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); assert.equal(await fs.readFile(source,'utf8'),original);assert.equal(await git('rev-parse','HEAD'),head); proof('native-mcp-documented-install-first-send',{installation,requestStartedAt:firstSendStartedAt,completedAt, elapsedSeconds,configuredSeconds:90,externalSeconds:150, @@ -1019,7 +1020,7 @@ exports.run = async (context,options={}) => { await require('./native-suite-chat-evidence.cjs').run(context,{calls:suiteCalls,receipts,providerErrors,state, setAttached:value=>{attached=value;}}); attached=savedEvidence; - await view.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(attached.id); + await view.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(attached.id,{timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return value.evidenceId===attached.id&&value.messages.length===0;},'The real evidence selector starts a conversation with the selected measured bundle'); await send('Inspect the intermediate allocation in sum_squares using this measured evidence without editing. What should I verify?',2); await send('Continue this conversation: how should empty inputs and signed Float64 values be checked?', 4); @@ -1044,13 +1045,13 @@ exports.run = async (context,options={}) => { await context.windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT, `native-${process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION}-${process.platform}-${vscode.version}-mcp-controlled-provider.png`)}); await hold(); - await view.getByRole('tab', {name: '02 · Implementation', exact: true}).click(); - assert.match(await view.locator('.warning').innerText(), /Git checkpoint.*isolated copy.*diff review/); + await view.getByRole('tab', {name: '02 · Implementation', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await view.locator('.warning').innerText({timeout:NATIVE_READINESS_MS}), /Git checkpoint.*isolated copy.*diff review/); if(!stdio){ - await view.getByText('Configure the MCP implementation tool', {exact: true}).click(); - await view.getByRole('textbox', {name: 'Implementation tool name', exact: true}).fill(implementationTool); - await view.getByRole('textbox',{name:'Other implementation tool arguments (JSON)',exact:true}).fill(JSON.stringify(implementationAdditional)); - await view.getByRole('button', {name: 'Save implementation tool', exact: true}).click(); + await view.getByText('Configure the MCP implementation tool', {exact: true}).click({timeout:NATIVE_READINESS_MS}); + await view.getByRole('textbox', {name: 'Implementation tool name', exact: true}).fill(implementationTool,{timeout:NATIVE_READINESS_MS}); + await view.getByRole('textbox',{name:'Other implementation tool arguments (JSON)',exact:true}).fill(JSON.stringify(implementationAdditional),{timeout:NATIVE_READINESS_MS}); + await view.getByRole('button', {name: 'Save implementation tool', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async () => /Implementation tool saved/.test((await state()).status), 'The native tool configuration is saved'); assert.deepEqual(settings().inspect('advisorImplementationMcpArguments')?.workspaceFolderValue,implementationAdditional, 'The real implementation form persists its explicit independent arguments in the owning folder'); @@ -1062,8 +1063,8 @@ exports.run = async (context,options={}) => { // editor activity; earlier stat-cache changes are not part of this action. const index = await fs.readFile(path.join(workspace, '.git', 'index')); log('native-ui-action',{surface:'MCP implementation',action:'Prepare implementation after review'}); - await view.getByRole('button', {name: 'I reviewed the advice · Prepare implementation', exact: true}).click(); - await eventually(async () => {const value = await state(); return !value.busy && value.proposal?.files.includes('src/PerfCheckerNativeFixture.jl');}, 'The real implementation worker returns its Git proposal', 240000); + await view.getByRole('button', {name: 'I reviewed the advice · Prepare implementation', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => {const value = await state(); return !value.busy && value.proposal?.files.includes('src/PerfCheckerNativeFixture.jl');}, 'The real implementation worker returns its Git proposal', NATIVE_READINESS_MS); const proposal=(await state()).proposal; await fs.writeFile(path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,'native-mcp-reviewed-proposal.patch'),proposal.patch); log('native-mcp-reviewed-proposal',{files:proposal.files,patchBytes:Buffer.byteLength(proposal.patch),patchSha256:hash(Buffer.from(proposal.patch))}); @@ -1074,8 +1075,8 @@ exports.run = async (context,options={}) => { await eventually(async()=>!await context.windowPage.locator('[id="workbench.parts.sidebar"]').isVisible()&&!await context.windowPage.locator('[id="workbench.parts.panel"]').isVisible(), 'Native workbench actions expose the full editor review viewport'); const proposedChanges=view.getByLabel('Proposed changes',{exact:true}); - assert((await proposedChanges.innerText()).includes('init=zero'),'The real proposed patch is populated before the review capture'); - await proposedChanges.scrollIntoViewIfNeeded(); + assert((await proposedChanges.innerText({timeout:NATIVE_READINESS_MS})).includes('init=zero'),'The real proposed patch is populated before the review capture'); + await proposedChanges.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); const reviewGeometry=()=>proposedChanges.evaluate(element=>({rectangle:element.getBoundingClientRect().toJSON(), viewport:{width:innerWidth,height:innerHeight},documentScroll:{x:scrollX,y:scrollY}, content:{height:element.scrollHeight,clientHeight:element.clientHeight,scrollTop:element.scrollTop}})); @@ -1096,17 +1097,17 @@ exports.run = async (context,options={}) => { await view.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))); await context.windowPage.screenshot({path:path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,`native-${process.platform}-${vscode.version}-mcp-reviewed-proposal.png`)});await hold(); log('native-ui-action',{surface:'MCP implementation',action:'Open full diff'}); - await view.getByRole('button', {name: 'Open full diff', exact: true}).click(); + await view.getByRole('button', {name: 'Open full diff', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => vscode.workspace.textDocuments.some(document => document.languageId === 'diff' && document.getText().includes('init=zero')), 'The actual diff editor opens'); await hold(); await vscode.commands.executeCommand('perfchecker.openChat'); view = await findFrame('#chat-root'); log('native-ui-action',{surface:'MCP implementation',action:'Apply reviewed changes'}); - await view.getByRole('button', {name: 'Apply reviewed changes', exact: true}).click(); + await view.getByRole('button', {name: 'Apply reviewed changes', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async () => !((await state()).busy) && (await fs.readFile(source, 'utf8')) === proposed, 'Apply changes the original only after the native user click'); assert.equal(await probe(workspace), implementationBytes); log('native-mcp-applied-oracle',{allocationBytes:implementationBytes,baselineBytes,originalChangedAfterReview:true});await hold(); log('native-ui-action',{surface:'MCP implementation',action:'Restore previous code'}); - await view.getByRole('button', {name: 'Restore previous code', exact: true}).click(); + await view.getByRole('button', {name: 'Restore previous code', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async () => !((await state()).busy) && (await fs.readFile(source, 'utf8')) === original, 'Restore returns the exact original bytes'); assert.deepEqual(await fs.readFile(path.join(workspace, '.git', 'index')), index); assert.equal(await git('rev-parse', 'HEAD'), head); @@ -1115,11 +1116,11 @@ exports.run = async (context,options={}) => { allocationBaselineBytes:baselineBytes,allocationCandidateBytes:implementationBytes,indexSha256:hash(index)}); if(custom)proof('native-mcp-custom-arguments',{adviceArgument,implementationArgument,workspaceArgument,additionalArgumentsVerified:true, actualHttpCalls:calls.length,measuredEvidenceIds:attached.evidence.map(row=>row.id),configurationRestoredInFinally:true}); - await view.getByRole('button', {name: 'Discard proposal', exact: true}).click(); + await view.getByRole('button', {name: 'Discard proposal', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async () => !(await state()).proposal, 'Discard closes the recovery proposal'); - await view.getByRole('tab', {name: '01 · Advice', exact: true}).click(); - await view.locator('#chat-question').fill('native cancellation probe'); - await view.getByRole('button', {name: 'Send question', exact: true}).click(); + await view.getByRole('tab', {name: '01 · Advice', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await view.locator('#chat-question').fill('native cancellation probe',{timeout:NATIVE_READINESS_MS}); + await view.getByRole('button', {name: 'Send question', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => pending.size > 0, 'The actual MCP request reached the provider'); const ownershipSnapshot=async()=>({pending:pending.size,uiBusy:(await state()).busy}); const owned=stdio?await ownedStdioJuliaProcesses():await ownedChatProcesses(log,[...pendingSockets.values()],ownershipSnapshot,'before-folder-switch'); @@ -1145,7 +1146,7 @@ exports.run = async (context,options={}) => { }); try{ assert(vscode.workspace.updateWorkspaceFolders(vscode.workspace.workspaceFolders.length,0,{uri:alternateFolder,name:'Chat alternate folder'})); - await eventually(()=>alternateAcknowledged,'The workbench acknowledges the alternate folder and its configuration model',30000); + await eventually(()=>alternateAcknowledged,'The workbench acknowledges the alternate folder and its configuration model',NATIVE_READINESS_MS); }finally{alternateChanged.dispose();} assert.equal(vscode.workspace.getWorkspaceFolder(alternateFolder)?.uri.toString(),alternateFolder.toString()); log('native-mcp-alternate-workspace-acknowledged',{resource:alternateFolder.toString(),actualWorkspaceEvent:true}); @@ -1171,12 +1172,12 @@ exports.run = async (context,options={}) => { } await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',alternateFolder); const alternateStudio=await findFrame('#studio-root'); - await eventually(async()=>await alternateStudio.locator('.workspace strong').innerText()==='Chat alternate folder', + await eventually(async()=>await alternateStudio.locator('.workspace strong').innerText({timeout:NATIVE_READINESS_MS})==='Chat alternate folder', 'Studio selects the second folder while the first chat request is still active'); const owningChatTab=()=>context.windowPage.locator('.tabs-container .tab').filter({ has:context.windowPage.locator('.label-name').filter({hasText:/^PerfChecker · Chat$/})}).filter({visible:true}).first(); - await owningChatTab().click(); - await eventually(async()=>await owningChatTab().getAttribute('aria-selected')==='true', + await owningChatTab().click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>await owningChatTab().getAttribute('aria-selected',{timeout:NATIVE_READINESS_MS})==='true', 'The exact owning Chat tab is active, rather than the alternate Studio with a similar title'); view=await findFrame('#chat-root'); const requestedOwner=vscode.Uri.file(workspace),owner=vscode.workspace.getWorkspaceFolder(requestedOwner); @@ -1193,7 +1194,7 @@ exports.run = async (context,options={}) => { assert.equal(calls.length,callsBeforeSwitch,'Selecting another Studio must not send another provider request'); log('native-mcp-cancel-before',{...owned,foreign:foreign.pid,heldResponses:pending.size,uiBusy:(await state()).busy}); const cancelUntil=Date.now()+60000; - await view.getByRole('button', {name: 'Cancel request', exact: true}).click(); + await view.getByRole('button', {name: 'Cancel request', exact: true}).click({timeout:30000}); await eventually(async () => { const idle=!(await state()).busy; const localGone=!stdio||await sameNativeIdentityGone(activeStdio.identity)&&await sameNativeIdentityGone(stdioDescendant)&& @@ -1225,11 +1226,11 @@ exports.run = async (context,options={}) => { originalOwnedProcessesPreservedBeforeCancel:true,owningEvidenceInventoryPreserved:true, additionalProviderRequestsOnWorkspaceSwitch:calls.length-callsBeforeSwitch}); await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',vscode.Uri.file(workspace)); - await owningChatTab().click(); - await eventually(async()=>await owningChatTab().getAttribute('aria-selected')==='true','The exact Chat tab is active after restoring Studio selection'); + await owningChatTab().click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>await owningChatTab().getAttribute('aria-selected',{timeout:NATIVE_READINESS_MS})==='true','The exact Chat tab is active after restoring Studio selection'); view=await findFrame('#chat-root'); assert.equal(await fs.readFile(source, 'utf8'), original); - await view.getByRole('button', {name: 'New conversation', exact: true}).click(); + await view.getByRole('button', {name: 'New conversation', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async () => (await state()).messages.length === 0, 'The native clear command removes the conversation'); proof('native-mcp-advice-implementation-restore', {provider: providerLabel,protocolRevision:'2026-07-28', adviceTurns: 2, checkpoint: true, diffEditor: true, apply: true, exactRestore: true, cancellation: true, @@ -1243,7 +1244,7 @@ exports.run = async (context,options={}) => { assert.deepEqual(await readSavedConfiguration(),savedConfig); view=await connectStdio('2026-07-28');const disconnected=activeStdio; const disconnectUntil=Date.now()+60000; - await view.getByRole('button',{name:'Disconnect local MCP server',exact:true}).click(); + await view.getByRole('button',{name:'Disconnect local MCP server',exact:true}).click({timeout:30000}); await eventually(async()=>await sameNativeIdentityGone(disconnected.identity)&& (!disconnected.windowsJobOwner||await sameNativeIdentityGone(disconnected.windowsJobOwner))&&!(await state()).connectionKind&&Date.now()({nonce,isDetached:()=>detached,parentFrame:()=>null, locator:selector=>selector.startsWith('button')?{count:async()=>{if(readError)throw readError;return 1;},first:()=>({isVisible:async()=>visible})}: @@ -207,8 +209,9 @@ function resultsScenario({snapshots=[],fallback=[],clickError,readError,detached const frames=()=>{reads++;const value=snapshots.length?snapshots.shift():fallback;return value.map(row=>frame(...row));}; const context={browser:{contexts:()=>[{pages:()=>[{frames}]}]},log:(name,value)=>observations.push({name,...value})}; class Clock extends Date{static now(){return elapsed;}} - const output=new Function('assert','clickStudioAction','Date','setTimeout',resultsSource+';return output;')( - assert,async(_,action)=>{assert.equal(action,'results');clicks++;if(clickError)throw clickError;},Clock,(resolve,ms)=>{elapsed+=ms;resolve();}); + const output=new Function('assert','clickStudioAction','Date','setTimeout','NATIVE_READINESS_MS',resultsSource+';return output;')( + assert,async(_,action)=>{assert.equal(action,'results');clicks++;if(clickError)throw clickError;},Clock, + (resolve,ms)=>{assert.equal(ms,100);elapsed+=Math.max(ms,clockAdvance);resolve();},NATIVE_READINESS_MS); return{run:()=>output(context),observations,snapshot:()=>({elapsed,clicks,reads})}; } @@ -223,10 +226,15 @@ test('Results does not accept an unseen new document until it becomes visible',a assert.equal((await value.run()).nonce,'new');assert.equal(value.snapshot().clicks,1);assert.equal(value.snapshot().elapsed,100); }); -test('Results times out after 60 seconds without replaying its native click',async()=>{ - const value=resultsScenario({fallback:[['old',true]]}); +test('Results waits beyond the former bound for its exact new document without replaying its native click',async()=>{ + const value=resultsScenario({snapshots:[[['old',true]],[['old',true]],[['new',true]]],clockAdvance:30*60000}); + assert.equal((await value.run()).nonce,'new');assert.equal(value.snapshot().elapsed,30*60000);assert.equal(value.snapshot().clicks,1); +}); + +test('Results times out after one hour without replaying its native click',async()=>{ + const value=resultsScenario({fallback:[['old',true]],clockAdvance:60000}); await assert.rejects(value.run(),/single Results click renders its new visible document/); - assert.equal(value.snapshot().elapsed,60000);assert.equal(value.snapshot().clicks,1); + assert.equal(value.snapshot().elapsed,3600000);assert.equal(value.snapshot().clicks,1); }); test('Results preserves a native click failure instead of retrying the action',async()=>{ @@ -240,7 +248,7 @@ test('Results preserves an inspection failure on an attached document',async()=> }); test('Results records detached reads and never treats them as a completed action',async()=>{ - const value=resultsScenario({snapshots:[[['old',true]]],readError:new Error('Frame was detached'),detached:true}); + const value=resultsScenario({snapshots:[[['old',true]]],readError:new Error('Frame was detached'),detached:true,clockAdvance:60000}); await assert.rejects(value.run(),/single Results click renders its new visible document/); assert.equal(value.snapshot().clicks,1);assert(value.observations.every(row=>row.name==='native-results-document-observation-invalidated')); }); diff --git a/test/native-narrative-controls.cjs b/test/native-narrative-controls.cjs index caadd99..2872be7 100644 --- a/test/native-narrative-controls.cjs +++ b/test/native-narrative-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Native installed-VSIX clicks and real Julia evidence; HTTP replies are a declared protocol fixture. const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); @@ -11,7 +12,7 @@ const customInstructions = 'Native qualification instruction: distinguish measur const names = {discover: 'discovery', run: 'run', narrate: 'narrative'}; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); -async function eventually(read, description, timeout = 180000) { +async function eventually(read, description, timeout = NATIVE_READINESS_MS) { const end = Date.now() + timeout; let last; while (Date.now() < end) { @@ -25,15 +26,15 @@ function configuration(context) { return context.vscode.workspace.getConfiguration('perfchecker', context.vscode.Uri.file(context.workspace)); } -async function view(context) { +async function view(context,timeout=NATIVE_READINESS_MS) { await context.vscode.commands.executeCommand('perfchecker.openInvestigations'); - return context.findFrame('#app nav[aria-label="Investigation views"]'); + return context.findFrame('#app nav[aria-label="Investigation views"]',{timeout}); } async function tab(context, name) { const frame = await view(context); - await frame.getByRole('button', {name, exact: true}).click(); - await eventually(async () => await frame.getByRole('button', {name, exact: true}).getAttribute('aria-current') === 'page', name); + await frame.getByRole('button', {name, exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => await frame.getByRole('button', {name, exact: true}).getAttribute('aria-current',{timeout:NATIVE_READINESS_MS}) === 'page', name); return frame; } @@ -45,11 +46,11 @@ async function reloadMeasuredEvidence(context, measured, stage) { context.log('native-narrative-history-reload-start', {stage, reportId: measured.id}); const frame = await tab(context, 'Saved evidence'); const card = frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^run ·/})}).first(); - const heading = await card.getByRole('heading', {name: /^run ·/}).innerText(); + const heading = await card.getByRole('heading', {name: /^run ·/}).innerText({timeout:NATIVE_READINESS_MS}); assert.match(heading, /^run · \d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/); const created = heading.slice('run · '.length); context.log('native-narrative-history-open-click', {stage, created, reportId: measured.id}); - await card.getByRole('button', {name: 'Open evidence', exact: true}).click(); + await card.getByRole('button', {name: 'Open evidence', exact: true}).click({timeout:NATIVE_READINESS_MS}); // The click changes tabs before loadHistory has read and published its report. // Match both the selected history timestamp and every rendered measured run. await eventually(async () => { @@ -83,13 +84,13 @@ async function directories(root) { } async function idle(context) { - const frame = await view(context); - await eventually(async () => !(await frame.locator('#app .status').getAttribute('class')).includes('busy'), 'native controller finishes cleanup', 240000); - assert(await frame.getByRole('button', {name: 'Cancel', exact: true}).isDisabled()); + const frame = await view(context,120000); + await eventually(async () => !(await frame.locator('#app .status').getAttribute('class',{timeout:30000})).includes('busy'), 'native controller finishes cleanup', 240000); + assert(await frame.getByRole('button', {name: 'Cancel', exact: true}).isDisabled({timeout:30000})); return frame; } -async function reportAfter(context, root, before, action) { +async function reportAfter(context, root, before, action,timeout=NATIVE_READINESS_MS) { const result = await eventually(async () => { for (const id of await directories(root)) { if (before.has(id)) continue; @@ -98,7 +99,7 @@ async function reportAfter(context, root, before, action) { catch (error) {if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;} } return false; - }, `real Julia ${action} report`, 360000); + }, `real Julia ${action} report`, timeout); await idle(context); return result; } @@ -106,7 +107,7 @@ async function reportAfter(context, root, before, action) { async function action(context, root, label, name) { const before = await directories(root); const frame = await view(context); - await frame.getByRole('button', {name: label, exact: true}).click(); + await frame.getByRole('button', {name: label, exact: true}).click({timeout:NATIVE_READINESS_MS}); return reportAfter(context, root, before, name); } @@ -177,22 +178,22 @@ async function remoteConfiguration(context, fixture, settings, keyEnvironment, t await context.vscode.commands.executeCommand('perfchecker.configureAdvisor'); let frame = await context.findFrame('#advisor-root'); assert.equal(await frame.locator('#advisor-allow_remote').isChecked(), false); - assert.equal(await frame.locator('#advisor-instructions').inputValue(), customInstructions); - assert.equal(await frame.locator('#advisor-api_key_env').inputValue(), keyEnvironment); + assert.equal(await frame.locator('#advisor-instructions').inputValue({timeout:NATIVE_READINESS_MS}), customInstructions); + assert.equal(await frame.locator('#advisor-api_key_env').inputValue({timeout:NATIVE_READINESS_MS}), keyEnvironment); const original = await fs.readFile(file), calls = fixture.requests.length; const save = async expression => { - await frame.getByRole('button', {name: 'Save configuration', exact: true}).click(); - await eventually(async () => await frame.locator('#advisor-root').getAttribute('aria-busy') === 'false' && - expression.test(await frame.getByRole('status').innerText()), 'The native remote configuration action finishes'); + await frame.getByRole('button', {name: 'Save configuration', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => await frame.locator('#advisor-root').getAttribute('aria-busy',{timeout:NATIVE_READINESS_MS}) === 'false' && + expression.test(await frame.getByRole('status').innerText({timeout:NATIVE_READINESS_MS})), 'The native remote configuration action finishes'); }; - await frame.locator('#advisor-endpoint').fill('https://perfchecker-native-fixture.invalid/v1/chat/completions'); + await frame.locator('#advisor-endpoint').fill('https://perfchecker-native-fixture.invalid/v1/chat/completions',{timeout:NATIVE_READINESS_MS}); await save(/remote evidence transmission requires allow_remote=true and HTTPS/); assert.deepEqual(await fs.readFile(file), original, 'Refused remote configuration preserves the saved local provider'); - await frame.locator('#advisor-allow_remote').check(); - await frame.locator('#advisor-endpoint').fill('http://perfchecker-native-fixture.invalid/v1/chat/completions'); + await frame.locator('#advisor-allow_remote').check({timeout:NATIVE_READINESS_MS}); + await frame.locator('#advisor-endpoint').fill('http://perfchecker-native-fixture.invalid/v1/chat/completions',{timeout:NATIVE_READINESS_MS}); await save(/remote evidence transmission requires allow_remote=true and HTTPS/); assert.deepEqual(await fs.readFile(file), original, 'Opt-in still refuses unencrypted remote HTTP'); - await frame.locator('#advisor-endpoint').fill('https://perfchecker-native-fixture.invalid/v1/chat/completions'); + await frame.locator('#advisor-endpoint').fill('https://perfchecker-native-fixture.invalid/v1/chat/completions',{timeout:NATIVE_READINESS_MS}); await save(/Configuration saved/); const saved = await fs.readFile(file, 'utf8'), config = JSON.parse(saved); assert.equal(config.allow_remote, true); assert.equal(config.instructions, customInstructions); @@ -202,7 +203,7 @@ async function remoteConfiguration(context, fixture, settings, keyEnvironment, t await context.vscode.commands.executeCommand('perfchecker.configureAdvisor'); frame = await context.findFrame('#advisor-root'); assert.equal(await frame.locator('#advisor-allow_remote').isChecked(), true); - assert.equal(await frame.locator('#advisor-endpoint').inputValue(), config.endpoint); + assert.equal(await frame.locator('#advisor-endpoint').inputValue({timeout:NATIVE_READINESS_MS}), config.endpoint); assert.equal(fixture.requests.length, calls, 'Validation and saving make no generation request'); context.proof('native-advisor-remote-opt-in-contract', {nativeUncheckedRefusal: true, nativeHttpOptInRefusal: true, explicitHttpsOptInSavedAndReopened: true, noGenerationRequests: true, credentialValueNotPersisted: true, @@ -223,7 +224,7 @@ async function exportAndVerify(context, result) { // its real frame and current card for each click, including the next export. const frame = await tab(context, 'Saved evidence'); const card = frame.locator('article.card').filter({has: frame.getByRole('heading', {name: /^narrate ·/})}).first(); - await card.getByRole('button', {name: label, exact: true}).click({timeout: 30000}); + await card.getByRole('button', {name: label, exact: true}).click({timeout: NATIVE_READINESS_MS}); const expected = result.location.replace(/\.json$/, `.${extension}`); await eventually(() => context.vscode.window.activeTextEditor?.document.uri.fsPath === context.vscode.Uri.file(expected).fsPath, `${label} export opens in the actual native editor`); assert(context.vscode.window.activeTextEditor.document.getText().includes(marker)); @@ -261,11 +262,11 @@ exports.run = async context => { assert.equal(await fs.realpath(declared[0].source),await fs.realpath(path.join(context.workspace,'perf','cases.jl'))); assert.deepEqual(selectedScenarios(discovery.report.declared,[wantedKey]),declared); let frame = await tab(context, 'Scenarios'); - await frame.getByRole('button', {name: 'Clear selection', exact: true}).click(); + await frame.getByRole('button', {name: 'Clear selection', exact: true}).click({timeout:NATIVE_READINESS_MS}); const card = frame.locator('article.card').filter({has: frame.locator('.scenario-title strong', {hasText: /^sum_squares$/})}) .filter({has: frame.locator('.implementation', {hasText: /^allocating$/})}); assert.equal(await card.count(),1,'Only the exact narrative scenario card is selected'); - await card.locator('.scenario-title input').check(); + await card.locator('.scenario-title input').check({timeout:NATIVE_READINESS_MS}); const selected=await frame.locator('article.card').evaluateAll(cards=>cards.filter(card=>card.querySelector('.scenario-title input')?.checked) .map(card=>({id:card.querySelector('.scenario-title strong')?.textContent,implementation:card.querySelector('.implementation')?.textContent}))); assert.deepEqual(selected.map(scenarioKey),[wantedKey],'The native checkbox selects exactly the canonical narrative scenario'); @@ -283,7 +284,7 @@ exports.run = async context => { assert(advice.recommendations.some(item => item.rule_id === 'evidence.samples'), 'Two real samples trigger the deterministic ten-sample policy'); const runBytes = await fs.readFile(measured.location); frame = await tab(context, 'Findings & advice'); - assert.equal(await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).isEnabled(), true); + assert.equal(await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).isEnabled({timeout:NATIVE_READINESS_MS}), true); const narrative = await action(context, root, 'Explain with configured model', 'narrate'); assert.deepEqual(fixture.errors, []); assert.equal(fixture.requests.length, 1); @@ -305,9 +306,9 @@ exports.run = async context => { outboundSyntheticBearerExact: true, realJuliaWorkerEnvironment: true, credentialValueNotPersisted: true, scope: 'Owned loopback fixture only; synthetic token, no human authentication or inference'}); frame = await tab(context, 'Findings & advice'); - await frame.getByRole('heading', {name: 'Optional model explanation', exact: true}).waitFor(); - assert((await frame.locator('#app').innerText()).includes(marker)); - assert((await frame.locator('#app').innerText()).includes('Generated prose needs review')); + await frame.getByRole('heading', {name: 'Optional model explanation', exact: true}).waitFor({timeout:NATIVE_READINESS_MS}); + assert((await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS})).includes(marker)); + assert((await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS})).includes('Generated prose needs review')); context.proof('native-narrative-controlled-response-visible', {provider: 'local controlled HTTP protocol fixture; no inference or human authentication', protocol: settings.advisorProtocol, requests: fixture.requests.length, scenario: 'sum_squares', evidenceIds: request.ids, samples: 2, status: narrative.report.status, authority: narrative.report.authority, visible: true, exportsStillRequired: true}); @@ -321,18 +322,18 @@ exports.run = async context => { frame = await reloadMeasuredEvidence(context, measured, 'before-cancel'); fixture.mode = 'hold'; const before = await directories(root); - await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click(); - await eventually(() => fixture.blocked, 'second actual HTTP generation request arrives', 240000); + await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(() => fixture.blocked, 'second actual HTTP generation request arrives', NATIVE_READINESS_MS); context.proof('native-narrative-http-held', {stage: 'before-cancel', requests: fixture.requests.length}); assert.equal(fixture.requests.length, 2); assert.equal(fixture.blocked.closed, false, 'Generation is actually in flight when Cancel is clicked'); - assert.equal(await frame.getByRole('button', {name: 'Cancel', exact: true}).isEnabled(), true); + assert.equal(await frame.getByRole('button', {name: 'Cancel', exact: true}).isEnabled({timeout:30000}), true); context.log('native-narrative-cancel-click', {stage: 'active-http'}); - await frame.getByRole('button', {name: 'Cancel', exact: true}).click(); + await frame.getByRole('button', {name: 'Cancel', exact: true}).click({timeout:30000}); await eventually(() => fixture.blocked.closed, 'cancellation closes the active model HTTP socket before fixture teardown', 180000); context.proof('native-narrative-http-closed', {stage: 'cancelled'}); frame = await idle(context); - assert.match(await frame.locator('#app .status').innerText(), /Cancelled after controller cleanup/); + assert.match(await frame.locator('#app .status').innerText({timeout:NATIVE_READINESS_MS}), /Cancelled after controller cleanup/); assert.deepEqual(fixture.errors, []); const added = [...await directories(root)].filter(id => !before.has(id)); assert.equal(added.length, 1); @@ -353,16 +354,16 @@ exports.run = async context => { fixture.blocked = undefined; const timeoutBefore = await directories(root); frame = await reloadMeasuredEvidence(context, measured, 'before-deadline'); - await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click(); - await eventually(() => fixture.blocked, 'The real third HTTP request reaches the deadline fixture', 180000); + await frame.getByRole('button', {name: 'Explain with configured model', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(() => fixture.blocked, 'The real third HTTP request reaches the deadline fixture', NATIVE_READINESS_MS); context.proof('native-narrative-http-held', {stage: 'before-deadline', requests: fixture.requests.length}); const held = fixture.blocked; assert.equal(held.closed, false); - const expired = await reportAfter(context, root, timeoutBefore, 'narrate'); + const expired = await reportAfter(context, root, timeoutBefore, 'narrate',360000); assert.equal(expired.report.status, 'timeout', 'The Core request deadline expires without a Cancel click'); - await eventually(() => held.closed, 'The expired model socket closes before fixture teardown'); + await eventually(() => held.closed, 'The expired model socket closes before fixture teardown',180000); assert.equal(configuration(context).get('advisorTimeout'), 45); frame = await tab(context, 'Findings & advice'); - assert.match(await frame.locator('#app').innerText(), /timeout|timed out|isolated worker stopped/i); + assert.match(await frame.locator('#app').innerText({timeout:NATIVE_READINESS_MS}), /timeout|timed out|isolated worker stopped/i); assert.deepEqual(await fs.readFile(measured.location), runBytes); context.proof('native-narrative-configured-deadline', {configuredSeconds: 45, requestReachedOwnedFixture: true, noCancelClick: true, coreStatus: expired.report.status, socketClosedBeforeFixtureCleanup: true, @@ -371,9 +372,9 @@ exports.run = async context => { context.log('native-narrative-finally-entered'); // Only the sacrificial workspace-folder settings are restored; global/user configuration is never written. try { - const frame = await view(context); - if (await frame.getByRole('button', {name: 'Cancel', exact: true}).isEnabled()) { - await frame.getByRole('button', {name: 'Cancel', exact: true}).click(); + const frame = await view(context,120000); + if (await frame.getByRole('button', {name: 'Cancel', exact: true}).isEnabled({timeout:30000})) { + await frame.getByRole('button', {name: 'Cancel', exact: true}).click({timeout:30000}); await idle(context); } } finally { diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs index 144deb1..d75534a 100644 --- a/test/native-platform-contract.test.mjs +++ b/test/native-platform-contract.test.mjs @@ -1,8 +1,13 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import {EventEmitter} from 'node:events'; +import readiness from './native-readiness.cjs'; +import ts from 'typescript'; import {readFileSync} from 'node:fs'; import {assertNativeCampaignSelection,nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition,nativeFullFunctionalGroups,nativeWorkflowMatrix} from './native-platform-contract.mjs'; +const {NATIVE_READINESS_MS,createNativeActivityBudget,waitForNativeAvailability}=readiness; + const historicalCases=['native-all-studio-controls','native-workbench-controls','native-investigation-controls', 'native-investigation-limits','native-enabled-narrative-protocol','native-advisor-controls', 'native-mcp-controls','native-pluto-controls','native-discovery-and-sync-after-pluto']; @@ -80,7 +85,7 @@ test('the real host dispatches every advertised shared-prefix case before its pa assert.deepEqual(calls,['controller-visible-in-studio','native-julia-terminal','native-pluto-without-jupyter']); const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); assert(suiteStart>=0&&suiteEnd>suiteStart); - const suite=Function('discoveryDiagnostic',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false); + const suite=Function('discoveryDiagnostic','NATIVE_READINESS_MS',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false,NATIVE_READINESS_MS); await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture'},runCase,{testItem:true,complete:true}); assert.deepEqual(calls,historicalPrefix); assert.deepEqual(nativeFullCasePartition({artifact:'candidate'}).sharedPrefix,historicalPrefix); @@ -99,9 +104,9 @@ test('each real full dispatcher executes eight prefixes followed by only its fun assert.equal(depth,0); const AsyncFunction=Object.getPrototypeOf(async function(){}).constructor; const prefix=new AsyncFunction('runCase','process',source.slice(prefixStart,prefixEnd)); - const suite=Function('discoveryDiagnostic',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false); + const suite=Function('discoveryDiagnostic','NATIVE_READINESS_MS',`return (${source.slice(suiteStart,suiteEnd).trim()})`)(false,NATIVE_READINESS_MS); const tail=new AsyncFunction('process','discoveryDiagnostic','runCase','context','fullCasePartition','assert', - 'controls','workbench','investigations','require','advisor','mcp','pluto','vscode',source.slice(tailStart,end+1)); + 'controls','workbench','investigations','require','advisor','mcp','pluto','vscode','waitForNativeAvailability',source.slice(tailStart,end+1)); for(const platform of ['linux','win32','darwin'])for(const group of nativeFullFunctionalGroups){ const calls=[],effects=[],context={vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture'}; const noEffects=async name=>{calls.push(name);return true;}; @@ -116,7 +121,7 @@ test('each real full dispatcher executes eight prefixes followed by only its fun nativeFullCasePartition({partition:group.id,platform,arch:'x64',stage:'full',artifact:'candidate'}),assert, {run:effect('native-all-studio-controls')},{run:effect('native-workbench-controls')}, {run:effect('native-investigation-controls')},name=>{assert(Object.hasOwn(modules,name));return modules[name];}, - {run:effect('native-advisor-controls')},{run:effect('native-mcp-controls')},{run:effect('native-pluto-controls')},vscode); + {run:effect('native-advisor-controls')},{run:effect('native-mcp-controls')},{run:effect('native-pluto-controls')},vscode,waitForNativeAvailability); assert.deepEqual(calls,[...historicalPrefix,...group.cases]); assert.deepEqual(effects,group.id==='mcp-pluto'? ['native-mcp-controls','native-pluto-controls','perfchecker.discoverScenarios','perfchecker.syncScenarios']:group.cases); @@ -127,8 +132,8 @@ test('the real diagnostic host keeps the seventh prefix and explicitly excludes const source=readFileSync(new URL('./public-vsix-host.cjs',import.meta.url),'utf8'); const suiteStart=source.indexOf('async function runSuiteCases('),suiteEnd=source.indexOf('async function measureNativeTestItem(',suiteStart); assert(suiteStart>=0&&suiteEnd>suiteStart); - const suite=Function('discoveryDiagnostic','process',`return (${source.slice(suiteStart,suiteEnd).trim()})`) - (true,{env:{PERFCHECKER_NATIVE_PHASE:'prepared'}}); + const suite=Function('discoveryDiagnostic','process','NATIVE_READINESS_MS',`return (${source.slice(suiteStart,suiteEnd).trim()})`) + (true,{env:{PERFCHECKER_NATIVE_PHASE:'prepared'}},NATIVE_READINESS_MS); const calls=[],logs=[]; await suite({vscode:{Uri:{file:value=>value}},workspace:'/disposable-fixture',log:(name,detail)=>logs.push({name,...detail})}, async name=>{calls.push(name);return true;},{testItem:true,complete:true}); @@ -203,15 +208,204 @@ test('the actual package output selects unique functional jobs for every complet assert.throws(()=>nativeWorkflowMatrix({event:'workflow_dispatch',scope:'macos-stable',stage:'full',artifact:'candidate',core:'candidate',discoveryDiagnostic:true,...change})); }); -test('functional splitting preserves the real three-phase preparation and all existing time limits',()=>{ +test('functional splitting preserves all three phases under the shared one-hour availability policy',()=>{ const workflow=readFileSync(new URL('../.github/workflows/native-vsix.yml',import.meta.url),'utf8'); const driver=readFileSync(new URL('./run-public-vsix-host.mjs',import.meta.url),'utf8'); - assert(workflow.includes("timeout-minutes: ${{ (github.event_name != 'workflow_dispatch' || inputs.stage == 'full') && 180 || 150 }}")); + assert(workflow.includes("timeout-minutes: 360")); + assert(workflow.includes("PERFCHECKER_NATIVE_ACTIVITY_STARTED_AT=$(( $(date +%s) * 1000 ))")); assert(driver.includes("const completeCampaign=['smoke','full'].includes(stage);")); assert(driver.includes("if(completeCampaign||bootstrapOnly)await launch('fresh');")); assert(driver.includes("if(completeCampaign)await launch('configured');")); assert(driver.includes("else await launch('prepared');")); - assert(driver.includes("phase==='prepared'&&environment.PERFCHECKER_NATIVE_STAGE==='full'?130")); + assert(driver.includes("const remainingActivityMs=nativeActivity.remaining(),activationWindowMs=Math.min(NATIVE_READINESS_MS,remainingActivityMs);")); + assert(driver.includes("PERFCHECKER_NATIVE_ACTIVITY_DEADLINE_AT:String(nativeActivity.deadlineAt)")); + assert(driver.includes("Historical discovery diagnostics retain their original branch and explicit contract")); assert(workflow.includes("if: runner.os == 'Linux' && matrix.vscode == 'stable' && (env.PERFCHECKER_NATIVE_STAGE == 'full' || env.PERFCHECKER_NATIVE_CASE_GROUP == 'mcp-pluto')"), 'Every complete group preserves the actual host prerequisite condition without a partition exception'); }); + +test('preparation and successive phases consume one activity deadline, leaving the technical reserve',()=>{ + let now=1000; + const budget=createNativeActivityBudget({startedAt:now,now:()=>now}); + const deadline=budget.deadlineAt; + now+=75*60000; // Installation and preparation are already part of the job. + assert.equal(budget.remaining(),255*60000); + assert.equal(budget.timeout(),3600000); + now+=220*60000; // Fresh/configured/prepared never renew the activity window. + assert.equal(budget.deadlineAt,deadline); + assert.equal(budget.timeout(),35*60000); + assert.equal(budget.timeout(3000),3000,'Short inspections are never expanded'); + now=deadline; + assert.throws(()=>budget.timeout(),{code:'NATIVE_INFRASTRUCTURE_ACTIVITY_LIMIT'}); + assert.throws(()=>budget.remaining(),{code:'NATIVE_INFRASTRUCTURE_ACTIVITY_LIMIT'}); + assert.equal(readiness.nativeReadinessPolicy.jobMinutes-readiness.nativeReadinessPolicy.activityMinutes,30); +}); + +function actualFunction(file,name,bindings,exposeRetention=false){ + const source=readFileSync(new URL(file,import.meta.url),'utf8'); + const tree=ts.createSourceFile(file,source,ts.ScriptTarget.Latest,true,ts.ScriptKind.JS); + const node=tree.statements.find(node=>ts.isFunctionDeclaration(node)&&node.name?.text===name); + assert(node,`The real ${name} implementation remains present`); + const body=exposeRetention?`let sessionSafeToRemove=true; return {run:(${node.getText(tree)}),retained:()=>!sessionSafeToRemove}`: + `return (${node.getText(tree)})`; + return Function(...Object.keys(bindings),body)(...Object.values(bindings)); +} + +test('the real host availability helper waits beyond the former bound while explicit cancellation stays short',async()=>{ + let now=0,polls=0; + const eventually=actualFunction('./public-vsix-host.cjs','eventually',{ + NATIVE_READINESS_MS,Date:{now:()=>now},delay:async milliseconds=>{ + assert.equal(milliseconds,150,'The existing polling interval is retained'); + now+=30*60000;polls++; + } + }); + assert.equal(await eventually(()=>now>=30*60000&&'ready','Positive readiness'),'ready'); + assert.equal(polls,1); + now=0; + await assert.rejects(eventually(()=>false,'Protected cancellation',120000),/Protected cancellation/); + assert.equal(polls,2,'The explicit short cancellation bound does not become an hour'); +}); + +test('real Testing availability delegates both UI actions once and preserves its failure before RunTest',async()=>{ + const primary=new Error('controlled row availability failure'),actions=[],waits=[]; + const measure=actualFunction('./public-vsix-host.cjs','measureNativeTestItemBody',{ + NATIVE_READINESS_MS,discoveryDiagnostic:false,controls:{clickStudioAction:async(_,action)=>actions.push(action)}, + process:{env:{}},redact:value=>value,assert + }); + const row={waitFor:async options=>{waits.push(options);throw primary;}}; + const context={vscode:{commands:{executeCommand:()=>assert.fail('RunTest must not execute after the availability failure')}}, + windowPage:{locator:()=>({filter:()=>({first:()=>row})})},workspace:'/fixture'}; + await assert.rejects(measure(context,true),error=>error===primary); + assert.deepEqual(actions,['items','testing']); + assert.deepEqual(waits,[{state:'visible',timeout:3600000}]); +}); + +test('native operation observation preserves results and errors, and expiry neither retries nor stops the operation',async()=>{ + const timers=new Map();let sequence=0,delegations=0,complete; + const wait=actualFunction('./native-readiness.cjs','waitForNativeAvailability',{ + NATIVE_READINESS_MS,setTimeout:(callback,milliseconds)=>{ + assert.equal(milliseconds,3600000);const id=++sequence;timers.set(id,callback);return id; + },clearTimeout:id=>timers.delete(id) + }); + const result={original:true},primary=new Error('original command failure'); + assert.equal(await wait(Promise.resolve(result),'Ready command'),result); + await assert.rejects(wait(Promise.reject(primary),'Failed command'),error=>error===primary); + assert.equal(timers.size,0); + const operation=()=>{delegations++;return new Promise(resolve=>{complete=resolve;});}; + const pending=wait(operation(),'Slow command'); + const rejected=assert.rejects(pending,{code:'NATIVE_AVAILABILITY_LIMIT'}); + [...timers.values()][0]();await rejected; + assert.equal(delegations,1); + complete(result);await Promise.resolve(); + assert.equal(timers.size,0,'The still-owned original command may settle later without a retry or unhandled rejection'); +}); + +test('real preparation execution clips activity but permits bounded technical teardown after exhaustion',async()=>{ + let now=1000; + const nativeActivity=createNativeActivityBudget({startedAt:now,now:()=>now}),calls=[]; + const execute=actualFunction('./run-public-vsix-host.mjs','execute',{ + NATIVE_READINESS_MS,nativeActivity,runnerEnvironment:{fixture:true},landscapeOnly:false,landscapeAbort:{}, + path:{basename:value=>value},process:{stdout:{write(){}},stderr:{write(){}}}, + spawn:(file,args,options)=>{calls.push({file,args,options});const child=new EventEmitter(); + child.stdout=new EventEmitter();child.stderr=new EventEmitter();queueMicrotask(()=>child.emit('close',0));return child;} + }); + now=nativeActivity.deadlineAt-2500; + await execute('prepare',['fixture']); + assert.equal(calls[0].options.timeout,2500); + now=nativeActivity.deadlineAt; + await assert.rejects(execute('prepare-again',[]),{code:'NATIVE_INFRASTRUCTURE_ACTIVITY_LIMIT'}); + assert.equal(calls.length,1,'An expired positive preparation cannot spawn'); + await execute('technical-drain',[],{technical:true,timeout:10000}); + assert.equal(calls[1].options.timeout,10000); + assert(!Object.hasOwn(calls[1].options,'technical'),'The internal phase flag is not passed to spawn'); +}); + +test('real failed or interrupted preparations retain their session without claiming descendant shutdown',async()=>{ + for(const event of ['error','signal']){ + const primary=Object.assign(new Error('original spawn/abort failure'),{name:'AbortError'}),logs=[]; + const execution=actualFunction('./run-public-vsix-host.mjs','execute',{ + NATIVE_READINESS_MS,nativeActivity:{timeout:value=>value},runnerEnvironment:{},landscapeOnly:false,landscapeAbort:{}, + path:{basename:value=>value},process:{stdout:{write(){}},stderr:{write(){}}},console:{error:value=>logs.push(value)}, + spawn:()=>{const child=new EventEmitter();child.pid=123;child.stdout=new EventEmitter();child.stderr=new EventEmitter(); + queueMicrotask(()=>event==='error'?child.emit('error',primary):child.emit('close',null,'SIGTERM'));return child;} + },true); + assert.equal(execution.retained(),false); + await assert.rejects(execution.run('julia',[],{timeout:10}),error=>event==='error'?error===primary: + error.exitCode===null&&error.message==='julia exited null\n'); + assert.equal(execution.retained(),true); + const report=JSON.parse(logs[0].slice(logs[0].indexOf('{'))); + assert.equal(report.treeCleanupQualified,false); + assert.equal(report.physicalCloseObserved,event==='signal'); + assert.equal(report.ownerPid,123); + } +}); + +test('real VS Code preparation has an absolute clipped wait despite continued download progress',async()=>{ + for(const remaining of [3600000,2500]){ + let now=1000,complete,delegations=0,progress=0;const timers=new Map(),logs=[]; + const wait=actualFunction('./native-readiness.cjs','waitForNativeAvailability',{ + NATIVE_READINESS_MS,setTimeout:(callback,milliseconds)=>{timers.set(1,{callback,at:now+milliseconds});return 1;}, + clearTimeout:id=>timers.delete(id) + }); + const download=actualFunction('./run-public-vsix-host.mjs','downloadNativeVSCode',{ + nativeActivity:{timeout:()=>remaining},waitForNativeAvailability:wait,console:{error:value=>logs.push(value)}, + downloadAndUnzipVSCode:options=>{delegations++;assert.equal(options.timeout,remaining); + return new Promise(resolve=>{complete=resolve;});} + },true); + const result=download.run({version:'stable'}); + const rejected=assert.rejects(result,{code:'NATIVE_AVAILABILITY_LIMIT'}); + const deadline=timers.get(1).at; + for(let index=0;index<5;index++){progress++;now+=Math.floor(remaining/6);assert.equal(timers.get(1).at,deadline);} + now=deadline;timers.get(1).callback();await rejected; + assert.equal(progress,5);assert.equal(delegations,1);assert.equal(download.retained(),true); + assert.equal(JSON.parse(logs[0].slice(logs[0].indexOf('{'))).physicalCloseObserved,false); + complete('/late-extraction');await Promise.resolve(); + assert.equal(download.retained(),true,'A late resolved download does not retroactively qualify teardown or change the failure'); + } +}); + +test('the real Advisor Cancel callsite keeps its short click, completion and internal read bounds',async()=>{ + const waits=[],clicks=[],reads=[];let status='Ready'; + const frame={getByRole:role=>({click:async options=>clicks.push(options.timeout), + innerText:async options=>{reads.push(options.timeout);return status;}}), + locator:()=>({getAttribute:async(_,options)=>{reads.push(options.timeout);return 'false';}})}; + const finished=actualFunction('./native-advisor-controls.cjs','finished',{ + NATIVE_READINESS_MS,eventually:async(read,_,timeout)=>{waits.push(timeout);assert(await read());} + }); + const click=actualFunction('./native-advisor-controls.cjs','click',{NATIVE_READINESS_MS}); + await finished(frame,/Ready/); + assert.deepEqual(reads,[3600000,3600000]);reads.length=0;status='Operation cancelled'; + const source=readFileSync(new URL('./native-advisor-controls.cjs',import.meta.url),'utf8'); + const start=source.indexOf("await click(frame, 'Cancel operation'"); + const end=source.indexOf('\n',start);assert(start>=0&&end>start); + const AsyncFunction=Object.getPrototypeOf(async function(){}).constructor; + await new AsyncFunction('frame','click','finished',source.slice(start,end))(frame,click,finished); + assert.deepEqual(clicks,[30000]);assert.deepEqual(waits,[3600000,180000]);assert.deepEqual(reads,[30000,30000]); +}); + +test('real reactive Cancel refresh retains its ninety-second window across nested auto waits',async()=>{ + let now=0;const calls=[]; + const refresh=actualFunction('./native-pluto-controls.cjs','refresh',{ + NATIVE_READINESS_MS,Date:{now:()=>now},assert,idle:async(_,timeout)=>{calls.push(['idle',timeout]);now+=90000;} + }); + const frame={getByRole:()=>({click:async options=>calls.push(['click',options.timeout])}), + locator:()=>({innerText:()=>assert.fail('An exhausted Cancel window cannot start another auto wait')})}; + await assert.rejects(refresh(frame,'Refresh','status',/cancelled/,90000),/existing reactive-result window/); + assert.deepEqual(calls,[['click',90000],['idle',90000]]); +}); + +test('native browser callbacks never capture the Node-only readiness constant',()=>{ + for(const file of ['public-vsix-host.cjs','native-workbench-controls.cjs','native-studio-controls.cjs', + 'native-pluto-controls.cjs','native-advisor-controls.cjs','native-investigation-controls.cjs', + 'native-mcp-controls.cjs','native-narrative-controls.cjs']){ + const source=readFileSync(new URL('./'+file,import.meta.url),'utf8'); + const tree=ts.createSourceFile(file,source,ts.ScriptTarget.Latest,true,ts.ScriptKind.JS); + const inspect=node=>{ + if(ts.isCallExpression(node)&&ts.isPropertyAccessExpression(node.expression)&&/^evaluate(All)?$/.test(node.expression.name.text)){ + assert(!/\bNATIVE_READINESS_MS\b/.test(node.arguments[0]?.getText(tree)||''),`${file}: browser callbacks have no Node scope`); + } + ts.forEachChild(node,inspect); + }; + inspect(tree); + } +}); diff --git a/test/native-pluto-controls.cjs b/test/native-pluto-controls.cjs index c5ac68d..d146a2a 100644 --- a/test/native-pluto-controls.cjs +++ b/test/native-pluto-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); // Actual VS Code webviews and Pluto workers in disposable CI workspaces only. const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); @@ -9,7 +10,7 @@ const execute=require('node:util').promisify(execFile); const {inflateSync}=require('node:zlib'); const {clickStudioAction}=require('./native-studio-controls.cjs'); -async function eventually(read, name, timeout = 180000) { +async function eventually(read, name, timeout = NATIVE_READINESS_MS) { const until = Date.now() + timeout; let last; while (Date.now() < until) { @@ -48,10 +49,10 @@ async function fingerprint(directory) { async function view(context) { const parent = await context.findFrame('iframe.perfchecker-pluto-frame'); - const element = await parent.locator('iframe.perfchecker-pluto-frame').elementHandle(); + const element = await parent.locator('iframe.perfchecker-pluto-frame').elementHandle({timeout:NATIVE_READINESS_MS}); const frame = await element.contentFrame(); assert(frame, 'The notebook must be an interactive Pluto iframe'); - await frame.locator('pluto-notebook').waitFor({state: 'visible', timeout: 180000}); + await frame.locator('pluto-notebook').waitFor({state: 'visible', timeout: NATIVE_READINESS_MS}); return {parent, frame}; } @@ -73,16 +74,16 @@ async function interactiveEditor(context,state,cell,label){ const observe=async stage=>context.log('pluto-editor-readiness',{label,stage,state:await geometry(),frontendErrors:[...errors]}); page.on('pageerror',onError); try{ - await cell.scrollIntoViewIfNeeded();await observe('before-gesture'); + await cell.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS});await observe('before-gesture'); const shown=cell.locator('pluto-input .cm-editor'); - if(!await shown.isVisible())await cell.locator('.foldcode').click(); + if(!await shown.isVisible())await cell.locator('.foldcode').click({timeout:NATIVE_READINESS_MS}); await observe('after-fold-gesture'); - await shown.waitFor({state:'visible'}); + await shown.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); // Pluto replaces the static placeholder with CodeMirror when its input // intersects the viewport. Scrolling the cell can leave that input clipped. - await shown.scrollIntoViewIfNeeded(); + await shown.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); const editor=cell.locator('pluto-input .cm-editor:not(.cm-ssr-fake) .cm-content[contenteditable="true"]'); - await eventually(async()=>await editor.isVisible()&&await editor.isEditable(),'The real UI exposes an interactive CodeMirror input'); + await eventually(async()=>await editor.isVisible()&&await editor.isEditable({timeout:NATIVE_READINESS_MS}),'The real UI exposes an interactive CodeMirror input'); await observe('interactive');return editor; }catch(error){ try{await observe('failed');}catch(diagnosticError){ @@ -93,9 +94,9 @@ async function interactiveEditor(context,state,cell,label){ finally{page.off('pageerror',onError);} } -async function idle(frame) { +async function idle(frame,timeout=NATIVE_READINESS_MS) { await eventually(async () => await frame.locator('pluto-cell.running, pluto-cell.queued').count() === 0, - 'Pluto reactive cells finish'); + 'Pluto reactive cells finish',timeout); } async function ready(frame, button) { @@ -106,16 +107,17 @@ async function ready(frame, button) { if (errors.length) throw Object.assign(new Error(`Pluto reactive errors: ${errors.join('\n').slice(0, 4000)}`),{name:'PlutoReactiveError'}); return await frame.getByRole('button', {name: button, exact: true}).isVisible() && await frame.locator('pluto-editor.loading, pluto-editor.disconnected, pluto-cell.running, pluto-cell.queued').count() === 0; - }, `The connected notebook finishes evaluation and renders ${button}`, 240000); + }, `The connected notebook finishes evaluation and renders ${button}`, NATIVE_READINESS_MS); } -async function refresh(frame, button, selector, expected, timeout = 360000) { +async function refresh(frame, button, selector, expected, timeout = NATIVE_READINESS_MS) { const until = Date.now() + timeout; let last = ''; while (Date.now() < until) { - await frame.getByRole('button', {name: button, exact: true}).click(); - await idle(frame); - last = await frame.locator(selector).innerText(); + const remaining=()=>{const value=until-Date.now();assert(value>0,'The existing reactive-result window remains available');return value;}; + await frame.getByRole('button', {name: button, exact: true}).click({timeout:remaining()}); + await idle(frame,remaining()); + last = await frame.locator(selector).innerText({timeout:remaining()}); if (expected.test(last)) return last; if (/"status"\s+"(?:error|failed|timeout)"|Status:\s*(?:failed|timeout)/.test(last)) { throw new Error(`Pluto worker failed: ${last.slice(0, 1500)}`); @@ -138,18 +140,18 @@ async function stop(context, state, closePanel = false,deadline) { assert(port > 0, 'This disposable desktop test has a real loopback Pluto server'); context.log('native-ui-action',{surface:'Pluto session',action:closePanel?'Close notebook view':'Stop session'}); if (closePanel) await context.vscode.commands.executeCommand('workbench.action.closeActiveEditor'); - else await state.parent.locator('#pluto-stop').click(); + else await state.parent.locator('#pluto-stop').click({timeout:30000}); const remaining=deadline===undefined?45000:deadline-Date.now();assert(remaining>0,'The existing 45 second Close budget remains available'); await eventually(async () => !await portOpen(port), 'Closing Pluto also closes its server and workers', remaining); if(!closePanel)await eventually(async()=>{ - const parent=await context.findFrame('#pluto-restart'); + const parent=await context.findFrame('#pluto-restart',{timeout:120000}); return await parent.locator('iframe.perfchecker-pluto-frame').count()===0&& - /session stopped/i.test(await parent.locator('[role="status"]').innerText()); - },'The explicit Stop transition finishes before another session action'); + /session stopped/i.test(await parent.locator('[role="status"]').innerText({timeout:30000})); + },'The explicit Stop transition finishes before another session action',180000); } async function create(context, file, kind) { - const uri = await context.vscode.commands.executeCommand('perfchecker.newNotebook', context.vscode.Uri.file(file), {kind}); + const uri = await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.newNotebook', context.vscode.Uri.file(file), {kind}),"perfchecker.newNotebook"); assert.equal(uri.fsPath, context.vscode.Uri.file(file).fsPath); const source = await fs.readFile(file, 'utf8'); assert(source.startsWith('### A Pluto.jl notebook ###')); @@ -268,21 +270,21 @@ async function studioNotebookButtons(context,directory){ // VS Code declares this as application scope. Only the disposable profile is changed. await settings.update('simpleDialog.enable',true,context.vscode.ConfigurationTarget.Global); await clickStudioAction(context,'notebook'); - let picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible'}); - let input=picker.locator('input[type="text"]');await input.fill(file);await new Promise(resolve=>setTimeout(resolve,300));await input.press('Enter'); + let picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + let input=picker.locator('input[type="text"]');await input.fill(file,{timeout:NATIVE_READINESS_MS});await new Promise(resolve=>setTimeout(resolve,300));await input.press('Enter',{timeout:NATIVE_READINESS_MS}); // Filter the real QuickPick, then accept its active option with the native keyboard. // A pointer target may disappear while the file dialog transitions on macOS. picker=context.windowPage.locator('.quick-input-widget').filter({has:context.windowPage.locator('.quick-input-title').filter({hasText:/^PerfChecker · Pluto dashboard$/})}); - await picker.waitFor({state:'visible'}); - const kindInput=picker.locator('input[type="text"]');await kindInput.fill('Investigation'); - await picker.locator('.monaco-list-row').filter({hasText:'Investigation'}).first().waitFor({state:'visible'}); - await kindInput.press('Enter'); + await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + const kindInput=picker.locator('input[type="text"]');await kindInput.fill('Investigation',{timeout:NATIVE_READINESS_MS}); + await picker.locator('.monaco-list-row').filter({hasText:'Investigation'}).first().waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await kindInput.press('Enter',{timeout:NATIVE_READINESS_MS}); const state=await view(context);await idle(state.frame); assert((await fs.readFile(file,'utf8')).startsWith('### A Pluto.jl notebook ###')); await stop(context,state,true); await clickStudioAction(context,'openNotebook'); - picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible'}); - input=picker.locator('input[type="text"]');await input.fill(file);await new Promise(resolve=>setTimeout(resolve,300));await input.press('Enter'); + picker=context.windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + input=picker.locator('input[type="text"]');await input.fill(file,{timeout:NATIVE_READINESS_MS});await new Promise(resolve=>setTimeout(resolve,300));await input.press('Enter',{timeout:NATIVE_READINESS_MS}); await stop(context,await view(context),true); context.proof('pluto-studio-file-dialog-buttons',{nativeNewClick:true,nativeOpenClick:true,realFileDialog:true,file:path.basename(file),jupyter:false}); }finally{await settings.update('simpleDialog.enable',previous,context.vscode.ConfigurationTarget.Global);} @@ -296,7 +298,7 @@ async function restartFailureCleanup(context,directory){ if(initialOwner)await eventually(async()=>{ const actual=await sessionInventory(context,'after-initial-stop',undefined,initialOwner.descendants); return actual.direct.length===0&&initialOwner.descendants.every(prior=>identityGone(actual,prior)); - },'Initial Stop leaves no selected-Julia child or observed session descendant before the failed startup'); + },'Initial Stop leaves no selected-Julia child or observed session descendant before the failed startup',180000); const settings=context.vscode.workspace.getConfiguration('perfchecker',context.vscode.Uri.file(context.workspace)); const previous=settings.get('plutoProject','perf/pluto'),before=await serverPids(context);let startingPids=[]; const startingIdentities=new Map(); @@ -305,7 +307,7 @@ async function restartFailureCleanup(context,directory){ startingIdentities.set(`${row.pid}:${row.createdAt}`,row); }; try{ - const parent=await context.findFrame('#pluto-restart');await parent.locator('#pluto-restart').click(); + const parent=await context.findFrame('#pluto-restart');await parent.locator('#pluto-restart').click({timeout:NATIVE_READINESS_MS}); if(['win32','darwin'].includes(process.platform)){ assert.equal(before.size,0,'The failed-startup baseline contains no direct selected-Julia child'); const inventory=await eventually(async()=>{ @@ -317,12 +319,12 @@ async function restartFailureCleanup(context,directory){ startingPids=[leader.pid];startingIdentities.set(`${leader.pid}:${leader.createdAt}`,leader);remember(inventory); const status=await eventually(async()=>{ const current=await context.findFrame('#pluto-restart'); - const value=await current.locator('[role="status"]').innerText(); + const value=await current.locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS}); assert.match(value,/Starting Pluto/); assert.equal(await current.locator('iframe.perfchecker-pluto-frame').count(),0, 'The configuration change must occur during actual nonREADY startup'); return value; - },'The current re-rendered parent witnesses actual nonREADY startup before the configuration change',10000); + },'The current re-rendered parent witnesses actual nonREADY startup before the configuration change',NATIVE_READINESS_MS); context.log('pluto-failed-startup-physical-owner',{leader,descendants:[...startingIdentities.values()], status,iframePublished:false,aliveBeforeConfigurationChange:true, observedListeners:inventory.listeners.filter(row=>startingPids.includes(row.pid)),listenerScope:'diagnostic only'}); @@ -330,8 +332,8 @@ async function restartFailureCleanup(context,directory){ await settings.update('plutoProject','perf/changed-pluto-environment',context.vscode.ConfigurationTarget.WorkspaceFolder); await eventually(async()=>{ if(['win32','darwin'].includes(process.platform))remember(await sessionInventory(context,'startup-configuration-change',undefined,[...startingIdentities.values()])); - return /environment changed|Start the action again/.test(await (await context.findFrame('#pluto-restart')).locator('[role="status"]').innerText()); - },'Restart reports a changed environment after starting',240000); + return /environment changed|Start the action again/.test(await (await context.findFrame('#pluto-restart')).locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS})); + },'Restart reports a changed environment after starting',NATIVE_READINESS_MS); if(['win32','darwin'].includes(process.platform))await eventually(async()=>{ const actual=await sessionInventory(context,'failed-startup-cleanup',undefined,[...startingIdentities.values()]);remember(actual); return [...startingIdentities.values()].every(prior=>identityGone(actual,prior)); @@ -341,10 +343,10 @@ async function restartFailureCleanup(context,directory){ }catch(error){ try{ const current=await eventually(async()=>{ - const parent=await context.findFrame('#pluto-restart'),status=await parent.locator('[role="status"]').innerText(); + const parent=await context.findFrame('#pluto-restart'),status=await parent.locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS}); const iframe=parent.locator('iframe.perfchecker-pluto-frame'); - return {status,src:await iframe.count()?await iframe.getAttribute('src'):undefined}; - },'Read the actual current failed-startup parent before teardown',10000); + return {status,src:await iframe.count()?await iframe.getAttribute('src',{timeout:NATIVE_READINESS_MS}):undefined}; + },'Read the actual current failed-startup parent before teardown',NATIVE_READINESS_MS); const port=current.src?Number(new URL(current.src).port):undefined; let listeners; if(process.platform==='win32'&&port)listeners=JSON.parse((await execute('powershell.exe',['-NoProfile','-Command', @@ -356,7 +358,7 @@ async function restartFailureCleanup(context,directory){ }catch(secondary){context.log('pluto-restart-diagnostic-error',{primary:String(error),secondary:String(secondary)});} throw error; }finally{await settings.update('plutoProject',previous,context.vscode.ConfigurationTarget.WorkspaceFolder);} - const parent=await context.findFrame('#pluto-restart');await parent.locator('#pluto-restart').click(); + const parent=await context.findFrame('#pluto-restart');await parent.locator('#pluto-restart').click({timeout:NATIVE_READINESS_MS}); state=await view(context);await idle(state.frame); assert(await portOpen(Number(new URL(state.frame.url()).port)),'A later successful Restart remains alive'); const finalOwner=['win32','darwin'].includes(process.platform)?await sessionOwner(context,state):undefined; @@ -364,7 +366,7 @@ async function restartFailureCleanup(context,directory){ if(finalOwner)await eventually(async()=>{ const actual=await sessionInventory(context,'after-final-close',undefined,finalOwner.descendants); return finalOwner.descendants.every(prior=>identityGone(actual,prior)); - },'Final Close stops the exact listening session owner and observed descendants before teardown'); + },'Final Close stops the exact listening session owner and observed descendants before teardown',180000); context.proof('pluto-failed-restart-real-worker-cleanup',{failedPids:startingPids.length, failedIdentities:[...startingIdentities.values()].map(row=>({pid:row.pid,createdAt:row.createdAt})), nonReadyPhysicalOwnerVerified:['win32','darwin'].includes(process.platform),exactSuccessfulListenerOwnerVerified:!!finalOwner, @@ -382,18 +384,18 @@ async function investigation(context, directory) { 'Execute selected setup action', 'Cancel advisor setup', 'Refresh advisor setup / model inventory', 'Compare saved measurements']) { assert(await state.frame.getByRole('button', {name, exact: true}).isVisible(), `Real Pluto control: ${name}`); } - await state.frame.locator('bond[def="action"] select').selectOption('run'); - await state.frame.locator('bond[def="selected"] select').selectOption({label:'sampled_sum_squares / sampled'}); + await state.frame.locator('bond[def="action"] select').selectOption('run',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="selected"] select').selectOption({label:'sampled_sum_squares / sampled'},{timeout:NATIVE_READINESS_MS}); await idle(state.frame); assert.deepEqual(await fingerprint(reportRoot), before, 'Opening and selecting cannot execute measurements'); context.log('native-ui-action',{surface:'Pluto investigation',action:'Launch selected action'}); - await state.frame.getByRole('button', {name: 'Launch selected action', exact: true}).click(); + await state.frame.getByRole('button', {name: 'Launch selected action', exact: true}).click({timeout:NATIVE_READINESS_MS}); const report = await eventually(async () => { for (const name of await files(reportRoot)) { if (!name.endsWith(`${path.sep}run.json`) || before[path.relative(reportRoot, name)]) continue; return JSON.parse(await fs.readFile(name, 'utf8')); } - }, 'Launch writes a real scenario report', 360000); + }, 'Launch writes a real scenario report', NATIVE_READINESS_MS); assert.equal(report.schema_version, 'perfchecker-scenario-run/1'); assert.deepEqual(new Set(report.runs.map(run => run.collector)), new Set(['benchmark', 'chairmark', 'profile', 'profile_alloc'])); for (const run of report.runs) { @@ -407,7 +409,7 @@ async function investigation(context, directory) { await refresh(state.frame, 'Refresh status and evidence', snapshot, /\bcomplete\b/); await capture(context,'pluto-investigation'); const completed = await fingerprint(reportRoot); - await state.frame.locator('bond[def="action"] select').selectOption('tools'); + await state.frame.locator('bond[def="action"] select').selectOption('tools',{timeout:NATIVE_READINESS_MS}); await idle(state.frame); assert.deepEqual(await fingerprint(reportRoot), completed, 'Changing selectors after completion cannot rerun the check'); context.proof('pluto-investigation-real-run', {collectors: [...new Set(report.runs.map(run => run.collector))], core: context.core}); @@ -417,43 +419,43 @@ async function investigation(context, directory) { const cell = state.frame.locator(`pluto-cell[id="${id}"]`); const title = 'PerfChecker native reactive qualification'; const editor = await interactiveEditor(context,state,cell,'investigation reactive title'); - await editor.click(); await editor.press('ControlOrMeta+A'); + await editor.click({timeout:NATIVE_READINESS_MS}); await editor.press('ControlOrMeta+A',{timeout:NATIVE_READINESS_MS}); context.log('native-ui-action',{surface:'Pluto investigation',action:'Edit reactive cell and evaluate'}); - await editor.pressSequentially(`md"# ${title}: $(4 + 5)"`); await editor.press('ControlOrMeta+Enter'); + await editor.pressSequentially(`md"# ${title}: $(4 + 5)"`,{timeout:NATIVE_READINESS_MS}); await editor.press('ControlOrMeta+Enter',{timeout:NATIVE_READINESS_MS}); await eventually(async () => (await fs.readFile(file, 'utf8')).includes(`md"# ${title}: $(4 + 5)"`), 'Reactive cell is saved as real Julia code'); - await eventually(async () => (await cell.locator('pluto-output').innerText()).includes(title), 'Reactive output changes'); - await state.frame.goto(state.frame.url()); + await eventually(async () => (await cell.locator('pluto-output').innerText({timeout:NATIVE_READINESS_MS})).includes(title), 'Reactive output changes'); + await state.frame.goto(state.frame.url(),{timeout:NATIVE_READINESS_MS}); state = await view(context); - await eventually(async () => (await state.frame.locator(`pluto-cell[id="${id}"] pluto-output`).innerText()).includes(title), 'Saved cell survives a real iframe reload'); - await state.parent.locator('#pluto-source').click(); + await eventually(async () => (await state.frame.locator(`pluto-cell[id="${id}"] pluto-output`).innerText({timeout:NATIVE_READINESS_MS})).includes(title), 'Saved cell survives a real iframe reload'); + await state.parent.locator('#pluto-source').click({timeout:NATIVE_READINESS_MS}); await eventually(() => context.vscode.window.activeTextEditor?.document.uri.fsPath === context.vscode.Uri.file(file).fsPath, 'Open source opens the actual generated .jl'); const saved = await fs.readFile(file); - await context.vscode.commands.executeCommand('perfchecker.openNotebook', context.vscode.Uri.file(file)); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.openNotebook', context.vscode.Uri.file(file)),"perfchecker.openNotebook"); state = await view(context); assert.deepEqual(await fs.readFile(file), saved, 'Open reuses the saved notebook without regeneration'); const port = Number(new URL(state.frame.url()).port); - await context.vscode.commands.executeCommand('perfchecker.openNotebook', context.vscode.Uri.file(file)); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.openNotebook', context.vscode.Uri.file(file)),"perfchecker.openNotebook"); assert.equal(Number(new URL((await view(context)).frame.url()).port), port, 'Reopening reuses the same server'); if(process.env.PERFCHECKER_NATIVE_BROWSER_ORACLE){ const endpoint=process.env.PERFCHECKER_NATIVE_BROWSER_ORACLE,token=process.env.PERFCHECKER_NATIVE_BROWSER_TOKEN; const events=async()=>{const response=await fetch(`${endpoint}/events`,{headers:{authorization:`Bearer ${token}`}});assert.equal(response.status,200);return response.json();}; const expect=async(kind)=>eventually(async()=>{const values=await events();assert(!values.some(value=>value.failed),JSON.stringify(values));return values.find(value=>value.kind===kind);},`Real default browser completes ${kind}`); const editorUrl=state.frame.url(); - await state.frame.locator('#at_the_top button.toggle_export').click(); - await state.frame.locator('#export a[href*="notebookfile?"]').click(); + await state.frame.locator('#at_the_top button.toggle_export').click({timeout:NATIVE_READINESS_MS}); + await state.frame.locator('#export a[href*="notebookfile?"]').click({timeout:NATIVE_READINESS_MS}); const julia=await expect('julia-source-browser');assert(julia.editedExpression&&julia.credentialAbsent); assert.equal(state.frame.url(),editorUrl,'Julia export leaves the original VS Code editor in place'); - if(!((await state.frame.locator('#pluto-nav').getAttribute('class'))||'').includes('show_export'))await state.frame.locator('#at_the_top button.toggle_export').click(); - await state.frame.locator('#pluto-nav.show_export #export a[href*="notebookexport?"]').click(); - await state.frame.locator('.export-html-dialog .ple-download a[download]').click(); + if(!((await state.frame.locator('#pluto-nav').getAttribute('class',{timeout:NATIVE_READINESS_MS}))||'').includes('show_export'))await state.frame.locator('#at_the_top button.toggle_export').click({timeout:NATIVE_READINESS_MS}); + await state.frame.locator('#pluto-nav.show_export #export a[href*="notebookexport?"]').click({timeout:NATIVE_READINESS_MS}); + await state.frame.locator('.export-html-dialog .ple-download a[download]').click({timeout:NATIVE_READINESS_MS}); const html=await expect('html-download');assert(html.embeddedJulia&&html.editedExpression&&html.credentialAbsent); assert.equal(state.frame.url(),editorUrl,'HTML export leaves the original VS Code editor in place'); - await state.frame.locator('img#logo-big').locator('..').click();await state.frame.locator('#recent').waitFor(); - await state.frame.locator('#recent li.new a').click({modifiers:['Control']}); + await state.frame.locator('img#logo-big').locator('..').click({timeout:NATIVE_READINESS_MS});await state.frame.locator('#recent').waitFor({timeout:NATIVE_READINESS_MS}); + await state.frame.locator('#recent li.new a').click({modifiers:['Control'],timeout:NATIVE_READINESS_MS}); const modified=await expect('new-context-editor');assert.equal(modified.route,'/new');assert(modified.authenticatedWebSocket); assert.equal(new URL(state.frame.url()).pathname,'/','Modified New opens a separate real browser editor'); const originalId=new URL(editorUrl).searchParams.get('id'); - await state.frame.locator(`#recent a[href*="id=${originalId}"]`).click();await state.frame.locator('pluto-notebook').waitFor(); + await state.frame.locator(`#recent a[href*="id=${originalId}"]`).click({timeout:NATIVE_READINESS_MS});await state.frame.locator('pluto-notebook').waitFor({timeout:NATIVE_READINESS_MS}); assert.equal(new URL(state.frame.url()).searchParams.get('id'),originalId,'Normal Recent navigation remains inside VS Code'); await capture(context,'pluto-browser-exports-completed'); context.proof('pluto-default-browser-exports',{source:'actual-installed-VSIX-env.openExternal-and-disposable-xdg-default-browser', @@ -461,9 +463,9 @@ async function investigation(context, directory) { }else context.log('pluto-default-browser-exports',{status:'skipped',reason:'Representative actual OS default-browser fixture requires disposable Linux stable; no human browser profile changes.'}); await stop(context, state); const stopped = await context.findFrame('#pluto-restart'); - await stopped.locator('#pluto-restart').click(); + await stopped.locator('#pluto-restart').click({timeout:NATIVE_READINESS_MS}); state = await view(context); - await eventually(async () => (await state.frame.locator(`pluto-cell[id="${id}"] pluto-output`).innerText()).includes(title), 'Restart retains saved cells'); + await eventually(async () => (await state.frame.locator(`pluto-cell[id="${id}"] pluto-output`).innerText({timeout:NATIVE_READINESS_MS})).includes(title), 'Restart retains saved cells'); assert.deepEqual(await fingerprint(reportRoot), completed, 'Restart is not a measurement request'); await stop(context, state, true); context.proof('pluto-reactive-save-reload-close', {source: path.basename(file), workersClosed: true}); @@ -477,12 +479,12 @@ async function cancellation(context, directory) { try { await settings.update('scenarioCatalog', catalog, context.vscode.ConfigurationTarget.WorkspaceFolder); const state = await create(context, path.join(directory, 'NativeCancellation.jl'), 'investigation'); - await state.frame.locator('bond[def="action"] select').selectOption('run'); + await state.frame.locator('bond[def="action"] select').selectOption('run',{timeout:NATIVE_READINESS_MS}); await idle(state.frame); assert.equal(await fs.stat(marker).then(() => true).catch(() => false), false); - await state.frame.getByRole('button', {name: 'Launch selected action', exact: true}).click(); - await eventually(() => fs.readFile(marker, 'utf8').then(value => value === 'running'), 'An actual measured workload reaches its active-worker marker', 360000); - await state.frame.getByRole('button', {name: 'Cancel active investigation', exact: true}).click(); + await state.frame.getByRole('button', {name: 'Launch selected action', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(() => fs.readFile(marker, 'utf8').then(value => value === 'running'), 'An actual measured workload reaches its active-worker marker', NATIVE_READINESS_MS); + await state.frame.getByRole('button', {name: 'Cancel active investigation', exact: true}).click({timeout:30000}); const snapshot = `pluto-cell[id="${cellId(state.source, 'snapshot = (refresh_click;')}"] pluto-output`; await refresh(state.frame, 'Refresh status and evidence', snapshot, /\bcancelled\b/, 90000); await stop(context, state, true); @@ -527,9 +529,9 @@ async function ownedWorkerClose(context,directory){ try{ await settings.update('scenarioCatalog',catalog,context.vscode.ConfigurationTarget.WorkspaceFolder); let state=await create(context,path.join(directory,'StopActiveInvestigation.jl'),'investigation'); - await state.frame.locator('bond[def="action"] select').selectOption('run');await idle(state.frame); - await state.frame.getByRole('button',{name:'Launch selected action',exact:true}).click(); - const info=await eventually(async()=>{const lines=(await fs.readFile(marker,'utf8')).split('\n');return lines.length===3?lines:false;},'The detached investigation reaches its real PID and private controller-directory marker',360000); + await state.frame.locator('bond[def="action"] select').selectOption('run',{timeout:NATIVE_READINESS_MS});await idle(state.frame); + await state.frame.getByRole('button',{name:'Launch selected action',exact:true}).click({timeout:NATIVE_READINESS_MS}); + const info=await eventually(async()=>{const lines=(await fs.readFile(marker,'utf8')).split('\n');return lines.length===3?lines:false;},'The detached investigation reaches its real PID and private controller-directory marker',NATIVE_READINESS_MS); const [pid,userDirectory,controllerDirectory]=info; fixtureWorkerPid=Number(pid);fixtureUserDirectory=userDirectory; for(const name of ['request.toml','worker.log'])assert(await fs.stat(path.join(controllerDirectory,name)).then(stat=>stat.isFile()), @@ -554,16 +556,16 @@ async function ownedWorkerClose(context,directory){ const suiteMarker=path.join(context.workspace,'perf','owned-suite-worker.marker'); await fs.rm(suiteMarker,{force:true}); state=await create(context,path.join(directory,'CancelActiveAllocation.jl'),'suite'); - await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc');await idle(state.frame); - await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click(); - const cancelInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The Suite Cancel test reaches its active allocation worker',360000); + await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc',{timeout:NATIVE_READINESS_MS});await idle(state.frame); + await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click({timeout:NATIVE_READINESS_MS}); + const cancelInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The Suite Cancel test reaches its active allocation worker',NATIVE_READINESS_MS); const cancelOwned=await allocationOwned(cancelInfo); - await state.frame.getByRole('button',{name:'Cancel active job',exact:true}).click(); + await state.frame.getByRole('button',{name:'Cancel active job',exact:true}).click({timeout:30000}); let workerAliveBeforeSecondClick; try{process.kill(Number(cancelInfo[0]),0);workerAliveBeforeSecondClick=true;}catch(error){if(error.code!=='ESRCH')throw error;workerAliveBeforeSecondClick=false;} - await state.frame.getByRole('button',{name:'Cancel active job',exact:true}).click(); + await state.frame.getByRole('button',{name:'Cancel active job',exact:true}).click({timeout:30000}); await refresh(state.frame,'Refresh status','[data-suite-state]',/\bcancelled\b/,90000); - await eventually(()=>{try{process.kill(Number(cancelInfo[0]),0);return false;}catch(error){if(error.code==='ESRCH')return true;throw error;}},'The actual Suite Cancel button terminates the detached allocation worker'); + await eventually(()=>{try{process.kill(Number(cancelInfo[0]),0);return false;}catch(error){if(error.code==='ESRCH')return true;throw error;}},'The actual Suite Cancel button terminates the detached allocation worker',180000); await eventually(()=>fs.stat(cancelOwned).then(()=>false).catch(error=>{if(error.code==='ENOENT')return true;throw error;}),'Suite Cancel cleans the actual private allocation environment and its journal',45000); assert.deepEqual(await fs.readFile(preserved),sentinel); assert.deepEqual(new Set((await files(context.workspace)).filter(file=>file.endsWith('.mem'))),beforeMem); @@ -571,37 +573,37 @@ async function ownedWorkerClose(context,directory){ context.proof('pluto-suite-cancel-active-allocation',{nativeClick:true,nativeCancelClicks:2,workerAliveBeforeSecondClick,secondNativeCancelDuringCleanup:'not-observed; the Core regression separately forces cancellation during cleanup',workerTerminated:true,ownedDirectoryRemoved:true,noNewMem:true,preExistingMemPreserved:true}); await fs.rm(suiteMarker,{force:true}); state=await create(context,path.join(directory,'CloseActiveAllocation.jl'),'suite'); - await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc');await idle(state.frame); - await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click(); - const suiteInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The allocation worker reaches its real PID marker',360000); + await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc',{timeout:NATIVE_READINESS_MS});await idle(state.frame); + await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click({timeout:NATIVE_READINESS_MS}); + const suiteInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The allocation worker reaches its real PID marker',NATIVE_READINESS_MS); const suiteOwned=await allocationOwned(suiteInfo); const suiteElapsed=await assertStopped(state,Number(suiteInfo[0]),suiteOwned,true); context.proof('pluto-close-active-allocation-worker',{kind:'SuiteJob',cleanupMilliseconds:suiteElapsed,pidTerminated:true,ownedFilesRemoved:true,preExistingMemPreserved:true,noNewMem:true}); await fs.rm(suiteMarker,{force:true}); state=await create(context,path.join(directory,'ShutdownActiveAllocation.jl'),'suite'); - await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc');await idle(state.frame); - await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click(); - const shutdownInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The Pluto homepage Shutdown test reaches its active allocation worker',360000); + await state.frame.locator('bond[def="selected_collector"] select').selectOption('alloc',{timeout:NATIVE_READINESS_MS});await idle(state.frame); + await state.frame.getByRole('button',{name:'Launch selected checks',exact:true}).click({timeout:NATIVE_READINESS_MS}); + const shutdownInfo=await eventually(async()=>{const lines=(await fs.readFile(suiteMarker,'utf8')).split('\n');return lines.length===2?lines:false;},'The Pluto homepage Shutdown test reaches its active allocation worker',NATIVE_READINESS_MS); const shutdownOwned=await allocationOwned(shutdownInfo); let confirmed=false; const shutdownElapsed=await assertStopped(state,Number(shutdownInfo[0]),shutdownOwned,false,async()=>{ - await state.frame.locator('img#logo-big').locator('..').click(); + await state.frame.locator('img#logo-big').locator('..').click({timeout:NATIVE_READINESS_MS}); await eventually(()=>new URL(state.frame.url()).pathname==='/'&&state.frame.locator('#recent').isVisible(),'The real Pluto logo opens its authenticated homepage'); const running=state.frame.locator('#recent li.running').filter({hasText:'ShutdownActiveAllocation.jl'}); - await running.locator('button').first().click(); + await running.locator('button').first().click({timeout:NATIVE_READINESS_MS}); const confirmation=state.frame.getByRole('dialog',{name:'Pluto confirmation'}); - await confirmation.waitFor({state:'visible'}); - assert.equal(await confirmation.locator('p').innerText(),'Shut down notebook process?', + await confirmation.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + assert.equal(await confirmation.locator('p').innerText({timeout:NATIVE_READINESS_MS}),'Shut down notebook process?', 'The real confirmation preserves Pluto 1.0.4\'s exact current translation'); await capture(context,'pluto-homepage-shutdown-confirmation'); - await confirmation.getByRole('button',{name:'Cancel',exact:true}).click(); + await confirmation.getByRole('button',{name:'Cancel',exact:true}).click({timeout:30000}); assert.equal(await running.count(),1,'Cancelling the actual confirmation preserves the notebook'); assert.doesNotThrow(()=>process.kill(Number(shutdownInfo[0]),0),'Cancelling preserves the active allocation worker'); assert(await fs.stat(shutdownOwned).then(stat=>stat.isDirectory()),'Cancelling preserves the active owned allocation directory'); - await running.locator('button').first().click();await confirmation.waitFor({state:'visible'}); - await confirmation.getByRole('button',{name:'Confirm',exact:true}).click();confirmed=true; - await eventually(async()=>await running.count()===0,'The native Pluto homepage removes the stopped notebook'); + await running.locator('button').first().click({timeout:NATIVE_READINESS_MS});await confirmation.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await confirmation.getByRole('button',{name:'Confirm',exact:true}).click({timeout:NATIVE_READINESS_MS});confirmed=true; + await eventually(async()=>await running.count()===0,'The native Pluto homepage removes the stopped notebook',180000); }); assert(confirmed,'The actual visible confirmation was accepted through its native button'); assert(await portOpen(Number(new URL(state.frame.url()).port)),'The homepage shuts down its notebook without stopping the shared Pluto server'); @@ -692,15 +694,15 @@ async function drawnCanvas(context,canvas,label,movedFrom){ assertDrawnFigure(pixels); if(movedFrom)assertMovedHighlight(movedFrom,pixels); return pixels; - },`The native ${label} WebGL figure draws measured evidence`,180000); + },`The native ${label} WebGL figure draws measured evidence`,NATIVE_READINESS_MS); } async function plotDocument(state,previous){ const element=await eventually(async()=>{ const frames=state.frame.locator('pluto-output iframe[data-perfchecker-plot-frame]'); if(await frames.count()!==1)return false; - const handle=await frames.elementHandle(); - const token=await handle.getAttribute('data-perfchecker-plot-frame'); + const handle=await frames.elementHandle({timeout:NATIVE_READINESS_MS}); + const token=await handle.getAttribute('data-perfchecker-plot-frame',{timeout:NATIVE_READINESS_MS}); if(previous&&token===previous.token){await handle.dispose();return false;} return handle; },'The actual generator displays one fresh SuitePlotFrame document'); @@ -717,7 +719,7 @@ async function plotDocument(state,previous){ await eventually(()=>previous.frame.isDetached(),'Pluto removes the previous plot document'); } const canvas=frame.locator('#offline-figure canvas'); - await canvas.waitFor({state:'visible',timeout:360000});await canvas.scrollIntoViewIfNeeded(); + await canvas.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await canvas.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); return {frame,canvas,token:attributes.token,title:attributes.title, htmlSha256:createHash('sha256').update(attributes.srcdoc).digest('hex')}; } @@ -739,7 +741,7 @@ function observePlotFrontend(context,state){ try{ const detail=await Promise.race([(async()=>{ const inspect=()=>{ - const ids=[...document.querySelectorAll('[data-jscall-id]')].map(node=>node.getAttribute('data-jscall-id')); + const ids=[...document.querySelectorAll('[data-jscall-id]')].map(node=>node.getAttribute('data-jscall-id',{timeout:NATIVE_READINESS_MS})); const counts=new Map();for(const id of ids)counts.set(id,(counts.get(id)||0)+1); const sessions=window.Bonito?.Sessions,queue=window.Bonito?.OBJECT_FREEING_LOCK; return {readyState:document.readyState,viewport:{width:innerWidth,height:innerHeight},bonitoLoaded:!!window.Bonito,queue:queue?{size:queue.size,pending:queue.pending,isPaused:queue.isPaused}:null, @@ -756,7 +758,7 @@ function observePlotFrontend(context,state){ }; const notebook=await state.frame.evaluate(inspect),documents=[]; for(const element of await state.frame.locator('iframe[data-perfchecker-plot-frame]').elementHandles()){ - const token=await element.getAttribute('data-perfchecker-plot-frame'),frame=await element.contentFrame(); + const token=await element.getAttribute('data-perfchecker-plot-frame',{timeout:NATIVE_READINESS_MS}),frame=await element.contentFrame(); if(frame&&!frame.isDetached())documents.push({token,...await frame.evaluate(inspect)}); } return {notebook,documents}; @@ -783,7 +785,7 @@ async function renderedPlots(context,state,selector,completedRoot){ const distribution=options.find(option=>option.label.endsWith(' · Sample distribution')); const trajectory=options.find(option=>option.label.endsWith(' · Version trajectory')); assert(distribution&&trajectory,'Real measured samples provide distribution and version-series entries'); - await selector.selectOption(distribution.value);await idle(state.frame); + await selector.selectOption(distribution.value,{timeout:NATIVE_READINESS_MS});await idle(state.frame); // Inspect the actual running notebook through a user-edited diagnostic cell. // The generated rendering cell and measured bundle remain untouched. const cell=state.frame.locator(`pluto-cell[id="${cellId(state.source,'## Performance curves')}"]`); @@ -806,15 +808,15 @@ end`; const clipboard=await context.vscode.env.clipboard.readText(); try{ await context.vscode.env.clipboard.writeText(diagnostic); - await editor.click();await editor.press('ControlOrMeta+A');await editor.press('ControlOrMeta+V'); + await editor.click({timeout:NATIVE_READINESS_MS});await editor.press('ControlOrMeta+A',{timeout:NATIVE_READINESS_MS});await editor.press('ControlOrMeta+V',{timeout:NATIVE_READINESS_MS}); await eventually(async()=>await editor.evaluate(node=>[...node.querySelectorAll('.cm-line')].map(line=>line.textContent).join('\n'))===diagnostic, - 'Native clipboard paste inserts the complete diagnostic into the real editor',30000); + 'Native clipboard paste inserts the complete diagnostic into the real editor',NATIVE_READINESS_MS); }finally{await context.vscode.env.clipboard.writeText(clipboard);} - await editor.press('ControlOrMeta+Enter'); + await editor.press('ControlOrMeta+Enter',{timeout:NATIVE_READINESS_MS}); await ready(state.frame,'Launch selected checks'); await eventually(async()=>(await fs.readFile(path.join(context.workspace,'perf','notebooks','NativeSuite.jl'),'utf8')).includes(diagnostic), - 'The submitted diagnostic is present in the actual Pluto-saved notebook',30000); - const evidence=async()=>eventually(async()=>{const raw=await cell.locator('#native-plot-evidence').textContent();return raw&&JSON.parse(raw);},'The real Pluto worker exposes loaded providers and measured plot data',360000); + 'The submitted diagnostic is present in the actual Pluto-saved notebook',NATIVE_READINESS_MS); + const evidence=async()=>eventually(async()=>{const raw=await cell.locator('#native-plot-evidence').textContent({timeout:NATIVE_READINESS_MS});return raw&&JSON.parse(raw);},'The real Pluto worker exposes loaded providers and measured plot data',NATIVE_READINESS_MS); const data=await evidence();assert.equal(data.kind,'distribution');assert(data.selected.startsWith('distribution-'));assert.equal(data.selectedLabel,distribution.label);assert(data.values.length>=2); const providerProvenance=JSON.parse(await fs.readFile(path.join(process.env.PERFCHECKER_NATIVE_OUTPUT,'pluto-plot-provider-provenance.json'),'utf8')); assert.equal(providerProvenance.pins.makieCommit,'ffbf33f0bda61dfc84adfb8e8e6dfd8a404d0642'); @@ -845,9 +847,9 @@ end`; context.log('pluto-plot-diagnostic-viewport',await diagnosticViewport('before-native-fold')); // Pluto follows its focused CodeMirror caret and compensates output height changes. // Fold the completed diagnostic through its real control before viewing the plot. - await cell.locator('.foldcode').click(); + await cell.locator('.foldcode').click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await cell.locator('.cm-content:visible').count()===0,'The native fold control hides the completed diagnostic input'); - await selector.focus(); + await selector.focus({timeout:NATIVE_READINESS_MS}); await eventually(async()=>!(await diagnosticViewport('focus-check')).focusedWithin,'Plot selection owns focus before its separate canvas is scrolled'); context.log('pluto-plot-diagnostic-viewport',await diagnosticViewport('after-native-fold')); const firstDocument=await plotDocument(state),canvas=firstDocument.canvas; @@ -864,29 +866,29 @@ end`; }; const distinct=distinctMeasuredPoint(data); const selectedIndex=distinct<0?1:distinct; - await pointInput.focus();for(let step=0;steprecordedPoint(await firstDocument.frame.locator('#point-readout').innerText(),selectedIndex,data), + await pointInput.focus({timeout:NATIVE_READINESS_MS});for(let step=0;steprecordedPoint(await firstDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}),selectedIndex,data), 'A real native number-input gesture inspects the exact chosen version, value and unit'); - const expected=await firstDocument.frame.locator('#point-readout').innerText(); + const expected=await firstDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}); const after=await drawnCanvas(context,canvas,'highlight',distinct<0?undefined:before); await frontend.snapshot('distribution-point-inspected'); await capture(context,'pluto-rendered-distribution'); - await selector.selectOption(trajectory.value);await ready(state.frame,'Launch selected checks'); + await selector.selectOption(trajectory.value,{timeout:NATIVE_READINESS_MS});await ready(state.frame,'Launch selected checks'); const second=await eventually(async()=>{const current=await evidence();return current.selectedLabel===trajectory.label&¤t;},'The real plot selector regenerates the version-series figure'); assert.equal(second.kind,'version_series');assert(second.selected.startsWith('version-series-')); const secondDocument=await plotDocument(state,firstDocument); - await eventually(async()=>recordedPoint(await secondDocument.frame.locator('#point-readout').innerText(),0,second), + await eventually(async()=>recordedPoint(await secondDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}),0,second), 'The newly rendered trajectory inspector identifies its exact version, measured value and unit'); await frontend.snapshot('trajectory-readout-ready'); const trajectoryPixels=await drawnCanvas(context,secondDocument.canvas,'trajectory'); await capture(context,'pluto-rendered-version-series'); - await selector.selectOption(distribution.value);await ready(state.frame,'Launch selected checks'); + await selector.selectOption(distribution.value,{timeout:NATIVE_READINESS_MS});await ready(state.frame,'Launch selected checks'); const returned=await eventually(async()=>{const current=await evidence();return current.selectedLabel===distribution.label&¤t;},'The real selector returns to the original measured distribution'); assert.deepEqual(returned.values,data.values);assert.deepEqual(returned.versions,data.versions); const returnedDocument=await plotDocument(state,secondDocument); assert.notEqual(returnedDocument.token,firstDocument.token,'Even a cached plot creates a new document identity'); assert.equal(returnedDocument.htmlSha256,firstDocument.htmlSha256,'The unchanged cached HTML is reused without patching queues or providers'); - await eventually(async()=>recordedPoint(await returnedDocument.frame.locator('#point-readout').innerText(),0,data), + await eventually(async()=>recordedPoint(await returnedDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}),0,data), 'The returned document starts with its own exact version, measured value and unit'); const returnedPixels=await drawnCanvas(context,returnedDocument.canvas,'distribution-returned'); await frontend.snapshot('distribution-returned'); @@ -896,7 +898,7 @@ end`; assert(timing,'Interactive plot qualification requires the real measured wall-time distribution'); let temporalInteraction; { - await selector.selectOption(timing.value);await ready(state.frame,'Launch selected checks'); + await selector.selectOption(timing.value,{timeout:NATIVE_READINESS_MS});await ready(state.frame,'Launch selected checks'); const temporal=await eventually(async()=>{const current=await evidence();return current.selectedLabel===timing.label&¤t;},'The actual measured wall-time distribution is selected'); assert.equal(temporal.kind,'distribution');assert(temporal.selected.startsWith('distribution-'));assert(temporal.values.length>=2); const temporalDocument=await plotDocument(state,returnedDocument),temporalCanvas=temporalDocument.canvas; @@ -905,10 +907,10 @@ end`; assert(point>=0,'The measured wall-time coordinates coincide; required pixel movement cannot be qualified'); const index=point; const temporalInput=temporalDocument.frame.getByRole('spinbutton',{name:'Recorded point index'}); - await temporalInput.focus();for(let step=0;steprecordedPoint(await temporalDocument.frame.locator('#point-readout').innerText(),index,temporal), + await temporalInput.focus({timeout:NATIVE_READINESS_MS});for(let step=0;steprecordedPoint(await temporalDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}),index,temporal), 'A native number-input gesture inspects the exact wall-time version, measured value and unit'); - const readout=await temporalDocument.frame.locator('#point-readout').innerText(); + const readout=await temporalDocument.frame.locator('#point-readout').innerText({timeout:NATIVE_READINESS_MS}); const moved=await drawnCanvas(context,temporalCanvas,'wall-time-highlight',initial); await capture(context,'pluto-rendered-wall-time-highlight'); temporalInteraction={available:true,selected:temporal.selected,measuredSamples:temporal.values.length,nativeKeyboard:true, @@ -933,22 +935,22 @@ async function suite(context, directory, requirePlots=false,observation) { for (const name of ['Launch selected checks', 'Cancel active job', 'Refresh status', 'Save completed reports']) { assert(await state.frame.getByRole('button', {name, exact: true}).isVisible(), `Real suite control: ${name}`); } - await state.frame.locator('bond[def="selected_collector"] select').selectOption('benchmark'); - await state.frame.locator('bond[def="selected_package"] select').selectOption('PerfCheckerNativeFixture'); - await state.frame.locator('bond[def="selected_workload"] select').selectOption('sum_squares'); - await state.frame.locator('bond[def="selected_target"] select').selectOption('baseline'); - await state.frame.locator('bond[def="samples"] input').fill('2'); - await state.frame.locator('bond[def="samples"] input').press('Tab'); - await state.frame.locator('bond[def="seconds"] input').fill('0.1'); - await state.frame.locator('bond[def="seconds"] input').press('Tab'); + await state.frame.locator('bond[def="selected_collector"] select').selectOption('benchmark',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="selected_package"] select').selectOption('PerfCheckerNativeFixture',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="selected_workload"] select').selectOption('sum_squares',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="selected_target"] select').selectOption('baseline',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="samples"] input').fill('2',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="samples"] input').press('Tab',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="seconds"] input').fill('0.1',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="seconds"] input').press('Tab',{timeout:NATIVE_READINESS_MS}); await idle(state.frame); assert.deepEqual(await fingerprint(root), before, 'Suite selectors and sample inputs do not execute workers'); context.log('native-ui-action',{surface:'Pluto suite',action:'Launch selected checks'}); - await state.frame.getByRole('button', {name: 'Launch selected checks', exact: true}).click(); + await state.frame.getByRole('button', {name: 'Launch selected checks', exact: true}).click({timeout:NATIVE_READINESS_MS}); await refresh(state.frame, 'Refresh status', '[data-suite-state]', /\bcomplete\b/); assert.deepEqual(await fingerprint(root), before, 'Reports are saved only on the explicit Save action'); context.log('native-ui-action',{surface:'Pluto suite',action:'Save completed reports'}); - await state.frame.getByRole('button', {name: 'Save completed reports', exact: true}).click(); + await state.frame.getByRole('button', {name: 'Save completed reports', exact: true}).click({timeout:NATIVE_READINESS_MS}); const saved = await eventually(async () => { for (const name of await files(root)) { if (!name.endsWith(`${path.sep}suite-result.json`) || before[path.relative(root, name)]) continue; @@ -956,7 +958,7 @@ async function suite(context, directory, requirePlots=false,observation) { } }, 'Save completed reports writes an actual measured bundle'); const savedOutput=state.frame.locator(`pluto-cell[id="${cellId(state.source,'saved_reports = begin')}"] pluto-output`); - await eventually(async()=>/Last saved reports:/.test(await savedOutput.innerText()), + await eventually(async()=>/Last saved reports:/.test(await savedOutput.innerText({timeout:NATIVE_READINESS_MS})), 'The Save cell finishes writing every report format'); await idle(state.frame); for(const name of ['version-series.json','version-comparison.json','version-comparison.md']) @@ -981,8 +983,8 @@ async function suite(context, directory, requirePlots=false,observation) { context.log('pluto-rendered-plot', {available: true, canvas: true,documentToken:document.token}); } const completed = await fingerprint(root); - await state.frame.locator('bond[def="samples"] input').fill('3'); - await state.frame.locator('bond[def="samples"] input').press('Tab'); + await state.frame.locator('bond[def="samples"] input').fill('3',{timeout:NATIVE_READINESS_MS}); + await state.frame.locator('bond[def="samples"] input').press('Tab',{timeout:NATIVE_READINESS_MS}); await idle(state.frame); assert.deepEqual(await fingerprint(root), completed, 'Changing a completed suite does not rerun or resave results'); const closeDeadline=observation?Date.now()+45000:undefined; @@ -1014,7 +1016,7 @@ exports.run = async context => { context.log('pluto-failure-before-stop', {case:name,reactiveErrors:reactive.map(value=>value.slice(0,3000)), renderedLaunch:await state.frame.getByRole('button',{name:/Launch selected/}).count(), cellCount:await state.frame.locator('pluto-cell').count(), - editorStatus:await state.frame.locator('pluto-editor').getAttribute('class')}); + editorStatus:await state.frame.locator('pluto-editor').getAttribute('class',{timeout:NATIVE_READINESS_MS})}); } catch (diagnostic) {context.log('pluto-failure-view-unavailable',{case:name,message:String(diagnostic).replace(/([?&]secret=)[^&\s"<>]+/g,'$1[redacted]')});} await capture(context,`pluto-${name}-failed-before-stop`).catch(()=>{}); } @@ -1043,21 +1045,21 @@ exports.runFreshInstall = async context => { await filesSettings.update('simpleDialog.enable',true,vscode.ConfigurationTarget.Global); await clickStudioAction(context,'notebook'); const dialog=windowPage.locator('.quick-input-widget').filter({has:windowPage.locator('.quick-input-title').filter({hasText:/^PerfChecker · Create Pluto notebook$/})}); - await dialog.waitFor({state:'visible',timeout:30000}); - const input=dialog.locator('input[type="text"]');await input.fill(file);await input.press('Enter'); + await dialog.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + const input=dialog.locator('input[type="text"]');await input.fill(file,{timeout:NATIVE_READINESS_MS});await input.press('Enter',{timeout:NATIVE_READINESS_MS}); const picker=windowPage.locator('.quick-input-widget');let folderCreationConfirmed=false; await eventually(async()=>{ const createFolder=picker.filter({hasText:'The folder notebooks does not exist. Would you like to create it?'}); if(!folderCreationConfirmed&&await createFolder.isVisible()){ - await createFolder.getByRole('button',{name:'OK',exact:true}).click();folderCreationConfirmed=true; + await createFolder.getByRole('button',{name:'OK',exact:true}).click({timeout:NATIVE_READINESS_MS});folderCreationConfirmed=true; context.log('native-ui-action',{surface:'Published Pluto save dialog',action:'Confirm creation of notebooks folder'}); } return await picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}).isVisible(); - },'The real Save dialog finishes folder creation and shows notebook kinds',30000); - await picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}).click(); + },'The real Save dialog finishes folder creation and shows notebook kinds',NATIVE_READINESS_MS); + await picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}).click({timeout:NATIVE_READINESS_MS}); const install=windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}); - await install.waitFor({state:'visible',timeout:30000});await install.click(); - const source=await eventually(()=>fs.readFile(file,'utf8'),'The real published installer generates the first native notebook',600000); + await install.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await install.click({timeout:NATIVE_READINESS_MS}); + const source=await eventually(()=>fs.readFile(file,'utf8'),'The real published installer generates the first native notebook',NATIVE_READINESS_MS); assert(source.startsWith('### A Pluto.jl notebook ###')); const state={source,...await view(context)};await ready(state.frame,'Launch selected checks'); await capture(context,'fresh-published-pluto'); @@ -1068,7 +1070,7 @@ exports.runFreshInstall = async context => { for(const [name,directory]of [['controller',controller],['pluto',project]]){ const result=await execute(process.env.PERFCHECKER_NATIVE_JULIA,['--startup-file=no','--history-file=no',`--project=${directory}`,'-e', 'using Pkg,PerfChecker; modules=Module[PerfChecker];if ARGS[1]=="pluto";using PerfCheckerPluto,Pluto;append!(modules,[PerfCheckerPluto,Pluto]);end;print("PUBLISHED_BOOTSTRAP_PROVENANCE ");PerfChecker.JSON.print(Dict(string(nameof(m))=>begin;i=Pkg.dependencies()[Base.PkgId(m).uuid];Dict("version"=>string(Base.pkgversion(m)),"tree"=>string(i.tree_hash),"registered"=>i.is_tracking_registry,"revision"=>i.git_revision,"source"=>i.git_source,"pathof"=>pathof(m),"pkgdir"=>pkgdir(m));end for m in modules));println()',name], - {cwd:workspace,env:{...process.env,JULIA_LOAD_PATH:['@','@stdlib'].join(path.delimiter)},timeout:180000,maxBuffer:1000000}); + {cwd:workspace,env:{...process.env,JULIA_LOAD_PATH:['@','@stdlib'].join(path.delimiter)},timeout:NATIVE_READINESS_MS,maxBuffer:1000000}); const line=result.stdout.split(/\r?\n/).find(value=>value.startsWith('PUBLISHED_BOOTSTRAP_PROVENANCE '));assert(line,'The actual installed modules report their source provenance'); environments[name]=JSON.parse(line.slice('PUBLISHED_BOOTSTRAP_PROVENANCE '.length)); assert.equal(environments[name].PerfChecker.version,process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_VERSION);assert.equal(environments[name].PerfChecker.registered,true); @@ -1186,10 +1188,10 @@ exports.runStartStop = async context => { }}; observer=(async()=>{while(observing){try{await inspect('first-open-live',port);await observeLogs('first-open-live');await observeUI();}catch(error){observationFailure ||= error;observing=false;}if(observing)await wait(500);}})(); try{ - const created=await context.vscode.commands.executeCommand('perfchecker.newNotebook',context.vscode.Uri.file(file),{kind:'suite'});assert.equal(created.fsPath,file); + const created=await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.newNotebook',context.vscode.Uri.file(file),{kind:'suite'}),"perfchecker.newNotebook");assert.equal(created.fsPath,file); state=await view(context);port=Number(new URL(state.frame.url()).port);assert(Number.isInteger(port)&&port>0&&port<65536); context.log('pluto-first-start-ready-iframe',{observedAt:new Date().toISOString(),port,notebookSha256:createHash('sha256').update(await fs.readFile(file)).digest('hex'),sessionCredentialsOmitted:true}); - await eventually(async()=>/idle/.test(await state.frame.locator('[data-suite-state]').innerText()),'The first prepared dashboard is idle without launching checks',180000); + await eventually(async()=>/idle/.test(await state.frame.locator('[data-suite-state]').innerText({timeout:NATIVE_READINESS_MS})),'The first prepared dashboard is idle without launching checks',NATIVE_READINESS_MS); observing=false;await observer;if(observationFailure)throw observationFailure; const active=await inspect('idle-before-first-stop',port);assert.equal(active.listeners.length,1);assert.equal(active.listeners[0].owners.length,1); owner=active.rows.find(row=>row.pid===active.listeners[0].owners[0]);assert(owner&&owner.parent===process.pid&&owner.canonicalExecutable===executable&&owner.serverReadyMarker&&owner.projects.includes(project)); diff --git a/test/native-readiness.cjs b/test/native-readiness.cjs new file mode 100644 index 0000000..e229720 --- /dev/null +++ b/test/native-readiness.cjs @@ -0,0 +1,28 @@ +// Native qualification availability, explicitly requested for the 1.0.3 campaign. +// Execution-limit, cancellation, ownership and teardown tests keep their own bounds. +const NATIVE_READINESS_MS=3600000; +const NATIVE_JOB_MINUTES=360; +const NATIVE_ACTIVITY_MINUTES=330; +const NATIVE_TECHNICAL_RESERVE_MINUTES=NATIVE_JOB_MINUTES-NATIVE_ACTIVITY_MINUTES; +const nativeReadinessPolicy=Object.freeze({availabilityMilliseconds:NATIVE_READINESS_MS, + activityMinutes:NATIVE_ACTIVITY_MINUTES,jobMinutes:NATIVE_JOB_MINUTES, + technicalReserveMinutes:NATIVE_TECHNICAL_RESERVE_MINUTES, + scope:'Availability only; intentional execution limits, RPC, cancellation, ownership and teardown are unchanged', + limitation:'The shared activity budget includes preparation and all phases; a late wait is clipped by the remaining activity budget'}); +function createNativeActivityBudget({startedAt=Date.now(),now=Date.now}={}){ + if(!Number.isSafeInteger(startedAt)||startedAt<=0)throw new Error('The native activity start must be an explicit epoch'); + const deadlineAt=startedAt+NATIVE_ACTIVITY_MINUTES*60000; + const remaining=()=>{const value=deadlineAt-now();if(value<=0){const error=new Error('The shared native infrastructure activity budget is exhausted'); + error.code='NATIVE_INFRASTRUCTURE_ACTIVITY_LIMIT';throw error;}return value;}; + return Object.freeze({startedAt,deadlineAt,remaining,timeout:(requested=NATIVE_READINESS_MS)=>Math.min(requested,remaining())}); +} +// The command is delegated by the caller before arming this observation timer. +// Expiry does not signal, retry, replace or re-execute the product operation. +function waitForNativeAvailability(result,label,{timeout=NATIVE_READINESS_MS}={}){ + return new Promise((resolve,reject)=>{ + const timer=setTimeout(()=>reject(Object.assign(new Error(`${label} exceeded the one-hour native availability window`), + {code:'NATIVE_AVAILABILITY_LIMIT'})),timeout); + Promise.resolve(result).then(value=>{clearTimeout(timer);resolve(value);},error=>{clearTimeout(timer);reject(error);}); + }); +} +module.exports={NATIVE_READINESS_MS,NATIVE_JOB_MINUTES,NATIVE_ACTIVITY_MINUTES,nativeReadinessPolicy,createNativeActivityBudget,waitForNativeAvailability}; diff --git a/test/native-reload-host.cjs b/test/native-reload-host.cjs index b263933..c5a4bb0 100644 --- a/test/native-reload-host.cjs +++ b/test/native-reload-host.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); // A real single-folder -> saved multi-root conversion restarts the extension // host. The handoff is test evidence, not a fallback in the product. const assert=require('node:assert/strict'); @@ -6,7 +7,7 @@ const path=require('node:path'); const os=require('node:os'); const {createHash}=require('node:crypto'); -async function eventually(read,description,timeout=90000){ +async function eventually(read,description,timeout=NATIVE_READINESS_MS){ const until=Date.now()+timeout;let last; while(Date.now(){ await fs.writeFile(preserved,'Existing allocation evidence owned by the fixture\n'); const beforeMem=await allocationInventory(workspace); const beforeChecks=new Set(await privateChecks()); - await vscode.commands.executeCommand('perfchecker.refresh'); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.refresh'),"perfchecker.refresh"); const running=vscode.commands.executeCommand('perfchecker.runAll').then(()=>({status:'finished'}),error=>({status:'rejected',error:String(error)})); let outcome;void running.then(result=>{outcome=result;}); const info=await eventually(async()=>{ @@ -99,7 +100,7 @@ exports.run=async context=>{ const text=await fs.readFile(marker,'utf8').catch(error=>{if(error.code==='ENOENT')return '';throw error;}); const [pid,directory]=text.split('\n'); return /^\d+$/.test(pid)&&directory&&path.isAbsolute(directory)?{pid:Number(pid),directory}:false; - },'A real allocation workload reaches its owned PID/directory marker',360000); + },'A real allocation workload reaches its owned PID/directory marker',NATIVE_READINESS_MS); assert(alive(info.pid));assert(await fs.stat(path.join(info.directory,'owned.tmp'))); assert(path.basename(path.dirname(info.directory)).startsWith('perfchecker-check-'),'The workload marker must point inside this allocation worker\'s private Core environment'); const privateCheckDirectories=[]; diff --git a/test/native-restricted-controls.cjs b/test/native-restricted-controls.cjs index 47c16ba..55ca134 100644 --- a/test/native-restricted-controls.cjs +++ b/test/native-restricted-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // A real untrusted workspace in the runner's private VS Code profile. const assert=require('node:assert/strict'); const fs=require('node:fs/promises'); @@ -39,15 +40,15 @@ exports.run=async({vscode,windowPage,workspace,log,proof,eventually})=>{ const project=await fs.readFile(path.join(workspace,'Project.toml')); const settings=await fs.readFile(path.join(workspace,'.vscode','settings.json')); const prompt=windowPage.getByRole('button',{name:"No, I don't trust the authors",exact:true}); - await prompt.waitFor({state:'visible',timeout:180000}); + await prompt.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); await windowPage.screenshot({path:path.join(output,'native-restricted-trust-prompt.png')}); log('native-ui-action',{surface:'Workspace Trust',action:"No, I don't trust the authors"}); - await prompt.click(); - await prompt.waitFor({state:'hidden'}); + await prompt.click({timeout:NATIVE_READINESS_MS}); + await prompt.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS}); await eventually(()=>vscode.workspace.isTrusted===false,'The real workspace remains untrusted'); const indicator=windowPage.locator('[id="status.workspaceTrust"]'); - await indicator.waitFor({state:'visible'}); - assert.match(await indicator.innerText(),/Restricted Mode/); + await indicator.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + assert.match(await indicator.innerText({timeout:NATIVE_READINESS_MS}),/Restricted Mode/); assert(inactive(),'VS Code has not activated the installed product'); assert.equal(manifest.capabilities.untrustedWorkspaces.supported,false); const commands=await vscode.commands.getCommands(true); @@ -56,10 +57,10 @@ exports.run=async({vscode,windowPage,workspace,log,proof,eventually})=>{ scope:'VS Code may retain contribution placeholders while the product is inactive'}); await vscode.commands.executeCommand('workbench.extensions.action.showExtensionsWithIds',[id]); const row=windowPage.locator('.monaco-list-row').filter({has:windowPage.locator('.name').filter({hasText:/^PerfChecker$/})}); - await row.first().click(); + await row.first().click({timeout:NATIVE_READINESS_MS}); const status=windowPage.locator('.extension-editor .actions-status-container .status'); - await status.waitFor({state:'visible'}); - await eventually(async()=>(await status.innerText()).trim()===manifest.capabilities.untrustedWorkspaces.description, + await status.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await eventually(async()=>(await status.innerText({timeout:NATIVE_READINESS_MS})).trim()===manifest.capabilities.untrustedWorkspaces.description, 'The actual installed-extension UI explains its Workspace Trust requirement'); assert(await status.locator('.codicon-extension-workspace-trust').count()>0, 'The official extension status displays the Workspace Trust restriction icon'); diff --git a/test/native-studio-controls.cjs b/test/native-studio-controls.cjs index 340137f..50ebf15 100644 --- a/test/native-studio-controls.cjs +++ b/test/native-studio-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); // Exercise the installed public VSIX through its real VS Code webviews in disposable CI profiles. // No intercepted VS Code API, replaced process, fabricated plan message or model credential is used. const assert = require('node:assert/strict'); @@ -24,7 +25,7 @@ const studioActions = { const checkTypes = ['BenchmarkTools', 'Chairmarks', 'Line allocations', 'Allocation profile', 'CPU profile', 'Wall-time profile', 'Network workload', 'Network interface', 'Isolated network']; -async function eventually(read, description, timeout = 60000) { +async function eventually(read, description, timeout = NATIVE_READINESS_MS) { const deadline = Date.now() + timeout; let last; while (Date.now() < deadline) { @@ -44,7 +45,7 @@ function assertDisposable(context) { async function frame(context, selector) { return eventually(async () => { const result = await context.findFrame(selector); - await result.locator(selector).first().waitFor({state: 'visible', timeout: 1000}); + await result.locator(selector).first().waitFor({state: 'visible', timeout: NATIVE_READINESS_MS}); return result; }, `Reacquire the current native webview ${selector}`); } @@ -55,7 +56,7 @@ async function studio(context) { if(!context.openStudioCommandVerified){ await context.vscode.commands.executeCommand('perfchecker.openStudio',context.vscode.Uri.file(context.workspace)); const opened=await frame(context,'#studio-root'); - await eventually(async()=>(await opened.locator('.workspace strong').innerText())===context.vscode.workspace.getWorkspaceFolder(context.vscode.Uri.file(context.workspace)).name,'The generic Studio command displays the explicit workspace'); + await eventually(async()=>(await opened.locator('.workspace strong').innerText({timeout:NATIVE_READINESS_MS}))===context.vscode.workspace.getWorkspaceFolder(context.vscode.Uri.file(context.workspace)).name,'The generic Studio command displays the explicit workspace'); context.proof('native-open-studio-command',{command:'perfchecker.openStudio',explicitWorkspace:true,currentPanelVisible:true}); context.openStudioCommandVerified=true; } @@ -69,12 +70,12 @@ async function clickStudioAction(context, action) { ? view.getByRole('button', {name: studioActions[action], exact: true}) : view.locator(`[data-action="${action}"]`); context.log('native-ui-action',{surface:'Studio',action:studioActions[action]}); - await selector.click(); + await selector.click({timeout:NATIVE_READINESS_MS}); } exports.clickStudioAction = clickStudioAction; async function selectionCount(view) { - const match = (await view.locator('#count').innerText()).match(/^(\d+) selected/); + const match = (await view.locator('#count').innerText({timeout:NATIVE_READINESS_MS})).match(/^(\d+) selected/); assert(match, 'The selection counter describes the exact selection'); return Number(match[1]); } @@ -85,7 +86,7 @@ async function options(view, selector) { } async function designer(context) { - return context.observeOperation?context.observeOperation('plan',()=>openDesigner(context),Date.now()+600000):openDesigner(context); + return context.observeOperation?context.observeOperation('plan',()=>openDesigner(context),Date.now()+NATIVE_READINESS_MS):openDesigner(context); } async function openDesigner(context) { @@ -105,12 +106,12 @@ async function openDesigner(context) { plan}); }; await observe('attached'); - try{await eventually(async()=>await view.locator('#check-types label').count()>0,'The controller supplies real check types before any filter gesture',360000);} + try{await eventually(async()=>await view.locator('#check-types label').count()>0,'The controller supplies real check types before any filter gesture',NATIVE_READINESS_MS);} catch(error){try{await observe('failed');}catch(secondary){context.log('native-designer-readiness-diagnostic-error',{message:String(secondary),primary:String(error)});}throw error;} await observe('plan-ready'); - await view.locator('#reset-filters').click(); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); await eventually(async () => (await view.locator('#cards .card').count()) > 0, - 'General-backed suite plan renders workload cards', 120000); + 'General-backed suite plan renders workload cards', NATIVE_READINESS_MS); return view; } @@ -119,7 +120,7 @@ async function savedConfiguration(context, view) { context.vscode.Uri.file(context.workspace)); const destination = path.resolve(context.workspace, settings.get('uiConfiguration', 'perf/perfchecker-ui.json')); const previous = await fs.stat(destination,{bigint:true}).catch(() => undefined); - await view.locator('#save').click(); + await view.locator('#save').click({timeout:NATIVE_READINESS_MS}); const config = await eventually(async () => { const stat = await fs.stat(destination,{bigint:true}); if (previous && stat.mtimeNs === previous.mtimeNs) return false; @@ -212,7 +213,7 @@ async function output(context) { // report read and HTML replacement, so visibility alone can still select its old frame. const previous=new Set((await resultDocuments(context)).map(document=>document.nonce)); await clickStudioAction(context, 'results'); - const deadline=Date.now()+60000; + const deadline=Date.now()+NATIVE_READINESS_MS; while(Date.now()document.visible&&!previous.has(document.nonce)); assert(current.length<=1,'Only one new results document belongs to this request'); @@ -227,8 +228,8 @@ async function output(context) { } async function resetDesigner(view) { - await view.locator('#reset-filters').click(); - await view.locator('#select-visible').click(); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#select-visible').click({timeout:NATIVE_READINESS_MS}); await eventually(async () => (await selectionCount(view)) > 0, 'Restore all suite selections'); } @@ -245,7 +246,7 @@ async function testSuiteSelection(context) { const ids=new Set(expected.map(run=>run.id)),keys=[...new Set(expected.map(groupKey))].sort(); assert(keys.length<=120,'This filter fixture fits entirely within the native page'); await eventually(async()=>{ - const counter=await view.locator('#count').innerText(),match=/· (\d+)\/(\d+) visible/.exec(counter); + const counter=await view.locator('#count').innerText({timeout:NATIVE_READINESS_MS}),match=/· (\d+)\/(\d+) visible/.exec(counter); const groups=await renderedGroups(); return match&&Number(match[1])===expected.length&&Number(match[2])===plan.runs.length&& groups.every(group=>ids.has(group.id))&&groups.length===keys.length&& @@ -260,69 +261,69 @@ async function testSuiteSelection(context) { for (const type of checkTypes) { assert(labels.some(label => label.startsWith(`${type} ·`)), `Native suite includes ${type}`); const input = view.locator('#check-types label').filter({hasText: `${type} ·`}).locator('input'); - await input.uncheck(); + await input.uncheck({timeout:NATIVE_READINESS_MS}); assert((await selectionCount(view)) < all, `${type} removes matching runs, including hidden runs`); - await input.check(); + await input.check({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), all, `${type} re-includes every matching run`); } for (const unavailable of await view.locator('.check-option.unavailable .check-status').all()) { - assert((await unavailable.getAttribute('title'))?.trim(), 'An unavailable check explains its prerequisite'); + assert((await unavailable.getAttribute('title',{timeout:NATIVE_READINESS_MS}))?.trim(), 'An unavailable check explains its prerequisite'); } context.proof('suite-check-types', {types: checkTypes, runs: all, unavailableReasonsChecked: true}); const targets = await options(view, '#target-filter'); assert(targets.length >= 2, 'At least two real targets are needed to qualify hidden selections and versions'); for (const target of targets) { - await view.locator('#target-filter').selectOption(target.value); + await view.locator('#target-filter').selectOption(target.value,{timeout:NATIVE_READINESS_MS}); await assertFiltered(plan.runs.filter(run=>run.version===target.value),`target=${target.value}`); assert.equal(await selectionCount(view), all, 'Target filtering preserves hidden selected runs'); } - await view.locator('#clear-visible').click(); + await view.locator('#clear-visible').click({timeout:NATIVE_READINESS_MS}); const hidden = await selectionCount(view); assert(hidden > 0 && hidden < all, 'Clear visible preserves selected runs belonging to other targets'); - assert.match(await view.locator('#count').innerText(), /selected outside filters/); - await view.locator('#reset-filters').click(); + assert.match(await view.locator('#count').innerText({timeout:NATIVE_READINESS_MS}), /selected outside filters/); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), hidden, 'Reset filters does not secretly replace the selection'); - await view.locator('#select-visible').click(); + await view.locator('#select-visible').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), all); - await view.locator('#clear-all').click(); + await view.locator('#clear-all').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), 0); - assert(await view.locator('#run').isDisabled(), 'No selection cannot launch a run'); - await view.locator('#select-visible').click(); + assert(await view.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS}), 'No selection cannot launch a run'); + await view.locator('#select-visible').click({timeout:NATIVE_READINESS_MS}); - await view.locator('#search').fill('a-query-that-matches-no-fixture'); + await view.locator('#search').fill('a-query-that-matches-no-fixture',{timeout:NATIVE_READINESS_MS}); await eventually(async () => (await view.locator('#cards .card').count()) === 0, 'Empty workload filter'); - assert.match(await view.locator('#cards').innerText(), /No workloads match/); + assert.match(await view.locator('#cards').innerText({timeout:NATIVE_READINESS_MS}), /No workloads match/); assert.equal(await selectionCount(view), all); - await view.locator('#clear-visible').click(); + await view.locator('#clear-visible').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), all, 'Clearing an empty visible set preserves all hidden selections'); - await view.locator('#reset-filters').click(); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); for (const selector of ['#package', '#target-kind']) { for (const option of await options(view, selector)) { - await view.locator(selector).selectOption(option.value); + await view.locator(selector).selectOption(option.value,{timeout:NATIVE_READINESS_MS}); const key=selector==='#package'?'package':'target_kind'; await assertFiltered(plan.runs.filter(run=>run[key]===option.value),`${key}=${option.value}`); assert.equal(await selectionCount(view), all); } - await view.locator(selector).selectOption(''); + await view.locator(selector).selectOption('',{timeout:NATIVE_READINESS_MS}); } const releaseTargets = targets.filter(item => /^v?\d+\.\d+\.\d+/.test(item.value)); if (releaseTargets.length) { - await view.locator('#from').fill(releaseTargets[0].value); - await view.locator('#to').fill(releaseTargets.at(-1).value); + await view.locator('#from').fill(releaseTargets[0].value,{timeout:NATIVE_READINESS_MS}); + await view.locator('#to').fill(releaseTargets.at(-1).value,{timeout:NATIVE_READINESS_MS}); const version=value=>value.replace(/^v/,'').split('.').slice(0,3).map(Number); const comparison=(a,b)=>{const x=version(a),y=version(b);return x[0]-y[0]||x[1]-y[1]||x[2]-y[2];}; await assertFiltered(plan.runs.filter(run=>run.target_kind!=='release'|| comparison(run.version,releaseTargets[0].value)>=0&&comparison(run.version,releaseTargets.at(-1).value)<=0),'inclusive release bounds'); assert.equal(await selectionCount(view), all, 'Release bounds preserve selected Git targets'); - await view.locator('#from').fill('999.0.0'); - await view.locator('#to').fill('0.0.0'); + await view.locator('#from').fill('999.0.0',{timeout:NATIVE_READINESS_MS}); + await view.locator('#to').fill('0.0.0',{timeout:NATIVE_READINESS_MS}); await assertFiltered(plan.runs.filter(run=>run.target_kind!=='release'),'inverted release bounds'); assert.equal(await selectionCount(view), all, 'Inverted release bounds do not discard hidden selections'); } else context.log('release-range-prerequisite', 'Fixture has only Git/dev targets; a registered-release fixture is still required'); - await view.locator('#reset-filters').click(); - await view.locator('#sort').selectOption('suite'); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#sort').selectOption('suite',{timeout:NATIVE_READINESS_MS}); const {config:orderConfiguration}=await savedConfiguration(context,view); assert.equal(orderConfiguration.selection.run_ids.length,plan.runs.length,'The real saved order includes every selected fixture run'); const byId=new Map(plan.runs.map(run=>[run.id,run])); @@ -336,7 +337,7 @@ async function testSuiteSelection(context) { const versionRank=new Map(versionOrder.map((value,index)=>[value,index])); let previousOrder=(await renderedGroups()).map(group=>group.key),observedSortChange=false; for (const sort of ['package', 'feature', 'version', 'suite']) { - await view.locator('#sort').selectOption(sort); + await view.locator('#sort').selectOption(sort,{timeout:NATIVE_READINESS_MS}); let observed=[]; try{ await eventually(async()=>{ @@ -359,23 +360,23 @@ async function testSuiteSelection(context) { } assert(observedSortChange,'At least one discriminating native sort gesture changes the displayed group order'); const group = view.locator('#cards .card').first(); - await group.locator('.pick').uncheck(); + await group.locator('.pick').uncheck({timeout:NATIVE_READINESS_MS}); assert((await selectionCount(view)) < all); - await group.locator('.pick').check(); - await group.locator('.check-option input').first().uncheck(); + await group.locator('.pick').check({timeout:NATIVE_READINESS_MS}); + await group.locator('.check-option input').first().uncheck({timeout:NATIVE_READINESS_MS}); assert.equal(await view.locator('.card.partial').count(), 1); - if(await view.locator('#selection-preview').locator('..').getAttribute('open')===null) - await view.locator('#selection-summary').click(); + if(await view.locator('#selection-preview').locator('..').getAttribute('open',{timeout:NATIVE_READINESS_MS})===null) + await view.locator('#selection-summary').click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await view.locator('#selection-preview li').count()===Math.min(all-1,80), 'The native details toggle renders the exact selected runs before their count is read'); - await view.locator('#open-after-run').uncheck(); + await view.locator('#open-after-run').uncheck({timeout:NATIVE_READINESS_MS}); await group.locator('.label').evaluate(input => { input.value = '#1266aa'; input.dispatchEvent(new Event('input', {bubbles: true})); }); - await view.locator('#doc-id').fill('native-checks'); - await view.locator('#doc-title').fill('Native control qualification'); - await view.locator('#doc-url').fill('https://example.invalid/qualified-report'); - for (const checkbox of await view.locator('input[name="view"]').all()) await checkbox.check(); + await view.locator('#doc-id').fill('native-checks',{timeout:NATIVE_READINESS_MS}); + await view.locator('#doc-title').fill('Native control qualification',{timeout:NATIVE_READINESS_MS}); + await view.locator('#doc-url').fill('https://example.invalid/qualified-report',{timeout:NATIVE_READINESS_MS}); + for (const checkbox of await view.locator('input[name="view"]').all()) await checkbox.check({timeout:NATIVE_READINESS_MS}); const {config} = await savedConfiguration(context, view); assert.equal(config.selection.run_ids.length, all - 1); assert.equal(new Set(config.selection.run_ids).size, all - 1); @@ -384,15 +385,15 @@ async function testSuiteSelection(context) { assert.deepEqual(config.documentation.blocks[0].views, ['summary', 'comparison', 'plots', 'observations', 'diagnostics', 'artifacts']); assert.equal(config.documentation.blocks[0].interactive_url, 'https://example.invalid/qualified-report'); - await view.locator('#refresh').click(); - await eventually(async () => !(await view.locator('#refresh').isDisabled()), 'Refresh native Julia plan', 120000); + await view.locator('#refresh').click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => !(await view.locator('#refresh').isDisabled({timeout:NATIVE_READINESS_MS})), 'Refresh native Julia plan', NATIVE_READINESS_MS); assert.equal(await selectionCount(view), all - 1, 'Native refresh preserves the saved and current selection'); context.proof('suite-selection-and-save', {runs: all, selected: all - 1, hiddenSelectionsPreserved: true, views: 6}); - await view.locator('h1').first().scrollIntoViewIfNeeded(); + await view.locator('h1').first().scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,'suite-designer'); // Exercise the script button and assert its real editor document, then return to the retained view. - await view.locator('#cards .card .check-option .open').first().click(); + await view.locator('#cards .card .check-option .open').first().click({timeout:NATIVE_READINESS_MS}); await eventually(() => context.vscode.window.activeTextEditor?.document.languageId === 'julia', 'Open workload Julia source'); assert(path.isAbsolute(context.vscode.window.activeTextEditor.document.uri.fsPath)); await studio(context); @@ -415,7 +416,7 @@ async function testGitTargetsAndComparisons(context) { await eventually(async()=>{ const state=await idleState(); return state.progressHidden&&Object.values(state.controls).every(disabled=>!disabled); - },'The preceding actual plan finishes before Git target gestures',120000); + },'The preceding actual plan finishes before Git target gestures',NATIVE_READINESS_MS); }catch(error){ try{context.log('git-target-idle-timeout',{state:await idleState(),primary:String(error)});} catch(secondary){context.log('git-target-idle-diagnostic-error',{message:String(secondary),primary:String(error)});} @@ -425,13 +426,13 @@ async function testGitTargetsAndComparisons(context) { const previousTargets = settings().inspect('gitTargets')?.workspaceFolderValue; const previousPolicies = settings().inspect('comparisonPolicies')?.workspaceFolderValue; try { - await view.locator('#target-package').selectOption('PerfCheckerNativeFixture'); - await eventually(async () => !(await view.locator('#target-reference').isDisabled()), 'Discover real local Git references'); + await view.locator('#target-package').selectOption('PerfCheckerNativeFixture',{timeout:NATIVE_READINESS_MS}); + await eventually(async () => !(await view.locator('#target-reference').isDisabled({timeout:NATIVE_READINESS_MS})), 'Discover real local Git references'); const groups = await view.locator('#target-reference optgroup').evaluateAll(items => items.map(item => item.label)); assert(groups.includes('Branches') && groups.includes('Tags') && groups.includes('Recent commits'), 'Disposable Git fixture contains branch, tag and commit references'); await context.vscode.commands.executeCommand('workbench.action.closePanel'); - await view.locator('#target-reference').scrollIntoViewIfNeeded(); + await view.locator('#target-reference').scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await view.evaluate(()=>new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))); const geometry=()=>view.evaluate(()=>{ const rect=node=>{const value=node.getBoundingClientRect();return {left:value.left,right:value.right,width:value.width, @@ -472,54 +473,54 @@ async function testGitTargetsAndComparisons(context) { for (const kind of ['branch', 'tag', 'commit']) { const candidate = refs.find(item => item.kind === kind); assert(candidate?.commit?.match(/^[a-f0-9]{40}$/), `Pinned ${kind} SHA supplied by actual Git discovery`); - await view.locator('#target-reference').selectOption(candidate.ref); - assert(await view.locator('#target-label').inputValue(), 'Automatic display label follows selected reference'); + await view.locator('#target-reference').selectOption(candidate.ref,{timeout:NATIVE_READINESS_MS}); + assert(await view.locator('#target-label').inputValue({timeout:NATIVE_READINESS_MS}), 'Automatic display label follows selected reference'); } const chosen = refs.find(item => item.kind === 'branch'); - await view.locator('#target-reference').selectOption(chosen.ref); - await view.locator('#target-label').fill('native-click-target'); - await view.locator('#add-target').click(); + await view.locator('#target-reference').selectOption(chosen.ref,{timeout:NATIVE_READINESS_MS}); + await view.locator('#target-label').fill('native-click-target',{timeout:NATIVE_READINESS_MS}); + await view.locator('#add-target').click({timeout:NATIVE_READINESS_MS}); await eventually(() => settings().get('gitTargets', []).some(item => item.label === 'native-click-target'), 'Persist target from button'); assert.equal(settings().get('gitTargets', []).find(item => item.label === 'native-click-target').revision, chosen.commit); - await eventually(async () => !(await view.locator('#add-target').isDisabled()), 'Replan with new Git target', 120000); - await view.locator('#target-list .target').filter({hasText: 'native-click-target'}).getByRole('button', {name: 'Remove comparison target'}).click(); + await eventually(async () => !(await view.locator('#add-target').isDisabled({timeout:NATIVE_READINESS_MS})), 'Replan with new Git target', NATIVE_READINESS_MS); + await view.locator('#target-list .target').filter({hasText: 'native-click-target'}).getByRole('button', {name: 'Remove comparison target'}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => !settings().get('gitTargets', []).some(item => item.label === 'native-click-target'), 'Remove target via native button'); - await eventually(async () => !(await view.locator('#add-target').isDisabled()), 'Replan after target removal', 120000); - await view.locator('#target-revision').fill('refs/heads/nonexistent-native-fixture'); - await view.locator('#target-label').fill('invalid-native-target'); - await view.locator('#add-target').click(); - await view.locator('#target-error').waitFor({state: 'visible'}); - assert((await view.locator('#target-error').innerText()).trim()); + await eventually(async () => !(await view.locator('#add-target').isDisabled({timeout:NATIVE_READINESS_MS})), 'Replan after target removal', NATIVE_READINESS_MS); + await view.locator('#target-revision').fill('refs/heads/nonexistent-native-fixture',{timeout:NATIVE_READINESS_MS}); + await view.locator('#target-label').fill('invalid-native-target',{timeout:NATIVE_READINESS_MS}); + await view.locator('#add-target').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#target-error').waitFor({state: 'visible',timeout:NATIVE_READINESS_MS}); + assert((await view.locator('#target-error').innerText({timeout:NATIVE_READINESS_MS})).trim()); assert(!settings().get('gitTargets', []).some(item => item.label === 'invalid-native-target')); - await view.locator('#refresh-targets').click(); - await eventually(async () => !(await view.locator('#target-reference').isDisabled()), 'Refresh actual Git inventory'); + await view.locator('#refresh-targets').click({timeout:NATIVE_READINESS_MS}); + await eventually(async () => !(await view.locator('#target-reference').isDisabled({timeout:NATIVE_READINESS_MS})), 'Refresh actual Git inventory'); context.proof('git-reference-controls', {groups, pinnedBranch: chosen.commit, addRemove: true, invalidReferenceRejected: true}); - await view.locator('#add-comparison').click(); - await view.locator('#comparison-error').waitFor({state: 'visible'}); + await view.locator('#add-comparison').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#comparison-error').waitFor({state: 'visible',timeout:NATIVE_READINESS_MS}); const candidates = await view.locator('#candidate-targets input').evaluateAll(items => items.map(item => item.value)); assert(candidates.length >= 2, 'Comparison fixture supplies two targets'); - for (const input of await view.locator('#baseline-targets input, #candidate-targets input').all()) await input.uncheck(); - await view.locator('#baseline-targets input').first().check(); - await view.locator('#candidate-targets input').first().check(); - await view.locator('#add-comparison').click(); - assert.match(await view.locator('#comparison-error').innerText(), /different targets/); - await view.locator('#candidate-targets input').first().uncheck(); - await view.locator('#candidate-targets input').nth(1).check(); - for (const value of ['median', 'mean', 'minimum', 'maximum']) await view.locator('#comparison-aggregation').selectOption(value); - await view.locator('#comparison-aggregation').selectOption('median'); + for (const input of await view.locator('#baseline-targets input, #candidate-targets input').all()) await input.uncheck({timeout:NATIVE_READINESS_MS}); + await view.locator('#baseline-targets input').first().check({timeout:NATIVE_READINESS_MS}); + await view.locator('#candidate-targets input').first().check({timeout:NATIVE_READINESS_MS}); + await view.locator('#add-comparison').click({timeout:NATIVE_READINESS_MS}); + assert.match(await view.locator('#comparison-error').innerText({timeout:NATIVE_READINESS_MS}), /different targets/); + await view.locator('#candidate-targets input').first().uncheck({timeout:NATIVE_READINESS_MS}); + await view.locator('#candidate-targets input').nth(1).check({timeout:NATIVE_READINESS_MS}); + for (const value of ['median', 'mean', 'minimum', 'maximum']) await view.locator('#comparison-aggregation').selectOption(value,{timeout:NATIVE_READINESS_MS}); + await view.locator('#comparison-aggregation').selectOption('median',{timeout:NATIVE_READINESS_MS}); await context.vscode.commands.executeCommand('workbench.action.closePanel'); const matrix=view.locator('.aside-panel').filter({has:view.getByRole('heading',{name:'Comparison matrix',exact:true})}); - await matrix.scrollIntoViewIfNeeded(); + await matrix.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,'comparison-matrix'); const before = settings().get('comparisonPolicies', []).length; - await view.locator('#add-comparison').click(); + await view.locator('#add-comparison').click({timeout:NATIVE_READINESS_MS}); await eventually(() => settings().get('comparisonPolicies', []).length === before + 1, 'Persist comparison from actual button'); - await eventually(async () => !(await view.locator('#add-comparison').isDisabled()), 'Replan after comparison', 120000); + await eventually(async () => !(await view.locator('#add-comparison').isDisabled({timeout:NATIVE_READINESS_MS})), 'Replan after comparison', NATIVE_READINESS_MS); const policy = settings().get('comparisonPolicies', []).at(-1); assert.equal(policy.aggregation, 'median'); assert.equal(policy.baselines.length, 1); assert.equal(policy.candidates.length, 1); - await view.locator('#comparison-list .target').last().getByRole('button', {name: 'Remove comparison', exact: true}).click(); + await view.locator('#comparison-list .target').last().getByRole('button', {name: 'Remove comparison', exact: true}).click({timeout:NATIVE_READINESS_MS}); await eventually(() => settings().get('comparisonPolicies', []).length === before, 'Remove comparison via actual button'); context.proof('comparison-controls', {invalidEmpty: true, overlapRejected: true, aggregationSelections: 4, aggregationsComputed: false, addRemove: true}); } finally { @@ -536,22 +537,22 @@ async function testCancelGitDiscovery(context){ let view; try{ view=await designer(context); - await view.locator('#target-package').selectOption('PerfCheckerNativeFixture'); - await eventually(async()=>!(await view.locator('#target-reference').isDisabled()),'Initial local Git inventory finishes'); + await view.locator('#target-package').selectOption('PerfCheckerNativeFixture',{timeout:NATIVE_READINESS_MS}); + await eventually(async()=>!(await view.locator('#target-reference').isDisabled({timeout:NATIVE_READINESS_MS})),'Initial local Git inventory finishes'); const advanced=view.locator('details').filter({hasText:'Advanced target options'}); - if(await advanced.getAttribute('open')===null)await advanced.locator('summary').click(); - await view.locator('#target-source').fill(`http://127.0.0.1:${server.address().port}/qualification.git`); - await view.locator('#refresh-targets').click(); - await eventually(()=>requested&&sockets.size>0,'The actual Git process reaches its pending HTTP discovery request',30000); - await view.locator('#cancel-targets').click(); + if(await advanced.getAttribute('open',{timeout:NATIVE_READINESS_MS})===null)await advanced.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#target-source').fill(`http://127.0.0.1:${server.address().port}/qualification.git`,{timeout:NATIVE_READINESS_MS}); + await view.locator('#refresh-targets').click({timeout:NATIVE_READINESS_MS}); + await eventually(()=>requested&&sockets.size>0,'The actual Git process reaches its pending HTTP discovery request',NATIVE_READINESS_MS); + await view.locator('#cancel-targets').click({timeout:30000}); await eventually(()=>sockets.size===0,'The native Cancel discovery button terminates Git and closes its real connection',30000); - await eventually(async()=>!(await view.locator('#cancel-targets').isVisible()),'The cancelled Git scan returns to idle'); - assert.match(await view.locator('#target-error').innerText(),/cancel/i); - await view.locator('#target-source').fill('');await view.locator('#refresh-targets').click(); + await eventually(async()=>!(await view.locator('#cancel-targets').isVisible()),'The cancelled Git scan returns to idle',60000); + assert.match(await view.locator('#target-error').innerText({timeout:NATIVE_READINESS_MS}),/cancel/i); + await view.locator('#target-source').fill('',{timeout:NATIVE_READINESS_MS});await view.locator('#refresh-targets').click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await view.locator('#target-reference optgroup[label="Branches"]').count()>0,'A fresh real local Git discovery works after cancellation'); context.proof('git-cancel-discovery',{nativeClick:true,actualRemoteGitRequest:true,connectionClosedBeforeHarnessCleanup:true,freshScanSucceeded:true}); }finally{ - if(view){await view.locator('#cancel-targets').click({timeout:1000}).catch(()=>{});await view.locator('#target-source').fill('').catch(()=>{});} + if(view){await view.locator('#cancel-targets').click({timeout:1000}).catch(()=>{});await view.locator('#target-source').fill('',{timeout:30000}).catch(()=>{});} server.closeAllConnections();await new Promise(resolve=>server.close(resolve)); } } @@ -559,11 +560,11 @@ async function testCancelGitDiscovery(context){ async function testFlameControls(context,view) { const graphs=view.locator('.flame-view');assert((await graphs.count())>0); for(let index=0;index{ const actual=await svg.evaluate(svg=>({width:svg.viewBox.baseVal.width, @@ -579,25 +580,25 @@ async function testFlameControls(context,view) { }; const original=await expectedGeometry(); const narrow=model.frames.reduce((best,frame)=>frame.x1-frame.x0(await detail.innerText()).startsWith(`Frame ${narrow.index} /`),'Native frame index updates the real readout'); + await indexInput.fill(String(narrow.index),{timeout:NATIVE_READINESS_MS});await indexInput.press('Tab',{timeout:NATIVE_READINESS_MS}); + const detailId=await graph.getAttribute('data-detail-target',{timeout:NATIVE_READINESS_MS}),detail=view.locator(`[id="${detailId}"]`); + await eventually(async()=>(await detail.innerText({timeout:NATIVE_READINESS_MS})).startsWith(`Frame ${narrow.index} /`),'Native frame index updates the real readout'); const callPath=[];for(let frame=narrow;frame;frame=frame.parent===null?undefined:model.frames[frame.parent-1])callPath.push(frame.name); - const text=await detail.innerText();assert(text.includes(callPath.reverse().join(' → '))); + const text=await detail.innerText({timeout:NATIVE_READINESS_MS});assert(text.includes(callPath.reverse().join(' → '))); assert(text.includes(`Inclusive weight: ${narrow.value} ${model.unit}`)); const colorMode=graph.getByRole('combobox',{name:'Color by',exact:true}),labelMode=graph.getByRole('combobox',{name:'Labels',exact:true}); - assert.equal(await colorMode.inputValue(),'function');assert.equal(await labelMode.inputValue(),'function'); + assert.equal(await colorMode.inputValue({timeout:NATIVE_READINESS_MS}),'function');assert.equal(await labelMode.inputValue({timeout:NATIVE_READINESS_MS}),'function'); const functionLabels=model.frames.map(frame=>frame.name.replace(/ \([^()\r\n]+\.jl:[1-9]\d*\)$/u,'')); const labelsText=()=>svg.locator('.flame-node text').allTextContents(); assert.deepEqual(await labelsText(),functionLabels); const functionColors=await svg.locator('.flame-node rect').evaluateAll(nodes=>nodes.map(node=>getComputedStyle(node).fill)); for(const [mode,expected]of [['full',model.frames.map(frame=>frame.name)],['function',functionLabels]]){ - await labelMode.selectOption(mode);assert.deepEqual(await labelsText(),expected); - assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame'),payload); - assert.equal(await detail.innerText(),text,'Changing labels retains the selected full call path and measured weight'); + await labelMode.selectOption(mode,{timeout:NATIVE_READINESS_MS});assert.deepEqual(await labelsText(),expected); + assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame',{timeout:NATIVE_READINESS_MS}),payload); + assert.equal(await detail.innerText({timeout:NATIVE_READINESS_MS}),text,'Changing labels retains the selected full call path and measured weight'); } - await colorMode.selectOption('diagnostics'); - assert.match(await graph.locator('.flame-legend').innerText(),/Unknown or missing inference information is not a warning/); + await colorMode.selectOption('diagnostics',{timeout:NATIVE_READINESS_MS}); + assert.match(await graph.locator('.flame-legend').innerText({timeout:NATIVE_READINESS_MS}),/Unknown or missing inference information is not a warning/); const diagnostics=await svg.locator('.flame-node').evaluateAll(nodes=>nodes.map(node=>({index:Number(node.dataset.frameIndex), dynamic:node.classList.contains('dynamic'),unstable:node.classList.contains('unstable'),gc:node.classList.contains('gc')}))); for(const actual of diagnostics){ @@ -607,42 +608,42 @@ async function testFlameControls(context,view) { assert.deepEqual(actual,{index:saved.index,dynamic:!saved.gc&&saved.dynamic, unstable:!saved.gc&&!saved.dynamic&&saved.unstable,gc:saved.gc}); } - assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame'),payload); - await colorMode.selectOption('function'); + assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame',{timeout:NATIVE_READINESS_MS}),payload); + await colorMode.selectOption('function',{timeout:NATIVE_READINESS_MS}); assert.deepEqual(await svg.locator('.flame-node rect').evaluateAll(nodes=>nodes.map(node=>getComputedStyle(node).fill)),functionColors); - assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame'),payload); - assert.equal(await detail.innerText(),text); + assert.deepEqual(await expectedGeometry(),original);assert.equal(await graph.getAttribute('data-flame',{timeout:NATIVE_READINESS_MS}),payload); + assert.equal(await detail.innerText({timeout:NATIVE_READINESS_MS}),text); context.proof('native-flame-presentation-modes',{source:'actual-installed-VSIX',colorModes:['function','diagnostics'], labelModes:['function','full'],nativeSelectChanges:4,frames:model.frames.length,diagnosticFlagsChecked:diagnostics.length, unknownInferenceFramesChecked:model.frames.filter(frame=>frame.inferenceStatuses.includes('unknown')&&!frame.unstable).length, fullCallPathPreserved:true,modelByteExact:true,geometryExact:true}); - await graph.getByRole('button',{name:'Zoom in flame graph',exact:true}).click(); - await eventually(async()=>Number(await svg.getAttribute('data-current-max'))-Number(await svg.getAttribute('data-current-min'))<1,'Native flame Zoom narrows only the viewport'); + await graph.getByRole('button',{name:'Zoom in flame graph',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>Number(await svg.getAttribute('data-current-max',{timeout:NATIVE_READINESS_MS}))-Number(await svg.getAttribute('data-current-min',{timeout:NATIVE_READINESS_MS}))<1,'Native flame Zoom narrows only the viewport'); await expectedGeometry(); const pan=graph.getByRole('button',{name:'Pan flame graph left',exact:true}); const otherPan=graph.getByRole('button',{name:'Pan flame graph right',exact:true}); - if(await pan.isEnabled())await pan.click();else await otherPan.click(); - await expectedGeometry();await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click(); - await eventually(async()=>await svg.getAttribute('data-current-min')==='0'&&await svg.getAttribute('data-current-max')==='1','Native flame Fit restores the complete original range'); + if(await pan.isEnabled({timeout:NATIVE_READINESS_MS}))await pan.click({timeout:NATIVE_READINESS_MS});else await otherPan.click({timeout:NATIVE_READINESS_MS}); + await expectedGeometry();await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>await svg.getAttribute('data-current-min',{timeout:NATIVE_READINESS_MS})==='0'&&await svg.getAttribute('data-current-max',{timeout:NATIVE_READINESS_MS})==='1','Native flame Fit restores the complete original range'); const restored=await expectedGeometry();assert.deepEqual(restored,original); - await graph.locator('.flame-range summary').click(); + await graph.locator('.flame-range summary').click({timeout:NATIVE_READINESS_MS}); const start=graph.locator('[data-flame-bound="min"]'),end=graph.locator('[data-flame-bound="max"]'); for(const [input,value]of [[start,'-1'],[end,'101'],[start,'100'],[start,'']]){ - await input.fill(value);await input.press('Tab'); - assert.match(await graph.locator('[role="alert"]').innerText(),/from 0 to 100/); + await input.fill(value,{timeout:NATIVE_READINESS_MS});await input.press('Tab',{timeout:NATIVE_READINESS_MS}); + assert.match(await graph.locator('[role="alert"]').innerText({timeout:NATIVE_READINESS_MS}),/from 0 to 100/); assert.deepEqual(await expectedGeometry(),original,'Invalid percentage input must not silently move the viewport'); } - await start.fill('90');await start.press('Tab');const beforeReversed=await expectedGeometry(); - await end.fill('10');await end.press('Tab'); - assert.match(await graph.locator('[role="alert"]').innerText(),/from 0 to 100/); + await start.fill('90',{timeout:NATIVE_READINESS_MS});await start.press('Tab',{timeout:NATIVE_READINESS_MS});const beforeReversed=await expectedGeometry(); + await end.fill('10',{timeout:NATIVE_READINESS_MS});await end.press('Tab',{timeout:NATIVE_READINESS_MS}); + assert.match(await graph.locator('[role="alert"]').innerText({timeout:NATIVE_READINESS_MS}),/from 0 to 100/); assert.deepEqual(await expectedGeometry(),beforeReversed,'Reversed user bounds leave the previous valid viewport unchanged'); - await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click(); - await start.fill('10');await end.fill('90');await end.press('Tab'); + await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await start.fill('10',{timeout:NATIVE_READINESS_MS});await end.fill('90',{timeout:NATIVE_READINESS_MS});await end.press('Tab',{timeout:NATIVE_READINESS_MS}); // Normalized bounds may differ by a few floating-point ULPs, never by a displayed percentage. const rangeTolerance=Number.EPSILON*8; const explicitRange=await eventually(async()=>{ - const actual={from:Number(await svg.getAttribute('data-current-min')),to:Number(await svg.getAttribute('data-current-max')), - start:await start.inputValue(),end:await end.inputValue()}; + const actual={from:Number(await svg.getAttribute('data-current-min',{timeout:NATIVE_READINESS_MS})),to:Number(await svg.getAttribute('data-current-max',{timeout:NATIVE_READINESS_MS})), + start:await start.inputValue({timeout:NATIVE_READINESS_MS}),end:await end.inputValue({timeout:NATIVE_READINESS_MS})}; assert(Math.abs(actual.from-.1)<=rangeTolerance&&Math.abs(actual.to-.9)<=rangeTolerance, `Expected normalized bounds 0.1–0.9 within ${rangeTolerance}; received ${JSON.stringify(actual)}`); assert.equal(actual.start,'10',`Start must be readable: ${JSON.stringify(actual)}`); @@ -650,9 +651,9 @@ async function testFlameControls(context,view) { return actual; }, 'A valid explicit percentage range changes only the viewport'); - await expectedGeometry();await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click(); - await graph.locator('.flame-range summary').click(); - assert.equal(await graph.getAttribute('data-flame'),payload,'Flame interaction leaves the measured model byte-exact'); + await expectedGeometry();await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await graph.locator('.flame-range summary').click({timeout:NATIVE_READINESS_MS}); + assert.equal(await graph.getAttribute('data-flame',{timeout:NATIVE_READINESS_MS}),payload,'Flame interaction leaves the measured model byte-exact'); const labels=await svg.evaluate(svg=>[...svg.querySelectorAll('.flame-node text')].filter(text=>text.style.display!=='none').map(text=>{ const rect=text.parentElement.querySelector('rect'),left=Number(rect.getAttribute('x')),width=Number(rect.getAttribute('width')); return {length:text.getComputedTextLength(),available:Math.min(svg.viewBox.baseVal.width,left+width)-Math.max(0,left)-8}; @@ -671,7 +672,7 @@ async function testFlamePresentation(context,view,graph){ const workbench=context.vscode.workspace.getConfiguration('workbench'); const previousTheme=workbench.inspect('colorTheme')?.globalValue; const previousViewport=await context.windowPage.evaluate(()=>({width:innerWidth,height:innerHeight})); - const colorMode=graph.getByRole('combobox',{name:'Color by',exact:true}),previousColor=await colorMode.inputValue(); + const colorMode=graph.getByRole('combobox',{name:'Color by',exact:true}),previousColor=await colorMode.inputValue({timeout:NATIVE_READINESS_MS}); const audits=[]; const audit=async()=>{ const presentation=await graph.locator('svg.flame').evaluate(svg=>({ @@ -720,11 +721,11 @@ async function testFlamePresentation(context,view,graph){ if(commands.includes(command))await context.vscode.commands.executeCommand(command); } for(const mode of ['function','diagnostics']){ - await colorMode.selectOption(mode); + await colorMode.selectOption(mode,{timeout:NATIVE_READINESS_MS}); await eventually(audit,'The rendered flame labels update their actual contrast after the theme and color mode change'); const desktop=await audit(); - await graph.locator('.flame-node').first().focus();const focused=await audit(); - await graph.locator('.flame-node').first().hover();const hovered=await audit(); + await graph.locator('.flame-node').first().focus({timeout:NATIVE_READINESS_MS});const focused=await audit(); + await graph.locator('.flame-node').first().hover({timeout:NATIVE_READINESS_MS});const hovered=await audit(); await context.windowPage.setViewportSize({width:390,height:844}); await eventually(async()=>await view.evaluate(()=>innerWidth)<=390,'The actual webview adopts the narrow workbench viewport'); const geometry=await graph.evaluate(node=>({viewport:innerWidth,width:node.getBoundingClientRect().width, @@ -733,11 +734,11 @@ async function testFlamePresentation(context,view,graph){ assert(geometry.width>=230&&geometry.width<=390,'The narrow qualification uses a readable editor after hiding disposable side panels');assert(geometry.scrollWidth<=geometry.clientWidth); assert(geometry.buttons.every(control=>control.height>=44),'The narrow native flame buttons and selects retain touch-sized targets'); const mobile=await audit(); - await graph.getByRole('button',{name:'Zoom in flame graph',exact:true}).click(); - await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click(); - await graph.locator('.flame-presentation').scrollIntoViewIfNeeded(); + await graph.getByRole('button',{name:'Zoom in flame graph',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await graph.getByRole('button',{name:'Fit all flame frames',exact:true}).click({timeout:NATIVE_READINESS_MS}); + await graph.locator('.flame-presentation').scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,`flame-mobile-${uiTheme}-${mode}-controls`); - await graph.locator('.flame-detail').scrollIntoViewIfNeeded(); + await graph.locator('.flame-detail').scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,`flame-mobile-${uiTheme}-${mode}-readout`); audits.push({theme:theme.id,colorMode:mode,desktop,focused,hovered,mobile,geometry}); await context.windowPage.setViewportSize(previousViewport); @@ -746,7 +747,7 @@ async function testFlamePresentation(context,view,graph){ context.proof('native-flame-theme-and-mobile-presentation',{source:'actual-installed-VSIX-and-built-in-VSCode-themes',audits}); }finally{ await context.windowPage.setViewportSize(previousViewport); - await colorMode.selectOption(previousColor); + await colorMode.selectOption(previousColor,{timeout:30000}); await workbench.update('colorTheme',previousTheme,context.vscode.ConfigurationTarget.Global); } } @@ -755,18 +756,18 @@ async function testResults(context) { const commands=await context.vscode.commands.getCommands(true); if(commands.includes('workbench.action.closePanel'))await context.vscode.commands.executeCommand('workbench.action.closePanel'); let view = await output(context); - assert.match(await view.locator('body').innerText(), /PERFCHECKER OUTPUT/); + assert.match(await view.locator('body').innerText({timeout:NATIVE_READINESS_MS}), /PERFCHECKER OUTPUT/); assert.equal(await view.locator('.empty').count(), 0, 'Real suite measurements were supplied to the installed VSIX'); const total = await view.locator('[data-result-item]').count(); assert(total > 0); for (const id of ['package', 'workload', 'backend', 'kind', 'status']) { for (const option of await options(view, `#result-${id}`)) { const expected=await view.locator('[data-result-item]').evaluateAll((items,{id,value})=>items.filter(item=>item.dataset[id]===value).length,{id,value:option.value}); - await view.locator(`#result-${id}`).selectOption(option.value); + await view.locator(`#result-${id}`).selectOption(option.value,{timeout:NATIVE_READINESS_MS}); const visible = view.locator('[data-result-item]:visible'); assert.equal(await visible.count(),expected,`The ${id}=${option.value} filter shows exactly the matching real results, including an empty status`); assert(await visible.evaluateAll((items, {id, value}) => items.every(item => item.dataset[id] === value), {id, value: option.value})); } - await view.locator(`#result-${id}`).selectOption(''); + await view.locator(`#result-${id}`).selectOption('',{timeout:NATIVE_READINESS_MS}); } const resultGroups=()=>view.locator('.result-list,.chart-grid,.flame-grid,tbody').evaluateAll(containers=>containers.map(container=> [...container.children].filter(item=>item.matches('[data-result-item]')).map(item=>({kind:item.dataset.kind,package:item.dataset.package, @@ -775,7 +776,7 @@ async function testResults(context) { assert(originalGroups.some(items=>items.length>=2),'The real Results fixture contains a group with multiple sortable entries'); let previousResults=originalGroups,observedResultSortChange=false; for (const sort of await options(view, '#result-sort')) { - await view.locator('#result-sort').selectOption(sort.value); + await view.locator('#result-sort').selectOption(sort.value,{timeout:NATIVE_READINESS_MS}); const key=sort.value==='name'?'search':sort.value; await eventually(async()=>{ const groups=await resultGroups(); @@ -790,31 +791,31 @@ async function testResults(context) { previousResults=groups; } assert(observedResultSortChange,'At least one discriminating Results sort gesture changes a rendered group order'); - await view.locator('#result-search').fill('there-is-no-matching-native-result'); + await view.locator('#result-search').fill('there-is-no-matching-native-result',{timeout:NATIVE_READINESS_MS}); assert.equal(await view.locator('[data-result-item]:visible').count(), 0); - await view.locator('#result-search').fill(''); assert.equal(await view.locator('[data-result-item]').count(), total); + await view.locator('#result-search').fill('',{timeout:NATIVE_READINESS_MS}); assert.equal(await view.locator('[data-result-item]').count(), total); const chartFamilies = {}; for (const [name, selector] of Object.entries({distribution: '.distribution .sample', allocation: '.pie-slice', normalized: '.normalized-chart .hover-value', flame: '.flame-node', series: '.spark .hover-value'})) { const points = view.locator(selector); const count = await points.count(); chartFamilies[name] = count; if (!count) {context.log(`plot-prerequisite-${name}`, 'No corresponding observations in this real run; collector fixture must qualify this family separately'); continue;} - const point = points.first(); await point.focus(); - const target = await point.getAttribute('data-target'); - if (target) assert((await view.locator(`[id="${target}"]`).innerText()).trim(), `${name} keyboard focus shows its evidence`); + const point = points.first(); await point.focus({timeout:NATIVE_READINESS_MS}); + const target = await point.getAttribute('data-target',{timeout:NATIVE_READINESS_MS}); + if (target) assert((await view.locator(`[id="${target}"]`).innerText({timeout:NATIVE_READINESS_MS})).trim(), `${name} keyboard focus shows its evidence`); } assert(chartFamilies.flame>0,'The actual profile reports render nonempty, focusable flame frames'); await testFlameControls(context,view); const overlay=view.locator('.normalized-plot').first(); assert.equal(await overlay.count(),1,'Measured version series supplies the real comparison controls'); const seriesFile=path.join(context.results,'version-series.json'),seriesBytes=await fs.readFile(seriesFile); - const payload=await overlay.getAttribute('data-normalized'),data=JSON.parse(payload); + const payload=await overlay.getAttribute('data-normalized',{timeout:NATIVE_READINESS_MS}),data=JSON.parse(payload); const inspected=overlay.locator('.normalized-chart .hover-value').first(); - const metric=await inspected.getAttribute('data-metric'),detailTarget=await inspected.getAttribute('data-target'); + const metric=await inspected.getAttribute('data-metric',{timeout:NATIVE_READINESS_MS}),detailTarget=await inspected.getAttribute('data-target',{timeout:NATIVE_READINESS_MS}); const beforeCount=await overlay.locator('.normalized-chart .hover-value').count();assert(beforeCount>0); - await inspected.focus();assert.match(await view.locator(`[id="${detailTarget}"]`).innerText(),/ratio/); - await overlay.locator(`[data-normalized-metric="${metric}"]`).uncheck(); + await inspected.focus({timeout:NATIVE_READINESS_MS});assert.match(await view.locator(`[id="${detailTarget}"]`).innerText({timeout:NATIVE_READINESS_MS}),/ratio/); + await overlay.locator(`[data-normalized-metric="${metric}"]`).uncheck({timeout:NATIVE_READINESS_MS}); assert.equal(await overlay.locator(`[data-metric="${metric}"]`).count(),0); - assert.match(await view.locator(`[id="${detailTarget}"]`).innerText(),/^Hover or focus/); - await overlay.getByRole('button',{name:'Reset chart',exact:true}).click(); + assert.match(await view.locator(`[id="${detailTarget}"]`).innerText({timeout:NATIVE_READINESS_MS}),/^Hover or focus/); + await overlay.getByRole('button',{name:'Reset chart',exact:true}).click({timeout:NATIVE_READINESS_MS}); assert.equal(await overlay.locator('.normalized-chart .hover-value').count(),beforeCount); const retained=await overlay.locator('.normalized-chart .hover-value').evaluateAll((points,versions)=>{ const point=points.find(point=>versions.indexOf(point.dataset.version)>0); @@ -822,34 +823,34 @@ async function testResults(context) { },data.versions); assert(retained,'The actual measured series has a finite point beyond its first version'); const index=String(data.versions.indexOf(retained.version)); - await overlay.locator('[data-normalized-from]').selectOption(index);await overlay.locator('[data-normalized-to]').selectOption(index); + await overlay.locator('[data-normalized-from]').selectOption(index,{timeout:NATIVE_READINESS_MS});await overlay.locator('[data-normalized-to]').selectOption(index,{timeout:NATIVE_READINESS_MS}); const filtered=await overlay.locator('.normalized-chart .hover-value').evaluateAll(points=>points.map(point=>({ version:point.dataset.version,metric:point.dataset.metric,y:point.getAttribute('cy'),detail:point.dataset.detail, }))); assert(filtered.length>0&&filtered.every(point=>point.version===retained.version)); assert.deepEqual(filtered.find(point=>point.metric===retained.metric),retained,'View filtering preserves the original ratio, raw value and Y'); - assert.equal(await overlay.getAttribute('data-normalized'),payload); - await overlay.getByRole('button',{name:'Reset chart',exact:true}).click(); - assert.equal(await overlay.locator('[data-normalized-from]').inputValue(),'0'); - assert.equal(await overlay.locator('[data-normalized-to]').inputValue(),String(data.versions.length-1)); + assert.equal(await overlay.getAttribute('data-normalized',{timeout:NATIVE_READINESS_MS}),payload); + await overlay.getByRole('button',{name:'Reset chart',exact:true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await overlay.locator('[data-normalized-from]').inputValue({timeout:NATIVE_READINESS_MS}),'0'); + assert.equal(await overlay.locator('[data-normalized-to]').inputValue({timeout:NATIVE_READINESS_MS}),String(data.versions.length-1)); assert.equal(await overlay.locator('.normalized-chart .hover-value').count(),beforeCount); - assert.match(await view.locator(`[id="${detailTarget}"]`).innerText(),/^Hover or focus/); + assert.match(await view.locator(`[id="${detailTarget}"]`).innerText({timeout:NATIVE_READINESS_MS}),/^Hover or focus/); assert.deepEqual(await fs.readFile(seriesFile),seriesBytes,'Comparison gestures do not rewrite measured reports'); context.proof('native-results-comparison-controls',{actualMeasuredVersions:data.versions.length,visibleMetrics:data.metrics.length, metricCheckbox:true,measuredVersionRange:true,reset:true,staleReadoutCleared:true, fixedYAndOriginalRatioPreserved:true,measurementFileUnchanged:true}); context.proof('result-controls', {items: total, filters: 7, chartFamilies}); const chart=view.locator('.normalized-chart,.distribution,.spark').first(); - if(await chart.count())await chart.scrollIntoViewIfNeeded(); + if(await chart.count())await chart.scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,'measured-results'); for (const [name, report] of [['Summary', 'suite-report.md'], ['JSON', 'suite-result.json'], ['Comparisons', 'version-comparison.md'], ['Series JSON', 'version-series.json']]) { view = await output(context); - await view.getByRole('button', {name, exact: true}).click(); + await view.getByRole('button', {name, exact: true}).click({timeout:NATIVE_READINESS_MS}); const file = path.join(context.results, report); const exists = await fs.access(file).then(() => true, () => false); if (exists) await eventually(() => context.vscode.window.tabGroups.all.some(group => group.tabs.some(tab => tab.input?.uri?.fsPath === context.vscode.Uri.file(file).fsPath || (report.endsWith('.md') && tab.label.includes(report)))), `Open actual ${report}`); - else await eventually(async () => (await context.windowPage.locator('body').innerText()).includes(`PerfChecker has not produced ${report} yet.`), `Missing ${report} explains its prerequisite`); + else await eventually(async () => (await context.windowPage.locator('body').innerText({timeout:NATIVE_READINESS_MS})).includes(`PerfChecker has not produced ${report} yet.`), `Missing ${report} explains its prerequisite`); context.log(`result-report-${name}`, {exists, openedOrExplained: true}); } } @@ -867,26 +868,26 @@ async function testComputedAggregations(context){ const reports=path.join(context.workspace,'perf','results','aggregations',aggregation); await settings().update('reports',reports,context.vscode.ConfigurationTarget.WorkspaceFolder); let view=await designer(context); - await view.locator('#comparison-package').selectOption('Example'); - await view.locator('#comparison-feature').selectOption('hello'); - for(const input of await view.locator('#baseline-targets input,#candidate-targets input').all())await input.uncheck(); - for(const version of baselines)await view.locator(`#baseline-targets input[value="${version}"]`).check(); - await view.locator(`#candidate-targets input[value="${candidate}"]`).check(); - await view.locator('#comparison-aggregation').selectOption(aggregation); - await view.locator('#add-comparison').click(); + await view.locator('#comparison-package').selectOption('Example',{timeout:NATIVE_READINESS_MS}); + await view.locator('#comparison-feature').selectOption('hello',{timeout:NATIVE_READINESS_MS}); + for(const input of await view.locator('#baseline-targets input,#candidate-targets input').all())await input.uncheck({timeout:NATIVE_READINESS_MS}); + for(const version of baselines)await view.locator(`#baseline-targets input[value="${version}"]`).check({timeout:NATIVE_READINESS_MS}); + await view.locator(`#candidate-targets input[value="${candidate}"]`).check({timeout:NATIVE_READINESS_MS}); + await view.locator('#comparison-aggregation').selectOption(aggregation,{timeout:NATIVE_READINESS_MS}); + await view.locator('#add-comparison').click({timeout:NATIVE_READINESS_MS}); await eventually(()=>settings().get('comparisonPolicies',[]).length===1,`The native ${aggregation} comparison policy is persisted`); - await eventually(async()=>!(await view.locator('#run').isDisabled()),'The policy replan finishes',120000); - await view.locator('#reset-filters').click();await view.locator('#clear-all').click(); - await view.locator('#package').selectOption('Example');await view.locator('#select-visible').click(); + await eventually(async()=>!(await view.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS})),'The policy replan finishes',NATIVE_READINESS_MS); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS});await view.locator('#clear-all').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#package').selectOption('Example',{timeout:NATIVE_READINESS_MS});await view.locator('#select-visible').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view),4,'Three registered references and a registered candidate are really measured'); - await view.locator('#open-after-run').uncheck();context.log('native-ui-action',{surface:'Suite designer',action:'Run comparison',aggregation});await view.locator('#run').click(); + await view.locator('#open-after-run').uncheck({timeout:NATIVE_READINESS_MS});context.log('native-ui-action',{surface:'Suite designer',action:'Run comparison',aggregation});await view.locator('#run').click({timeout:NATIVE_READINESS_MS}); const comparison=await eventually(async()=>{ const suite=JSON.parse(await fs.readFile(path.join(reports,'suite-result.json'),'utf8')); if(suite.runs.length!==4)return false; assert(suite.runs.every(run=>run.status==='pass'&&run.qualification.correctness.status==='passed')); return JSON.parse(await fs.readFile(path.join(reports,'version-comparison.json'),'utf8')); - },`Actual benchmark reports complete for ${aggregation}`,360000); - await eventually(async()=>!(await view.locator('#run').isDisabled()),'The measured comparison returns to idle'); + },`Actual benchmark reports complete for ${aggregation}`,NATIVE_READINESS_MS); + await eventually(async()=>!(await view.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS})),'The measured comparison returns to idle'); const records=comparison.records.filter(record=>record.package==='Example'&&record.candidate_version===candidate); assert(records.length>0,'The measured policy produces comparisons, rather than merely saving an option'); for(const record of records){ @@ -907,7 +908,7 @@ async function testComputedAggregations(context){ if(saved)await fs.writeFile(configFile,saved);else await fs.rm(configFile,{force:true}); await settings().update('reports',previous.reports,context.vscode.ConfigurationTarget.WorkspaceFolder); await settings().update('comparisonPolicies',previous.policies,context.vscode.ConfigurationTarget.WorkspaceFolder); - await context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri),"perfchecker.openDesignerForWorkspace"); } } @@ -915,37 +916,37 @@ async function testAdvisorAndTools(context) { await clickStudioAction(context, 'advisor'); let view = await frame(context, '#advisor-root'); for (const mode of ['none', 'ollama', 'chat_completions', 'chat_completions_schema', 'mcp_http']) { - await view.locator('#advisor-protocol').selectOption(mode); - assert.equal(await view.getByRole('button', {name: 'Test connection / discover', exact: true}).isDisabled(), mode === 'none'); + await view.locator('#advisor-protocol').selectOption(mode,{timeout:NATIVE_READINESS_MS}); + assert.equal(await view.getByRole('button', {name: 'Test connection / discover', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS}), mode === 'none'); assert.equal(await view.locator('#advisor-mcp_tool').isVisible(), mode === 'mcp_http'); assert.equal(await view.locator('#advisor-download-model').isVisible(), mode === 'ollama'); } - await view.locator('#advisor-mcp_arguments').fill('{ invalid JSON'); - await view.getByRole('button', {name: 'Save configuration', exact: true}).click(); - assert.match(await view.locator('[role="status"]').innerText(), /JSON|property|Unexpected/i); - await view.locator('#advisor-mcp_arguments').fill('{}'); - await view.locator('#advisor-investigates').check(); + await view.locator('#advisor-mcp_arguments').fill('{ invalid JSON',{timeout:NATIVE_READINESS_MS}); + await view.getByRole('button', {name: 'Save configuration', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await view.locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS}), /JSON|property|Unexpected/i); + await view.locator('#advisor-mcp_arguments').fill('{}',{timeout:NATIVE_READINESS_MS}); + await view.locator('#advisor-investigates').check({timeout:NATIVE_READINESS_MS}); assert(await view.locator('#advisor-max_experiments').isVisible()); - await view.getByRole('button', {name: 'Save configuration', exact: true}).click(); - assert.match(await view.locator('[role="status"]').innerText(), /structured response/); - await view.locator('#advisor-investigates').uncheck(); - assert(await view.getByRole('button', {name: 'Cancel operation', exact: true}).isDisabled()); + await view.getByRole('button', {name: 'Save configuration', exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.match(await view.locator('[role="status"]').innerText({timeout:NATIVE_READINESS_MS}), /structured response/); + await view.locator('#advisor-investigates').uncheck({timeout:NATIVE_READINESS_MS}); + assert(await view.getByRole('button', {name: 'Cancel operation', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS})); context.proof('provider-controls', {modes: 5, malformedArgumentsRejected: true, textInvestigationRejected: true, noNetworkRequest: true}); await clickStudioAction(context, 'tools'); view = await frame(context, '#app'); - await view.getByRole('button', {name: 'Findings & advice', exact: true}).click(); - await view.getByRole('heading', {name: 'Tool catalogue', exact: true}).waitFor({timeout: 120000}); + await view.getByRole('button', {name: 'Findings & advice', exact: true}).click({timeout:NATIVE_READINESS_MS}); + await view.getByRole('heading', {name: 'Tool catalogue', exact: true}).waitFor({timeout: NATIVE_READINESS_MS}); const search = view.getByRole('searchbox', {name: 'Search tool catalogue'}); - await search.fill('nonexistent-native-catalogue-tool'); + await search.fill('nonexistent-native-catalogue-tool',{timeout:NATIVE_READINESS_MS}); const catalogueContainer = search.locator('..'); assert.equal(await catalogueContainer.locator('article.card').count(), 0); - await search.fill(''); assert((await catalogueContainer.locator('article.card').count()) > 0); + await search.fill('',{timeout:NATIVE_READINESS_MS}); assert((await catalogueContainer.locator('article.card').count()) > 0); for (const tab of ['Scenarios', 'Findings & advice', 'Before / after', 'Saved evidence']) { - await view.getByRole('button', {name: tab, exact: true}).click(); - assert.equal(await view.getByRole('button', {name: tab, exact: true}).getAttribute('aria-current'), 'page'); + await view.getByRole('button', {name: tab, exact: true}).click({timeout:NATIVE_READINESS_MS}); + assert.equal(await view.getByRole('button', {name: tab, exact: true}).getAttribute('aria-current',{timeout:NATIVE_READINESS_MS}), 'page'); } - await view.getByRole('button', {name: 'Worker log', exact: true}).click(); + await view.getByRole('button', {name: 'Worker log', exact: true}).click({timeout:NATIVE_READINESS_MS}); context.proof('native-tool-catalogue', {backend: context.core, filter: true, investigationTabs: 4, workerLog: true}); } @@ -954,8 +955,8 @@ async function testSavePalette(context) { const {destination} = await savedConfiguration(context, view); assert((await fs.stat(destination)).size > 0, 'Editor Save works before testing the independent palette command'); const title = `Unsaved palette qualification ${Date.now()}`; - await view.locator('#doc-title').fill(title); - await view.locator('#cards .card .check-option input').first().uncheck(); + await view.locator('#doc-title').fill(title,{timeout:NATIVE_READINESS_MS}); + await view.locator('#cards .card .check-option input').first().uncheck({timeout:NATIVE_READINESS_MS}); const count = await selectionCount(view); await view.locator('#cards .card .label').first().evaluate(input => { input.value = '#a12655'; input.dispatchEvent(new Event('input', {bubbles: true})); @@ -978,18 +979,18 @@ async function testLargePlanAndOrdering(context) { try{ await fs.rm(configFile,{force:true}); await settings().update('suite','perf/large-suite.jl',context.vscode.ConfigurationTarget.WorkspaceFolder); - await context.vscode.commands.executeCommand('perfchecker.refresh'); - await context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.refresh'),"perfchecker.refresh"); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri),"perfchecker.openDesignerForWorkspace"); let view=await frame(context,'#cards'); - await eventually(async()=>/Showing 120 of 125 workload groups/.test(await view.locator('#rendered-count').innerText()),'The real large plan paginates 125 workload groups',120000); + await eventually(async()=>/Showing 120 of 125 workload groups/.test(await view.locator('#rendered-count').innerText({timeout:NATIVE_READINESS_MS})),'The real large plan paginates 125 workload groups',NATIVE_READINESS_MS); assert.equal(await view.locator('#cards .card').count(),120); - await view.locator('#reset-filters').click(); - await view.locator('#select-visible').click(); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#select-visible').click({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view),125,'Bulk selection includes the five groups not rendered yet'); - await view.locator('#show-more').click(); + await view.locator('#show-more').click({timeout:NATIVE_READINESS_MS}); assert.equal(await view.locator('#cards .card').count(),125); assert(!await view.locator('#show-more').isVisible()); - await view.locator('#sort').selectOption('suite'); + await view.locator('#sort').selectOption('suite',{timeout:NATIVE_READINESS_MS}); const before=await savedConfiguration(context,view); const first=before.config.selection.run_ids[0]; // Start on the workload title rather than a checkbox/color input in the card's @@ -1029,9 +1030,9 @@ async function testLargePlanAndOrdering(context) { assert(events.some(event=>event.type==='dragstart')&&events.some(event=>event.type==='drop'),'The real webview receives native dragstart and drop before checking order'); assert.notEqual(after.config.selection.run_ids[0],first,'Native drag changes the persisted execution order'); assert.deepEqual(new Set(after.config.selection.run_ids),new Set(before.config.selection.run_ids)); - await view.locator('#search').fill('workload_125'); + await view.locator('#search').fill('workload_125',{timeout:NATIVE_READINESS_MS}); await eventually(async()=>await view.locator('#cards .card').count()===1,'The debounced workload search has rendered the one matching group'); - await view.locator('#clear-visible').click(); + await view.locator('#clear-visible').click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await selectionCount(view)===124,'Clearing the one filtered group preserves all124hidden selections'); context.proof('large-plan-pagination-and-drag',{groups:125,initiallyRendered:120,bulkIncludesHidden:true,nativeDrag:true,executionsRequested:0}); }catch(error){ @@ -1043,10 +1044,10 @@ async function testLargePlanAndOrdering(context) { }finally{ if(saved)await fs.writeFile(configFile,saved);else await fs.rm(configFile,{force:true}); await settings().update('suite',previous,context.vscode.ConfigurationTarget.WorkspaceFolder); - await context.vscode.commands.executeCommand('perfchecker.refresh'); - await context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.refresh'),"perfchecker.refresh"); + await waitForNativeAvailability(context.vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri),"perfchecker.openDesignerForWorkspace"); const view=await frame(context,'#cards'); - await eventually(async()=>!(await view.locator('#show-more').isVisible())&&!/workload_125/.test(await view.locator('#cards').innerText()),'Restore the real measurement suite after plan-only pagination',120000); + await eventually(async()=>!(await view.locator('#show-more').isVisible())&&!/workload_125/.test(await view.locator('#cards').innerText({timeout:30000})),'Restore the real measurement suite after plan-only pagination',NATIVE_READINESS_MS); } } @@ -1076,7 +1077,7 @@ exports.runColour = async context => { const planFile=path.join(process.env.PERFCHECKER_NATIVE_PROFILE,'User','globalStorage', 'mirage-interactive-fr.perfchecker-vscode','suite-plan.json'); const planBytes=await fs.readFile(planFile),plan=JSON.parse(planBytes); - const group=view.locator('#cards .card').first(),leader=await group.getAttribute('data-id'); + const group=view.locator('#cards .card').first(),leader=await group.getAttribute('data-id',{timeout:NATIVE_READINESS_MS}); const first=plan.runs.find(run=>run.id===leader);assert(first,'The visible group belongs to the real controller plan'); const ids=plan.runs.filter(run=>run.package===first.package&&run.workload===first.workload&&run.version===first.version).map(run=>run.id); assert(ids.length>1,'The picker labels a real group of multiple checks'); @@ -1087,11 +1088,11 @@ exports.runColour = async context => { cards:[...document.querySelectorAll('#cards .card')].map(card=>card.dataset.id),checkTypes:document.querySelectorAll('#check-types label').length})); const captureGroup=async name=>{ await context.vscode.commands.executeCommand('workbench.action.closePanel'); - await view.locator(`#cards .card[data-id="${leader}"]`).scrollIntoViewIfNeeded(); + await view.locator(`#cards .card[data-id="${leader}"]`).scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); await capture(context,name); }; await context.vscode.commands.executeCommand('workbench.action.closePanel'); - await group.locator('input.label').scrollIntoViewIfNeeded(); + await group.locator('input.label').scrollIntoViewIfNeeded({timeout:NATIVE_READINESS_MS}); const previous=await eventually(async()=>{ const cards=await colours();assert(cards.length>0,'The real plan has visible groups'); for(const card of cards){ @@ -1104,11 +1105,11 @@ exports.runColour = async context => { context.log('native-colour-initial-borders',{cards:previous}); const windowIds=new Set((await windows()).map(window=>window.id)); context.log('native-ui-action',{surface:'Suite designer',action:'Open real colour picker'}); - await group.locator('input.label').click(); + await group.locator('input.label').click({timeout:NATIVE_READINESS_MS}); const popup=await eventually(async()=>{ const opened=(await windows()).filter(window=>!windowIds.has(window.id)&&window.width>=220&&window.width<=260&&window.height>=240&&window.height<=280); return opened.length===1&&opened[0]; - },'The trusted colour click creates its actual X11 picker popup',10000); + },'The trusted colour click creates its actual X11 picker popup',NATIVE_READINESS_MS); const output=process.env.PERFCHECKER_NATIVE_OUTPUT,temporary=path.join(output,'native-colour-popup.xwd'); const image=path.join(output,`native-${process.platform}-vscode-${context.vscode.version}-colour-picker.png`); let painted; @@ -1117,7 +1118,7 @@ exports.runColour = async context => { await execute('convert',[temporary,image],{timeout:10000}); painted=Number((await execute('identify',['-format','%k',image],{timeout:10000})).stdout.trim()); return painted>20; - },'The actual OS picker is painted before its review capture',10000);} + },'The actual OS picker is painted before its review capture',NATIVE_READINESS_MS);} finally{await fs.rm(temporary,{force:true});} context.log('native-colour-picker-before-input',{popup,paintedColors:painted,file:path.basename(image), sha256:createHash('sha256').update(await fs.readFile(image)).digest('hex')}); @@ -1131,7 +1132,7 @@ exports.runColour = async context => { await native(['type','--clearmodifiers',value]); } await native(['key','--clearmodifiers','Return']); - await eventually(async()=>!(await windows()).some(window=>window.id===popup.id),'Return dismisses the real colour picker',10000); + await eventually(async()=>!(await windows()).some(window=>window.id===popup.id),'Return dismisses the real colour picker',NATIVE_READINESS_MS); const verify=async()=>{ const current=await colours(),changed=current.find(card=>card.id===leader); assert.equal(changed.colour,'#1266aa');assert.equal(changed.border,'rgb(18, 102, 170)'); @@ -1174,18 +1175,18 @@ exports.runColour = async context => { exports.runSelection = async context => { assertDisposable(context); let view = await designer(context); - await view.locator('#results').click(); + await view.locator('#results').click({timeout:NATIVE_READINESS_MS}); const empty = await frame(context, 'button[data-report="suite-result.json"]'); - assert.match(await empty.locator('.empty').innerText(), /No persisted output yet/); + assert.match(await empty.locator('.empty').innerText({timeout:NATIVE_READINESS_MS}), /No persisted output yet/); context.log('empty-result-controls', {native: true, noFabricatedEvidence: true}); view = await designer(context); - await view.locator('#reset-filters').click(); - await view.locator('#clear-all').click(); + await view.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS}); + await view.locator('#clear-all').click({timeout:NATIVE_READINESS_MS}); const ready = view.locator('.check-option.ready').filter({hasText: 'BenchmarkTools'}).first(); assert.equal(await ready.count(), 1, 'A real runnable benchmark is in the fixture'); - await ready.locator('input').check(); + await ready.locator('input').check({timeout:NATIVE_READINESS_MS}); assert.equal(await selectionCount(view), 1, 'The Run button receives exactly one selected run'); - await view.locator('#open-after-run').uncheck(); + await view.locator('#open-after-run').uncheck({timeout:NATIVE_READINESS_MS}); const settings = context.vscode.workspace.getConfiguration('perfchecker', context.vscode.Uri.file(context.workspace)); const reports = path.resolve(context.workspace, settings.get('reports', 'perf/results/vscode')); const reportPath = path.join(reports, 'suite-result.json'); @@ -1202,7 +1203,7 @@ exports.runSelection = async context => { const idleState=await state(),beforeLaunches=await controllerLaunches(); const launchCount=entries=>entries.reduce((sum,item)=>sum+item.commands.length,0); context.log('native-ui-action',{surface:'Suite designer',action:'Run selected check'}); - await view.locator('#run').click(); + await view.locator('#run').click({timeout:NATIVE_READINESS_MS}); const freshReport=async()=>{ try{const text=await fs.readFile(reportPath,'utf8');if(text===old)return false;const value=JSON.parse(text); return value.schema_version==='perfchecker-suite-result/1'&&value.runs.length===1&&value.runs[0].status==='pass'&&value; @@ -1212,7 +1213,7 @@ exports.runSelection = async context => { const worker=await nativeSuiteWorker(context),snapshot=await state(); if(worker&&Object.values(snapshot.controls).every(Boolean))return {worker,snapshot}; if(await freshReport())return {finished:true};return false; - },'The existing selected suite has an owned active controller and busy controls, or completes with fresh real evidence',30000); + },'The existing selected suite has an owned active controller and busy controls, or completes with fresh real evidence',NATIVE_READINESS_MS); const runningLaunches=await eventually(async()=>{ const value=await controllerLaunches();return launchCount(value)===launchCount(beforeLaunches)+1&&value; },'The real output log witnesses exactly the intended suite launch'); @@ -1252,9 +1253,9 @@ exports.runSelection = async context => { const value = JSON.parse(text); if (value.schema_version !== 'perfchecker-suite-result/1' || value.runs.length !== 1) return false; return value; - }, 'The native Run button completes the registered Julia backend and writes current evidence', 240000); + }, 'The native Run button completes the registered Julia backend and writes current evidence', NATIVE_READINESS_MS); assert.equal(report.runs[0].status, 'pass', 'Actual correctness-checked benchmark succeeds'); - await eventually(async () => !(await view.locator('#run').isDisabled()), 'Run control returns to idle'); + await eventually(async () => !(await view.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS})), 'Run control returns to idle'); assert.deepEqual(await state(),idleState,'The same reloaded run finishes with the original exact idle controls and inventory'); assert.deepEqual(await controllerLaunches(),runningLaunches,'No additional controller launches occur through completion of the same selected run'); assert.deepEqual(await fs.readFile(planFile),planBytes,'Active reload preserves the exact canonical real plan'); @@ -1262,7 +1263,7 @@ exports.runSelection = async context => { context.proof('native-designer-active-reload-completed',{workerPid:active.worker?.pid,sameRunCompleted:true,activeBusyHydrationQualified, planSha256:createHash('sha256').update(planBytes).digest('hex'),planUnchanged:true,idleControlsRestored:true, inventorySelectionOrderPreserved:true,additionalControllerLaunches:0}); - await view.locator('#results').click(); + await view.locator('#results').click({timeout:NATIVE_READINESS_MS}); const result = await frame(context, 'button[data-report="suite-result.json"]'); assert.equal(await result.locator('.empty').count(), 0); assert((await result.locator('[data-result-item]').count()) > 0); @@ -1277,8 +1278,8 @@ exports.run = async context => { assert.equal(await view.locator('.lab button').count(), 5); assert(await view.locator('.hero img').evaluate(image => image.complete && image.naturalWidth > 0)); const environment=view.locator('details.environment'); - if(await environment.getAttribute('open')===null)await environment.locator('summary').click(); - await eventually(async()=>(await environment.locator('p').innerText()).includes(context.controller),'Studio displays the selected controller after its state arrives'); + if(await environment.getAttribute('open',{timeout:NATIVE_READINESS_MS})===null)await environment.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>(await environment.locator('p').innerText({timeout:NATIVE_READINESS_MS})).includes(context.controller),'Studio displays the selected controller after its state arrives'); context.proof('studio-inventory', {cards: 9, workbenchButtons: 5, controllerShown: true}); const failures = []; for (const [name, run] of [['suite-selection', testSuiteSelection], ['git-and-comparisons', testGitTargetsAndComparisons],['git-cancel-discovery',testCancelGitDiscovery], @@ -1300,20 +1301,20 @@ exports.runFresh = async context => { assertDisposable(context); const view = await studio(context); assert.equal(await view.locator('.card').count(), 9); - assert.match(await view.locator('.status').innerText(), /Install the Julia extension/); + assert.match(await view.locator('.status').innerText({timeout:NATIVE_READINESS_MS}), /Install the Julia extension/); await clickStudioAction(context, 'suite'); const missingSetup = /suite file not found.*perfchecker\.suite|controller Project\.toml not found.*perfchecker\.runnerProject/i; - await eventually(async () => missingSetup.test(await (await frame(context, '#studio-root')).locator('.status').innerText()) || - missingSetup.test(await context.windowPage.locator('body').innerText()), + await eventually(async () => missingSetup.test(await (await frame(context, '#studio-root')).locator('.status').innerText({timeout:NATIVE_READINESS_MS})) || + missingSetup.test(await context.windowPage.locator('body').innerText({timeout:NATIVE_READINESS_MS})), 'Fresh install explains the missing controller or suite file'); await clickStudioAction(context, 'results'); - await eventually(async () => missingSetup.test(await (await frame(context, '#studio-root')).locator('.status').innerText()) || - missingSetup.test(await context.windowPage.locator('body').innerText()), + await eventually(async () => missingSetup.test(await (await frame(context, '#studio-root')).locator('.status').innerText({timeout:NATIVE_READINESS_MS})) || + missingSetup.test(await context.windowPage.locator('body').innerText({timeout:NATIVE_READINESS_MS})), 'Fresh results explain the missing controller or suite file'); await clickStudioAction(context, 'advisor'); const advisor = await frame(context, '#advisor-root'); assert(await advisor.locator('#advisor-protocol').isVisible()); - await advisor.locator('#advisor-protocol').selectOption('none'); - assert(await advisor.getByRole('button', {name: 'Test connection / discover', exact: true}).isDisabled()); + await advisor.locator('#advisor-protocol').selectOption('none',{timeout:NATIVE_READINESS_MS}); + assert(await advisor.getByRole('button', {name: 'Test connection / discover', exact: true}).isDisabled({timeout:NATIVE_READINESS_MS})); context.proof('fresh-studio-and-advisor', {visibleControls: 14, exercisedActions:['suite','results','advisor'], missingSuiteExplained: true, noJuliaExplained: true, ruleBasedAvailable: true}); }; diff --git a/test/native-suite-chat-evidence.cjs b/test/native-suite-chat-evidence.cjs index c538094..a41c45a 100644 --- a/test/native-suite-chat-evidence.cjs +++ b/test/native-suite-chat-evidence.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,waitForNativeAvailability}=require('./native-readiness.cjs'); const assert=require('node:assert/strict'); const fs=require('node:fs/promises'); const path=require('node:path'); @@ -6,7 +7,7 @@ const {execFile}=require('node:child_process'); const {promisify}=require('node:util'); const execute=promisify(execFile),hash=bytes=>createHash('sha256').update(bytes).digest('hex'); const documents=['manifest.json','measurement-definitions.json','observations.jsonl','diagnostics.jsonl','artifacts.json','integrity.json']; -async function eventually(callback,label,timeout=180000){ +async function eventually(callback,label,timeout=NATIVE_READINESS_MS){ const until=Date.now()+timeout; do {const result=await callback();if(result)return result;await new Promise(resolve=>setTimeout(resolve,100));}while(Date.now(){ const reports=path.resolve(workspace,settings.get('reports','perf/results/vscode')); assert(!path.relative(workspace,reports).startsWith('..')&&!path.isAbsolute(path.relative(workspace,reports))); const previous=await fs.readFile(path.join(reports,'suite-result.json')).catch(error=>{if(error.code==='ENOENT')return undefined;throw error;}); - await vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace',uri),"perfchecker.openDesignerForWorkspace"); const designer=await findFrame('#cards',{allowAttached:true}); await eventually(async()=>await designer.locator('#check-types label').count()>0&&await designer.locator('#cards .card').count()>0, - 'The real controller supplies the suite plan before Run selection',360000); - await designer.locator('#reset-filters').click();await designer.locator('#clear-all').click(); + 'The real controller supplies the suite plan before Run selection',NATIVE_READINESS_MS); + await designer.locator('#reset-filters').click({timeout:NATIVE_READINESS_MS});await designer.locator('#clear-all').click({timeout:NATIVE_READINESS_MS}); const card=designer.locator('#cards .card') .filter({has:designer.locator('.package',{hasText:/^PerfCheckerNativeFixture$/})}) .filter({has:designer.locator('strong',{hasText:/^sum_squares$/})}).first(); assert.equal(await card.count(),1); const check=card.locator('.check-option.ready').filter({hasText:'BenchmarkTools'}); - assert.equal(await check.count(),1);await check.locator('input').check(); - assert.match(await designer.locator('#count').innerText(),/^1 selected/); - await designer.locator('#open-after-run').uncheck();await designer.locator('#run').click(); + assert.equal(await check.count(),1);await check.locator('input').check({timeout:NATIVE_READINESS_MS}); + assert.match(await designer.locator('#count').innerText({timeout:NATIVE_READINESS_MS}),/^1 selected/); + await designer.locator('#open-after-run').uncheck({timeout:NATIVE_READINESS_MS});await designer.locator('#run').click({timeout:NATIVE_READINESS_MS}); let suite; await eventually(async()=>{ const bytes=await fs.readFile(path.join(reports,'suite-result.json')).catch(error=>{if(error.code==='ENOENT')return undefined;throw error;}); if(!bytes||bytes.equals(previous??Buffer.alloc(0)))return false; try{suite=JSON.parse(bytes);}catch(error){if(error instanceof SyntaxError)return false;throw error;} - return suite.schema_version==='perfchecker-suite-result/1'&&suite.runs.length===1&&suite.runs[0].status==='pass'&&!(await designer.locator('#run').isDisabled()); - },'The one selected real BenchmarkTools suite check generates its saved report',360000); + return suite.schema_version==='perfchecker-suite-result/1'&&suite.runs.length===1&&suite.runs[0].status==='pass'&&!(await designer.locator('#run').isDisabled({timeout:NATIVE_READINESS_MS})); + },'The one selected real BenchmarkTools suite check generates its saved report',NATIVE_READINESS_MS); assert.equal(suite.runs[0].package,'PerfCheckerNativeFixture'); assert.equal(suite.runs[0].feature,'sum_squares_benchmark'); const series=JSON.parse(await fs.readFile(path.join(reports,'version-series.json'))); @@ -63,7 +64,7 @@ exports.run=async(context,{calls,receipts,providerErrors,state,setAttached})=>{ // measurement row is manufactured by the host qualification. const projected=await execute(process.env.PERFCHECKER_NATIVE_JULIA,['--startup-file=no',`--project=${context.controller}`,'-e', 'using PerfChecker; bundle=read_run_bundle(ARGS[1];require_integrity=true); advice=advise(bundle); @assert haskey(advice,"measurement_summaries"); config=PerfChecker.AdvisorConfig(protocol=:mcp_http,mcp_tool="ask_perfchecker",mcp_response=:text); evidence=PerfChecker._advisor_evidence(advice,config); serialized=PerfChecker.JSON.json(evidence); PerfChecker.JSON.print(Dict("version"=>string(Base.pkgversion(PerfChecker)),"advice"=>advice,"evidence"=>evidence,"characters"=>length(serialized),"budget"=>config.max_evidence_chars))',directory], - {windowsHide:true,timeout:180000,env:{...process.env,JULIA_LOAD_PATH:'@'+path.delimiter+'@stdlib'}}); + {windowsHide:true,timeout:NATIVE_READINESS_MS,env:{...process.env,JULIA_LOAD_PATH:'@'+path.delimiter+'@stdlib'}}); const expected=JSON.parse(projected.stdout);assert.equal(expected.version,context.core.version); assert.equal(expected.budget,12000);assert(expected.characters<=expected.budget); const measured=expected.evidence.filter(row=>row.kind==='measurement');assert(measured.length>0); @@ -104,19 +105,19 @@ exports.run=async(context,{calls,receipts,providerErrors,state,setAttached})=>{ let chat,originalObservations,corrupted=false,primaryError; try{ await vscode.commands.executeCommand('perfchecker.openChat');chat=await findFrame('#chat-root'); - await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click(); + await chat.getByRole('tab',{name:'01 · Advice',exact:true}).click({timeout:NATIVE_READINESS_MS}); const entry=await eventually(async()=>(await state()).evidence.find(item=>item.id.startsWith('suite:')&&item.id.includes(series.run_id)&&!item.unavailable), 'Chat refresh exposes the exact generated Suite run before its selection'); assert(entry&&!entry.unavailable,'The real picker exposes the generated bundle by explicit workspace/run identity'); assert(entry.id.includes(uri.toString())); assert.equal(receipts.length,beforeOpenReceipts,'Open Chat contacts no provider'); - await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(entry.id); + await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(entry.id,{timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return value.evidenceId===entry.id&&!value.busy&&value.messages.length===0;},'Native Suite selection stays idle'); assert.equal(calls.length,beforeCalls);assert.equal(receipts.length,beforeOpenReceipts); assert.deepEqual(await sourceHashes(),before); setAttached({evidence:expected.evidence,evidenceBudget:expected.budget,inspectPayload}); - await chat.locator('#chat-question').fill(question); - await chat.getByRole('button',{name:'Send question',exact:true}).click(); + await chat.locator('#chat-question').fill(question,{timeout:NATIVE_READINESS_MS}); + await chat.getByRole('button',{name:'Send question',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{assert.deepEqual(providerErrors,[]);const value=await state();return !value.busy&&value.messages.length===2;},'Actual Suite projection and the real MCP request complete'); assert.equal(calls.length,beforeCalls+1);assert.deepEqual(calls.at(-1).evidenceIds,expected.evidence.map(row=>row.id)); assert.equal(inspectedPayloads,1,'The complete real request was inspected before its controlled reply'); @@ -130,7 +131,7 @@ exports.run=async(context,{calls,receipts,providerErrors,state,setAttached})=>{ completePayloadSha256:payloadSha256,completePayloadNoPathsOrRawSource:true,uniqueEvidenceIds:true, originalProtocolBytesPreserved:true,originalIdentityReportBytesPreserved:true, provider:'Controlled actual MCP transport; no agent inference or performance gain claimed'}); - await chat.getByRole('button',{name:'New conversation',exact:true}).click(); + await chat.getByRole('button',{name:'New conversation',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>!(await state()).busy&&(await state()).messages.length===0,'Clear Suite conversation before integrity refusal'); originalObservations=await fs.readFile(path.join(directory,'observations.jsonl')); const changedObservations=Buffer.concat([originalObservations,Buffer.from('\n')]); @@ -138,8 +139,8 @@ exports.run=async(context,{calls,receipts,providerErrors,state,setAttached})=>{ const refusedHashes={...before,'bundle/observations.jsonl':hash(changedObservations)}; assert.deepEqual(await sourceHashes(),refusedHashes,'Only the deliberate disposable observation corruption changes any source or identity report'); const beforeRejectedCalls=calls.length,beforeRejectedReceipts=receipts.length; - await chat.locator('#chat-question').fill('Do not send this modified bundle.'); - await chat.getByRole('button',{name:'Send question',exact:true}).click(); + await chat.locator('#chat-question').fill('Do not send this modified bundle.',{timeout:NATIVE_READINESS_MS}); + await chat.getByRole('button',{name:'Send question',exact:true}).click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{const value=await state();return !value.busy&&/byte integrity check.*No measurements were sent/.test(value.status);},'A modified real bundle is refused before contacting MCP'); assert.equal(calls.length,beforeRejectedCalls);assert.equal(receipts.length,beforeRejectedReceipts); assert.deepEqual((await state()).messages,[]); @@ -152,8 +153,8 @@ exports.run=async(context,{calls,receipts,providerErrors,state,setAttached})=>{ const errors=[]; try{if(corrupted)await fs.writeFile(path.join(directory,'observations.jsonl'),originalObservations);}catch(error){errors.push(error);} setAttached(undefined); - try{if(chat){await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption(''); - await eventually(async()=>!(await state()).busy&&(await state()).evidenceId===''&&(await state()).messages.length===0,'Restore configuration-only conversation after the Suite test');}} + try{if(chat){await chat.getByRole('combobox',{name:'Attach saved evidence',exact:true}).selectOption('',{timeout:30000}); + await eventually(async()=>!(await state()).busy&&(await state()).evidenceId===''&&(await state()).messages.length===0,'Restore configuration-only conversation after the Suite test',180000);}} catch(error){errors.push(error);} if(errors.length)throw new AggregateError([...(primaryError?[primaryError]:[]),...errors],'Native Suite evidence failed or could not restore its disposable fixture'); } diff --git a/test/native-workbench-controls.cjs b/test/native-workbench-controls.cjs index 278bef9..0c62b84 100644 --- a/test/native-workbench-controls.cjs +++ b/test/native-workbench-controls.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS}=require('./native-readiness.cjs'); // Real workbench buttons, Julia processes and resource-scoped settings in disposable CI. const assert=require('node:assert/strict'); const fs=require('node:fs/promises'); @@ -7,7 +8,7 @@ const {clickStudioAction}=require('./native-studio-controls.cjs'); const delay=ms=>new Promise(resolve=>setTimeout(resolve,ms)); const preCleanupCaptures=new WeakSet(); class DebugExecutionFailed extends Error {} -async function eventually(read,description,timeout=180000){ +async function eventually(read,description,timeout=NATIVE_READINESS_MS){ const end=Date.now()+timeout;let last; while(Date.now()nodes.map(node=>({ariaLabel:node.getAttribute('aria-label'), placeholder:node.getAttribute('placeholder'),readOnly:node.readOnly,disabled:node.disabled}))), selectorContract:'Official SimpleFileDialog title; placeholder not required'}); - await folderInput.waitFor({state:'visible',timeout:60000}); - assert(await folderInput.isEditable(),'The actual controller folder dialog accepts a path'); - await folderInput.fill(controller+path.sep); - await eventually(async()=>vscode.Uri.file(path.resolve(await folderInput.inputValue())).fsPath===vscode.Uri.file(controller).fsPath, - 'The real folder dialog contains the requested controller before confirmation',60000); - await folderInput.press('Enter'); - await folderInput.waitFor({state:'hidden',timeout:60000}); + await folderInput.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + assert(await folderInput.isEditable({timeout:NATIVE_READINESS_MS}),'The actual controller folder dialog accepts a path'); + await folderInput.fill(controller+path.sep,{timeout:NATIVE_READINESS_MS}); + await eventually(async()=>vscode.Uri.file(path.resolve(await folderInput.inputValue({timeout:NATIVE_READINESS_MS}))).fsPath===vscode.Uri.file(controller).fsPath, + 'The real folder dialog contains the requested controller before confirmation',NATIVE_READINESS_MS); + await folderInput.press('Enter',{timeout:NATIVE_READINESS_MS}); + await folderInput.waitFor({state:'hidden',timeout:NATIVE_READINESS_MS}); await eventually(async()=>{ const current=settings(),persisted=JSON.parse(await fs.readFile(settingsFile,'utf8')); return current.inspect('runnerProject')?.workspaceFolderValue===selected&&persisted['perfchecker.runnerProject']===selected&& vscode.Uri.file(path.resolve(workspace,current.get('runnerProject'))).fsPath===vscode.Uri.file(controller).fsPath; }, - 'The real controller chooser verifies the existing Core environment and saves its selected path',180000); + 'The real controller chooser verifies the existing Core environment and saves its selected path',NATIVE_READINESS_MS); frame=await context.findFrame('#studio-root'); const environment=frame.locator('details.environment'); - if(await environment.getAttribute('open')===null)await environment.locator('summary').click(); + if(await environment.getAttribute('open',{timeout:NATIVE_READINESS_MS})===null)await environment.locator('summary').click({timeout:NATIVE_READINESS_MS}); let displayedIdentity; const expectedController=await fs.realpath(controller); const comparable=value=>process.platform==='win32'?value.toLowerCase():value; await eventually(async()=>{ - const displayed=(await environment.locator('p.subtle').innerText()).trim(); + const displayed=(await environment.locator('p.subtle').innerText({timeout:NATIVE_READINESS_MS})).trim(); let canonical; try{canonical=await fs.realpath(displayed);}catch(error){if(error.code==='ENOENT')return false;throw error;} if(displayedIdentity!==displayed+'\0'+canonical){ @@ -90,7 +91,7 @@ async function existingController(context){ try{ const visible=await folderInput.isVisible(); context.log('existing-controller-before-cleanup',{message:String(error),stack:error.stack, - folderDialogVisible:visible,folderPath:visible?await folderInput.inputValue():undefined, + folderDialogVisible:visible,folderPath:visible?await folderInput.inputValue({timeout:NATIVE_READINESS_MS}):undefined, runnerProject:settings().get('runnerProject'),workspaceFolderValue:settings().inspect('runnerProject')?.workspaceFolderValue}); }catch{} throw error; @@ -98,7 +99,7 @@ async function existingController(context){ const cleanupErrors=[]; try{ if(await folderInput.isVisible()){ - await folderInput.press('Escape'); + await folderInput.press('Escape',{timeout:30000}); await folderInput.waitFor({state:'hidden',timeout:30000}); } }catch(error){cleanupErrors.push(error);} @@ -124,7 +125,7 @@ async function repl(context){ const [coreVersion,juliaVersion,activeProject]=(await fs.readFile(marker,'utf8')).split('\n'); return coreVersion===context.coreVersion&&juliaVersion===process.env.PERFCHECKER_NATIVE_OFFICIAL_JULIA_VERSION&& activeProject&&vscode.Uri.file(activeProject).fsPath===vscode.Uri.file(path.join(controller,'Project.toml')).fsPath; - },'The official Julia REPL evaluates code with its documented runtime in the explicitly activated controller',360000); + },'The official Julia REPL evaluates code with its documented runtime in the explicitly activated controller',NATIVE_READINESS_MS); const firstImportMeasuredElapsedMs=Date.now()-started; if(process.env.PERFCHECKER_NATIVE_VIDEO==='1')await delay(3000); await vscode.commands.executeCommand('language-julia.stopREPL'); @@ -151,10 +152,10 @@ async function debug(context){ }}); try{ await clickStudioAction(context,'debug'); - await eventually(()=>stopped,'The Studio debugger uses the real Julia debug adapter and stops on entry',240000); + await eventually(()=>stopped,'The Studio debugger uses the real Julia debug adapter and stops on entry',NATIVE_READINESS_MS); assert.equal(vscode.Uri.file(session.configuration.project).fsPath,vscode.Uri.file(context.controller).fsPath); const button=windowPage.locator('.debug-toolbar .action-label[title^="Continue"],.debug-toolbar .action-label[aria-label^="Continue"]').first(); - await button.click(); + await button.click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{ if(terminated&&!await fs.stat(marker).then(()=>true).catch(()=>false))throw new DebugExecutionFailed(`The real Julia adapter terminated without the program marker: ${JSON.stringify(protocol).slice(-6000)}`); const lines=(await fs.readFile(marker,'utf8')).split('\n'); @@ -164,8 +165,8 @@ async function debug(context){ assert.equal(Number(lines[2]),333833500); } catch(error) {throw new DebugExecutionFailed(`The executed debugger marker diverges from its requested runtime/controller/oracle: ${error.message}`);} return true; - },'The actual Continue button executes the saved Julia file in the selected controller',240000); - await eventually(()=>terminated||!vscode.debug.activeDebugSession,'The Julia debug session terminates after execution'); + },'The actual Continue button executes the saved Julia file in the selected controller',NATIVE_READINESS_MS); + await eventually(()=>terminated||!vscode.debug.activeDebugSession,'The Julia debug session terminates after execution',180000); context.proof('official-julia-debug',{nativeStudioClick:true,nativeContinueClick:true,stopOnEntry:true,controller:context.controller,workspace:path.basename(workspace),julia:process.env.PERFCHECKER_NATIVE_OFFICIAL_JULIA_VERSION,targetSourceExecuted:true,activeProjectVerified:true,coreImportUnderInterpreter:false}); }catch(error){primaryError=error;context.log('julia-debug-protocol',{events:protocol,error:String(error)});throw error;} finally{ @@ -186,14 +187,14 @@ async function tasks(context){ const previous=await fs.readFile(file).catch(()=>undefined); try{ await fs.writeFile(file,JSON.stringify({version:'2.0.0',tasks:[{label:'PerfChecker native fixture',type:'process',command:process.env.PERFCHECKER_NATIVE_JULIA,args:['--startup-file=no',`--project=${context.controller}`,'-e',`using PerfChecker;write(${JSON.stringify(marker)},string(Base.pkgversion(PerfChecker)))`],problemMatcher:[]}]})); - await eventually(async()=>(await vscode.tasks.fetchTasks()).some(task=>task.name==='PerfChecker native fixture'),'The native task provider has loaded the on-disk task before its picker opens',60000); + await eventually(async()=>(await vscode.tasks.fetchTasks()).some(task=>task.name==='PerfChecker native fixture'),'The native task provider has loaded the on-disk task before its picker opens',NATIVE_READINESS_MS); await clickStudioAction(context,'tasks'); - const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible'}); - await picker.locator('.monaco-list-row').filter({hasText:'PerfChecker native fixture'}).first().click(); + const picker=windowPage.locator('.quick-input-widget');await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await picker.locator('.monaco-list-row').filter({hasText:'PerfChecker native fixture'}).first().click({timeout:NATIVE_READINESS_MS}); // VS Code may ask for the problem-matcher choice even for an explicitly empty matcher. await delay(300); const noMatcher=picker.locator('.monaco-list-row').filter({hasText:'Continue without scanning the task output'}); - if(await noMatcher.isVisible().catch(()=>false))await noMatcher.click(); + if(await noMatcher.isVisible().catch(()=>false))await noMatcher.click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>await fs.readFile(marker,'utf8')===context.coreVersion,'The Project tasks button runs the real Julia task'); context.proof('project-task',{nativeStudioClick:true,core:context.core}); }finally{if(previous)await fs.writeFile(file,previous);else await fs.rm(file,{force:true});} @@ -222,9 +223,9 @@ async function multiRoot(context){ await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',uri); const studio=await context.findFrame('#studio-root'); const environment=studio.locator('details.environment'); - if(!await environment.evaluate(element=>element.open))await environment.locator('summary').click(); - await eventually(async()=>(await studio.locator('.workspace strong').innerText())==='Second native workspace'&& - (await environment.innerText()).includes(secondController),'The Studio identifies the selected second folder and its expanded controller'); + if(!await environment.evaluate(element=>element.open))await environment.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>(await studio.locator('.workspace strong').innerText({timeout:NATIVE_READINESS_MS}))==='Second native workspace'&& + (await environment.innerText({timeout:NATIVE_READINESS_MS})).includes(secondController),'The Studio identifies the selected second folder and its expanded controller'); await clickStudioAction({...context,workspace:second},'terminal'); const terminal=await eventually(()=>vscode.window.terminals.find(item=>item.name==='PerfChecker · Second native workspace'),'Studio routes its terminal to the explicitly selected second folder'); const marker=path.join(second,'terminal-version.txt'); diff --git a/test/public-vsix-host.cjs b/test/public-vsix-host.cjs index ae8b133..5b20ff6 100644 --- a/test/public-vsix-host.cjs +++ b/test/public-vsix-host.cjs @@ -1,3 +1,4 @@ +const {NATIVE_READINESS_MS,nativeReadinessPolicy,waitForNativeAvailability}=require('./native-readiness.cjs'); // Runs in an actual Electron extension host, using an independently installed VSIX. const vscode = require('vscode'); const assert = require('node:assert/strict'); @@ -190,7 +191,7 @@ function configurationCoverage(properties,checks){ } const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); -async function eventually(read, description, timeout = 120000) { +async function eventually(read, description, timeout = NATIVE_READINESS_MS) { const deadline = Date.now() + timeout; let last; while (Date.now() < deadline) { @@ -389,12 +390,12 @@ async function runSuiteCases(context,runCase,options={}){ const selected=await runCase('native-suite-run-button',async()=>{context.results=await controls.runSelection(context);}); await runCase('native-suite-save-palette', async () => { if(options.blockDependents&&!selected)throw Object.assign(new Error('Blocked by native-suite-run-button; no dependent action was executed'),{blockedBy:'native-suite-run-button'}); - await vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace', uri); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.openDesignerForWorkspace', uri),"perfchecker.openDesignerForWorkspace"); const frame = await findFrame('#save'); - await frame.locator('#save').click(); + await frame.locator('#save').click({timeout:NATIVE_READINESS_MS}); await eventually(() => fs.readFile(path.join(workspace, 'perf', 'perfchecker-ui.json'), 'utf8').then(JSON.parse), 'Native Save button writes the shared configuration'); const title = `Unsaved palette state ${Date.now()}`; - await frame.locator('#doc-title').fill(title); + await frame.locator('#doc-title').fill(title,{timeout:NATIVE_READINESS_MS}); await vscode.commands.executeCommand('perfchecker.saveConfiguration'); const saved = JSON.parse(await fs.readFile(path.join(workspace, 'perf', 'perfchecker-ui.json'), 'utf8')); assert.equal(saved.documentation.blocks[0].title, title, 'Palette Save persists current unsaved editor state'); @@ -406,7 +407,7 @@ async function runSuiteCases(context,runCase,options={}){ await vscode.commands.executeCommand('perfchecker.openEntrypoint',run); assert.equal(vscode.window.activeTextEditor.document.uri.fsPath,vscode.Uri.file(run.entrypoint).fsPath); proof('native-workload-command',{command:'perfchecker.openEntrypoint',entrypoint:run.entrypoint}); - await vscode.commands.executeCommand('perfchecker.runNode',{runs:[run]}); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.runNode',{runs:[run]}),"perfchecker.runNode"); const report=JSON.parse(await fs.readFile(path.join(context.results,'suite-result.json'),'utf8')); assert.equal(report.runs.length,1);assert.equal(report.runs[0].status,'pass'); }); @@ -418,7 +419,7 @@ async function runSuiteCases(context,runCase,options={}){ if(options.complete){ await runCase('native-complete-suite-measurements',async()=>{ if(options.blockDependents&&!selected)throw Object.assign(new Error('Blocked by native-suite-run-button; no dependent action was executed'),{blockedBy:'native-suite-run-button'}); - await vscode.commands.executeCommand('perfchecker.runAll'); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.runAll'),"perfchecker.runAll"); const report=JSON.parse(await fs.readFile(path.join(context.results,'suite-result.json'),'utf8')); const plan=JSON.parse(await fs.readFile(path.join(process.env.PERFCHECKER_NATIVE_PROFILE,'User','globalStorage','mirage-interactive-fr.perfchecker-vscode','suite-plan.json'),'utf8')); const backend=run=>plan.runs.find(item=>item.package===run.package && item.feature===run.feature && item.version===run.version)?.backend; @@ -463,7 +464,7 @@ async function measureNativeTestItemBody(context,expectedPassed,options={}){ const row=windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).first(); if(discoveryDiagnostic&&process.env.PERFCHECKER_NATIVE_PHASE==='prepared')context.log('native-discovery-ui-oracle-start', {uiOracleMs:120000,diagnosticOnly:true,qualifiesCanonicalProduct:false}); - try{await row.waitFor({state:'visible',timeout:120000});} + try{await row.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});} catch(error){ try{await context.observeWork?.('testitems-discovery-not-ready');} catch(diagnosticError){context.log('native-work-observation-error',{case:'testitems-discovery-not-ready',error:redact(diagnosticError)});} @@ -471,9 +472,9 @@ async function measureNativeTestItemBody(context,expectedPassed,options={}){ } if(discoveryDiagnostic&&process.env.PERFCHECKER_NATIVE_PHASE==='prepared')context.log('native-discovery-ui-row-observed', {surface:'visible-Testing-row',testControllerPublication:'unknown',diagnosticOnly:true,qualifiesCanonicalProduct:false}); - await row.hover(); + await row.hover({timeout:NATIVE_READINESS_MS}); if(context.editorQualification)context.log('native-editor-visible-testing-item',{name,file:filename,matchingRows:await windowPage.locator(`.monaco-list-row[aria-label*="${filename}"]`).filter({hasText:name}).count(), - visible:await row.isVisible(),ariaLabel:await row.getAttribute('aria-label'),runButtons:await row.locator('.action-label[title="Run Test"],.action-label[aria-label="Run Test"],.action-label.codicon-testing-run-icon').count()}); + visible:await row.isVisible(),ariaLabel:await row.getAttribute('aria-label',{timeout:NATIVE_READINESS_MS}),runButtons:await row.locator('.action-label[title="Run Test"],.action-label[aria-label="Run Test"],.action-label.codicon-testing-run-icon').count()}); await context.editorQualification?.surface('tagged-testing-before-run'); await context.observeWork?.('testitems-real-tree-ready'); const button=row.locator('.action-label[title="Run Test"],.action-label[aria-label="Run Test"],.action-label.codicon-testing-run-icon').first(); @@ -481,9 +482,9 @@ async function measureNativeTestItemBody(context,expectedPassed,options={}){ const before=new Set(await fs.readdir(storage).catch(()=>[])); await vscode.window.showTextDocument(await vscode.workspace.openTextDocument(vscode.Uri.file(path.join(workspace,filename))),{preview:false}); if(process.env.PERFCHECKER_NATIVE_VIDEO==='1')await delay(2500); - await row.hover(); + await row.hover({timeout:NATIVE_READINESS_MS}); context.log('native-ui-action',{surface:'Test Explorer',action:'Run Test',expectedPassed}); - await button.click(); + await button.click({timeout:NATIVE_READINESS_MS}); const measured=await eventually(async()=>{ for(const name of await fs.readdir(storage)){ if(before.has(name))continue; @@ -491,7 +492,7 @@ async function measureNativeTestItemBody(context,expectedPassed,options={}){ const payload=await fs.readFile(file,'utf8').then(JSON.parse).catch(()=>undefined); if(payload?.schema_version==='perfchecker-testitem-run/1')return {file,payload}; } - },'The actual Test Explorer Run button returns fresh worker evidence',180000); + },'The actual Test Explorer Run button returns fresh worker evidence',NATIVE_READINESS_MS); assert.equal(measured.payload.passed,expectedPassed);assert.equal(measured.payload.runs.length,1); await context.observeWork?.('testitems-current-evidence-before-teardown'); if(options.samples!==undefined)assert.equal(measured.payload.runs[0].samples.length,options.samples); @@ -500,18 +501,18 @@ async function measureNativeTestItemBody(context,expectedPassed,options={}){ await fs.mkdir(path.dirname(retained),{recursive:true});await fs.copyFile(measured.file,retained); if(expectedPassed){ assert.equal(measured.payload.runs[0].status,'validated'); - await eventually(async()=>/passed/i.test(await row.getAttribute('aria-label')||'')||await row.locator('.codicon-testing-passed-icon').count()>0,'The actual Testing tree reports Passed'); + await eventually(async()=>/passed/i.test(await row.getAttribute('aria-label',{timeout:NATIVE_READINESS_MS})||'')||await row.locator('.codicon-testing-passed-icon').count()>0,'The actual Testing tree reports Passed'); context.proof(options.proofName||'testitem-current-evidence',{items:1,core:context.core,nativeClick:true,studioDiscoverAndTestingButtons:true,report:measured.file,retainedReport:path.relative(process.env.PERFCHECKER_NATIVE_OUTPUT,retained),samples:measured.payload.runs[0].samples}); await vscode.window.showTextDocument(await vscode.workspace.openTextDocument(vscode.Uri.file(measured.file)),{preview:false}); if(process.env.PERFCHECKER_NATIVE_VIDEO==='1')await delay(3000); }else{ assert.equal(measured.payload.runs[0].samples[0].passes,0);assert.equal(measured.payload.runs[0].samples[0].errors,1); - await eventually(async()=>/failed|errored/i.test(await row.getAttribute('aria-label')||''),'The tree does not present missing target dependencies as passed'); - await row.click();await vscode.commands.executeCommand('testing.openOutputPeek'); + await eventually(async()=>/failed|errored/i.test(await row.getAttribute('aria-label',{timeout:NATIVE_READINESS_MS})||''),'The tree does not present missing target dependencies as passed'); + await row.click({timeout:NATIVE_READINESS_MS});await vscode.commands.executeCommand('testing.openOutputPeek'); const results=windowPage.locator('.part.panel .pane-body:visible').filter({has:windowPage.locator('.test-output-peek-tree')}); - await results.waitFor({state:'visible'}); + await results.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); const failedResult=results.locator('.monaco-list-row').filter({hasText:name}).first(); - await failedResult.waitFor({state:'visible'});await failedResult.click(); + await failedResult.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await failedResult.click({timeout:NATIVE_READINESS_MS}); await eventually(async()=>{ // Monaco soft-wraps even within words. Read only the visible result editor, // retaining the actual UI oracle while removing those rendering breaks. @@ -531,6 +532,9 @@ exports.run = async () => { assert(['x64','arm64'].includes(expectedArch),'The runner declares the expected native architecture'); assert.equal(process.arch,expectedArch,'The actual Electron extension host must use the native architecture'); const phase = process.env.PERFCHECKER_NATIVE_PHASE; + assert(!discoveryDiagnostic,'Historical discovery diagnostics retain their original source and explicit oracle contract'); + const activityDeadlineAt=Number(process.env.PERFCHECKER_NATIVE_ACTIVITY_DEADLINE_AT); + assert(Number.isFinite(activityDeadlineAt)&&activityDeadlineAt>Date.now(),'The runner supplies the remaining shared activity budget'); const {nativeFullCasePartition}=await import('./native-platform-contract.mjs'); const fullCasePartition=nativeFullCasePartition({partition:process.env.PERFCHECKER_NATIVE_FULL_PARTITION||'all', stage:process.env.PERFCHECKER_NATIVE_STAGE,artifact:process.env.PERFCHECKER_NATIVE_MODE}); @@ -564,6 +568,7 @@ exports.run = async () => { const report=JSON.stringify({status,phase,platform:process.platform,processArch:process.arch,vscode:vscode.version,extension:process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION,invocation:process.env.PERFCHECKER_NATIVE_INVOCATION, coverage:discoveryDiagnostic?'nonqualifying-full-prelude-testitem-discovery':process.env.PERFCHECKER_NATIVE_STAGE==='focused'?`focused-native-${process.env.PERFCHECKER_NATIVE_CASE_GROUP}`:process.env.PERFCHECKER_NATIVE_STAGE==='targeted'?'actual-workspace-reload-and-controlled-narrative-protocol':process.env.PERFCHECKER_NATIVE_STAGE==='full'?(fullCasePartition.name==='all'?'first-install-studio-investigations-mcp-pluto':`full-native-${fullCasePartition.name}`):'first-install-and-first-run-smoke', ...(discoveryDiagnostic?{discoveryObservation}:{}), + readinessPolicy:{...nativeReadinessPolicy,activityDeadlineAt:new Date(activityDeadlineAt).toISOString()}, core:phase==='fresh'?{mode:'production-first-install',registry:'General',version:minimumVersion,available:process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE==='true'}:JSON.parse(process.env.PERFCHECKER_NATIVE_CORE_PROVENANCE), fullCasePartition:process.env.PERFCHECKER_NATIVE_STAGE==='full'?fullCasePartition:undefined, commands:commandCoverage(commands,checks),configuration:configurationCoverage(configurationProperties,checks),buttons:buttonCoverage(checks),activeCase,checks,failures},null,2); @@ -639,7 +644,7 @@ exports.run = async () => { finally{clearInterval(observer);await observation;activeCase=undefined;await persist();} }; try { - browser = await eventually(() => chromium.connectOverCDP('http://127.0.0.1:9222'), 'Connect to the disposable Electron host'); + browser = await eventually(() => chromium.connectOverCDP('http://127.0.0.1:9222',{timeout:NATIVE_READINESS_MS}), 'Connect to the disposable Electron host'); windowPage = await eventually(async () => browser.contexts().flatMap(context => context.pages()).find(page => page.url().includes('workbench')), 'Locate the actual VS Code workbench'); const allCommands=await vscode.commands.getCommands(true); if(allCommands.includes('workbench.action.closeAuxiliaryBar'))await vscode.commands.executeCommand('workbench.action.closeAuxiliaryBar'); @@ -658,12 +663,12 @@ exports.run = async () => { if(allCommands.includes('workbench.action.toggleFullScreen')){ await vscode.commands.executeCommand('workbench.action.toggleFullScreen').catch(error=>log('native-recording-layout-warning',{fallback:String(error)})); } - await eventually(async()=>await windowPage.evaluate(()=>innerWidth>=1800),'The asynchronous native fullscreen action reaches the recording width',5000) + await eventually(async()=>await windowPage.evaluate(()=>innerWidth>=1800),'The asynchronous native fullscreen action reaches the recording width',NATIVE_READINESS_MS) .catch(error=>log('native-recording-layout-warning',{fallback:String(error),functionalTestsContinue:true})); log('native-recording-viewport',{dimensions:await windowPage.evaluate(()=>({width:innerWidth,height:innerHeight})),fallback:'actual-workbench-fullscreen-command'}); }finally{await target?.detach().catch(()=>{});await cdp?.detach().catch(()=>{});} } - const findFrame = (selector,{allowAttached=false}={}) => eventually(async () => { + const findFrame = (selector,{allowAttached=false,timeout=NATIVE_READINESS_MS}={}) => eventually(async () => { for (const context of browser.contexts()) for (const page of context.pages()) for (const frame of page.frames()) { if (allowAttached?!await frame.locator(selector).count().catch(()=>0):!await frame.locator(selector).first().isVisible().catch(() => false)) continue; let visible=true; @@ -674,7 +679,7 @@ exports.run = async () => { } if(visible)return frame; } - }, `Locate the real webview ${selector}`); + }, `Locate the real webview ${selector}`,timeout); if(phase==='restricted'){ await runCase('native-restricted-mode',()=>require('./native-restricted-controls.cjs').run({ vscode,windowPage,workspace,log,proof,eventually})); @@ -732,8 +737,8 @@ exports.run = async () => { await runCase('native-mcp-controls',()=>mcp.run(context)); await runCase('native-pluto-controls',()=>pluto.run(context)); await runCase('native-discovery-and-sync-after-pluto',async()=>{ - const discovery=await vscode.commands.executeCommand('perfchecker.discoverScenarios'); - const sync=await vscode.commands.executeCommand('perfchecker.syncScenarios'); + const discovery=await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.discoverScenarios'),"perfchecker.discoverScenarios"); + const sync=await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.syncScenarios'),"perfchecker.syncScenarios"); assert.equal(discovery.schema_version,'perfchecker-discovery/1'); assert.equal(sync.schema_version,'perfchecker-scenario-sync/1'); assert.equal(sync.discovery.schema_version,'perfchecker-discovery/1'); @@ -772,7 +777,7 @@ exports.run = async () => { assert(run,`A real ${backend} workload is available for the focused Results controls`);return run; }); assert.equal(examples.length,4,'Four registered Example versions supply real series and distributions'); - await vscode.commands.executeCommand('perfchecker.runNode',{runs:[...sampled,...examples]}); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.runNode',{runs:[...sampled,...examples]}),"perfchecker.runNode"); const report=JSON.parse(await fs.readFile(path.join(context.results,'suite-result.json'),'utf8')); assert.equal(report.runs.length,8);assert(report.runs.every(run=>run.status==='pass')); proof('native-focused-result-measurements',{checks:8,collectors:['profile','wall_profile','profile_alloc','alloc','benchmark'],versions:examples.map(run=>run.version),source:'Actual Julia workers; no synthetic reports'}); @@ -797,7 +802,7 @@ exports.run = async () => { log('bootstrap-prerequisite', {controllerAbsent: true, initializeCurrentlyBlocked: true});return; } const notebook=vscode.commands.executeCommand('perfchecker.newNotebook',vscode.Uri.file(path.join(workspace,'perf','notebooks','Declined.jl')),{kind:'suite'}); - await windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}).waitFor({timeout:30000}); + await windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}).waitFor({timeout:NATIVE_READINESS_MS}); await windowPage.keyboard.press('Escape');assert.equal(await notebook,undefined); assert(!await fs.stat(path.join(workspace,'perf','pluto','Project.toml')).then(()=>true).catch(()=>false),'Opening alone never installs Pluto'); // A conventional perf project can exist while PerfChecker is absent. @@ -805,27 +810,27 @@ exports.run = async () => { await fs.writeFile(path.join(workspace,'perf','Project.toml'),'[deps]\n'); const declined=vscode.commands.executeCommand('perfchecker.initialize',uri); const picker=windowPage.locator('.quick-input-widget'); - await picker.waitFor({state:'visible',timeout:60000}); - await picker.locator('.monaco-list-row').filter({hasText:'Create controller environment'}).click(); - await windowPage.getByRole('button',{name:'Install controller',exact:true}).waitFor({timeout:30000}); + await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await picker.locator('.monaco-list-row').filter({hasText:'Create controller environment'}).click({timeout:NATIVE_READINESS_MS}); + await windowPage.getByRole('button',{name:'Install controller',exact:true}).waitFor({timeout:NATIVE_READINESS_MS}); await windowPage.keyboard.press('Escape');await declined; assert(!await fs.stat(path.join(workspace,'perf','controller','Project.toml')).then(()=>true).catch(()=>false)); await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace',uri); const studio=await findFrame('#setup-workspace'); const suitePlanFile=path.join(process.env.PERFCHECKER_NATIVE_PROFILE,'User','globalStorage','mirage-interactive-fr.perfchecker-vscode','suite-plan.json'); await assert.rejects(fs.access(suitePlanFile),{code:'ENOENT'},'Fresh setup must not reuse a stale plan from any workspace'); - const initializationStartedAt=Date.now(),initializationDeadline=initializationStartedAt+600000; - const initializationRemaining=()=>{assert(Date.now(){assert(Date.now(){ - await studio.locator('#setup-workspace').click(); - await picker.waitFor({state:'visible',timeout:60000}); - await picker.locator('.monaco-list-row').filter({hasText:'Create controller environment'}).click(); - await windowPage.getByRole('button',{name:'Install controller',exact:true}).click(); + await studio.locator('#setup-workspace').click({timeout:NATIVE_READINESS_MS}); + await picker.waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await picker.locator('.monaco-list-row').filter({hasText:'Create controller environment'}).click({timeout:NATIVE_READINESS_MS}); + await windowPage.getByRole('button',{name:'Install controller',exact:true}).click({timeout:NATIVE_READINESS_MS}); if(process.env.PERFCHECKER_NATIVE_GENERAL_MINIMUM_AVAILABLE!=='true'){ await eventually(async()=>{ - const text=(await studio.locator('body').innerText())+'\n'+(await windowPage.locator('body').innerText()); + const text=(await studio.locator('body').innerText({timeout:NATIVE_READINESS_MS}))+'\n'+(await windowPage.locator('body').innerText({timeout:NATIVE_READINESS_MS})); return text.includes(`requires registered PerfChecker ${minimumVersion}`)&&/Install or upgrade it explicitly/.test(text); - },'Production bootstrap explains the registered minimum and upgrade action',180000); + },'Production bootstrap explains the registered minimum and upgrade action',NATIVE_READINESS_MS); assert.equal(vscode.workspace.getConfiguration('perfchecker',uri).get('runnerProject','perf'),'perf','Failed production setup preserves the selected controller'); assert(!await fs.stat(path.join(workspace,'perf','suite.jl')).then(()=>true).catch(()=>false)); proof('bootstrap-awaiting-registration',{status:'prerequisite',minimum:minimumVersion,registry:'General',positiveBootstrapQualified:false,explicitConfirmation:true,nativeStudioClick:true}); @@ -841,7 +846,8 @@ exports.run = async () => { bootstrapPlan={sha256:createHash('sha256').update(bytes).digest('hex'),runs:plan.runs.length};return true; }catch(error){if(error.code==='ENOENT'||error instanceof SyntaxError)return false;throw error;} },'The real initialization plan finishes before opening Feature suite',initializationRemaining()); - log('native-bootstrap-plan-ready',{...bootstrapPlan,elapsedMs:Date.now()-initializationStartedAt,maximumMs:600000,noAdditionalPlanRequested:true}); + log('native-bootstrap-plan-ready',{...bootstrapPlan,elapsedMs:Date.now()-initializationStartedAt,maximumMs:NATIVE_READINESS_MS, + deadlineAt:new Date(initializationDeadline).toISOString(),noAdditionalPlanRequested:true}); return true; },initializationDeadline); if(!initialized)return; @@ -869,28 +875,28 @@ exports.run = async () => { await vscode.commands.executeCommand('perfchecker.openStudioForWorkspace', uri); const frame = await findFrame('#studio-root'); const environment=frame.locator('details.environment'); - if(!await environment.evaluate(element=>element.open))await environment.locator('summary').click(); - await eventually(async()=>(await environment.innerText()).includes(context.controller),'The expanded Studio environment shows the selected controller'); + if(!await environment.evaluate(element=>element.open))await environment.locator('summary').click({timeout:NATIVE_READINESS_MS}); + await eventually(async()=>(await environment.innerText({timeout:NATIVE_READINESS_MS})).includes(context.controller),'The expanded Studio environment shows the selected controller'); }); await runCase('native-julia-terminal', async () => { - const terminal = await vscode.commands.executeCommand('perfchecker.openTerminal', uri); + const terminal = await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.openTerminal', uri),"perfchecker.openTerminal"); assert(terminal && await terminal.processId, 'Julia runs in the actual integrated terminal'); const marker = path.join(workspace, 'terminal-version.txt'); terminal.sendText(`using PerfChecker; write(${JSON.stringify(marker)}, string(Base.pkgversion(PerfChecker)))`); - await eventually(async () => (await fs.readFile(marker, 'utf8')) === context.coreVersion, 'The integrated terminal imports the explicitly qualified Core version', 180000); - assert.strictEqual(await vscode.commands.executeCommand('perfchecker.openTerminal', uri), terminal, 'The project terminal is reused'); + await eventually(async () => (await fs.readFile(marker, 'utf8')) === context.coreVersion, 'The integrated terminal imports the explicitly qualified Core version', NATIVE_READINESS_MS); + assert.strictEqual(await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.openTerminal', uri),"perfchecker.openTerminal"), terminal, 'The project terminal is reused'); terminal.dispose(); }); if(process.env.PERFCHECKER_NATIVE_MODE==='candidate'){ await runCase('native-pluto-without-jupyter',async()=>{ assert(!vscode.extensions.getExtension('ms-toolsai.jupyter')); const file=path.join(workspace,'perf','notebooks',`${phase}-first.jl`); - const result=await vscode.commands.executeCommand('perfchecker.newNotebook',vscode.Uri.file(file),{kind:'suite'}); + const result=await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.newNotebook',vscode.Uri.file(file),{kind:'suite'}),"perfchecker.newNotebook"); assert.equal(result.fsPath,vscode.Uri.file(file).fsPath); const parent=await findFrame('iframe.perfchecker-pluto-frame'); - const frame=await(await parent.locator('iframe.perfchecker-pluto-frame').elementHandle()).contentFrame(); - await frame.locator('pluto-notebook').waitFor({state:'visible',timeout:180000}); - await eventually(async()=>/idle/.test(await frame.locator('[data-suite-state]').innerText()),'Opening Pluto initializes a live dashboard without launching checks',180000); + const frame=await(await parent.locator('iframe.perfchecker-pluto-frame').elementHandle({timeout:NATIVE_READINESS_MS})).contentFrame(); + await frame.locator('pluto-notebook').waitFor({state:'visible',timeout:NATIVE_READINESS_MS}); + await eventually(async()=>/idle/.test(await frame.locator('[data-suite-state]').innerText({timeout:NATIVE_READINESS_MS})),'Opening Pluto initializes a live dashboard without launching checks',NATIVE_READINESS_MS); await windowPage.screenshot({path:path.join(output,`native-${process.env.PERFCHECKER_NATIVE_EXPECTED_VERSION}-${process.platform}-${vscode.version}-${phase}-pluto.png`)}); await vscode.commands.executeCommand('perfchecker.stopNotebookSession',uri); await vscode.commands.executeCommand('workbench.action.closeActiveEditor'); @@ -901,7 +907,7 @@ exports.run = async () => { await runCase('notebook-missing-prerequisite', async () => { assert(!vscode.extensions.getExtension('ms-toolsai.jupyter')); try { - const notebookUri = await vscode.commands.executeCommand('perfchecker.newNotebook', uri); + const notebookUri = await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.newNotebook', uri),"perfchecker.newNotebook"); const document = vscode.workspace.notebookDocuments.find(document => document.uri.toString() === notebookUri.toString()); assert(document, 'The available native serializer opens the actual notebook'); assert(document.getCells().some(cell => /Select the \*\*Julia\*\* kernel/.test(cell.document.getText())), 'The notebook explains the Julia kernel prerequisite'); @@ -917,7 +923,7 @@ exports.run = async () => { } else if(phase==='prepared') { if(process.env.PERFCHECKER_NATIVE_MODE==='public')await runCase('native-notebook-with-jupyter', async () => { assert(vscode.extensions.getExtension('ms-toolsai.jupyter')); - const notebookUri = await vscode.commands.executeCommand('perfchecker.newNotebook', uri); + const notebookUri = await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.newNotebook', uri),"perfchecker.newNotebook"); const document = vscode.workspace.notebookDocuments.find(document => document.uri.toString() === notebookUri.toString()); assert(document && document.cellCount > 0, 'A real investigation notebook was opened'); assert.equal(document.notebookType, 'jupyter-notebook'); @@ -935,8 +941,8 @@ exports.run = async () => { 'native-mcp-controls':()=>mcp.run(context), 'native-pluto-controls':()=>pluto.run(context), 'native-discovery-and-sync-after-pluto':async()=>{ - await vscode.commands.executeCommand('perfchecker.discoverScenarios'); - await vscode.commands.executeCommand('perfchecker.syncScenarios'); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.discoverScenarios'),"perfchecker.discoverScenarios"); + await waitForNativeAvailability(vscode.commands.executeCommand('perfchecker.syncScenarios'),"perfchecker.syncScenarios"); } }; for(const name of fullCasePartition.cases){assert.equal(typeof callbacks[name],'function');await runCase(name,callbacks[name]);} diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index 83ae24a..c86e44b 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -8,6 +8,7 @@ import {promisify} from 'node:util'; import {pipeline} from 'node:stream/promises'; import {fileURLToPath} from 'node:url'; import {createRequire} from 'node:module'; +import readiness from './native-readiness.cjs'; import {downloadAndUnzipVSCode, resolveCliArgsFromVSCodeExecutablePath} from '@vscode/test-electron'; import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; import {nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; @@ -16,6 +17,8 @@ import {createCoalescingObserver,beforeDeadline} from './native-landscape-contro import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,assertNativeDiscoveryDiagnostic,verifyNativeSampleSyntax} from './native-controller-preflight.mjs'; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const {NATIVE_READINESS_MS,NATIVE_ACTIVITY_MINUTES,nativeReadinessPolicy,createNativeActivityBudget,waitForNativeAvailability}=readiness; +const nativeActivity=createNativeActivityBudget({startedAt:Number(process.env.PERFCHECKER_NATIVE_ACTIVITY_STARTED_AT)||Date.now()}); const output = path.join(repository, 'native-qualification-results'); // Darwin's TMPDIR is long; per-phase profiles otherwise exceed its 103-byte // Unix-domain socket limit before VS Code can start. This remains a disposable @@ -98,20 +101,51 @@ const publicSha = 'c4123271e71e4c4d148fe0e613ba260f4aeea6f28445338cab11d3fb9513d if (!['public', 'candidate'].includes(mode)) throw new Error('Choose public or candidate VSIX explicitly.'); async function execute(executable, args, options = {}) { + const {technical=false,...spawnOptions}=options; + // Technical teardown retains its explicit old bound even after activity expires. + const timeout=technical?(options.timeout??30000):nativeActivity.timeout(options.timeout??NATIVE_READINESS_MS); return await new Promise((resolve, reject) => { const child = spawn(executable, args, {windowsHide: true, env:runnerEnvironment, - ...(landscapeOnly?{signal:landscapeAbort.signal}:{}), ...options}); + ...(landscapeOnly?{signal:landscapeAbort.signal}:{}), ...spawnOptions,timeout}); let text = ''; + const preserve=(reason,closed,error,signal)=>{ + // Closing the preparation leader is not evidence that its compilation + // descendants are gone. Retention is conservative, never a leak claim. + sessionSafeToRemove=false; + try{console.error('NATIVE_PREPARATION_CLEANUP_INCONCLUSIVE '+JSON.stringify({reason,ownerPid:child.pid??null, + physicalCloseObserved:closed,treeCleanupQualified:false,errorClass:error?.name??null,signal:signal??null}));}catch{} + }; child.stdout?.on('data', chunk => {text += chunk; process.stdout.write(chunk);}); child.stderr?.on('data', chunk => {text += chunk; process.stderr.write(chunk);}); - child.once('error', reject); - child.once('close', code => code === 0 ? resolve(text) : reject(Object.assign(new Error(`${path.basename(executable)} exited ${code}\n${text.slice(-5000)}`), {commandOutput: text, exitCode: code}))); + child.once('error', error=>{preserve('spawn-error',false,error);reject(error);}); + child.once('close', (code,signal) => { + if(code===0)return resolve(text); + preserve('nonzero-or-signal',true,undefined,signal); + reject(Object.assign(new Error(`${path.basename(executable)} exited ${code}\n${text.slice(-5000)}`), {commandOutput: text, exitCode: code})); + }); }); } +async function downloadNativeVSCode(options){ + const timeout=nativeActivity.timeout(); + try{ + // The library's timeout is idle-only. This separate absolute observation + // bound cannot be renewed by transport progress or extraction activity. + return await waitForNativeAvailability(downloadAndUnzipVSCode({...options,timeout}), + 'VS Code download/extraction',{timeout}); + }catch(error){ + // The public API provides no proof that pending writes/extraction stopped. + sessionSafeToRemove=false; + try{console.error('NATIVE_PREPARATION_CLEANUP_INCONCLUSIVE '+JSON.stringify({reason:'download-or-extraction', + physicalCloseObserved:false,treeCleanupQualified:false,errorClass:error.name}));}catch{} + throw error; + } +} + // This preparation owns its compilation children independently of the import // timeout. No uncertain observation is accepted as proof of an empty tree. -async function executeControllerPreflight(executable,args,receipt,{timeoutMs=180000}={}){ +async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NATIVE_READINESS_MS}={}){ + timeoutMs=nativeActivity.timeout(timeoutMs); assert(Number.isSafeInteger(timeoutMs)&&timeoutMs>0); const windows=process.platform==='win32',expected=await fs.realpath(executable); const launcher=path.join(repository,'resources','windows-owned-process.ps1'); @@ -120,7 +154,7 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=180 const ownerArgs=windows?['-NoLogo','-NoProfile','-NonInteractive','-ExecutionPolicy','Bypass','-File',launcher, '-Executable',expected,'-WorkingDirectory',receipt.project,'-ArgumentsBase64',Buffer.from(JSON.stringify(args)).toString('base64'),'-ParentPid',String(process.pid)]:args; const child=spawn(ownerExecutable,ownerArgs,{cwd:receipt.project,env:runnerEnvironment,windowsHide:true,detached:!windows,stdio:['pipe','pipe','pipe']}); - const records=new Map(),errors=[];let text='',readyPid,started=false,expired=false,spawnError,lastRows=[],cleanupUntil=Date.now()+timeoutMs; + const records=new Map(),errors=[];let text='',readyPid,started=false,expired=false,spawnError,lastRows=[],cleanupUntil=Math.min(nativeActivity.deadlineAt,Date.now()+timeoutMs); receipt.deadlineAt=new Date(cleanupUntil).toISOString(); const limit=maximum=>{const remaining=cleanupUntil-Date.now();if(remaining<=0)throw new Error('Owned preflight inspection exceeded its current preparation or cleanup deadline');return Math.min(maximum,remaining);}; receipt.ownership={ownerPid:child.pid,identities:[],observations:[],errors,cleanupQualified:false}; @@ -375,12 +409,13 @@ async function prepareLandscapeFixture(runtime,artifactRecord){ projects:await Promise.all(['Project.toml','Manifest.toml'].map(async name=>[name,await fs.readFile(path.join(source,name)).then(hash).catch(error=>{if(error.code==='ENOENT')return null;throw error;})]))}); const archiveInto=async(source,commit,destination)=>{ await fs.mkdir(destination,{recursive:true}); + const availability=nativeActivity.timeout(); const archive=spawn('git',['-C',source,'archive','--format=tar',commit],{env:runnerEnvironment,signal:landscapeAbort.signal,stdio:['ignore','pipe','pipe']}); // GNU tar must consume the entire producer stream, including trailing archive padding. const unpack=spawn('tar',['--ignore-zeros','-xf','-','-C',destination],{env:runnerEnvironment,signal:landscapeAbort.signal,stdio:['pipe','ignore','pipe']}); let errors='';for(const child of [archive,unpack])child.stderr.on('data',data=>{errors=(errors+data.toString()).slice(-5000);}); const exits=[archive,unpack].map(child=>new Promise((resolve,reject)=>{child.once('error',reject);child.once('close',code=>code===0?resolve():reject(new Error(`Immutable source archive failed (${code}): ${errors}`)));})); - const timer=setTimeout(()=>{archive.kill('SIGKILL');unpack.kill('SIGKILL');},120000); + const timer=setTimeout(()=>{archive.kill('SIGKILL');unpack.kill('SIGKILL');},availability); try{await Promise.all([...exits,pipeline(archive.stdout,unpack.stdin)]);} finally{clearTimeout(timer);for(const child of [archive,unpack])if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');await Promise.allSettled(exits);} }; @@ -430,10 +465,10 @@ for (name,directory) in zip(names,ARGS[2:end]);@assert normpath(joinpath(root,so write(file,original) env=Pkg.Types.EnvCache(file);env.manifest.other["project_hash"]=Pkg.Types.workspace_resolve_hash(env) Pkg.Types.write_manifest(env.manifest,env.manifest_file) -`,root,...sdk.map(item=>item.directory)],{timeout:900000}); +`,root,...sdk.map(item=>item.directory)],{timeout:NATIVE_READINESS_MS}); assert.deepEqual(await fs.readFile(path.join(root,'Project.toml')),project,'Pkg preparation preserves the game tag Project.toml'); for(const item of sdk)assert.equal(hash((await sourceDigest(item.directory)).join('')),item.sha256,'SDK runtime source remains identical to its archived tag'); - // Install and compile this private SDK before the unchanged native 120-second + // Install and compile this private SDK before the one-hour native import // import observation. This qualifies prepared operation, never a cold launch. const preparedHashes=async()=>({ project:hash(await fs.readFile(path.join(root,'Project.toml'))), @@ -442,9 +477,9 @@ Pkg.Types.write_manifest(env.manifest,env.manifest_file) }); assert.equal(runnerEnvironment.JULIA_NUM_PRECOMPILE_TASKS,'1'); const preparation={status:'running',phase:'Landscape SDK cache preparation',startedAt:new Date().toISOString(),project:root, - packages:['EtenduBeautifulLandscape',...sdk.map(item=>item.name)],strict:true,timeoutMilliseconds:900000, + packages:['EtenduBeautifulLandscape',...sdk.map(item=>item.name)],strict:true,timeoutMilliseconds:NATIVE_READINESS_MS, precompileTasks:1,hashesBefore:await preparedHashes(), - scope:'Explicit private SDK cache preparation with owned compilation children; native import remains 120 seconds; no cold-launch qualification'}; + scope:'Explicit private SDK cache preparation with owned compilation children; native import availability is one hour, clipped by shared activity; no cold-launch qualification'}; artifactRecord.landscapeSdkCachePreparation=preparation; await fs.writeFile(path.join(output,'artifact.json'),JSON.stringify(artifactRecord,null,2)); console.log('NATIVE_LANDSCAPE_SDK_PREPARATION_START '+JSON.stringify(preparation)); @@ -478,11 +513,12 @@ println("LANDSCAPE_SDK_PREPARATION_COMPLETE_V1 ",(time_ns()-started)/1e9) for(const [source,before]of snapshots)assert.deepEqual(await sourceSnapshot(source),before,'The human HEAD, status and project files remain unchanged'); const executable=await fs.realpath(process.env.PERFCHECKER_TEST_XVFB||'/usr/bin/Xvfb'); const tools=Object.fromEntries(await Promise.all(['xwd','xwininfo','vulkaninfo'].map(async name=>[name,await fs.realpath(`/usr/bin/${name}`)]))); + const displayAvailability=nativeActivity.timeout(); const displayChild=spawn(executable,['-displayfd','3','-screen','0','1920x1080x24','-nolisten','tcp','-ac'],{env:runnerEnvironment,stdio:['ignore','ignore','pipe','pipe']}); const displayExit=new Promise(resolve=>{displayChild.once('close',resolve);displayChild.once('error',()=>resolve(-1));}); landscapeFixture={root,displayChild,displayExit};displayChild.stderr.resume(); const display=await new Promise((resolve,reject)=>{ - let text='';const timer=setTimeout(()=>reject(new Error('The owned Xvfb did not reserve a private display')),15000); + let text='';const timer=setTimeout(()=>reject(new Error('The owned Xvfb did not reserve a private display')),displayAvailability); const finish=(error,value)=>{clearTimeout(timer);error?reject(error):resolve(value);}; displayChild.once('error',error=>finish(error));displayChild.once('close',()=>finish(new Error('The owned Xvfb exited before native launch'))); displayChild.stdio[3].on('data',bytes=>{text+=bytes;const match=text.match(/^(\d+)\s*$/);if(match)finish(undefined,`:${match[1]}`);}); @@ -544,7 +580,7 @@ try { "print(\"{\\\"executable\\\":\", repr(joinpath(Sys.BINDIR, Base.julia_exename())), \",\\\"version\\\":\", repr(string(VERSION)), \",\\\"arch\\\":\", repr(string(Sys.ARCH)), \",\\\"cpuName\\\":\", repr(Sys.CPU_NAME), \",\\\"threads\\\":\", Threads.nthreads(), \",\\\"hardwareThreads\\\":\", length(Sys.cpu_info()), \"}\")"])).trim()); assertNativeJuliaArchitecture(officialRuntime,architecture,'Official Julia runtime'); const expectedVersion = mode === 'public' ? '1.0.0' : JSON.parse(await fs.readFile(path.join(repository, 'package.json'), 'utf8')).version; - const vscode = await downloadAndUnzipVSCode({version, cachePath: path.join(session, 'vscode')}); + const vscode = await downloadNativeVSCode({version, cachePath: path.join(session, 'vscode')}); const [cli, ...cliArgs] = resolveCliArgsFromVSCodeExecutablePath(vscode, {reuseMachineInstall: true}); const application=await nativeVSCodeApplication(vscode,cli,{expectedVersion:version}); const hostVersion=application.version,cliSource=application.cliSource; @@ -573,7 +609,7 @@ try { await fs.writeFile(path.join(workspace, 'test', 'performance.jl'), 'using TestItems\n@testitem "Vector reduction" tags=[:performance] begin\n data=collect(1:10_000)\n @test sum(data)==50_005_000\nend\n'); const vsix = path.join(session, 'perfchecker.vsix'); if (mode === 'public') { - const response = await fetch('https://github.com/Mirage-Interactive-Fr/PerfCheckerVSCode/releases/download/v1.0.0/perfchecker-vscode-1.0.0.vsix'); + const response = await fetch('https://github.com/Mirage-Interactive-Fr/PerfCheckerVSCode/releases/download/v1.0.0/perfchecker-vscode-1.0.0.vsix',{signal:AbortSignal.timeout(nativeActivity.timeout())}); if (!response.ok) throw new Error(`Public VSIX download failed: ${response.status}`); await fs.writeFile(vsix, Buffer.from(await response.arrayBuffer())); } else await fs.copyFile(path.join(repository, 'candidate.vsix'), vsix); @@ -592,6 +628,7 @@ try { await fs.writeFile(path.join(output,'candidate.provenance.json'),JSON.stringify(packageProvenance,null,2)); } let discoveryObservation; + assert(!discoveryDiagnostic,'Historical discovery diagnostics retain their original branch and explicit contract; this source qualifies the one-hour native campaign'); if(discoveryDiagnostic){ assertNativeDiscoveryDiagnostic({enabled:true,event:process.env.GITHUB_EVENT_NAME,artifact:mode,stage, scope:process.env.PERFCHECKER_NATIVE_SCOPE,core:coreMode,partition:fullCasePartition.name,platform:process.platform,arch:process.arch, @@ -616,19 +653,20 @@ try { const cliProfile = await privateProfile(profile); await execute(cli, [...cliArgs, ...cliProfile, '--install-extension', vsix, '--force'], {shell: process.platform === 'win32' && cli.endsWith('.cmd')}); const artifactRecord={mode,retainedVsix,sha256: sha,vscodeRequested: version,core:coreProvenance,packageProvenance, + readinessPolicy:{...nativeReadinessPolicy,activityStartedAt:new Date(nativeActivity.startedAt).toISOString(),activityDeadlineAt:new Date(nativeActivity.deadlineAt).toISOString()}, ...(discoveryDiagnostic?{discoveryObservation}:{}), fullCasePartition:stage==='full'?fullCasePartition:undefined, runner:{platform:process.platform,processArch:process.arch,runnerArch:process.env.RUNNER_ARCH||null},runtimes:{perfchecker:runtime,officialJulia:officialRuntime},hostPreferences:{scope:'disposable-application-profile',dialogStyle:'custom',dialogAPI:'real-VS-Code-no-interception',actualVersion:hostVersion, actualCommit:application.commit,applicationName:application.applicationName,applicationRelativePath:path.relative(path.dirname(vscode),application.application), cliRelativePath:path.relative(path.dirname(vscode),cli),privateDirectoryFlags}}; await fs.writeFile(path.join(output, 'artifact.json'), JSON.stringify(artifactRecord, null, 2)); - const minimumResponse=await fetch('https://raw.githubusercontent.com/JuliaRegistries/General/master/P/PerfChecker/Versions.toml'); + const minimumResponse=await fetch('https://raw.githubusercontent.com/JuliaRegistries/General/master/P/PerfChecker/Versions.toml',{signal:AbortSignal.timeout(nativeActivity.timeout())}); if(!minimumResponse.ok)throw new Error(`Cannot verify the production minimum in General: ${minimumResponse.status}`); const versionsText=await minimumResponse.text(),minimumSection=versionsText.split('["'+minimumVersion+'"]')[1]?.split(/^\[/m)[0]; const minimumTree=/^git-tree-sha1\s*=\s*"([a-f0-9]{40})"/m.exec(minimumSection||'')?.[1]; const minimumAvailable=Boolean(minimumTree); if(minimumAvailable&&mode==='candidate')assert.equal(minimumTree,CORE111_CANDIDATE.tree,'Published General 1.1.1 must match the independently pinned Core source tree'); - const plutoTagResponse=await fetch('https://raw.githubusercontent.com/Mirage-Interactive-Fr/PerfChecker.jl/v1.0.1/packages/PerfCheckerPluto/Project.toml'); + const plutoTagResponse=await fetch('https://raw.githubusercontent.com/Mirage-Interactive-Fr/PerfChecker.jl/v1.0.1/packages/PerfCheckerPluto/Project.toml',{signal:AbortSignal.timeout(nativeActivity.timeout())}); if(!plutoTagResponse.ok&&plutoTagResponse.status!==404)throw new Error(`Cannot verify published Pluto companion tag: ${plutoTagResponse.status}`); const plutoTagAvailable=plutoTagResponse.ok&&/^version\s*=\s*"1\.0\.1"\s*$/m.test(await plutoTagResponse.text()); await fs.writeFile(path.join(output,'production-bootstrap-gate.json'),JSON.stringify({minimum:minimumVersion,registry:'General',available:minimumAvailable, @@ -692,7 +730,8 @@ try { PERFCHECKER_NATIVE_PHASE: phase, PERFCHECKER_NATIVE_INVOCATION:randomUUID(),PERFCHECKER_NATIVE_OUTPUT: output, PERFCHECKER_NATIVE_PROFILE: phaseProfile, PERFCHECKER_NATIVE_SESSION:session,PERFCHECKER_NATIVE_WORKSPACE: workspace, PERFCHECKER_NATIVE_CONTROLLER: controller, ...(phase==='pluto-start-stop'?{PERFCHECKER_NATIVE_PLUTO_ENVIRONMENT:JSON.stringify(artifactRecord.plutoEnvironment)}:{}), - ...(phase==='editor'&&process.platform==='linux'?{PERFCHECKER_NATIVE_EDITOR_DEADLINE_AT:new Date(Date.now()+40*60*1000).toISOString()}:{}), + ...(phase==='editor'&&process.platform==='linux'?{PERFCHECKER_NATIVE_EDITOR_DEADLINE_AT:new Date(nativeActivity.deadlineAt).toISOString()}:{}), + PERFCHECKER_NATIVE_ACTIVITY_DEADLINE_AT:String(nativeActivity.deadlineAt), PERFCHECKER_NATIVE_TARGET: target, PERFCHECKER_NATIVE_JULIA: julia, ...(nativeNode?{PERFCHECKER_NATIVE_NODE:nativeNode,PERFCHECKER_NATIVE_NODE_PROVENANCE:JSON.stringify(nodeProvenance)}:{}), PERFCHECKER_NATIVE_OFFICIAL_JULIA:officialRuntime.executable,PERFCHECKER_NATIVE_OFFICIAL_JULIA_VERSION:officialRuntime.version, @@ -708,6 +747,7 @@ try { UV_THREADPOOL_SIZE: '4'}; const launchWorkspace=phase!=='fresh'&&(phase==='prepared'||stage==='focused')?workspaceFile:workspace; await fs.access(launchWorkspace); + const remainingActivityMs=nativeActivity.remaining(),activationWindowMs=Math.min(NATIVE_READINESS_MS,remainingActivityMs); await new Promise((resolve,reject)=>{ // Actual interactive host: no --extensionTestsPath or smoke driver, and // no intercepted APIs. All dialogs are clicked through the real UI. @@ -733,20 +773,21 @@ try { const stop=()=>{if(child.exitCode!==null)return;if(process.platform==='win32')spawn('taskkill',['/pid',String(child.pid),'/T','/F'],{windowsHide:true});else try{process.kill(-child.pid,'SIGTERM');}catch{}}; const activationTimer=setTimeout(()=>{if(activated)return;void(async()=>{ await retainHostLogs().catch(error=>console.error(`NATIVE_STARTUP_LOGS ${error.message}`));stop(); - reject(new Error(`The ${phase} helper did not activate within 3 minutes. Retained Electron/extension-host logs distinguish startup failure from campaign duration.`)); - })();},180000); - const minutes=phase==='prepared'&&environment.PERFCHECKER_NATIVE_STAGE==='full'?130:phase==='landscape'?45:40; + reject(new Error(`The ${phase} helper exceeded its one-hour availability or remaining shared infrastructure activity budget. Retained logs distinguish startup failure from campaign duration.`)); + })();},activationWindowMs); + const minutes=NATIVE_ACTIVITY_MINUTES; const deadlineStartedAt=new Date().toISOString(); let globalDeadlineExpired=false; const timer=setTimeout(()=>{globalDeadlineExpired=true;void(async()=>{ - const failure=new Error(`The ${phase} native phase exceeded its explicit ${minutes}-minute bound`); + const failure=Object.assign(new Error(`The shared ${minutes}-minute native infrastructure activity budget was exhausted during ${phase}`),{code:'NATIVE_INFRASTRUCTURE_ACTIVITY_LIMIT'}); try{await fs.writeFile(path.join(output,`${phase}-global-limit.json`),JSON.stringify({status:'failed',phase, stage:environment.PERFCHECKER_NATIVE_STAGE,limitMinutes:minutes,startedAt:deadlineStartedAt, + limitKind:'infrastructure-shared-activity',activityStartedAt:new Date(nativeActivity.startedAt).toISOString(),activityDeadlineAt:new Date(nativeActivity.deadlineAt).toISOString(), expiredAt:new Date().toISOString(),lastObservedCase:activeCase||null, hostStopRequested:true,productCleanupQualified:false,error:failure.message},null,2));} catch(error){console.error(`NATIVE_GLOBAL_LIMIT_REPORT ${error.message}`);} stop();reject(failure); - })();},minutes*60*1000); + })();},remainingActivityMs); const onInterrupt=()=>stop();process.once('SIGINT',onInterrupt);process.once('SIGTERM',onInterrupt); const clean=()=>{clearTimeout(timer);clearTimeout(activationTimer);clearInterval(progress);process.off('SIGINT',onInterrupt);process.off('SIGTERM',onInterrupt);}; child.once('error',error=>{clean();if(!globalDeadlineExpired)reject(error);}); @@ -766,7 +807,7 @@ try { const code=await recorded;clearTimeout(timeout); if(code!==0)phaseFailures.push({phase,kind:'recording',error:`Real Xvfb recording failed (${code}): ${recordingError}`}); else{ - const probe=JSON.parse(await execute('ffprobe',['-v','error','-show_entries','format=duration:stream=codec_name,width,height,avg_frame_rate','-of','json',video])); + const probe=JSON.parse(await execute('ffprobe',['-v','error','-show_entries','format=duration:stream=codec_name,width,height,avg_frame_rate','-of','json',video],{technical:true,timeout:30000})); await fs.writeFile(video.replace(/\.mp4$/,'.json'),JSON.stringify({file:path.basename(video),source:'actual-Xvfb-screen',phase, startedAt:videoStartedAt,timeline:`${phase}.json`,timestampPrecision:'Wall-clock offsets from recorder launch; correlate with native UI events and review frames before editing.', vscodeRequested:version,extension:expectedVersion,core:coreProvenance,fixtures:'Disposable deterministic qualification data; no human model authentication', @@ -781,7 +822,7 @@ try { const installationStartedAt=new Date().toISOString(); if(documentedMcpInstall){artifactRecord.documentedMcpInstallation={status:'running',startedAt:installationStartedAt}; await fs.writeFile(path.join(output,'artifact.json'),JSON.stringify(artifactRecord,null,2));} - const installation=await execute(julia, ['--startup-file=no', '-e', `using Pkg; Pkg.activate(ARGS[1]); ${installCore}; if ARGS[6]=="documented-mcp";Pkg.add("HTTP");elseif ARGS[6]=="standard";Pkg.add(["TestItemRunner","HTTP","BenchmarkTools","Chairmarks","JET","AllocCheck"]);end; using PerfChecker; @assert Base.pkgversion(PerfChecker)==VersionNumber(ARGS[4]); info=Pkg.dependencies()[Base.PkgId(PerfChecker).uuid]; if !isempty(ARGS[3]); @assert string(info.tree_hash)==ARGS[3]; else; @assert info.is_tracking_registry; end; print("QUALIFIED_CORE_PROVENANCE ");PerfChecker.JSON.print(Dict("version"=>string(Base.pkgversion(PerfChecker)),"tree"=>string(info.tree_hash),"registered"=>info.is_tracking_registry));println();if ARGS[6]=="documented-mcp";httpinfo=only(p for p in values(Pkg.dependencies()) if p.name=="HTTP");@assert httpinfo.is_tracking_registry;print("QUALIFIED_DOCUMENTED_HTTP ");PerfChecker.JSON.print(Dict("version"=>string(httpinfo.version),"tree"=>string(httpinfo.tree_hash),"registered"=>httpinfo.is_tracking_registry,"source"=>realpath(httpinfo.source),"autoPrecompile"=>ENV["JULIA_PKG_PRECOMPILE_AUTO"]));println();end;println("QUALIFIED_CORE_MODE=", ARGS[5], " VERSION=", Base.pkgversion(PerfChecker), " TREE=",info.tree_hash," SOURCE=", pathof(PerfChecker))`, controller,coreCommit,coreTree,expectedCoreVersion,coreMode,landscapeOnly?'landscape':savedReportOnly?'saved-report':documentedMcpInstall?'documented-mcp':'standard'],{timeout:landscapeOnly?900000:undefined}).catch(async error=>{ + const installation=await execute(julia, ['--startup-file=no', '-e', `using Pkg; Pkg.activate(ARGS[1]); ${installCore}; if ARGS[6]=="documented-mcp";Pkg.add("HTTP");elseif ARGS[6]=="standard";Pkg.add(["TestItemRunner","HTTP","BenchmarkTools","Chairmarks","JET","AllocCheck"]);end; using PerfChecker; @assert Base.pkgversion(PerfChecker)==VersionNumber(ARGS[4]); info=Pkg.dependencies()[Base.PkgId(PerfChecker).uuid]; if !isempty(ARGS[3]); @assert string(info.tree_hash)==ARGS[3]; else; @assert info.is_tracking_registry; end; print("QUALIFIED_CORE_PROVENANCE ");PerfChecker.JSON.print(Dict("version"=>string(Base.pkgversion(PerfChecker)),"tree"=>string(info.tree_hash),"registered"=>info.is_tracking_registry));println();if ARGS[6]=="documented-mcp";httpinfo=only(p for p in values(Pkg.dependencies()) if p.name=="HTTP");@assert httpinfo.is_tracking_registry;print("QUALIFIED_DOCUMENTED_HTTP ");PerfChecker.JSON.print(Dict("version"=>string(httpinfo.version),"tree"=>string(httpinfo.tree_hash),"registered"=>httpinfo.is_tracking_registry,"source"=>realpath(httpinfo.source),"autoPrecompile"=>ENV["JULIA_PKG_PRECOMPILE_AUTO"]));println();end;println("QUALIFIED_CORE_MODE=", ARGS[5], " VERSION=", Base.pkgversion(PerfChecker), " TREE=",info.tree_hash," SOURCE=", pathof(PerfChecker))`, controller,coreCommit,coreTree,expectedCoreVersion,coreMode,landscapeOnly?'landscape':savedReportOnly?'saved-report':documentedMcpInstall?'documented-mcp':'standard'],{timeout:NATIVE_READINESS_MS}).catch(async error=>{ if(documentedMcpInstall){Object.assign(artifactRecord.documentedMcpInstallation,{status:'failed',finishedAt:new Date().toISOString(), elapsedSeconds:(Date.now()-Date.parse(installationStartedAt))/1000,error:String(error).slice(0,5000)}); await fs.writeFile(path.join(output,'artifact.json'),JSON.stringify(artifactRecord,null,2)); @@ -816,10 +857,10 @@ try { if(prepareTestItemDiscovery||stage==='targeted'||stage==='full'||stage==='focused'&&(['general100','mcp','mcp-stdio','mcp-pluto','advisor','narrative'].includes(caseGroup)||caseGroup==='editor'&&process.platform==='linux')){ // This is installation/cache preparation, not first-Send or a cold-start // measurement. Keep its explicit compilation cost and physical ownership - // separate from the unchanged 180-second prepared-import qualification. + // separate from the prepared-import availability check (one hour, clipped by shared activity). const preparationBefore=Object.fromEntries(await Promise.all(['Project.toml','Manifest.toml'].map(async name=>[name,createHash('sha256').update(await fs.readFile(path.join(controller,name))).digest('hex')]))); const preparation={status:'running',phase:'cache preparation',startedAt:new Date().toISOString(),project:controller, - packages:['PerfChecker','HTTP',...(prepareTestItemDiscovery?['TestItemRunner']:[])],strict:true,timeoutMilliseconds:900000,hashesBefore:preparationBefore, + packages:['PerfChecker','HTTP',...(prepareTestItemDiscovery?['TestItemRunner']:[])],strict:true,timeoutMilliseconds:NATIVE_READINESS_MS,hashesBefore:preparationBefore, automaticPrecompile:runnerEnvironment.JULIA_PKG_PRECOMPILE_AUTO, scope:'Explicit selected-package cache preparation with owned compilation children; no cold-start or native first-Send claim'}; artifactRecord.controllerCachePreparation=preparation;await fs.writeFile(path.join(output,'artifact.json'),JSON.stringify(artifactRecord,null,2)); From b3ba1b7e32e50ed1582299fe8c1701eafda61733 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:59:41 +0000 Subject: [PATCH 18/22] Capture idle baseline from the snapshot that proved idle Co-authored-by: Azzaare <4535131+Azzaare@users.noreply.github.com> --- test/native-studio-controls.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/native-studio-controls.cjs b/test/native-studio-controls.cjs index 50ebf15..00c182e 100644 --- a/test/native-studio-controls.cjs +++ b/test/native-studio-controls.cjs @@ -1199,8 +1199,8 @@ exports.runSelection = async context => { .map(id=>[id,document.getElementById(id).disabled])),inventory:[...document.querySelectorAll('#cards .card')].map(card=>({id:card.dataset.id, colour:card.querySelector('input.label').value,checks:[...card.querySelectorAll('.check-option')].map(check=>({name:check.querySelector('.check-name').textContent,selected:check.querySelector('input').checked}))})), count:document.getElementById('count').textContent,sort:document.getElementById('sort').value,openAfterRun:document.getElementById('open-after-run').checked})); - await eventually(async()=>Object.values((await state()).controls).every(disabled=>!disabled),'The saved real run panel returns to idle before launching'); - const idleState=await state(),beforeLaunches=await controllerLaunches(); + const idleState=await eventually(async()=>{const snapshot=await state();return Object.values(snapshot.controls).every(disabled=>!disabled)&&snapshot;},'The saved real run panel returns to idle before launching'); + const beforeLaunches=await controllerLaunches(); const launchCount=entries=>entries.reduce((sum,item)=>sum+item.commands.length,0); context.log('native-ui-action',{surface:'Suite designer',action:'Run selected check'}); await view.locator('#run').click({timeout:NATIVE_READINESS_MS}); From ee886e02111972fe7530812656f68513d126dbd9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:03:23 +0000 Subject: [PATCH 19/22] Retry Feature suite selection in fresh Pluto install test instead of one hour click Co-authored-by: Azzaare <4535131+Azzaare@users.noreply.github.com> --- test/native-pluto-controls.cjs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/native-pluto-controls.cjs b/test/native-pluto-controls.cjs index d146a2a..7b2f061 100644 --- a/test/native-pluto-controls.cjs +++ b/test/native-pluto-controls.cjs @@ -1054,9 +1054,11 @@ exports.runFreshInstall = async context => { await createFolder.getByRole('button',{name:'OK',exact:true}).click({timeout:NATIVE_READINESS_MS});folderCreationConfirmed=true; context.log('native-ui-action',{surface:'Published Pluto save dialog',action:'Confirm creation of notebooks folder'}); } - return await picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}).isVisible(); - },'The real Save dialog finishes folder creation and shows notebook kinds',NATIVE_READINESS_MS); - await picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}).click({timeout:NATIVE_READINESS_MS}); + const row=picker.locator('.monaco-list-row').filter({hasText:'Feature suite'}); + if(!await row.isVisible())return false; + await row.click({timeout:10000}); + return true; + },'The real Save dialog finishes folder creation and selects the Feature suite notebook kind',NATIVE_READINESS_MS); const install=windowPage.getByRole('button',{name:'Install Pluto environment',exact:true}); await install.waitFor({state:'visible',timeout:NATIVE_READINESS_MS});await install.click({timeout:NATIVE_READINESS_MS}); const source=await eventually(()=>fs.readFile(file,'utf8'),'The real published installer generates the first native notebook',NATIVE_READINESS_MS); From 15a1a7c83bddfba010d6d773aec2f80360e8c1b4 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 19:37:30 +0200 Subject: [PATCH 20/22] Keep explicit test discovery current and freeze the qualified 1.0.3 archive Queue explicit Refresh behind physical child close, invalidate stale inventory, and stop queued discovery before extension shutdown. Preserve both Copilot fixes and add the real idle-snapshot regression. Freeze qualification to the existing H3 origin. Validation: compile and 373 Node tests (345 passed, 28 skipped, zero failures); compiled product modules are byte-identical to H3; 45 fresh bootstrap cases physically qualified. --- .github/workflows/native-vsix.yml | 4 +- src/extension.ts | 3 +- src/testitems.ts | 96 ++++++++++++--- test/native-items-host.test.mjs | 161 +++++++++++++++++++++++-- test/native-platform-contract.test.mjs | 37 ++++++ 5 files changed, 274 insertions(+), 27 deletions(-) diff --git a/.github/workflows/native-vsix.yml b/.github/workflows/native-vsix.yml index 11cd72b..9c42675 100644 --- a/.github/workflows/native-vsix.yml +++ b/.github/workflows/native-vsix.yml @@ -209,8 +209,8 @@ jobs: console.log('Release 1.0.2 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } if(version==='1.0.3'&&!requested&&!newOrigin){ - // Product commit 73b99deb87efaf45bd5e96036f755e42c748bdab, tree 3722727a870402497078e63da78be90d606f5ed5. - fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38653410110\nREUSE_NAME=native-candidate-73b99deb87efaf45bd5e96036f755e42c748bdab\nREUSE_SHA=bc61f9fa172069c61722e0691900d3ddf8a39ce143fd9d553b252304dbc55850\n'); + // Product commit 4a27ae0d76a297c58d141c8396808e5d86e61ef3, tree a20c19dc66670bf39bb3e203ede0081f6048ddcf. + fs.appendFileSync(e.GITHUB_ENV,'REUSE_RUN=38769699307\nREUSE_NAME=native-candidate-4a27ae0d76a297c58d141c8396808e5d86e61ef3\nREUSE_SHA=416ea51fa31471545b022ca37f7ab30901a2e36b871ba230b54bbeadb97943ec\n'); console.log('Release 1.0.3 qualification reuses the frozen archive; product changes require a declared new qualification origin.'); } NODE diff --git a/src/extension.ts b/src/extension.ts index df8e3f0..18bc35b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -8,7 +8,7 @@ import { spawn } from 'node:child_process'; import * as path from 'node:path'; import { promises as fs } from 'node:fs'; import {registerInvestigations} from './investigation'; -import {registerNativeTestItems} from './testitems'; +import {registerNativeTestItems, shutdownNativeTestItems} from './testitems'; import {registerStudio} from './studio'; import {executeLiveProvider, prepareLiveProvider} from './live-provider'; import {cancellableJulia, controllerCancellation, shutdownControllerProcesses} from './controllerCancellation'; @@ -1165,5 +1165,6 @@ export function activate(context: vscode.ExtensionContext): void { } export async function deactivate(): Promise { + shutdownNativeTestItems(); await Promise.all([shutdownCodexConnections(),shutdownMcpStdioConnections(),shutdownPlutoSessions(),shutdownWorkspaceSetup(),shutdownControllerProcesses()]); } diff --git a/src/testitems.ts b/src/testitems.ts index 1770707..f4beeae 100644 --- a/src/testitems.ts +++ b/src/testitems.ts @@ -12,23 +12,44 @@ interface NativeInvocation { readonly folder: string; readonly root: string; readonly project: string; readonly reason: string; readonly executable: string; readonly tags: readonly string[]; readonly excludeTags: readonly string[]; } +const discoveryShutdowns = new Set<() => void>(); + +/** Stop queued discoveries before deactivation snapshots the active controller processes. */ +export function shutdownNativeTestItems(): void { + for (const stop of [...discoveryShutdowns]) stop(); +} /** Native TestItemRunner declarations, with one controller per workspace folder. */ export function registerNativeTestItems(context: vscode.ExtensionContext): void { const output = vscode.window.createOutputChannel('PerfChecker test items'); context.subscriptions.push(output); - const controls = new Map Promise; run: (request: vscode.TestRunRequest, token: vscode.CancellationToken) => Promise}>(); + const controls = new Map Promise; run: (request: vscode.TestRunRequest, token: vscode.CancellationToken) => Promise; stop: () => void}>(); + let stopped = false; + const stoppedError = () => new Error('Native test item discovery has stopped.'); + const stop = () => { + if (stopped) return; + stopped = true; + for (const control of controls.values()) control.stop(); + discoveryShutdowns.delete(stop); + }; + discoveryShutdowns.add(stop); + context.subscriptions.push({dispose:stop}); const ensureController = (folder: vscode.WorkspaceFolder) => { + if (stopped) throw stoppedError(); const existing = controls.get(folder.uri.toString()); if (existing) return existing; const tests = vscode.tests.createTestController(`perfchecker.testitems.${folder.uri.toString()}`, `PerfChecker — measures · ${folder.name}`); context.subscriptions.push(tests); let declarations: NativeItem[] = []; let busy = false; - const discoveries = new Map}>(); + type Discovery = {invocation: NativeInvocation; generation: number; promise: Promise; + resolve: () => void; reject: (error: unknown) => void}; + type DiscoveryQueue = {running: Discovery; queued?: Discovery}; + const discoveries = new Map(); let latestDiscovery = 0; const setting = (key: string, fallback: T) => vscode.workspace.getConfiguration('perfchecker', folder.uri).get(key, fallback); const assertWorkspace = () => { + if (stopped) throw stoppedError(); if (!vscode.workspace.isTrusted || !vscode.workspace.workspaceFolders?.some(f => f.uri.toString() === folder.uri.toString())) throw new Error('Open the original trusted workspace before executing test items.'); }; @@ -44,7 +65,8 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void const invocationKey = (invocation: NativeInvocation) => JSON.stringify([ invocation.folder,invocation.root,invocation.project,invocation.executable,invocation.tags,invocation.excludeTags]); const invoke = async (args: string[], token?: vscode.CancellationToken, - invocation: NativeInvocation = captureInvocation()): Promise<{code: number; payload: any}> => { + invocation: NativeInvocation = captureInvocation(), + lifecycle?: {closed?: Promise; physicallyClosed?: boolean; beforeSpawn?: () => void}): Promise<{code: number; payload: any}> => { output.appendLine(`Controller project: ${invocation.project} (${invocation.reason})`); const directory = path.join(context.globalStorageUri.fsPath, 'native-testitems', randomUUID()); await fs.mkdir(directory, {recursive:true}); @@ -55,10 +77,12 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void const code = await new Promise((resolve,reject) => { if (token?.isCancellationRequested) {resolve(130); return;} assertWorkspace(); + lifecycle?.beforeSpawn?.(); const child = spawn(invocation.executable, ['--startup-file=no',`--project=${project}`, '-e',cancellableJulia('using PerfChecker, TestItemRunner; exit(perfchecker_main(ARGS))'),'--','testitems', `--root=${invocation.root}`,`--project=${project}`,`--output=${destination}`, ...args], {cwd:invocation.root,windowsHide:true,detached:process.platform!=='win32',env}); + if (lifecycle) lifecycle.closed = new Promise(resolve => child.once('close', () => {lifecycle.physicallyClosed = true; resolve();})); const stop = controllerCancellation(child, (message, forced) => { output.appendLine(message); if (forced) void vscode.window.showWarningMessage(message); }); @@ -81,17 +105,22 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void }; const selectionArgs = (invocation: NativeInvocation) => [`--tags=${invocation.tags.join(',')}`, `--exclude-tags=${invocation.excludeTags.join(',')}`]; - const refresh = (): Promise => { - const generation = {value:++latestDiscovery}; - let invocation: NativeInvocation; - try {invocation = captureInvocation();} catch (error) {return Promise.reject(error);} - const key = invocationKey(invocation); - const existing = discoveries.get(key); - if (existing) {existing.generation.value = generation.value;return existing.promise;} + const createDiscovery = (invocation: NativeInvocation, generation: number): Discovery => { + let resolve!: () => void, reject!: (error: unknown) => void; + const promise = new Promise((completed, failed) => {resolve = completed; reject = failed;}); + return {invocation,generation,promise,resolve,reject}; + }; + const startDiscovery = (key: string, queue: DiscoveryQueue, discovery: Discovery, deferred = false): void => { + const invocation = discovery.invocation; + const lifecycle: {closed?: Promise; physicallyClosed?: boolean; beforeSpawn?: () => void} = {}; + if (deferred) lifecycle.beforeSpawn = () => { + if (invocationKey(captureInvocation()) !== key) + throw new Error('The test item discovery settings changed before the queued refresh started. Refresh again.'); + }; const pending = (async () => { - const {code,payload} = await invoke(['--list',...selectionArgs(invocation)],undefined,invocation); + const {code,payload} = await invoke(['--list',...selectionArgs(invocation)],undefined,invocation,lifecycle); if (code || payload.schema_version !== 'perfchecker-testitems/1' || !Array.isArray(payload.items)) throw new Error('Unsupported test item listing.'); - if (latestDiscovery !== generation.value) return; + if (latestDiscovery !== discovery.generation) return; try {if (invocationKey(captureInvocation()) !== key) return;} catch {return;} declarations = payload.items; tests.items.replace(declarations.map(item=>{ @@ -100,9 +129,36 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void test.tags = item.tags.map(tag=>new vscode.TestTag(tag)); return test; })); - })().finally(() => {if (discoveries.get(key)?.promise === pending) discoveries.delete(key);}); - discoveries.set(key,{generation,promise:pending}); - return pending; + })(); + const release = () => { + if (discoveries.get(key) !== queue || queue.running !== discovery) return; + const next = queue.queued; + if (next) {queue.queued = undefined; queue.running = next; startDiscovery(key,queue,next,true);} + else discoveries.delete(key); + }; + void pending.then(() => {release(); discovery.resolve();}, error => { + // A spawn error can reject invoke before physical close. Preserve that + // error immediately, but keep the signature occupied until close. + if (!lifecycle.closed || lifecycle.physicallyClosed) release(); + else void lifecycle.closed.then(release); + discovery.reject(error); + }); + }; + const refresh = (force = false): Promise => { + const generation = ++latestDiscovery; + let invocation: NativeInvocation; + try {invocation = captureInvocation();} catch (error) {return Promise.reject(error);} + const key = invocationKey(invocation),existing = discoveries.get(key); + if (existing) { + const joined = existing.queued ?? (force ? undefined : existing.running); + if (joined) {joined.generation = generation; return joined.promise;} + const queued = createDiscovery(invocation,generation); + existing.queued = queued; + return queued.promise; + } + const running = createDiscovery(invocation,generation),queue = {running}; + discoveries.set(key,queue); startDiscovery(key,queue,running); + return running.promise; }; const run = async (request: vscode.TestRunRequest, token: vscode.CancellationToken) => { if (busy) throw new Error('A native test item run is already active in this folder.'); @@ -155,9 +211,15 @@ export function registerNativeTestItems(context: vscode.ExtensionContext): void } finally {execution.end();busy=false;} }; tests.resolveHandler = () => refresh(); - tests.refreshHandler = () => refresh(); + tests.refreshHandler = () => refresh(true); tests.createRunProfile('PerfChecker — measures',vscode.TestRunProfileKind.Run,run,true); - const control = {refresh,run}; + const control = {refresh: () => refresh(true),run,stop: () => { + ++latestDiscovery; + for (const queue of discoveries.values()) { + queue.queued?.reject(stoppedError()); + queue.queued = undefined; + } + }}; controls.set(folder.uri.toString(),control); return control; }; diff --git a/test/native-items-host.test.mjs b/test/native-items-host.test.mjs index d0889e0..5975314 100644 --- a/test/native-items-host.test.mjs +++ b/test/native-items-host.test.mjs @@ -6,6 +6,7 @@ import {PassThrough} from 'node:stream'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import {createHash} from 'node:crypto'; test('native item controller preserves selection, exclusions, cancellation and trusted workspace', async()=>{ const root=await fs.mkdtemp(path.join(os.tmpdir(),'perfchecker-native-')); @@ -84,7 +85,7 @@ test('native item controller preserves selection, exclusions, cancellation and t } finally {await fs.rm(root,{recursive:true,force:true});} }); -test('native discovery shares equivalent callbacks and only publishes the latest requested selection', {timeout:10000}, async()=>{ +test('native discovery shares equivalent resolves and only publishes the latest requested selection', {timeout:10000}, async()=>{ const root=await fs.mkdtemp(path.join(os.tmpdir(),'perfchecker-discovery-')); const uri=p=>({scheme:'file',fsPath:p,toString:()=>`file://${p}`}); const folders=['one','two'].map(name=>({name,uri:uri(path.join(root,name))})); @@ -135,8 +136,8 @@ test('native discovery shares equivalent callbacks and only publishes the latest const command=commands.get('perfchecker.discoverTestItems'); const tags=settings.testItemTags,excluded=settings.testItemExcludeTags; const first=track(command(folders[0].uri)),controller=controllers[0]; - const resolved=track(controller.resolveHandler()),refreshed=track(controller.refreshHandler()); - assert.equal(resolved,refreshed,'Resolve and Refresh share the actual in-flight promise'); + const resolved=track(controller.resolveHandler()),peerResolve=track(controller.resolveHandler()); + assert.equal(resolved,peerResolve,'Equivalent Resolve callbacks share the actual in-flight promise'); tags.push('changed-after-request');excluded.push('changed-after-request'); settings={runnerProject:'controller-b',juliaExecutable:'julia-b',testItemTags:['b'],testItemExcludeTags:['excluded-b']}; const middle=track(command(folders[0].uri)); @@ -152,7 +153,7 @@ test('native discovery shares equivalent callbacks and only publishes the latest assert(b.args.includes('--tags=b'));assert(b.args.includes('--exclude-tags=excluded-b')); assert.equal(a.options.cwd,folders[0].uri.fsPath);assert(a.args.includes(`--root=${folders[0].uri.fsPath}`)); assert(a.args.includes('--list'));assert.equal(a.options.env.JULIA_PROJECT,undefined); - await complete(a,'latest-a');await Promise.all([first,resolved,refreshed,latest]); + await complete(a,'latest-a');await Promise.all([first,resolved,peerResolve,latest]); await complete(b,'obsolete-b');await middle; assert.equal(calls.length,2);assert.deepEqual(controller.items.values.map(i=>i.id),['latest-a']); @@ -170,10 +171,10 @@ test('native discovery shares equivalent callbacks and only publishes the latest } settings=base();let before=calls.length; - const failedCommand=track(command(folders[0].uri)),failedResolve=track(controller.resolveHandler()),failedRefresh=track(controller.refreshHandler()); - assert.equal(failedResolve,failedRefresh); + const failedCommand=track(command(folders[0].uri)),failedResolve=track(controller.resolveHandler()),failedPeerResolve=track(controller.resolveHandler()); + assert.equal(failedResolve,failedPeerResolve); const originalError=new Error('owned discovery spawn failed'); - const failures=[failedCommand,failedResolve,failedRefresh].map(p=>assert.rejects(p,error=>error===originalError)); + const failures=[failedCommand,failedResolve,failedPeerResolve].map(p=>assert.rejects(p,error=>error===originalError)); await waitForCalls(before+1);calls[before].child.emit('error',originalError);close(calls[before],2); await Promise.all(failures);assert.equal(calls.length,before+1); before=calls.length;const recovery=track(controller.resolveHandler()); @@ -229,3 +230,149 @@ test('native discovery shares equivalent callbacks and only publishes the latest await fs.rm(root,{recursive:true,force:true}); } }); + +test('explicit native Refresh rescans a file saved after the in-flight scan', {timeout:10000}, async()=>{ + const root=await fs.mkdtemp(path.join(os.tmpdir(),'perfchecker-refresh-')); + const uri=p=>({scheme:'file',fsPath:p,toString:()=>`file://${p}`}),folder={name:'demo',uri:uri(root)}; + const sourceFile=path.join(root,'items.jl'),freshSource='@testitem "saved-after-scan" begin\n @test true\nend\n'; + const calls=[],controllers=[],commands=new Map(),requests=[],scans=[],events=new EventEmitter(),publications=[],contexts=[]; + const disposable=()=>({dispose(){}});let finishing=false,project='controller'; + class Items {values=[];replace(values){this.values=values;publications.push(values.map(item=>item.id));}forEach(fn){this.values.forEach(fn);}} + const mock={Uri:{file:uri},TestTag:class {constructor(id){this.id=id;}},TestRunProfileKind:{Run:1}, + workspace:{isTrusted:true,workspaceFolders:[folder],textDocuments:[],getConfiguration:()=>({get:(key,fallback)=>key==='runnerProject'?project:fallback})}, + window:{createOutputChannel:()=>({...disposable(),append(){},appendLine(){},show(){}})}, + commands:{registerCommand:(name,fn)=>{commands.set(name,fn);return disposable();}}, + tests:{createTestController:(id,label)=>{ + const controller={id,label,items:new Items(),...disposable(),createTestItem:(id,label,uri)=>({id,label,uri}),createRunProfile(){}}; + controllers.push(controller);return controller; + }}}; + const close=(call,code)=>{ + if(call.child.exitCode!==null)return; + call.child.exitCode=code;call.child.stdout.end();call.child.stderr.end();call.child.emit('close',code); + }; + const spawn=(executable,args,options)=>{ + assert(args.includes('--list'),'This regression never launches an item measurement'); + const child=new EventEmitter();child.stdout=new PassThrough();child.stderr=new PassThrough();child.exitCode=null;child.signalCode=null; + const call={executable,args,options,child,scanned:false};calls.push(call); + // Simulate only the CLI boundary: each invocation snapshots the actual saved + // file before close is released. This does not execute or validate Julia. + const scan=fs.readFile(sourceFile,'utf8').then(source=>{ + call.sourceSha256=createHash('sha256').update(source).digest('hex'); + call.rows=[...source.matchAll(/@testitem "([^"]+)"/g)].map(match=>({id:match[1],name:match[1],file:'items.jl',tags:[],source_sha256:call.sourceSha256})); + call.scanned=true;events.emit('scanned');if(finishing)close(call,2); + }).catch(error=>{child.emit('error',error);close(call,2);}); + scans.push(scan);return child; + }; + const track=promise=>{requests.push(promise);void promise.catch(()=>{});return promise;}; + const waitForScans=count=>calls.filter(call=>call.scanned).length>=count?Promise.resolve():new Promise(resolve=>{ + const ready=()=>{if(calls.filter(call=>call.scanned).length>=count){events.off('scanned',ready);resolve();}};events.on('scanned',ready); + }); + const complete=async call=>{ + assert(call.scanned);assert.equal(call.child.exitCode,null); + await fs.writeFile(call.args.find(arg=>arg.startsWith('--output=')).slice(9),JSON.stringify({schema_version:'perfchecker-testitems/1',items:call.rows})); + close(call,0); + }; + const nextTurn=()=>new Promise(resolve=>setImmediate(resolve)); + const original=Module._load,require=createRequire(import.meta.url);let modulePath; + try { + await fs.mkdir(path.join(root,'controller'),{recursive:true}); + await fs.writeFile(path.join(root,'controller','Project.toml'),'name = "Controller"\n'); + await fs.mkdir(path.join(root,'other-controller'),{recursive:true}); + await fs.writeFile(path.join(root,'other-controller','Project.toml'),'name = "OtherController"\n'); + modulePath=require.resolve('../dist/testitems.js');delete require.cache[modulePath]; + Module._load=function(name,...args){return name==='vscode'?mock:name==='node:child_process'?{spawn}:original.call(this,name,...args);}; + let register,shutdown;try {({registerNativeTestItems:register,shutdownNativeTestItems:shutdown}=require(modulePath));} finally {Module._load=original;} + await fs.writeFile(sourceFile,'# No TestItem declarations\n'); + const context={subscriptions:[],globalStorageUri:uri(path.join(root,'storage'))};contexts.push(context);register(context); + const commandRequest=track(commands.get('perfchecker.discoverTestItems')(folder.uri)),controller=controllers[0]; + const older=track(controller.resolveHandler());await waitForScans(1); + assert.deepEqual(calls[0].rows,[],'The in-flight scan has already read the old empty file'); + await fs.writeFile(sourceFile,freshSource); + const refreshed=track(controller.refreshHandler()); + assert.notEqual(refreshed,older,'Explicit Refresh after a saved edit must not join the already-scanned invocation'); + const resolved=track(controller.resolveHandler());assert.equal(resolved,refreshed,'Resolve joins the requested fresh scan'); + let settled=false;void refreshed.then(()=>{settled=true;},()=>{settled=true;}); + await nextTurn();assert.equal(calls.length,1,'The follow-up waits for physical close of the old child');assert.equal(settled,false); + await complete(calls[0]);await Promise.all([older,commandRequest]);await waitForScans(2);await nextTurn(); + assert.deepEqual(publications,[],'The invalidated empty inventory is never published'); + assert.equal(settled,false,'Refresh still waits for the new scan to close'); + assert.equal(calls[1].sourceSha256,createHash('sha256').update(freshSource).digest('hex')); + const selection=call=>[call.executable,call.options.cwd,...call.args.filter(arg=>/^--(?:root|project|tags|exclude-tags)=/.test(arg))]; + assert.deepEqual(selection(calls[1]),selection(calls[0]),'Saved content changed while the invocation selection stayed identical'); + await complete(calls[1]);await Promise.all([refreshed,resolved]);await nextTurn(); + assert.deepEqual(publications,[['saved-after-scan']]); + assert.deepEqual(controller.items.values.map(item=>item.id),['saved-after-scan']); + assert.equal(calls.length,2,'Exactly one fresh listing follows the old scan'); + + let before=calls.length,publicationCount=publications.length; + const firstScan=track(controller.resolveHandler());await waitForScans(before+1); + await fs.writeFile(sourceFile,'@testitem "queued-current" begin end\n'); + const queued=track(controller.refreshHandler()); + for(let i=0;i<3;i++)assert.equal(track(controller.refreshHandler()),queued,'Multiple queued Refresh requests share one successor'); + assert.equal(track(controller.resolveHandler()),queued); + await nextTurn();assert.equal(calls.length,before+1); + await complete(calls[before]);await firstScan;await waitForScans(before+2); + assert.equal(track(controller.resolveHandler()),queued,'Resolve joins the running successor'); + await fs.writeFile(sourceFile,'@testitem "queued-newest" begin end\n'); + const newest=track(controller.refreshHandler());assert.notEqual(newest,queued); + for(let i=0;i<3;i++)assert.equal(track(controller.refreshHandler()),newest,'A running successor gets only one further queued Refresh'); + assert.equal(track(controller.resolveHandler()),newest); + await nextTurn();assert.equal(calls.length,before+2); + await complete(calls[before+1]);await queued;await waitForScans(before+3); + assert.deepEqual(publications.slice(publicationCount),[],'Neither superseded scan publishes its inventory'); + await complete(calls[before+2]);await newest;await nextTurn(); + assert.equal(calls.length,before+3);assert.deepEqual(publications.slice(publicationCount),[['queued-newest']]); + + before=calls.length;publicationCount=publications.length; + const failed=track(controller.resolveHandler());await waitForScans(before+1); + await fs.writeFile(sourceFile,'@testitem "after-error-close" begin end\n'); + const recovery=track(controller.refreshHandler()),originalError=new Error('Original discovery error before physical close'); + const originalFailure=assert.rejects(failed,error=>error===originalError); + calls[before].child.emit('error',originalError);await originalFailure;await nextTurn(); + assert.equal(calls.length,before+1,'Rejection before physical close cannot start the queued child'); + close(calls[before],2);await waitForScans(before+2);await complete(calls[before+1]);await recovery; + assert.deepEqual(publications.slice(publicationCount),[['after-error-close']]); + + for(const revoked of ['trust','folder','settings']){ + before=calls.length;publicationCount=publications.length; + const active=track(controller.resolveHandler());await waitForScans(before+1); + const pending=track(controller.refreshHandler()); + if(revoked==='trust')mock.workspace.isTrusted=false; + else if(revoked==='folder')mock.workspace.workspaceFolders=[]; + else project='other-controller'; + const refusal=assert.rejects(pending,/trusted workspace|settings changed before the queued refresh/); + await complete(calls[before]);await active;await refusal; + assert.equal(calls.length,before+1,`${revoked} is checked again before a deferred spawn`); + assert.deepEqual(publications.slice(publicationCount),[]); + mock.workspace.isTrusted=true;mock.workspace.workspaceFolders=[folder];project='controller'; + } + + for(const stopBy of ['shutdown','dispose']){ + const nextContext={subscriptions:[],globalStorageUri:uri(path.join(root,'storage'))};contexts.push(nextContext);register(nextContext); + before=calls.length;publicationCount=publications.length; + const command=commands.get('perfchecker.discoverTestItems'),activeCommand=track(command(folder.uri)),current=controllers.at(-1); + const active=track(current.resolveHandler());await waitForScans(before+1); + const pending=track(current.refreshHandler()),refusal=assert.rejects(pending,/discovery has stopped/); + if(stopBy==='shutdown')shutdown();else for(const subscription of nextContext.subscriptions)subscription.dispose(); + await refusal; + const controllerCount=controllers.length; + await assert.rejects(command(folder.uri),/discovery has stopped/); + await assert.rejects(current.resolveHandler(),/discovery has stopped/); + await complete(calls[before]);await Promise.all([activeCommand,active]);await nextTurn(); + assert.equal(controllers.length,controllerCount,`${stopBy} prevents creating another controller`); + assert.equal(calls.length,before+1,`${stopBy} prevents a queued spawn after close even in the same trusted folder`); + assert.deepEqual(publications.slice(publicationCount),[],`${stopBy} prevents late publication`); + } + } finally { + finishing=true; + try { + for(const context of contexts)for(const subscription of context.subscriptions)subscription.dispose(); + for(const call of calls)if(call.child.exitCode===null)close(call,2); + await Promise.allSettled(requests);await Promise.allSettled(scans); + assert(calls.every(call=>call.child.exitCode!==null),'Every mock child is closed before removing its workspace'); + } finally { + events.removeAllListeners();if(modulePath)delete require.cache[modulePath]; + await fs.rm(root,{recursive:true,force:true}); + } + } +}); diff --git a/test/native-platform-contract.test.mjs b/test/native-platform-contract.test.mjs index d75534a..42a0448 100644 --- a/test/native-platform-contract.test.mjs +++ b/test/native-platform-contract.test.mjs @@ -394,6 +394,43 @@ test('real reactive Cancel refresh retains its ninety-second window across neste assert.deepEqual(calls,[['click',90000],['idle',90000]]); }); +test('real active-reload baseline keeps the validated idle snapshot across adversarial UI reads',async()=>{ + const source=readFileSync(new URL('./native-studio-controls.cjs',import.meta.url),'utf8'); + const tree=ts.createSourceFile('native-studio-controls.cjs',source,ts.ScriptTarget.Latest,true,ts.ScriptKind.JS); + const selection=tree.statements.find(node=>ts.isExpressionStatement(node)&& + ts.isBinaryExpression(node.expression)&&node.expression.left.getText(tree)==='exports.runSelection'); + assert(selection&&ts.isArrowFunction(selection.expression.right),'The real runSelection implementation remains present'); + const statements=selection.expression.right.body.statements; + const declarationIndex=name=>statements.findIndex(node=>ts.isVariableStatement(node)&& + node.declarationList.declarations.some(declaration=>declaration.name.getText(tree)===name)); + const start=declarationIndex('state'),end=declarationIndex('launchCount'); + assert(start>=0&&end>start,'The actual baseline capture remains between state and launchCount'); + const AsyncFunction=Object.getPrototypeOf(async function(){}).constructor; + const capture=new AsyncFunction('state','eventually','controllerLaunches', + statements.slice(start+1,end).map(node=>node.getText(tree)).join('\n')+'\nreturn {idleState,beforeLaunches};'); + for(const initiallyBusy of [false,true]){ + let now=0,polls=0; + const eventually=actualFunction('./native-studio-controls.cjs','eventually',{ + NATIVE_READINESS_MS,Date:{now:()=>now},setTimeout:(resolve,milliseconds)=>{ + assert.equal(milliseconds,100,'The real polling interval remains unchanged'); + assert(polls<4,'The simulated scheduler has a finite polling budget'); + polls++;now+=NATIVE_READINESS_MS/4;resolve(); + } + }); + const idle={controls:{run:false,refresh:false,save:false,'add-target':false,'add-comparison':false}, + inventory:[{id:'one',checks:[{name:'smoke',selected:true}]}],count:'1 selected',sort:'name',openAfterRun:false}; + const busy=structuredClone(idle);busy.controls.run=busy.controls.refresh=busy.controls.save=true; + const observations=initiallyBusy?[busy,idle,busy]:[idle,busy]; + let reads=0,launchReads=0;const launches=[{commands:['existing launch']}]; + const result=await capture(async()=>{assert(reads{launchReads++;return launches;}); + assert.equal(result.idleState,idle,'The baseline is the exact snapshot accepted by the real readiness callback'); + assert.equal(reads,initiallyBusy?2:1,'There is no unvalidated read after readiness succeeds'); + assert.equal(polls,initiallyBusy?1:0,'Only the simulated busy observation schedules a poll'); + assert.equal(result.beforeLaunches,launches);assert.equal(launchReads,1); + } +}); + test('native browser callbacks never capture the Node-only readiness constant',()=>{ for(const file of ['public-vsix-host.cjs','native-workbench-controls.cjs','native-studio-controls.cjs', 'native-pluto-controls.cjs','native-advisor-controls.cjs','native-investigation-controls.cjs', From 9622654a48910001657be14f4a047f0cee954d1c Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Thu, 15 Oct 2026 22:10:50 +0200 Subject: [PATCH 21/22] Resolve absent unqualified Linux images within their survey Keep an exact new-image /proc executable ENOENT provisional until the existing revalidation in the same completed survey proves that PID absent. Preserve the original failure and refuse all qualified, live, unrelated or incomplete-survey cases without adding reads, waits or signal authority. Validation: three Node syntax checks and the complete controller-preflight suite, 47 passed, no failures, skips or cancellations. Product and the qualified H3 archive are unchanged. --- test/native-controller-preflight.mjs | 38 ++++++++++++++- test/native-controller-preflight.test.mjs | 59 ++++++++++++++++++++++- test/run-public-vsix-host.mjs | 16 ++++-- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/test/native-controller-preflight.mjs b/test/native-controller-preflight.mjs index ec19bc7..fb1ffeb 100644 --- a/test/native-controller-preflight.mjs +++ b/test/native-controller-preflight.mjs @@ -553,7 +553,7 @@ export function parseControllerImportReceipt(text,{project,version,tree}){ return {elapsedSeconds,extension:fields[2],activeProject,packages}; } -// Inspection failures remain fatal. Retain the evidence execFile supplies, +// Retain inspection evidence without granting ownership or signal authority, // without its command/environment or unbounded stdout/stderr dumps. export function controllerInspectionFailure(error,{stage,pid,ownerPid}){ return {stage,pid,ownerPid,observedAt:new Date().toISOString(), @@ -562,6 +562,42 @@ export function controllerInspectionFailure(error,{stage,pid,ownerPid}){ stderr:String(error.stderr??'').slice(-4000)}; } +// A new private-group member can exit between its image read and the existing +// absence revalidation. Keep that exact error provisional only within one survey. +export function createControllerImageInspectionSurvey({platform,errors,diagnostics}){ + const origins=new WeakMap(),pending=[];let finished=false; + return { + markLinuxImageFailure(error,{row,before,current,qualified,ownerPid}){ + if(finished||platform!=='linux'||qualified||!Number.isSafeInteger(row.pid)||row.pid<=0|| + row.group!==ownerPid||before.group!==ownerPid||error?.code!=='ENOENT'|| + error.syscall!=='realpath'||error.path!==`/proc/${row.pid}/exe`|| + !current||['Z','X'].includes(current.state)|| + !['pid','parent','group','started'].every(key=>current[key]===before[key])||before.pid!==row.pid)return; + origins.set(error,{row,ownerPid}); + }, + capture(error,failure){ + const origin=origins.get(error); + if(finished||!origin||failure.stage!=='inspect'||failure.pid!==origin.row.pid||failure.ownerPid!==origin.ownerPid){errors.push(failure);return;} + const diagnostic={stage:'unqualified-linux-image-inspection',pid:failure.pid,ownerPid:failure.ownerPid, + initialFailure:failure,resolution:'pending'}; + diagnostics.push(diagnostic);pending.push({origin,failure,diagnostic}); + }, + resolveAbsent(row,observation,{qualified,live}){ + if(finished||qualified||live||observation.stage!=='unqualified-private-group-revalidation'|| + observation.pid!==row.pid||observation.current!==null||observation.resolution!=='proved-absent'|| + !['parent','group','started','state'].every(key=>observation.initial?.[key]===row[key]))return; + for(const entry of pending)if(entry.origin.row===row&&row.group===entry.origin.ownerPid)entry.proof=observation; + }, + finish(completed){ + if(finished)return;finished=true; + for(const entry of pending){ + if(completed===true&&entry.proof){entry.diagnostic.resolution='proved-absent-in-same-survey';entry.diagnostic.revalidation=entry.proof;} + else{entry.diagnostic.resolution=completed===true?'fatal-absence-unproved':'fatal-survey-incomplete';errors.push(entry.failure);} + } + } + }; +} + // Only an already qualified incarnation may resolve a transient /proc/exe // disappearance. These reads grant no signal authority while its image is unknown. export async function revalidateQualifiedLinuxExecutable({identity,error,deadlineAt,readCurrent,readExecutable,record, diff --git a/test/native-controller-preflight.test.mjs b/test/native-controller-preflight.test.mjs index 84e3d4b..26e422f 100644 --- a/test/native-controller-preflight.test.mjs +++ b/test/native-controller-preflight.test.mjs @@ -5,7 +5,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import {EventEmitter} from 'node:events'; import {writeSync} from 'node:fs'; -import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,nativeOperationSnapshot,observeNativeOperation,captureQualifiedNativeProgress,assertNativeDiscoveryDiagnostic,nativeDiscoverySpawnOutput,installNativeDiscoveryObserver,readBoundedNativeFile,nativeSampleProjection,nativeSampleManualContract,createNativeDiscoveryReaders,discoveryOwnedCommand} from './native-controller-preflight.mjs'; +import {controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,createControllerImageInspectionSurvey,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,nativeOperationSnapshot,observeNativeOperation,captureQualifiedNativeProgress,assertNativeDiscoveryDiagnostic,nativeDiscoverySpawnOutput,installNativeDiscoveryObserver,readBoundedNativeFile,nativeSampleProjection,nativeSampleManualContract,createNativeDiscoveryReaders,discoveryOwnedCommand} from './native-controller-preflight.mjs'; const nextObservationTurn=()=>new Promise(resolve=>setImmediate(resolve)); const discoveryExpected={executable:'/private/Julia/bin/julia',project:'/private/controller',root:'/private/workspace', @@ -372,6 +372,63 @@ test('failed inspection preserves target, timing and subprocess cause without we }); const incarnation={pid:5280,parent:4607,group:4607,started:'26774',state:'R',executable:path.resolve('julia','bin','julia')}; +function newLinuxImageSurvey(changes={}){ + const row={pid:4890,parent:4398,group:4398,started:'Thu Oct 15 19:31:06 2026',state:'Sl'}; + const before={pid:4890,parent:4398,group:4398,started:'26774',state:'S'}; + const error=Object.assign(new Error('The initial image read failed'),{code:'ENOENT',syscall:'realpath',path:'/proc/4890/exe'},changes.error); + const errors=[],diagnostics=[],survey=createControllerImageInspectionSurvey({platform:changes.platform??'linux',errors,diagnostics}); + const failure=controllerInspectionFailure(error,{stage:'inspect',pid:4890,ownerPid:4398}); + const context={row,before,current:{...before},qualified:false,ownerPid:4398,...changes.context}; + const absent={stage:'unqualified-private-group-revalidation',pid:4890,initial:{...row},current:null,resolution:'proved-absent'}; + const capture=()=>{survey.markLinuxImageFailure(error,context);survey.capture(error,failure);}; + return {row,error,failure,errors,diagnostics,survey,absent,capture}; +} +test('a new Linux image ENOENT resolves only through the existing absence proof in the same completed survey',()=>{ + const f=newLinuxImageSurvey();f.capture();assert.deepEqual(f.errors,[]); + assert.equal(f.diagnostics[0].initialFailure,f.failure);assert.equal(f.diagnostics[0].resolution,'pending'); + assert.equal(f.survey.resolveAbsent(f.row,f.absent,{qualified:false,live:false}),undefined); + f.survey.finish(true);assert.deepEqual(f.errors,[]); + assert.equal(f.diagnostics[0].resolution,'proved-absent-in-same-survey');assert.equal(f.diagnostics[0].revalidation,f.absent); + assert(!('executable'in f.diagnostics[0]));assert(!('identity'in f.diagnostics[0])); +}); +test('only the exact realpath failure of a new private Linux image can be provisional',()=>{ + for(const changes of [{platform:'darwin'},{error:{code:'ESRCH'}},{error:{code:'EACCES'}},{error:{syscall:'stat'}}, + {error:{path:'/proc/4890/stat'}},{error:{path:'/proc/4891/exe'}},{context:{qualified:true}},{context:{ownerPid:9999}}, + {context:{current:{pid:4890,parent:4398,group:4398,started:'26775',state:'S'}}}, + {context:{current:{pid:4890,parent:1,group:4398,started:'26774',state:'S'}}}, + {context:{current:{pid:4890,parent:4398,group:9999,started:'26774',state:'S'}}}]){ + const f=newLinuxImageSurvey(changes);f.capture();f.survey.resolveAbsent(f.row,f.absent,{qualified:false,live:false});f.survey.finish(true); + assert.deepEqual(f.errors,[f.failure]);assert.deepEqual(f.diagnostics,[]); + } +}); +test('live, zombie, changed, missing and unrelated absence evidence retain the original image failure',()=>{ + for(const change of [{current:{pid:4890,state:'S'},resolution:'live-unqualified'}, + {current:{pid:4890,state:'Z'},resolution:'proved-zombie-or-exited'},{current:undefined},{pid:4891}, + {stage:'qualified-executable-revalidation'},{resolution:'inspection-failed'},{initial:undefined}, + {initial:{parent:1,group:4398,started:'Thu Oct 15 19:31:06 2026',state:'Sl'}}, + {initial:{parent:4398,group:9999,started:'Thu Oct 15 19:31:06 2026',state:'Sl'}}, + {initial:{parent:4398,group:4398,started:'Thu Oct 15 19:31:07 2026',state:'Sl'}}]){ + const f=newLinuxImageSurvey();f.capture();f.survey.resolveAbsent(f.row,{...f.absent,...change},{qualified:false,live:false});f.survey.finish(true); + assert.deepEqual(f.errors,[f.failure]);assert.equal(f.diagnostics[0].resolution,'fatal-absence-unproved'); + } + for(const state of [{qualified:true,live:false},{qualified:false,live:true}]){ + const f=newLinuxImageSurvey();f.capture();f.survey.resolveAbsent(f.row,f.absent,state);f.survey.finish(true);assert.deepEqual(f.errors,[f.failure]); + } + const unrelated=newLinuxImageSurvey();unrelated.capture(); + unrelated.survey.resolveAbsent({...unrelated.row},unrelated.absent,{qualified:false,live:false});unrelated.survey.finish(true); + assert.deepEqual(unrelated.errors,[unrelated.failure]); + const missingProof=newLinuxImageSurvey();missingProof.capture();missingProof.survey.finish(true); + assert.deepEqual(missingProof.errors,[missingProof.failure]); +}); +test('a survey exception or later survey cannot clear an original provisional image failure',()=>{ + const f=newLinuxImageSurvey();f.capture();f.survey.resolveAbsent(f.row,f.absent,{qualified:false,live:false});f.survey.finish(false); + assert.deepEqual(f.errors,[f.failure]);assert.equal(f.diagnostics[0].resolution,'fatal-survey-incomplete'); + const other=createControllerImageInspectionSurvey({platform:'linux',errors:f.errors,diagnostics:f.diagnostics}); + other.resolveAbsent(f.row,f.absent,{qualified:false,live:false});other.finish(true);f.survey.finish(true);assert.deepEqual(f.errors,[f.failure]); + const unmarked=newLinuxImageSurvey();unmarked.survey.capture(unmarked.error,unmarked.failure); + unmarked.survey.resolveAbsent(unmarked.row,unmarked.absent,{qualified:false,live:false});unmarked.survey.finish(true); + assert.deepEqual(unmarked.errors,[unmarked.failure]); +}); const missing=code=>Object.assign(new Error('The qualified executable is temporarily unavailable'),{code}); function executableRevalidation({states=[incarnation],images=[incarnation.executable],code='ENOENT',deadlineAt=5000,identity=incarnation}={}){ let clock=0,stateReads=0,imageReads=0;const history=[]; diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index c86e44b..66586de 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -14,7 +14,7 @@ import {nativeCoreContract,CORE111_CANDIDATE} from './native-core-contract.mjs'; import {nativePlatformContract,assertNativeJuliaArchitecture,nativeFullCasePartition} from './native-platform-contract.mjs'; import {nativeVSCodeApplication} from './native-vscode-application.mjs'; import {createCoalescingObserver,beforeDeadline} from './native-landscape-controls.cjs'; -import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,assertNativeDiscoveryDiagnostic,verifyNativeSampleSyntax} from './native-controller-preflight.mjs'; +import {candidateGitSetupScript,candidateCheckoutPermissionsScript,controllerCachePreparationScript,controllerPreflightScript,parseControllerImportReceipt,parseControllerTestItemDiscoveryReceipt,controllerInspectionFailure,createControllerImageInspectionSurvey,revalidateQualifiedLinuxExecutable,captureControllerInventory,controllerReadinessObserved,assertNativeDiscoveryDiagnostic,verifyNativeSampleSyntax} from './native-controller-preflight.mjs'; const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const {NATIVE_READINESS_MS,NATIVE_ACTIVITY_MINUTES,nativeReadinessPolicy,createNativeActivityBudget,waitForNativeAvailability}=readiness; @@ -190,7 +190,7 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT assert(typeof current.started==='string'&&/^\d{4}-\d{2}-\d{2}T.*Z$/.test(current.started)&&Number.isFinite(Date.parse(current.started))); return current; }; - const inspect=async row=>{ + const inspect=async(row,imageFailures)=>{ if(windows){ assert(typeof row.started==='string'&&/^\d{4}-\d{2}-\d{2}T.*Z$/.test(row.started)&&Number.isFinite(Date.parse(row.started))); if(typeof row.executable==='string'&&row.executable)return {...row,executable:await fs.realpath(row.executable)}; @@ -218,7 +218,8 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT if(qualified)return revalidateQualifiedLinuxExecutable({identity:qualified,error,deadlineAt:cleanupUntil,readCurrent:linuxCurrent, readExecutable:pid=>fs.realpath(`/proc/${pid}/exe`),record:observation=>(receipt.ownership.revalidations??=[]).push(observation)}); const current=await linuxCurrent(row.pid); - if(!current)return undefined;same(current);if(['Z','X'].includes(current.state))return undefined;throw error;} + if(!current)return undefined;same(current);if(['Z','X'].includes(current.state))return undefined; + imageFailures.markLinuxImageFailure(error,{row,before,current,qualified:Boolean(qualified),ownerPid:child.pid});throw error;} const after=await linuxCurrent(row.pid);if(!after)return undefined;same(after);if(['Z','X'].includes(after.state))return undefined; if(after.parent!==before.parent||after.state!==before.state)(receipt.ownership.revalidations??=[]).push({stage:'mutable-process-fields',pid:row.pid, started:after.started,group:after.group,beforeParent:before.parent,currentParent:after.parent,beforeState:before.state,currentState:after.state,stillAlive:true,observedAt:new Date().toISOString()}); @@ -271,6 +272,9 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT return {...after,executable:mappings.includes(expected)?expected:mappings[0],executableMappings:mappings}; }; const survey=async()=>{ + const imageFailures=createControllerImageInspectionSurvey({platform:process.platform,errors, + diagnostics:receipt.ownership.imageInspectionFailures??=[]});let completed=false; + try{ const {readyPid:observedReadyPid,rows}=await captureControllerInventory(()=>readyPid,async()=>{ let rows; if(windows){const value=await inspectCommand(ownerExecutable,['-NoProfile','-Command',"$ErrorActionPreference='Stop'; @((Get-CimInstance Win32_Process -ErrorAction Stop) | ForEach-Object { @{pid=$_.ProcessId;parent=$_.ParentProcessId;started=$(if($_.CreationDate){$_.CreationDate.ToUniversalTime().ToString('o')}else{$null});executable=$_.ExecutablePath} }) | ConvertTo-Json -Compress"],{timeout:limit(10000),maxBuffer:4*1024*1024});rows=JSON.parse(value.stdout);assert(Array.isArray(rows));} @@ -279,7 +283,7 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT return rows; }); lastRows=rows;const inspected=new Map(); - const read=async row=>{if(inspected.has(row.pid))return inspected.get(row.pid);try{const current=await inspect(row);inspected.set(row.pid,current);return current;}catch(error){errors.push(controllerInspectionFailure(error,{stage:'inspect',pid:row.pid,ownerPid:child.pid}));inspected.set(row.pid,undefined);return undefined;}}; + const read=async row=>{if(inspected.has(row.pid))return inspected.get(row.pid);try{const current=await inspect(row,imageFailures);inspected.set(row.pid,current);return current;}catch(error){imageFailures.capture(error,controllerInspectionFailure(error,{stage:'inspect',pid:row.pid,ownerPid:child.pid}));inspected.set(row.pid,undefined);return undefined;}}; const original=rows.find(row=>row.pid===child.pid),owner=original&&await read(original); if(owner&&!records.size){assert.equal(owner.parent,process.pid);assert.equal(canonical(owner.executable),canonical(await fs.realpath(ownerExecutable)));records.set(owner.pid,owner);} const live=new Set();for(const prior of records.values()){ @@ -306,6 +310,7 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT try{ const current=await linuxCurrent(row.pid);observation.current=current??null; observation.resolution=!current?'proved-absent':['Z','X'].includes(current.state)?'proved-zombie-or-exited':'live-unqualified'; + imageFailures.resolveAbsent(row,observation,{qualified:records.has(row.pid),live:live.has(row.pid)}); if(!current||['Z','X'].includes(current.state))continue; // The ps member has no qualified incarnation yet. Even a reparented // or reused PID that is now alive remains unknown, never a gone proof. @@ -320,7 +325,8 @@ async function executeControllerPreflight(executable,args,receipt,{timeoutMs=NAT const signature=JSON.stringify({alive:observation.alive,unknown:observation.unknown}); if(receipt.ownership.observations.at(-1)?.signature!==signature)receipt.ownership.observations.push({...observation,signature}); if(controllerReadinessObserved({readyPid:observedReadyPid,started,identities:inspected,live,parent:windows?child.pid:process.pid,expected,canonical})){started=true;child.stdin.write('OBSERVED\n');} - return [...live].map(pid=>records.get(pid)); + const result=[...live].map(pid=>records.get(pid));completed=true;return result; + }finally{imageFailures.finish(completed);} }; let observation=Promise.resolve(),sampling=false; const sample=()=>{if(sampling)return observation;sampling=true;observation=survey().catch(error=>errors.push(controllerInspectionFailure(error,{stage:'survey',pid:readyPid??child.pid,ownerPid:child.pid}))).finally(()=>{sampling=false;});return observation;}; From 0649f92642d3a0f519ba45451a5a3dc29f730dd3 Mon Sep 17 00:00:00 2001 From: Jean-Francois Baffier Date: Fri, 16 Oct 2026 01:53:18 +0200 Subject: [PATCH 22/22] Keep the native wait-task wall profile in one continuous window Use state reuse only for the immutable wait_task wall-profile fixture. Preserve its oracle, profile duration and delay, target filtering, and all other collectors. Local Julia 1.13.1 fresh and reuse wall profiles passed with the exact fixture and roots; this does not establish the historical Windows cause. CPU-profile diagnostics remain inconclusive and their fixture behavior is unchanged. --- test/run-public-vsix-host.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/run-public-vsix-host.mjs b/test/run-public-vsix-host.mjs index 66586de..fe84e8e 100644 --- a/test/run-public-vsix-host.mjs +++ b/test/run-public-vsix-host.mjs @@ -984,6 +984,8 @@ function build_suite() features=FeatureSpec[] for (workload,file) in [(:sum_squares,"sum.jl"),(:wait_task,"wait.jl")], backend in [:benchmark,:chairmark,:profile,:wall_profile,:profile_alloc,:alloc] options=Dict{Symbol,Any}(:samples=>3,:seconds=>0.03,:evals=>1,:repeat=>false,:targets=>["PerfCheckerNativeFixture"],:profile_seconds=>0.5,:profile_delay=>0.0005,:profile_repetitions=>3,:sample_rate=>1.0) + # wait_task only reads its immutable input; reuse keeps one continuous wall-profiling window. + workload===:wait_task && backend===:wall_profile && (options[:state_policy]=:reuse) entrypoint=workload===:sum_squares && backend in (:profile,:wall_profile) ? "profile-sum.jl" : file push!(features,FeatureSpec(Symbol(workload,"_",backend);workload,backend,entrypoint=joinpath(@__DIR__,entrypoint),options,oracle=OracleSpec())) end