From 6fbe23ecd27015418eee0318d2cfc9a05d62447c Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 9 Nov 2026 07:55:22 +0800 Subject: [PATCH] Windows coverage --- .github/workflows/coverage.yaml | 72 ++++++++++++++++++++++++--------- .mise/config.coverage.toml | 54 +++++++++++++++++-------- .mise/cov-branch-assert.jq | 3 +- 3 files changed, 92 insertions(+), 37 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index dba46719..fdda9f77 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -29,12 +29,12 @@ jobs: contents: read # Codecov's tokenless upload uses an OIDC id-token instead of a repository upload token. id-token: write - # Two lanes, both uploading to Codecov and only one to DeepSource. + # Three lanes, every one uploading to both Codecov and DeepSource. # ubuntu-latest is the canonical lane: it is the only one that can run the wasm/browser half of the pipeline. - # macos-latest re-runs the native half as a second platform, which is what catches a branch reachable only - # under a different target's cfg. Codecov merges every upload on a commit, so the macOS report adds the paths - # only it reaches, and `os_flag` tags each upload with its lane. DeepSource instead keeps one report per - # `--key`, where the macOS lane's partial picture would replace the Linux one, so only Linux sends it there. + # macos-latest and windows-latest re-run the native half on two more platforms, which is what catches a branch + # reachable only under a different target's cfg. Both services merge every report sent for a commit rather than + # keeping the last one, so each partial lane adds the paths only it reaches. `os_flag` tags each Codecov upload + # with its lane; DeepSource has no such tag and folds every lane into the one report per `--key`. strategy: fail-fast: false matrix: @@ -48,9 +48,22 @@ jobs: - os: macos-latest os_flag: macos timeout: 150 + # gnullvm, the default Windows target, so et-ws-web-runner stays out of the build as on every gnullvm lane. + - os: windows-latest + os_flag: windows + timeout: 150 env: MISE_ENV: dart,dotnet,java,js,kotlin,python,r,rust,zig,coverage ET_TEST_COVERAGE: "true" + # Windows steps run under busybox-w32 ash, the shell the Windows mise tasks themselves run on. + # Git Bash's MSYS fork emulation breaks down on long task chains. A job's `shell` default is resolved before + # any step runs, so it cannot read the MISE_INSTALLS_DIR install-mise exports and spells its `C:\i` root out. + defaults: + run: + shell: >- + ${{ startsWith(matrix.os, 'windows') && + 'C:\i\http-busybox\1.37.0\ash.exe -euo pipefail {0}' + || 'bash --noprofile --norc -euo pipefail {0}' }} steps: - name: Checkout # DeepSource's test-coverage analyzer keys the uploaded artifact to the checked-out commit's SHA. @@ -77,7 +90,7 @@ jobs: # Running it early means the figures land in the job summary even when a later coverage step fails, and a # misclassification (the task's partition-sum check) fails in seconds rather than after the long run. # sloc-crosscheck depends on sloc, so this one invocation emits both the tables and the gocloc deltas. - # Linux only: the count is of the checked-out tree, so the second lane would recompute the same figures. + # Linux only: the count is of the checked-out tree, so the other lanes would recompute the same figures. - name: Count source lines (scc, cross-checked with gocloc) if: runner.os == 'Linux' timeout-minutes: 5 @@ -104,9 +117,22 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: mise run prefetch-ci + # MISE_JOBS=1 serializes mise's task scheduler, because parallel module builds race on Windows. + # Their cargo invocations contend for the package cache lock, which Windows cargo fails on rather than waits + # for. mise rejects an empty MISE_JOBS, so it is set from a conditional step rather than in `env:`. + - name: Serialize mise tasks on Windows + if: runner.os == 'Windows' + run: echo "MISE_JOBS=1" >> "$GITHUB_ENV" + + # Windows builds the modules uninstrumented, because nothing on that lane reads their coverage. + # Every consumer of the guests' profiles is Linux-only: the wasm merges and wasm-cov-branch-check below. + # The instrumented browser builds also look for their coverage wrapper under target/debug, which + # CARGO_BUILD_TARGET moves to target//debug on Windows. The `"false"` is read at render time by + # the module build vars, which instrument only on `"true"`; the coverage run below keeps the job's value. - name: Build WASM modules timeout-minutes: 25 env: + ET_TEST_COVERAGE: ${{ runner.os == 'Windows' && 'false' || 'true' }} GITHUB_TOKEN: ${{ github.token }} run: mise run build-modules @@ -114,12 +140,20 @@ jobs: timeout-minutes: 75 run: mise run cargo-llvm-cov - # Deliberately unguarded, unlike the merges and uploads below. - # Those carry `!cancelled()` so a failing test still publishes a report; this is a gate, and a gate has - # nothing to say about a run whose tests did not pass. Leaving it on the default "skip if an earlier step - # failed" means a red test run reports the test failure, not a coverage shortfall caused by it. It reads - # the profile data the step above already wrote, so it re-runs nothing. + # No `!cancelled()` here, unlike the merges and uploads below. + # Those carry it so a failing test still publishes a report; this is a gate, and a gate has nothing to say + # about a run whose tests did not pass. Leaving it on the default "skip if an earlier step failed" means a + # red test run reports the test failure, not a coverage shortfall caused by it. It reads the profile data + # the step above already wrote, so it re-runs nothing. + # + # Linux and macOS hold libs/ to 100%; the Windows lane's report is too partial to be held to it. + # Crates whose tests pass there contribute no records at all, and et-otlp falls a branch short: + # libs/et-otlp: D:/a/core/core/libs/et-otlp/src/lib.rs 7/8 branches + # libs/math1: no records in the report -- this run never exercised the crate + # on commit https://github.com/edge-toolkit/core/commit/65e78dc834455a2a24cfece0feb36c7e3352bbd9 at + # https://github.com/edge-toolkit/core/actions/runs/42537545904/job/128905082309. - name: Check native libs/ branch coverage + if: runner.os != 'Windows' timeout-minutes: 10 run: mise run cargo-llvm-cov-branch-check @@ -134,7 +168,8 @@ jobs: # # Linux only, here and for the two browser merges below. The instrumented guest builds need the # wasm-capable conda clang resolving as bare `clang`, and the headless-Chrome steps need the image's - # chromedriver; neither holds on the macOS lane, which is why wasm-cov-branch-check is Linux-gated too. + # chromedriver; neither holds on the macOS lane, and the Windows lane builds no instrumented guests at all. + # That is why wasm-cov-branch-check is Linux-gated too. - name: Merge wasm guest coverage into lcov.info if: ${{ !cancelled() && runner.os == 'Linux' && hashFiles('lcov.info') != '' }} timeout-minutes: 10 @@ -168,7 +203,7 @@ jobs: # After all three merges, because it reads what they leave behind. # That is the guests' profiles and objects under target/wasi-cov, and the two browser runs' under # target/agent-cov and target/pic-viewer-cov. - # The task is Linux-gated internally as well as here: this `if` keeps the step off the macOS lane at all, + # The task is Linux-gated internally as well as here: this `if` keeps the step off the other lanes at all, # and the gate inside it keeps a direct `mise run` on a workstation from passing on an empty report. - name: Check wasm libs/ branch coverage if: runner.os == 'Linux' @@ -183,7 +218,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: mise run pytest-cov - # Per-OS name because both lanes produce one and artifact names have to be unique within a run. + # Per-OS name because every lane produces one and artifact names have to be unique within a run. - name: Upload coverage reports as artifacts if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 @@ -200,8 +235,7 @@ jobs: # second, misleading failure: when an earlier step (a tool install, say) means nothing was ever produced, # these skip instead of erroring on a missing file and burying the real cause. # - # Every upload names its language and lane in `flags`, so Codecov can report each on its own. The two - # DeepSource uploads carry `runner.os == 'Linux'`, for the one-report-per-key reason given on the job. + # Every Codecov upload names its language and lane in `flags`, so Codecov can report each on its own. # # `disable_search` keeps each upload to the one report it names. With search on, the action also sends every # coverage-looking file in the workspace, including the unfiltered intermediates the coverage tasks leave @@ -232,11 +266,11 @@ jobs: fail_ci_if_error: true - name: Upload Rust coverage to DeepSource (OIDC) - if: ${{ !cancelled() && runner.os == 'Linux' && hashFiles('lcov.info') != '' }} + if: ${{ !cancelled() && hashFiles('lcov.info') != '' }} run: deepsource report --analyzer test-coverage --key rust --value-file lcov.info --use-oidc - name: Upload Python coverage to DeepSource (OIDC) - if: ${{ !cancelled() && runner.os == 'Linux' && hashFiles('coverage-python.xml') != '' }} + if: ${{ !cancelled() && hashFiles('coverage-python.xml') != '' }} run: deepsource report --analyzer test-coverage --key python --value-file coverage-python.xml --use-oidc # Upload test results even when tests failed. @@ -302,7 +336,7 @@ jobs: path: target/test-badge # Separate from the coverage job so only this one holds the Pages write permissions. - # `needs` waits on both matrix lanes, and the artifact exists only when the Linux lane got through the steps above. + # `needs` waits on every matrix lane, and the artifact exists only when the Linux lane got through the steps above. pages: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} needs: coverage diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 81a9cf17..3fd66176 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -17,8 +17,9 @@ # run the task body manually in a shell with the conda-clang bin dir genuinely first on PATH. CI (ubuntu) is # unaffected because the distro clang already has the wasm backend. # -# Windows is deliberately out of scope for this env (hence the os scoping here): coverage collection runs only -# on the ubuntu CI lane, and there is no intention to make the wasm coverage pipeline work on Windows. +# The wasm coverage pipeline is deliberately out of scope on Windows (hence the os scoping here). +# The Windows CI lane collects native Rust and Python coverage only, and there is no intention to make the wasm +# half work there. [tools] # scc is the primary line counter behind the sloc task. # Picked over the alternatives on three grounds: a prebuilt binary for every target platform, counting rules @@ -87,7 +88,7 @@ CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER = "/usr/bin/cc" CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER = "/usr/bin/cc" DYLD_LIBRARY_PATH = "{{ vars.ort_loc_unix }}/lib" LD_LIBRARY_PATH = "{{ vars.ort_loc_unix }}/lib" -_.path = "{{ env.HOME }}/.local/share/mise/installs/conda-clang/latest/bin" +_.path = "{{ vars.a_home }}/.local/share/mise/installs/conda-clang/latest/bin" [tasks.cargo-llvm-cov] description = "Rust coverage (cargo-llvm-cov) + JUnit test results (nextest); emits lcov.info + JUnit reports" @@ -106,8 +107,10 @@ cargo llvm-cov clean --workspace # The prior `build-modules` step compiled et-cli + its native dep tree (edge-toolkit, et-path, int-gen, ...) # NON-instrumented into target/{debug,release}, and `cargo llvm-cov clean` leaves those in place; without this the # coverage build reuses them uninstrumented so those crates report zero coverage (et-path was the symptom). The -# wasm guest .ll that wasm-cov consumes live under target/wasm32-wasip2 and are left untouched. -coreutils rm -rf target/debug target/release +# wasm guest .ll that wasm-cov consumes live under target/wasm32-wasip2 and are left untouched. Where +# CARGO_BUILD_TARGET is set (Windows), the native crates build under target// instead, so those go too. +native="target${CARGO_BUILD_TARGET:+/$CARGO_BUILD_TARGET}" +coreutils rm -rf target/debug target/release "$native/debug" "$native/release" # Apply cargo-llvm-cov's instrumentation env via `eval` of its `show-env` exports. # `--sh` is the current flag (the old `--export-prefix` alias is deprecated). This sets the rustc wrapper + # LLVM_PROFILE_FILE + profile dir so nextest and the regens instrument into one profile set that `report` merges. @@ -128,8 +131,9 @@ coreutils mkdir -p "$ort_build_dir/examples" "$ort_build_dir/deps" # RUSTFLAGS directly; `report`'s lcov export emits the BRDA records on its own once the covmap carries branch # regions. On Linux continuous mode additionally needs relocatable counters # (`-Cllvm-args=-runtime-counter-relocation`). -# macOS is the exception and keeps the profile pattern show-env emits, because continuous mode there writes -# nothing back at all. The run completes and the report generates, but every counter in it reads zero: +# Linux is the only OS that takes %c, and the others keep the profile pattern show-env emits. +# On macOS continuous mode writes nothing back at all. The run completes and the report generates, but every +# counter in it reads zero: # lines: 0/8677 hit # branches: 0/954 hit # from that lane's own artifact on commit @@ -137,15 +141,17 @@ coreutils mkdir -p "$ort_build_dir/examples" "$ort_build_dir/deps" # https://github.com/edge-toolkit/core/actions/runs/35032214650/job/104593077352, and reproduced on a # workstation by changing this one variable and nothing else -- the same crates and tests report real # numbers under the merge-pool pattern and zeros under %c. The relocation flag above is what makes %c work -# on Linux and has no Mach-O counterpart to reach for. The carve-out costs exactly what %c was added for: -# the pyo3 runner that nextest SIGKILLs contributes nothing on macOS, so its async paths read 0% there and -# the Linux lane is the one that measures them. The wrapper flags env is CARGO_ENCODED_RUSTFLAGS format (ASCII +# on Linux and has no Mach-O counterpart to reach for. Windows has never run %c here, and the way it fails on +# macOS is silent -- a report of zeros, not an error -- so that lane takes the pattern known to work rather +# than finding out from its numbers. The carve-out costs exactly what %c was added for: the pyo3 runner that +# nextest SIGKILLs contributes nothing on macOS or Windows, so its async paths read 0% there and the Linux lane +# is the one that measures them. The wrapper flags env is CARGO_ENCODED_RUSTFLAGS format (ASCII # unit separator 0x1f between flags), not space-separated -- a space append glues the llvm-arg onto the last # existing flag and rustc rejects `--cfg=coverage_nightly -Cllvm-args=...` as one invalid --cfg value. goawk # generates the separator byte at run time: bash dollar-quote hex is not a valid escape in this TOML basic # string, and a TOML unicode escape renders as an invisible byte in the body, so it is constructed visibly. host_os="$(coreutils uname -s)" -if [ "$host_os" != "Darwin" ]; then +if [ "$host_os" = "Linux" ]; then cov_dir="$(coreutils dirname "$LLVM_PROFILE_FILE")" export LLVM_PROFILE_FILE="$cov_dir/core-cont-%p%c.profraw" fi @@ -161,6 +167,16 @@ export __CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS="$flags" # et-ws-web-runner/coverage adds the browser-module capture (shims + the __et_capture_coverage PUT); # et-ws-wasi-runner/coverage adds the guest `/cov` preopen. ET_TEST_COVERAGE (set by the coverage job) still # gates capture at runtime within each build. +# The run takes the same workspace exclusions as cargo-test, and the web runner's feature goes with its crate. +# cargo_ws_excludes leaves et-ws-web-runner out on the gnullvm Windows target, and cargo refuses a feature of +# an unselected package outright: +# error: none of the selected packages contains this feature: et-ws-web-runner/coverage +excludes="{{ vars.cargo_ws_excludes }}" +features=et-ws-wasi-runner/coverage +case " $excludes " in + *" et-ws-web-runner "*) ;; + *) features="$features,et-ws-web-runner/coverage" ;; +esac # A failing test must not cost the whole report: capture the status and re-raise it once lcov.info is written. # Under `set -e` a nextest failure would abort the task here, so `cargo llvm-cov report` below never runs and # lcov.info is never produced -- the coverage upload steps then have nothing to send and the run reports no @@ -169,8 +185,11 @@ export __CARGO_LLVM_COV_RUSTC_WRAPPER_RUSTFLAGS="$flags" # The unit and integration passes run separately so each writes its own JUnit report, and both instrument into # the one profile set. The integration pass runs even when the unit pass failed, and the first failure is kept. nextest() { - local features=et-ws-web-runner/coverage,et-ws-wasi-runner/coverage status=0 - cargo nextest run --config-file config/nextest.toml --features "$features" --profile "$1" -E "$2" || status=$? + local profile="$1" filter="$2" status=0 + set -- --config-file config/nextest.toml --features "$features" --profile "$profile" -E "$filter" + # $excludes is a space-separated flag list; word-splitting is intentional. + # shellcheck disable=SC2086 + cargo nextest run --workspace $excludes "$@" || status=$? if [ "$test_status" -eq 0 ]; then test_status=$status fi @@ -189,8 +208,9 @@ mise run regen-verification # `.deepsource.toml` exclude only stops the analyzers, not the coverage aggregate. # Deliberately NOT anchored with `^`: llvm-cov matches this regex against the absolute path recorded in the # coverage map (`/home/runner/work/core/core/services/...`), which is what the lcov SF: lines then carry, so a -# leading `^` matches nothing and the file stays in the report. Match the path tail instead. -cov_ignore='services/ws-wasi-runner/src/bindings\\.rs$' +# leading `^` matches nothing and the file stays in the report. Match the path tail instead. Each separator +# matches either slash, because Windows records the path with backslashes. +cov_ignore='services[/\\\\]ws-wasi-runner[/\\\\]src[/\\\\]bindings\\.rs$' cargo llvm-cov report --lcov --include-build-script --ignore-filename-regex "$cov_ignore" --output-path lcov.info exit "$test_status" """ @@ -319,8 +339,8 @@ shell = "{{ vars.task_shell }}" # Linux-only, because the pipeline feeding it is. # The guest .profraw this reads exist only if the instrumented wasm builds ran, and those need the wasm-capable # conda clang genuinely first on PATH -- which this env's `_.path` prepend does not achieve on macOS (see the -# header). Windows never runs coverage at all. So on any other OS there is no data to judge and the check says -# so and stops, rather than passing on an empty report. +# header). Windows builds no instrumented guests at all. So on any other OS there is no data to judge and the +# check says so and stops, rather than passing on an empty report. # # One export per coverage source, folded into a union, rather than one export over everything. # libs/web and libs/wasi-guest are linked into several guests and into the browser test binaries, so every diff --git a/.mise/cov-branch-assert.jq b/.mise/cov-branch-assert.jq index c04f1e53..c59eb1e6 100644 --- a/.mise/cov-branch-assert.jq +++ b/.mise/cov-branch-assert.jq @@ -6,9 +6,10 @@ # vacuously: a crate dropping out of the report is how coverage silently stops being measured. # $nocode inverts that for the crates that are constants and macros only, which llvm-cov cannot record even # when their tests run -- an empty report is expected there, and records appearing mean the crate grew code. +# Windows records each filename with backslashes, so they are folded to `/` before the `/libs//` match. ($libs | split(" ") | map(select(length > 0))) as $names | ($nocode | split(" ") | map(select(length > 0))) as $constants -| .data[0].files as $files +| (.data[0].files | map(.filename |= (split("\\") | join("/")))) as $files | [ ( $names[] | . as $n | ("/libs/" + $n + "/") as $dir