diff --git a/.circleci/config.yml b/.circleci/config.yml index 610ddf508b..fef9d7e00a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,6 +201,58 @@ jobs: - save_cache: key: go-freebsd-v1-{{ checksum "third_party/go/BUILD" }} paths: [ ".plz-cache/third_party/go" ] + + build-windows: + working_directory: ~/please + docker: + - image: ghcr.io/thought-machine/please_windows_builder:20260910 + resource_class: large + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - restore_cache: + key: go-windows-v1-{{ checksum "third_party/go/BUILD" }} + - run: + name: Extract plz + command: tar -xzf /tmp/workspace/linux_amd64/please_*.tar.gz + - run: + name: Cross-compile + command: ./please/please build -p -v2 --profile ci --arch windows_amd64 //package:release_files + - persist_to_workspace: + root: plz-out/pkg + paths: + - windows_amd64/* + - store_artifacts: + path: plz-out/log + - save_cache: + key: go-windows-v1-{{ checksum "third_party/go/BUILD" }} + paths: [ ".plz-cache/third_party/go" ] + + # Runs the Windows binaries we just cross-built, under Wine, on Linux. This is the only + # thing in CI that checks the port's runtime behaviour rather than that it compiles; see + # docs/design/windows/05-testing-strategy.md for what it does and does not cover. + test-windows-wine: + working_directory: ~/please + docker: + - image: ghcr.io/thought-machine/please_windows_builder:20260910 + resource_class: large + steps: + - checkout + - attach_workspace: + at: /tmp/workspace + - restore_cache: + key: go-windows-v1-{{ checksum "third_party/go/BUILD" }} + - run: + name: Extract plz + command: tar -xzf /tmp/workspace/linux_amd64/please_*.tar.gz + - run: + name: Test under Wine + command: ./please/please test -p -v2 --profile ci --include=wine + - store_test_results: + path: plz-out/log/test_results.xml + - store_artifacts: + path: plz-out/log build-linux-arm64: working_directory: ~/please docker: @@ -515,6 +567,12 @@ workflows: - build-freebsd: requires: - build-alpine + - build-windows: + requires: + - build-alpine + - test-windows-wine: + requires: + - build-alpine - test-rex: requires: - build-alpine @@ -531,6 +589,8 @@ workflows: requires: - build-alpine - build-freebsd + - build-windows + - test-windows-wine - build-darwin-amd64 - build-linux-arm64 - build-linux diff --git a/.circleci/release.sh b/.circleci/release.sh index f51286bd49..86ed5fba23 100755 --- a/.circleci/release.sh +++ b/.circleci/release.sh @@ -50,6 +50,7 @@ release_folder /tmp/workspace/darwin_arm64 darwin_arm64/$VERSION release_folder /tmp/workspace/linux_amd64 linux_amd64/$VERSION release_folder /tmp/workspace/linux_arm64 linux_arm64/$VERSION release_folder /tmp/workspace/freebsd_amd64 freebsd_amd64/$VERSION +release_folder /tmp/workspace/windows_amd64 windows_amd64/$VERSION # Sign the download script with our release key /tmp/workspace/release_signer pgp -o get_plz.sh.asc -i tools/misc/get_plz.sh @@ -58,6 +59,14 @@ release_file tools/misc/get_plz.sh get_plz.sh text/x-shellscript release_file get_plz.sh.asc get_plz.sh.asc text/plain release_file get_plz.sh.sig get_plz.sh.sig application/octet-stream +# The Windows installer, served the same way and signed the same way. Windows has no shell to +# curl | sh with; this is run with irm ... | iex instead. +/tmp/workspace/release_signer pgp -o get_plz.ps1.asc -i tools/misc/get_plz.ps1 +/tmp/workspace/release_signer kms -o get_plz.ps1.sig -i tools/misc/get_plz.ps1 +release_file tools/misc/get_plz.ps1 get_plz.ps1 text/plain +release_file get_plz.ps1.asc get_plz.ps1.asc text/plain +release_file get_plz.ps1.sig get_plz.ps1.sig application/octet-stream + if [[ "$VERSION" == *"beta"* ]] || [[ "$VERSION" == *"alpha"* ]] || [[ "$VERSION" == *"prerelease"* ]]; then echo "$VERSION is a prerelease, only setting latest_prerelease_version" else diff --git a/.gitattributes b/.gitattributes index 050d731e99..1d6e67abec 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,7 @@ -*.build_defs linguist-language=Starlark diff=python -BUILD.plz linguist-language=Starlark diff=python -docs/* linguist-documentation -*_bindata.go linguist-generated -third_party/go/zip/* linguist-vendored +# Batch files are read by cmd.exe, which is not reliable on LF-only input, and the fixtures +# under test/windows are compared byte for byte against output the bundled busybox produced. +# Neither should be translated on checkout. +*.cmd -text +src/run/test_data/** -text +test/windows/smoke_repo/** -text +test/windows/offline_repo/** -text diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..10857e0e19 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,85 @@ +# Builds and publishes a release from this fork. +# +# Upstream publishes to a GCS bucket from CircleCI, which only runs on thought-machine/please. +# A fork cannot drive that, so nothing this branch produces was downloadable by pleasew.ps1, +# get_plz.ps1 or plz update - the Windows support was real and unreachable at the same time. +# +# This publishes the same artifacts as a GitHub Release instead. The asset names carry the +# platform, which is what gen_release.py does for the GitHub half of an upstream release, so the +# two layouts agree on names even though they disagree on paths. +# +# Delete this whole workflow if upstream ever starts publishing Windows builds. +name: Release + +on: + push: + tags: + - "v*" + # So a release can be cut without tagging, while this is still being worked out. + workflow_dispatch: + inputs: + tag: + description: "Tag to release, e.g. v17.34.0" + required: true + +permissions: + contents: write + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Please + run: | + ./pleasew --version + echo "$HOME/.please" >> "$GITHUB_PATH" + - name: Build this repo's Please + # The released Please is not this one, and it hangs parsing //test/windows. Everything + # below uses the one we just built, the same two-step test.sh insists on. + run: ./pleasew build -p -v2 --profile ci //src:please + - name: Work out what we are releasing + id: version + run: | + tag="${{ github.event.inputs.tag }}" + [ -n "$tag" ] || tag="${GITHUB_REF#refs/tags/}" + version="$(cat VERSION)" + # A tag that disagrees with VERSION would publish assets nobody can find, since every + # consumer builds the filename out of the version rather than out of the tag. + [ "$tag" = "v$version" ] || { echo "tag $tag does not match VERSION $version" >&2; exit 1; } + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Build the releases + run: | + plz-out/bin/src/please build -p -v2 --profile ci --arch windows_amd64 //package:release_files + plz-out/bin/src/please build -p -v2 --profile ci //package:release_files + - name: Name the assets by platform + id: assets + run: | + version="${{ steps.version.outputs.version }}" + mkdir -p assets + for arch in windows_amd64 linux_amd64; do + for f in plz-out/pkg/$arch/*; do + [ -f "$f" ] || continue + # The same rename gen_release.py does: please_1.2.3.zip -> please_1.2.3_arch.zip. + base="$(basename "$f")" + cp "$f" "assets/${base/$version/${version}_$arch}" + done + done + ls -l assets + - name: Publish + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ steps.version.outputs.tag }}" \ + --repo "${{ github.repository }}" \ + --title "Please ${{ steps.version.outputs.version }}" \ + --notes "Built from this fork, which carries Windows support that is not upstream yet. + + Install on Windows: + + irm https://raw.githubusercontent.com/${{ github.repository }}/${{ steps.version.outputs.tag }}/tools/misc/get_plz.ps1 | iex + + See docs/design/windows/07-state-of-play.md for what works and what does not." \ + assets/* diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000000..afead1826c --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,183 @@ +# The only thing anywhere that runs Please on a real Windows machine. +# +# Everything else the port claims rests on Wine, and docs/design/windows/05-testing-strategy.md +# is explicit that Wine passing is evidence rather than proof. The classes it cannot show are +# the ones Please works hardest in: sharing violations on teardown, path length, case-insensitive +# collisions, symlink privileges. +# +# CircleCI cross-builds the same artifacts, but its workspaces are scoped to one CircleCI run +# and cannot be read from here, so this cross-builds its own on a Linux runner and hands them +# over. That duplication is the price of there being no Windows runner in CircleCI's config. +name: Windows + +on: + push: + branches: + - master + - wine + pull_request: + +concurrency: + group: windows-${{ github.ref }} + cancel-in-progress: true + +jobs: + cross-build: + name: cross-build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Please + run: | + ./pleasew --version + echo "$HOME/.please" >> "$GITHUB_PATH" + - name: Build this repo's Please + # The released Please is not this one, and the difference is not cosmetic: it predates + # the parse-deadlock fix and hangs parsing //test/windows, silently and for ever. + # Everything below uses the one we just built, the same two-step test.sh insists on. + run: ./pleasew build -p -v2 --profile ci //src:please + - name: Cross-build the test bundle + run: plz-out/bin/src/please build -p -v2 --profile ci //test/windows:native_test_bundle + - name: Cross-build the release + # The same command CircleCI's build-windows job runs, so what the Windows job gets here + # is what a user would get. + run: plz-out/bin/src/please build -p -v2 --profile ci --arch windows_amd64 //package:release_files + - name: Extract the codelab plan + # Built here, on Linux, so the Windows job replays a plan that + # //test/windows/codelab_script/script:script_test has already checked against the same + # codelabs and the same sidecar. See test/windows/run_codelabs.ps1. + run: plz-out/bin/src/please build -p -v2 --profile ci //test/windows:codelab_plan + - name: Upload the codelab plan + uses: actions/upload-artifact@v4 + with: + name: windows-codelab-plan + path: plz-out/gen/test/windows/codelab_plan.json + retention-days: 7 + - name: Upload the test bundle + uses: actions/upload-artifact@v4 + with: + name: windows-test-bundle + path: plz-out/gen/test/windows/native_test_bundle + # The bundle's repo-root marker is a .plzconfig, and this action drops hidden files + # unless told not to. Without it the tests that look for a repo root find none and + # die, which nothing on the Linux side can catch - the Wine test runs the bundle + # directly, never through an artifact. + include-hidden-files: true + retention-days: 7 + - name: Upload the release + uses: actions/upload-artifact@v4 + with: + name: windows-release + path: plz-out/pkg/windows_amd64/please_*.zip + retention-days: 7 + - name: Upload the logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: cross-build-logs + path: plz-out/log + retention-days: 7 + + test: + name: test + needs: cross-build + runs-on: windows-latest + # Blocking. It was advisory for exactly one run, to produce a failure list without a red + # check nobody had read yet; everything that run found is fixed. Anything genuinely left + # over belongs in test/windows/known_failures.txt with a reason, not behind a flag that + # makes the whole job ignorable. + steps: + - name: Keep Unix line endings + # The fixtures are compared byte for byte against output the bundled busybox produced, + # and busybox does not translate line endings. Has to precede the checkout. + run: git config --global core.autocrlf input + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: windows-test-bundle + path: ${{ runner.temp }}\bundle + - uses: actions/download-artifact@v4 + with: + name: windows-release + path: ${{ runner.temp }}\release + - name: Report the machine + # Both of these change what is reachable, both differ between runner images, and + # guessing either has already cost time. Print them rather than assuming. + run: | + (Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -EA SilentlyContinue) | + Select-Object -ExpandProperty LongPathsEnabled -EA SilentlyContinue | + ForEach-Object { "LongPathsEnabled=$_" } | Tee-Object -Append $env:GITHUB_STEP_SUMMARY + whoami /priv | Select-String SeCreateSymbolicLinkPrivilege | + Tee-Object -Append $env:GITHUB_STEP_SUMMARY + - name: Run the cross-built unit tests + run: | + ./test/windows/run_native_tests.ps1 ` + -Bundle "$env:RUNNER_TEMP\bundle" ` + -Logs "$env:RUNNER_TEMP\logs" ` + -KnownFailures test/windows/known_failures.txt + - name: Build a repo with the release + if: ${{ !cancelled() }} + run: | + ./test/windows/run_native_probes.ps1 ` + -Release "$env:RUNNER_TEMP\release" ` + -Logs "$env:RUNNER_TEMP\logs" + - name: Upload the logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: windows-test-logs + path: ${{ runner.temp }}\logs + retention-days: 7 + + codelabs: + name: codelabs + needs: cross-build + runs-on: windows-latest + # A job of its own rather than a step in test. It needs the release but not the bundle, it + # runs for far longer - eight codelabs, plugin downloads, a Go toolchain - and a hung codelab + # should not eat the unit tests' time or delay the signal people actually read. + timeout-minutes: 90 + # Blocking from its first run, for the reason the test job gives. What the codelabs cannot do + # on Windows is expected, and the mechanism for expected already exists: every such step is + # in test/windows/codelab_known_failures.txt with a reason. There is no advisory mode to + # forget to take out. + steps: + - name: Keep Unix line endings + # The runner script and the failures list come from the checkout. Has to precede it. + run: git config --global core.autocrlf input + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: windows-release + path: ${{ runner.temp }}\release + - uses: actions/download-artifact@v4 + with: + name: windows-codelab-plan + path: ${{ runner.temp }}\plan + - name: Report the toolchains + # The codelabs assume Go, Python, git, Docker and kubectl, and which of those exist - and + # for Docker, which kind of containers it runs - decides what is skipped. Print it rather + # than inferring it from the results. + run: | + "pwsh = $($PSVersionTable.PSVersion)" | Tee-Object -Append $env:GITHUB_STEP_SUMMARY + foreach ($t in 'go', 'python', 'git', 'docker', 'kubectl') { + $c = Get-Command $t -EA SilentlyContinue + "$t = $(if ($c) { $c.Source } else { 'not installed' })" | Tee-Object -Append $env:GITHUB_STEP_SUMMARY + } + if (Get-Command docker -EA SilentlyContinue) { + "docker OSType = $(docker info --format '{{.OSType}}' 2>$null)" | Tee-Object -Append $env:GITHUB_STEP_SUMMARY + } + - name: Run the codelabs + run: | + ./test/windows/run_codelabs.ps1 ` + -Plan "$env:RUNNER_TEMP\plan\codelab_plan.json" ` + -Release "$env:RUNNER_TEMP\release" ` + -Logs "$env:RUNNER_TEMP\logs" ` + -KnownFailures test/windows/codelab_known_failures.txt + - name: Upload the logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: windows-codelab-logs + path: ${{ runner.temp }}\logs + retention-days: 7 diff --git a/.plzconfig b/.plzconfig index 3c69865cf8..433b91bea1 100644 --- a/.plzconfig +++ b/.plzconfig @@ -105,6 +105,7 @@ accept = Apache-2.0 accept = Apache License, Version 2.0 accept = The Apache Software License, Version 2.0 accept = PSF +accept = PSF-2.0 accept = ASL accept = MPL-2.0 accept = LGPL @@ -112,6 +113,10 @@ accept = Artistic License accept = ISC # Not really a licence, but Bazel projects commonly describe things this way. accept = notice +# Only for the busybox we ship in the Windows release. Please neither links it nor derives +# from it - it is execed as the shell - so the two are separately distributed works, and the +# release records the licence and where to get the source. +accept = GPL-2.0 [remote] url = diff --git a/.plzconfig_windows_amd64 b/.plzconfig_windows_amd64 new file mode 100644 index 0000000000..9b0646e79c --- /dev/null +++ b/.plzconfig_windows_amd64 @@ -0,0 +1,19 @@ +; Per-architecture config for windows_amd64, layered on .plzconfig by +; state.ForArch(). See docs/design/windows/ for the design notes behind these. + +[Plugin "go"] +; go-flags treats '/' as its option delimiter and ':' as its name/argument +; delimiter on Windows, which collides with Please's own label syntax: +; //pkg:target parses as option /pkg with argument target. The forceposix tag +; disables that behaviour. Without it every command taking a build label is +; broken. See docs/design/windows/00-overview.md, decision D5. +BuildTags = forceposix + +[build] +; Windows has no extended attributes; fall back to the sidecar-file mechanism. +xattrs = false + +[sandbox] +; No mount/network namespace equivalent yet. See M7. +build = false +test = false diff --git a/BUILD b/BUILD index 2cb4d48d9b..cf93654e8c 100644 --- a/BUILD +++ b/BUILD @@ -29,7 +29,10 @@ filegroup( filegroup( name = "pleasew", - srcs = ["pleasew"], + srcs = [ + "pleasew", + "pleasew.ps1", + ], binary = True, visibility = ["//src/assets/..."], ) diff --git a/ChangeLog b/ChangeLog index 888ed17376..5dc9c712bc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +Version 18.0.0 +-------------- + * Native Windows support on amd64. `plz` runs, builds and tests on Windows, + with busybox bundled as the build shell so a fresh install needs no + configuration. Go, C++ and Python targets all build; `sh_binary` produces a + `.cmd`, since Windows has no shebang mechanism. + * Windows installer at https://get.please.build/get_plz.ps1, and a + `pleasew.ps1` counterpart to `pleasew`. + * Releases for `windows_amd64` are a `.zip`, with a `plz.cmd` shim in place of + the `plz` symlink, which Windows will not create without Developer Mode. + * `plz clean` no longer fails on Windows. Please held its own log file open + inside the directory it was deleting. + * `plz` no longer hangs when run outside a repo on Windows. The walk towards + the filesystem root never terminated at a drive letter. + * `link:` labels fall back to copying where Windows refuses to create a + symlink, rather than silently doing nothing. + * `remote_file` with a `file://` URL is correctly refused when it points + inside the repo on Windows; the check compared path separators that never + matched. + * No build sandbox on Windows: it is built on Linux namespaces, and there is + no equivalent. + Version 17.33.0 --------------- * Fix macOS tmpdir cleanup race caused by system daemons creating `~/Library` (#3515) diff --git a/README.md b/README.md index 1850deb1cb..a6acfe7d36 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ nearly any aspect of your build process. See [please.build](https://please.build) for more information. -Currently Linux (tested on Ubuntu), macOS and FreeBSD are actively supported. +Currently Linux (tested on Ubuntu), macOS, FreeBSD and Windows (amd64) are actively supported. If you're a fan of Please, don't forget to add yourself to the [adopters](https://github.com/thought-machine/please/blob/master/ADOPTERS.md) diff --git a/VERSION b/VERSION index 1d6c72b5c7..7eae4e2e91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -17.33.0 +18.0.0 diff --git a/docs/codelabs/BUILD b/docs/codelabs/BUILD index c6c2985c20..72431714ca 100644 --- a/docs/codelabs/BUILD +++ b/docs/codelabs/BUILD @@ -31,3 +31,13 @@ genrule( tools = [":codelab_templator"], visibility = ["//docs/..."], ) + +# The codelab sources, for //test/windows/codelab_script, which checks they run on Windows. +filegroup( + name = "codelab_md", + srcs = glob(["*.md"]), + visibility = [ + "//docs/...", + "//test/windows/...", + ], +) diff --git a/docs/config.html b/docs/config.html index c7f8cb8b92..5645dc7094 100644 --- a/docs/config.html +++ b/docs/config.html @@ -816,6 +816,20 @@

ParallelDownloads {{ index .ConfigHelpText "build.paralleldownloads" }}

+
  • +
    +

    Shell

    + +

    {{ index .ConfigHelpText "build.shell" }}

    +
    +
    • +
  • +
    +

    ShellArgs (repeatable)

    + +

    {{ index .ConfigHelpText "build.shellargs" }}

    +
    +
    • diff --git a/docs/design/windows/00-overview.md b/docs/design/windows/00-overview.md new file mode 100644 index 0000000000..34904348f2 --- /dev/null +++ b/docs/design/windows/00-overview.md @@ -0,0 +1,182 @@ +# Windows Port — Overview + +Status: **Draft** · Owner: _unassigned_ · Last updated: 2026-09-10 + +This directory holds the engineering design documents for adding a Windows port to Please. +They are working documents for contributors, not user-facing documentation — the docs site +build (`docs/BUILD`) only globs `milestones/*.html` and does not pick this directory up. + +**Starting cold? Read `07-state-of-play.md`.** It is the short version: what works, where the +five repos are, what to pick up next, and the mistakes that have already cost time once each. +Then this file for the why, and `06-milestones.md` for the per-milestone detail. + +## Why + +Please ships binaries for `linux_amd64`, `linux_arm64`, `darwin_amd64`, `darwin_arm64` and +`freebsd_amd64`. There is no Windows build. `pleasew` bails out with *"Please does not +support the %s operating system"*, and `src/cli/winch_windows.go` is the only Windows-aware +file in 272 Go source files. + +The 15.9.1 milestone note said *"We're scoping out support for Windows"*. This is that +scoping, turned into a plan. + +## Goal + +`plz.exe` on Windows, at **full feature parity** with the Linux build, with **building C++ +projects as the primary driving use case**. + +Parity means sandboxing, remote execution and all four language plugins eventually work. +It does not mean they all arrive at once: the milestone sequence deliberately front-loads a +working C++ vertical slice, then fills in the rest. + +## Constraint that shapes everything + +**Development and CI stay on Linux.** The binary is cross-compiled. Real Windows testing is +a later stage (M9). + +This is workable because of two independent cross-compilation axes. Conflating them is the +most likely way to lose a week, so they get separate names throughout these documents. + +### Axis 1 — cross-building `plz.exe` + +`GOOS=windows go build`, driven from a Linux host by: + +```bash +plz build --arch windows_amd64 //package:release_files +``` + +Produces the Please binary we ship. Modelled on the existing FreeBSD release flow, which is +already a Linux-hosted cross build (`.circleci/config.yml`, the `build-freebsd` job). + +### Axis 2 — cross-building C++ *for* Windows + +`plz --arch windows_amd64` with `.plzconfig_windows_amd64` pointing `cpptool` at +`x86_64-w64-mingw32-g++`. Produces `.exe`/`.dll`/`.a` artifacts. + +The important property: **this exercises every cc-rules Windows codepath while plz itself +runs on Linux.** Roughly 80% of the C++ work is verifiable before any Windows machine is +involved. + +Layer Wine on top (M6) and `plz.exe` itself becomes testable on Linux too. See +`05-testing-strategy.md`. + +## Decisions + +Each decision has a fuller ADR in the linked document. D1–D4 were taken during planning; +D5 came out of the M0 investigation. + +### D1 — MinGW-w64 GCC first, MSVC/clang-cl later + +See `03-cc-toolchain.md`. + +MinGW reuses the existing GNU-driver flag logic (`-Wl,--start-group`, `--whole-archive`, +`-Wl,--gc-sections`) and the existing `please_cc` GCC matcher essentially unchanged. +Critically, `x86_64-w64-mingw32-g++` runs **on Linux**, so Axis 2 works from day one. + +MSVC would require a new flag dialect (`/c`, `/Fo`, `/EHsc`, `.obj`/`.lib`), `vcvarsall` +environment discovery, *and* a Windows host — three unknowns at once. It remains the +eventual target for most real-world Windows C++ projects, and `please_cc` is the designed +extension point for it. + +### D2 — Bundle a POSIX shell (busybox-w64) in the Windows release + +See `02-shell-and-build-actions.md`. + +Build actions run through `bash --noprofile --norc -e -u -o pipefail -c` +(`src/process/process.go`), and the cc rules emit genuine shell pipelines — `find | sort | +sed | tr`, backticks, `&&`. Bundling busybox pins the behaviour, requires nothing installed +on the user's machine, and avoids MSYS2's `/c/foo` ⇄ `C:\foo` path translation. + +The alternative considered and rejected: rewriting the cc rules to be shell-free. That is +architecturally cleaner and remains a good idea for its own sake, but it is a large change +in a second repo and would block the Windows port on it. + +### D3 — Full feature parity is the destination + +Sandboxing (M7), remote execution and the go/python/shell plugins (M8) are real milestones, +not a dropped backlog. The OS abstraction layer built in M1 is designed so they have +somewhere to land — in particular, the Job Object machinery introduced for process control +is also what a Windows sandbox will be built on. + +### D4 — Two repos, one programme + +The C/C++ rules are not in this repo. `plugins/BUILD` pins `please-build/cc-rules` at +`v0.7.3`, fetched as a plugin subrepo. + +- **Workstream A** — `thought-machine/please`: the core port, the OS abstraction layer, the + release pipeline. +- **Workstream B** — `please-build/cc-rules`: `please_cc`, output extensions, the MinGW + flag review. + +During development, point `plugins/BUILD` at a fork or branch revision. Upstream to +`please-build/cc-rules` as the final step of M5. + +### D5 — Build every go-flags binary with `-tags forceposix` + +Discovered by running the binary, not by reading the source. `go-flags` uses `/` as its option +delimiter and `:` as its name/argument delimiter on Windows, which collides with Please's +**entire label syntax**: `//pkg:target` parses as option `/pkg` with argument `target`, and +`//...` is rejected as an unknown flag. + +The library guards that file with `// +build !forceposix`, so the fix is a build tag. Verified: +label parsing works completely with it, and is completely broken without it. + +**It is set as config, not per-target.** `go_binary` has no `tags` parameter — the go plugin +takes build tags from `CONFIG.GO.BUILD_TAGS`, which also feeds the stdlib, every `go_repo` +and the source filter. So it belongs in `.plzconfig_windows_amd64`: + +```ini +[Plugin "go"] +BuildTags = forceposix +``` + +Scoping it to the arch config means it applies to every go-flags binary built for Windows — +`//src:please`, `//tools/please_shim` — without affecting any other platform. + +This must be recorded as a decision rather than a code comment, because it is invisible in +Please's own source and will silently regress if the tag is ever dropped. + +See R1 in `appendix-baseline-errors.md`. + +## Non-goals + +Explicitly out of scope for this programme: + +- **MSVC support.** Designed for (see `03-cc-toolchain.md`) but not built. +- **32-bit Windows.** `windows_amd64` only. `windows_arm64` is plausible later; nothing in + the design precludes it. +- **`pkg-config` on Windows.** The codepath stays, but it is documented as unsupported. + Users set flags explicitly. +- **Windows Containers for sandboxing.** M7 accepts the tmp-dir isolation Please already + does and documents the gap rather than taking on that dependency. +- **Native Windows as a development platform.** M9 adds native CI; the day-to-day loop + stays on Linux. + +## Document index + +| Document | Contents | +|---|---| +| `00-overview.md` | This file. Charter, decisions, axes, non-goals. | +| `01-os-abstraction.md` | The `_windows.go` convention, `ExecReplace`, Job Objects, file locking, xattrs. | +| `02-shell-and-build-actions.md` | ADR for D2. busybox applet audit. The path-format rule. | +| `03-cc-toolchain.md` | ADR for D1. MinGW flag matrix. Output extensions. The MSVC extension point. | +| `04-release-and-ci.md` | Cross-build and release pipeline, modelled on FreeBSD. | +| `05-testing-strategy.md` | MinGW for Axis 2, Wine for `plz.exe`, and what Wine misses. | +| `06-milestones.md` | The living tracker. Status, exit criteria, owners. | +| `07-state-of-play.md` | Where the port actually is, what to pick up next, and the standing traps. | +| `appendix-baseline-errors.md` | **Measured** M0 results: compile blockers, runtime findings, what already works. | +| `probe/` | Throwaway M0 artifacts, incl. `m1-skeleton.patch`. Not implementations. | + +## Process + +Per `CONTRIBUTING.md`: raise a GitHub issue for each milestone **before** writing code, and +keep PRs small and single-purpose — no refactors mixed with features. + +Two repo-specific hazards worth repeating here: + +- **Hash stability.** Please's cache is content-hash based over rule definition, config, + sources and secrets. Any change to command generation, environment variables or config + defaults changes target hashes and invalidates every user's cache. Confirm `plz hash + //...` is unchanged on Linux before merging anything in M1–M3. +- **Brittle e2e tests.** The tests in `test/` assert on exact output text. Expect to update + `.txt` golden files; treat any *unexpected* change there as a real regression. diff --git a/docs/design/windows/01-os-abstraction.md b/docs/design/windows/01-os-abstraction.md new file mode 100644 index 0000000000..eda68cef9d --- /dev/null +++ b/docs/design/windows/01-os-abstraction.md @@ -0,0 +1,302 @@ +# OS Abstraction Layer + +Status: **Draft** · Milestones: M1, M2 · Last updated: 2026-09-10 + +How platform-specific code is organised, and the design of each abstraction the Windows +port introduces. See `00-overview.md` for the programme charter. + +## The convention + +Please already has the pattern; it is just barely used. `src/cli/winch_windows.go` plus +`src/cli/winch_other.go` (`//go:build !windows`) is the template. Follow it: + +``` +foo_windows.go // no build tag needed — the filename suffix is the constraint +foo_other.go // //go:build !windows +``` + +Two rules that are easy to get wrong here: + +1. **The filename suffix is itself a build constraint.** `exec_linux.go` has no `//go:build` + line and does not need one. Adding a redundant one is harmless; omitting the constraint + on the `_other.go` sibling is not. +2. **`plz` lists `srcs` explicitly in BUILD files** (see `src/process/BUILD`). New files + must be added there. `plz puku sync` handles `third_party/go`, not first-party srcs. + +### Beware the `!linux` files + +`src/process/exec_other.go` and `src/sandbox/sandbox_other.go` are constrained `!linux`, +which means **they are selected on Windows**. One of them compiles there and one does not: + +- `sandbox_other.go` degrades to a plain `exec.Command(...).Run()` and compiles fine. Not a + blocker. +- `exec_other.go` sets `SysProcAttr{Setpgid, Foreground}` — fields that do not exist in + Windows' `SysProcAttr`. Narrow its constraint to `!linux && !windows`. + +## Inventory + +**Measured, not predicted** — see `appendix-baseline-errors.md` for method and evidence. +The original source survey over-stated this considerably. + +### Blocks compilation — the complete set + +Four layers, five sites, four packages. Each layer is only visible once the previous one is +fixed, because `src/process` is a dependency of nearly everything. + +| Layer | Site | Problem | Design | +|---|---|---|---| +| 1 | `src/process/exec_other.go:17,18` | `SysProcAttr{Setpgid, Foreground}` | New `exec_windows.go`; narrow `!linux` → `!linux && !windows` | +| 1 | `src/process/process.go:206` | `syscall.Kill(-pid, …)` | Job Objects — see below | +| 2 | `src/core/lock.go` ×10 | `syscall.Flock`, `LOCK_SH/EX/UN/NB` | `LockFileEx` — see below | +| 3 | `src/clean/clean.go:96` | `syscall.ForkExec` | Detached `exec.Command` | +| 3 | `src/output/shell_output.go:467` | `cmd.SysProcAttr.Setpgid` | **Abstraction leak** — see below | + +After these, every package compiles and `./src` links to a valid PE32+ binary. + +### The abstraction leak + +`src/output/shell_output.go:467` reaches into the process executor's platform-specific +attributes from *outside* `src/process`: + +```go +cmd := state.ProcessExecutor.ExecCommand(...) +// TODO(jpoole): Read the docs. Attaching stdin and out doesn't seem to work with this. +cmd.SysProcAttr.Setpgid = false +``` + +The fix is not a build tag at the call site — it is to expose the *intent* from `src/process` +(`process.ClearProcessGroup(cmd)`, or a parameter on `ExecCommand`) so the platform detail +stays in one package. Audit for other instances while doing M1. + +### Does NOT block compilation (corrections) + +Three predictions were wrong, all from the same mistake: assuming "Unix-only API" means "does +not compile on Windows". Go's `syscall` package ships Windows stubs. + +| Site | Reality | +|---|---| +| `syscall.Exec` ×5 (`src/please.go`, `src/run`, `src/tool`, `src/update`, `tools/please_shim`) | Defined in `syscall/exec_windows.go`, returns `EWINDOWS`. **Compiles; fails at runtime.** | +| `syscall.Chdir` (`src/run/run_step.go`) | Defined on Windows | +| `SIGHUP`/`SIGQUIT`/`SIGABRT` (`src/cli/process.go`) | All defined in `syscall/types_windows.go` | +| `github.com/pkg/xattr` (`src/fs/attr.go`) | Ships `xattr_unsupported.go`; returns `ENOTSUP` | + +**`syscall.Exec` being a silent runtime failure is worse than a compile error**, not better. +`plz run`, `plz tool`, `plz update`, `plz op` and the shim will build, ship, and then fail +with an opaque *"not supported by windows"*. The compiler cannot drive this work — it needs +tests. Likewise, narrowing the signal set and defaulting `Build.Xattrs = false` are +*correctness* changes with no build-time signal. + +### Compiles, behaves wrong + +Covered in M2. `PATH` split on literal `":"` (7 sites in `src/core/config.go` and +`src/core/utils.go`, plus `src/remote/action.go`); `src/fs/home.go` reading `$HOME` +directly; `/etc/please/plzconfig` and `DefaultPath`; `const SandboxDir = "/tmp/plz_sandbox"`; +the executable-bit model; `os.Symlink` privileges; `RemoveAll`'s chmod-to-force-delete; +hardcoded `sh -c` in `src/cache/cmd_cache.go`; `HOME=tmpDir` in `src/core/build_env.go`. + +One of these was **confirmed as a hard startup blocker**, not a cosmetic issue: +`src/cli/logging.go:64` uses `path.Dir(logFile)` where it needs `filepath.Dir`, so `plz` +cannot create its log directory and dies before doing anything. Fix it early in M1, not in M2 +— nothing can be tested under Wine until it is fixed. + +## Design: process control via Job Objects + +`src/process` is the highest-leverage package — every build action, test and `plz run` +funnels through it. It is also where Unix and Windows differ most. + +Three separate Unix mechanisms collapse into one Windows primitive: + +| Unix | Where | Purpose | +|---|---|---| +| `SysProcAttr{Setpgid: true}` | `exec_other.go`, `exec_linux.go` | Group the child and its descendants | +| `Pdeathsig: syscall.SIGHUP` | `exec_linux.go` | Kill orphans if plz dies | +| `syscall.Kill(-pid, sig)` | `process.go` | Signal the whole group | + +**On Windows all three are a Job Object** created with +`JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`. Assign the child to the job at creation; every +descendant inherits membership; closing the handle (including on abnormal plz exit) kills +the tree. `TerminateJobObject` is the group kill. + +### Graceful-then-forceful termination + +`killProcess` currently sends `SIGTERM`, waits 30ms, then `SIGKILL`, waits 1s. There is a +deliberate comment in `ExecWithTimeout` explaining why `exec.CommandContext` is *not* used: +it only sends `SIGKILL`, which children cannot handle. **Preserve that intent.** + +The Windows equivalent: + +1. `GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, pgid)` — the closest thing to `SIGTERM`. + Requires the child to have been created with `CREATE_NEW_PROCESS_GROUP`. +2. Wait the same 30ms. +3. `TerminateJobObject` — the `SIGKILL` equivalent. + +Ctrl-Break delivery is unreliable for GUI subsystem processes and for children that have +detached from the console. Treat step 1 as best-effort; step 3 is the guarantee. + +### Where this lands + +``` +src/process/exec_windows.go // ExecCommand: create job, CREATE_NEW_PROCESS_GROUP +src/process/kill_windows.go // killProcess, sendSignal +src/process/kill_unix.go // //go:build !windows — the existing signal path +``` + +`process.go` keeps the timing policy and the executor bookkeeping; only the two primitives +(`sendSignal`, group kill) move behind the build tag. + +## Design: `process.ExecReplace` + +`syscall.Exec` replaces the current process image. Windows has no equivalent — Go's +`syscall/exec_windows.go` defines it as a stub returning `EWINDOWS`. + +**None of the six call sites break compilation.** They build, ship, and fail at runtime with +an opaque error. That makes this the highest-risk item in M1: there is no compiler signal, so +it must be driven by tests. + +Introduce one helper rather than six ad-hoc fixes: + +```go +// ExecReplace replaces the current process with the given command where the OS supports +// it, and otherwise runs it as a child and exits with its status. It does not return on +// success. +func ExecReplace(argv []string, env []string) error +``` + +- **Unix** (`exec_replace_unix.go`): `syscall.Exec(argv[0], argv, env)`. Behaviour unchanged. +- **Windows** (`exec_replace_windows.go`): spawn, wait, `os.Exit(child.ExitCode())`. + +### The behavioural difference, and why it matters + +On Windows `plz run` and `plz tool` become a *parent process that outlives the child*. That +is not a transparent substitution, and three things follow: + +1. **Signal forwarding must be explicit.** Ctrl-C in the console reaches both processes; the + parent must not exit before the child has finished cleaning up, or the user sees plz's + exit code instead of the program's. +2. **The parent must not hold the repo lock** while waiting. `src/core/lock.go` writes the + PID into `plz-out/.lock`; a parent blocked in `Wait()` holding an exclusive lock + deadlocks any nested plz invocation. Release before spawning. +3. **Exit codes must round-trip exactly.** `ExitError.ExitCode()` on Windows returns the + raw process exit code, which for a crashing program is an `NTSTATUS` (e.g. + `0xC0000005`). Do not truncate it to 8 bits. + +Document all three in the code comment on `ExecReplace`, not just here. + +### Special case: `tools/please_shim` + +The shim's entire design is exec-replace: resolve `~/.please`, check the version, hand off. +On Windows it additionally needs `.exe`-aware path construction — +`filepath.Join(Location, "please")` must become `please.exe`. It is a separate binary with +its own `main`, so it needs its own copy of the helper or a shared package. + +## Design: file locking + +`src/core/lock.go` opens `plz-out/.lock` and holds an advisory `flock` on it, reusing one +file descriptor so the lock mode can be upgraded/downgraded in place. The file header says +so explicitly: *"The logic below relies heavily on flock (advisory locks)."* + +Split into: + +``` +src/core/lock_unix.go // //go:build !windows — syscall.Flock, unchanged +src/core/lock_windows.go // LockFileEx / UnlockFileEx +``` + +Mapping: + +| Unix | Windows | +|---|---| +| `LOCK_SH` | `LockFileEx` with no flags | +| `LOCK_EX` | `LockFileEx` with `LOCKFILE_EXCLUSIVE_LOCK` | +| `LOCK_NB` | `LOCKFILE_FAIL_IMMEDIATELY` | +| `LOCK_UN` | `UnlockFileEx` | + +Two semantic differences to handle: + +- **`LockFileEx` locks a byte range, not the file.** Lock `[0, 1)` consistently; the code + writes a PID into the file, so do not lock a range that the write touches, or use a + distinct offset well past any content. +- **Mode upgrade is not atomic.** `flock` can atomically convert shared → exclusive on the + same fd. On Windows you must unlock and relock, which opens a race window. The existing + callers (`AcquireSharedRepoLock` / `AcquireExclusiveRepoLock`) acquire once at startup, so + this is tolerable — but assert it rather than assuming it, and cover it in + `lock_test.go`. + +This is the most self-contained piece of M1 and the best place to start. + +## Design: signal handling + +`src/cli/process.go` registers `SIGHUP`, `SIGINT`, `SIGQUIT`, `SIGABRT`, `SIGTERM` and exits +with `128 + signum`. + +On Windows, Go's `signal` package delivers only `os.Interrupt` (Ctrl-C) and +`syscall.SIGTERM` (synthesised). The rest are unusable. Narrow the set behind a build tag, +and note that the `128 + signum` convention is a shell idiom with no meaning on Windows — +exit `1` instead. The `AtExit` handler machinery itself is portable and stays shared. + +## Design: xattrs + +`src/fs/attr.go` uses `github.com/pkg/xattr` to store content hashes as extended attributes, +consumed by `src/test/test_step.go` (`user.plz_test`) and `src/build`. + +**A fallback already exists and is well-factored.** `RecordAttr` takes an `xattrsEnabled +bool` and delegates to `RecordAttrFile` (a sidecar file) when false. The config knob is +`config.Build.Xattrs`. + +So the work is small: + +1. Default `Build.Xattrs` to `false` on Windows. +2. Confirm `pkg/xattr` compiles for `GOOS=windows`. It ships a stub returning `ENOTSUP`, in + which case no build tag is needed at all and step 1 is sufficient. **Verify this in M0 + rather than assuming it** — if the stub is absent, split `attr.go` into + `attr_unix.go`/`attr_windows.go`. + +Note the chmod-to-set-xattr dance in `RecordAttr` (chmod `|0200`, set, restore) becomes +dead code on Windows, which is fine — it is behind the `xattrsEnabled` branch. + +## Design: the `.exe` model + +The question with the widest blast radius, and the one most likely to be over-engineered. + +Today, executability is a mode bit: `core.BuildTarget.OutMode()` returns `0555` for binary +targets and `0444` otherwise, applied by `src/build/build_step.go`. `src/fs/executable.go` +checks `(mode & 0111) == 0`. Nothing anywhere handles `.exe`. + +**Recommendation: do not change the core model.** + +`OutMode()` stays as-is — the mode bits are simply ignored by Windows, which is harmless. +The `.exe` suffix becomes a *rule-level* concern, handled in cc-rules (M5) and the go +plugin (M8) where the output name is chosen. This keeps the `plz-out` layout identical +across platforms and avoids threading a platform flag through `BuildTarget`. + +Core only needs `.exe` awareness in the three places where it *looks up* an executable +rather than declaring one: + +- `src/fs/executable.go` — the `mode & 0111` check and the `$PATH` search in `Executable()`. + Use `PATHEXT` on Windows. +- `src/run/run_step.go` — `!strings.Contains(args[0], "/")` decides "is this a bare command + name". Needs to consider `\` too. +- `tools/please_shim/main.go` — `filepath.Join(Location, "please")`. + +If this turns out wrong — specifically, if `plz run` on a `cc_binary` cannot find its output +without core knowing about `.exe` — revisit before M6 rather than patching around it. + +## Reuse rather than reinvent + +Already in the tree, correct, and currently under-used: + +- **`src/fs/executable.go` `splitPathList`** — a correct `os.PathListSeparator`-based split, + used only by the FreeBSD `Executable()` fallback. Promote it to exported + `fs.SplitPathList`/`fs.JoinPathList` and use it for all 7 raw `":"` splits in M2. Do not + write a new one. +- **`rules/misc_rules.build_defs` `is_platform`** — the platform conditional for BUILD + files. `src/BUILD.plz` uses it for the Linux-only `ldd` static-link assertion; that is the + pattern to copy for Windows-conditional packaging. +- **`src/core/state.go` `ForArch`** — per-arch config layering (`.plzconfig__`). + No changes needed; `.plzconfig_windows_amd64` slots straight in. +- **`config.Build.Xattrs`** — the xattr fallback, above. +- **`Remote.Shell`** (`src/core/config.go`, `src/remote/remote.go`) — the only configurable + shell in the codebase today, and the precedent for the local `[build] Shell` knob in + `02-shell-and-build-actions.md`. +- **`cli.Arch`** (`src/cli/flags.go`) — generic `OS_ARCH` parsing. `windows_amd64` parses + today with no code change. diff --git a/docs/design/windows/02-shell-and-build-actions.md b/docs/design/windows/02-shell-and-build-actions.md new file mode 100644 index 0000000000..54807a0e54 --- /dev/null +++ b/docs/design/windows/02-shell-and-build-actions.md @@ -0,0 +1,237 @@ +# Build Actions and the Bundled Shell + +Status: **Draft** · Milestone: M3 · Last updated: 2026-09-10 + +ADR for decision **D2**: ship a POSIX shell inside the Windows release rather than depending +on one being installed. See `00-overview.md` for the decision summary. + +## The problem + +Every build action, test and `plz run --cmd` is a **shell string**, not an argv. It is +executed by `src/process/process.go`: + +```go +func BashCommand(binary, command string, exitOnError bool) []string { + if exitOnError { + return []string{binary, "--noprofile", "--norc", "-e", "-u", "-o", "pipefail", "-c", command} + } + return []string{binary, "--noprofile", "--norc", "-u", "-o", "pipefail", "-c", command} +} +``` + +`binary` is the literal string `"bash"` at every local call site +(`process.go`, `src/run/run_step.go`). Only the *remote* execution path is configurable +(`Remote.Shell`, `src/core/config.go`). + +The command strings are not incidentally shell-shaped — they are genuinely shell. The +built-in rules use `&&`, `>`, `echo`, `mkdir`, `cp -r`, `mv`, `xz`. The cc rules go much +further: backticks, `find`, `sort`, `sed`, `tr`, and `; R=$?; …; exit $R`. + +The worst single line, from `cc-rules` `build_defs/cc.build_defs` (`_binary_build_flags`): + +```sh +find . -name '*.o' -or -name '*.a' | sort \ + | sed -e 's/\(.*\)/"-Wl,-force_load","\1"/' | tr '\n' , | sed -e 's/.$//' +``` + +Windows has none of this. + +## Options considered + +| Option | Verdict | +|---|---| +| **Require MSYS2 / Git Bash on PATH** | Rejected. Every user needs an extra install, and MSYS2's `/c/foo` ⇄ `C:\foo` path translation is applied heuristically to arguments that look like paths — which silently mangles compiler flags. | +| **Rewrite cc rules to be shell-free** | Deferred, not rejected. Moving `find`/`sort`/whole-archive assembly into `please_cc` is architecturally cleaner and would benefit every platform. But it is a large change in a second repo, and blocking the Windows port on it inverts the priorities. Revisit after M6. | +| **Bundle busybox-w64** | **Chosen.** Pins behaviour, needs nothing installed, no path translation. Costs one vendored binary (~700KB) in the release. | + +## Verification results + +Two rounds: Linux BusyBox 1.36.1 first, then the real **busybox-w64 1.38.0-FRP-6075** +(, SHA-256 +`07bb1e5b095b00d68a695481f9240879f33c5724b40aa2308f999d54ed78f075`) under Wine 9.0. + +**The two rounds disagreed, and the w64 result is the one that counts.** The caution +originally written here — *"re-run every check against the actual busybox.exe"* — was +load-bearing. + +### busybox-w64 ships a `bash` applet + +Better than expected. Its applet list includes `bash` as well as `sh` and `ash`, so Please's +hardcoded `"bash"` resolves if `busybox.exe` is copied or hardlinked to `bash.exe`. No +indirection needed for a first cut. + +### Shell flags — `--noprofile`/`--norc` are rejected + +Linux busybox tolerated bash's full flag set. **busybox-w64 does not:** + +```console +$ wine bash.exe --noprofile --norc -e -u -o pipefail -c 'echo ok' +bash: bad option '--noprofile' +``` + +Everything else is accepted, with semantics identical to bash: + +```console +$ wine bash.exe -e -u -o pipefail -c 'false; echo REACHED' # exit=1, not reached +$ wine bash.exe -e -u -o pipefail -c 'echo "$NOPE"' # NOPE: parameter not set, exit=2 +$ wine bash.exe -u -o pipefail -c 'false | true' # exit=1 +``` + +**Consequence: `ShellArgs` is mandatory, not a hedge.** `BashCommand` must drop +`--noprofile --norc` on Windows. Those two flags exist to stop bash sourcing user rc files; +busybox's shell has no rc files to source, so dropping them loses no hermeticity. + +### The cc-rules pipeline works verbatim + +Under Wine, through `plz.exe`, as a real build action: + +```python +genrule( + name = "findpipe", + outs = ["found.txt"], + cmd = "mkdir -p d/e && touch d/a.o d/e/b.a && find . -name '*.o' -or -name '*.a' | sort | tr '\\n' ',' > $OUT", +) +``` + +produces `./d/a.o,./d/e/b.a,`. This is the construct `_binary_build_flags` depends on, and it +is the strongest evidence for D2. + +`cat $SRCS | sort > $OUT` also produces correctly sorted output across two source files. + +### Applet coverage + +Present in busybox-w64 and used by Please's rules: `sh`, `ash`, `bash`, `find`, `sort`, `sed`, +`tr`, `cat`, `cp`, `mv`, `rm`, `mkdir`, `echo`, `printf`, `test`, `dirname`, `basename`, +`xargs`, `cut`, `which`, `env`, `tar`, `gzip`, `unzip`, `head`, `tail`, `wc`, `tee`, `touch`, +`ln`, `readlink`, `realpath`, `grep`, `awk`, `flock`, `install`, `make`. + +**Gaps:** + +1. **`pkg-config` is absent.** No applet, no shim. Documented as unsupported on Windows — + see `03-cc-toolchain.md`. +2. **`zip` is absent** (only `unzip`). Relevant to the M4 `.zip` release target, which is + produced on Linux, so not a problem. +3. **`xz` compression.** Linux busybox `xz` is decompress-only (`xz -zc` → `invalid option + -- 'z'`). `rules/misc_rules.build_defs` uses `xz -zc -T 0 $SRCS > "$OUT"` for + `tarball(xzip = True)`. Release artifacts are produced on Linux so this is not on the + user's critical path — gate the rule on `is_platform(os = "linux")`. Re-verify against + busybox-w64, whose applet list does include `xz`. + +## Design + +### Config + +Add to `[build]`, mirroring the existing `Remote.Shell`: + +```ini +[build] +Shell = bash ; unix default +ShellArgs = --noprofile ; repeatable +ShellArgs = --norc +ShellArgs = -u +ShellArgs = -o +ShellArgs = pipefail +``` + +On Windows the default resolves to the bundled `bash.exe` (busybox) with +`ShellArgs = -u -o pipefail` — i.e. the same set **minus `--noprofile --norc`**, which +busybox-w64 rejects. The `-e` flag stays conditional on `target.ShouldExitOnError()` and is +appended by `BashCommand`, not configured. + +Verified working shape (from the probe): + +```go +func BashCommand(binary, command string, exitOnError bool) []string { + argv := append([]string{binary}, shellArgs...) // platform-specific + if exitOnError { + argv = append(argv, "-e") + } + return append(argv, "-u", "-o", "pipefail", "-c", command) +} +``` + +### Code changes + +- `src/process/process.go` — `ExecWithTimeoutShellStdStreams` takes the shell from config + instead of the literal `"bash"`. `BashCommand` gains an args parameter. +- `src/run/run_step.go` — same. +- `src/cache/cmd_cache.go` — replace hardcoded `exec.Command("sh", "-c", …)` (two sites) + with the same knob. + +### Packaging + +Vendor `busybox.exe` via a `remote_file` with a pinned SHA-256, and add it to +`//package:installed_files` under `is_platform(os = "windows")`. The pattern to copy is the +Linux-only `ldd` assertion in `src/BUILD.plz`. + +Pin an exact release. busybox-w64 is a third-party fork +(); record the source URL, version and hash in +`third_party/binary/BUILD` so the provenance is auditable, and note the licence (GPL-2.0) +in the release's licence file. + +## The path-format rule + +**Implemented and verified.** `BuildEnv.normalisePathSeparators` (`src/core/pathsep_windows.go`) +rewrites the paths Please generates; `TestBuildEnvironmentUsesForwardSlashes` asserts the +invariant. + +The hazard is narrower than first assumed, and worth stating precisely, because the obvious +mental model is wrong. Shell *variable expansion* does not reprocess escapes, so a backslash +path survives `echo "$TMP_DIR"` and `printf '%s' "$TMP_DIR"` intact. It is passing the value +to something that interprets its own arguments that destroys it. Measured under Wine: + +```console +$ # cmd = echo placeholder | sed -e "s#placeholder#$TMP_DIR#" > $OUT +Z:^Impclaude-1000-...scratchpadwinrepoplz-out^Impsedtest._build +``` + +`\t` became a literal tab and every other backslash was consumed. The C/C++ rules build their +link line with `sed`, so this is squarely on the path of the primary use case. + +Build actions receive paths through the environment — `$TMP_DIR`, `$OUT`, `$OUTS`, `$SRCS`, +`$SRCS_`, `$TOOLS_` — assembled in `src/core/build_env.go`. Those values are +interpolated into a **shell string**, where `\` is an escape character. A Windows path like +`C:\plz-out\tmp\foo` becomes `C:plz-outtmpfoo` after one round of shell processing. + +**The rule: Please uses forward slashes everywhere inside `plz-out` and everywhere in the +build environment, on every platform, including Windows.** + +Three reasons: + +1. **Win32 accepts forward slashes.** `CreateFileW` and the whole `Win32` file API treat `/` + and `\` interchangeably. So does MinGW GCC. So does busybox. +2. **It keeps hashes identical across platforms.** Please's cache is content-hash based over + the rule definition and environment. If `$OUT` is `a\b` on Windows and `a/b` on Linux, + every target hash diverges — which is correct but wasteful, and makes cross-platform + remote cache sharing impossible. +3. **It is the smaller change.** `filepath.Join` produces `\` on Windows, so the conversion + point is well-defined: normalise on the way *into* the build environment + (`BuildEnvironment`, `toolPath`) rather than auditing every producer. + +**Exceptions**, which must be explicit and commented: + +- Absolute paths with a drive letter (`C:/...`) are fine with forward slashes and should + keep the drive letter. +- UNC paths (`\\server\share`) cannot be normalised. Detect and reject them as a repo root + with a clear error rather than producing corrupt commands. +- Paths passed to Windows APIs directly (not through the shell) keep whatever + `filepath` produces. Only the *build environment* is normalised. + +Add a test in `src/core/build_env_test.go` asserting no `\` appears in any value returned by +`BuildEnvironment` on Windows. + +### Related: `HOME` and `TMPDIR` + +`src/core/build_env.go` sets `HOME=tmpDir` and `TMPDIR=tmpDir` for every action. On Windows, +tools look at `USERPROFILE` and `TEMP`/`TMP`. Set all of them (M2), pointing at the same +normalised tmp dir, so the hermetic-environment guarantee holds for Windows-native tools too. + +## Exit criterion + +```bash +# under Wine, with the bundled shell +wine plz-out/bin/windows_amd64/src/please.exe build //test/genrule:pipeline_test +``` + +where the target is a `genrule` with `cmd = "cat $SRCS | sort > $OUT"`. That exercises +argument interpolation, a pipe, a redirect and two applets in one action. diff --git a/docs/design/windows/03-cc-toolchain.md b/docs/design/windows/03-cc-toolchain.md new file mode 100644 index 0000000000..3acccc1fb1 --- /dev/null +++ b/docs/design/windows/03-cc-toolchain.md @@ -0,0 +1,304 @@ +# C/C++ Toolchain on Windows + +Status: **Draft** · Milestone: M5 · Workstream B (`please-build/cc-rules`) · Last updated: 2026-09-10 + +ADR for decision **D1**: target MinGW-w64 GCC first, MSVC later. Building C++ projects is the +programme's driving use case, so this is the document that matters most. + +## Where the rules live + +**Not in this repo.** `plugins/BUILD` pins `please-build/cc-rules` at `v0.7.3`, fetched as a +plugin subrepo via `plugin_repo` (`rules/subrepo_rules.build_defs`), from +`https://github.com/please-build/cc-rules/archive/v0.7.3.zip`. + +The only in-repo consumer of cc rules is `tools/sandbox/BUILD`. + +There is also a **dead** `[Cpp]` config section in `src/core/config.go` (`CCTool`, `CppTool`, +`LdTool`, `ArTool`, …). It is a no-op in plz v17+ — the code even warns *"You're overriding +field %s which is deprecated in plz v17+"*. Ignore it; the live config is the plugin's +`[PluginConfig …]` block, addressed as `CONFIG.CC.*` in BUILD files and `-o plugin.cc:…` on +the command line. + +## Decision: MinGW-w64 first + +### Why not MSVC first + +MSVC is what most real Windows C++ projects use, and it is the eventual target. But choosing +it first means taking on three unknowns simultaneously: + +- a completely different flag dialect (`/c`, `/Fo`, `/EHsc`, `/link`, `.obj`, `.lib`), +- `vcvarsall.bat` environment discovery (INCLUDE/LIB/PATH, SDK version selection), +- and a Windows host, because `cl.exe` does not run on Linux. + +That last point is disqualifying on its own. It would block every C++ change on the same +native-Windows CI that M9 exists to defer. + +### Why MinGW + +`x86_64-w64-mingw32-g++` **runs on Linux**. That is Axis 2 from `00-overview.md`: the entire +C++ codepath becomes testable on the development platform, producing real PE32+ binaries, +before any Windows machine exists. + +And the flag surface is almost entirely reusable. `_binary_build_flags` in +`build_defs/cc.build_defs` already branches Apple-vs-GNU throughout, via `please_cc`'s +expression language: + +```python +oflags += ["""'{{ !ld64 && !appleld ? ["-Wl,--start-group", "-Wl,--whole-archive"] }}'"""] +``` + +MinGW's `ld` is GNU ld. **It takes the GNU branch for free.** That is the entire payoff of +D1. + +## `please_cc` — the extension point + +The rules do not emit compiler command lines directly. They emit `please_cc` invocations: + +``` +"$TOOLS_PLEASE_CC" cc "$TOOLS_CC" -c -I . ${SRCS_SRCS} +``` + +`please_cc` (`tools/please_cc/`, ~1000 lines of Go) runs the compiler with `-v -Wl,-v`, +regex-matches the output to identify the compiler *and* the linker it will invoke, evaluates +any `{{ … }}` expressions in the arguments against that identity, and `exec`s the real tool. + +Known identities today (`tools/please_cc/cctool/tool.go`): GCC, Clang, Apple Clang, GNU ld, +GNU gold, LLD, ld64, Apple ld. + +### The assumption D1 rests on — **confirmed** + +Measured against two toolchains, both by regex and in the real pipeline: + +| Toolchain | Compiler line | Linker line | `please_cc` says | +|---|---|---|---| +| WinLibs GCC 16.2.0 (Windows-native, under Wine) | `gcc version 16.2.0 (MinGW-W64 …)` | `GNU ld (Binutils for MinGW-W64 …) 2.47.20260726` | GCC 16.2.0 / GNU ld 2.47.20260726 | +| Ubuntu `g++-mingw-w64-x86-64` 13 (Linux cross) | `gcc version 13-win32 (GCC)` | `GNU ld (GNU Binutils) 2.41.90.20240122` | GCC 13 / GNU ld 2.41.90 | + +Both match the existing GCC and GNU ld matchers, and the Clang matcher correctly does not. +**No new matchers are needed.** + +Note the Ubuntu build reports `13-win32`, so the captured version is a bare `13`. +`MustParseVersion` handles a single component, and `Compare` zero-pads the shorter of two +version numbers, so `gcc >= 9` style expressions still evaluate correctly. + +The original check, kept for reference: + +```bash +x86_64-w64-mingw32-g++ -v -Wl,-v 2>&1 | head -20 +``` + +Check the output against these two patterns from `cctool/tool.go`: + +``` +^gcc (?:version|\(GCC\)) (?P[\d.]+) +^GNU ld (?:\(.*\) |version )(?P\d+(?:\.\d+)*) +``` + +If either fails to match, `please_cc` exits with *"failed to identify C/C++ compiler; please +report the output of … to "* and **every cc target fails**. If that happens, the +fix is small (add a matcher) but it must be known up front, not discovered mid-milestone. + +Note the ordering constraint documented in `tool.go`: the Apple Clang matcher must run before +the Clang matcher because Go's `regexp` has no zero-length assertions. Any new matcher must be +placed with the same care. + +### `please_cc` needs a Windows build + +`tools/please_cc/please_cc.go` ends in: + +```go +func execvp(file string, args []string) error { + execFile, err := exec.LookPath(file) + ... + return syscall.Exec(execFile, append([]string{file}, args...), os.Environ()) +} +``` + +`syscall.Exec` does not exist on Windows. Add `execvp_windows.go` that spawns, waits and +propagates the exit code — the same shape as `process.ExecReplace` in `01-os-abstraction.md`. + +**Priority note:** under Please's cross-compilation model, `tools` are always built for the +*host* arch. So for Axis 2 (cross-building C++ from Linux) `please_cc` runs as a Linux +binary and this is not on the critical path. It is only required for a native Windows `plz`. +Sequence it accordingly. + +## Output extensions + +Current naming, all in `build_defs/cc.build_defs`: + +| Rule | Output | MinGW needs | +|---|---|---| +| `cc_object` | `.o` | unchanged — MinGW uses `.o` | +| `cc_library` | `lib.a` | unchanged — MinGW uses `.a` | +| `cc_static_library` | `lib.a` | unchanged | +| `cc_shared_object` | `lib.so` | **`.dll`**, plus `lib.dll.a` (import library) as an `optional_out` | +| `cc_binary` | `` (bare) | **`.exe`** | +| `cc_test` | `` (bare) | **`.exe`** | + +This is the one place MinGW's GNU-ness does not carry over, and it is why D1 saves work +rather than eliminating it: only two of six naming schemes change. + +Gate on `CONFIG.OS == "windows"`. The rules already precedent OS-conditional logic: + +```python +if CONFIG.TARGET_OS == "darwin" and static: + log.warning("%s: statically-linked binaries are unsupported on Darwin; ignoring 'static'") + static = False +``` + +Also add `windows_amd64` to `SUPPORTED_ARCHITECTURES` in `build_defs/arch.build_defs`, which +today lists only the five released platforms. + +### Consequence for `plz run` and `cc_test` + +`cc_binary` sets `binary = True` and `outs = [name]`. Changing the out to `name + ".exe"` +changes the path `plz run` and the test runner resolve. Per `01-os-abstraction.md`, core is +*not* being taught about `.exe`, so verify early that `plz run //some:cc_binary` works with +the renamed output — this is the specific case that would force a rethink. + +## Flag review + +Every flag in `_build_flags` and `_binary_build_flags`, assessed for PE/COFF via MinGW. + +| Flag | Where | Verdict | +|---|---|---| +| `-c`, `-I .` | `_library_cmds` | Fine | +| `-fPIC` | `_build_flags` | **Remove for Windows.** Meaningless for PE; MinGW emits *"-fPIC ignored for target"* on every compile. Noise, not breakage — but it pollutes every build log. | +| `-fdata-sections`, `-ffunction-sections` | `_build_flags` (LdGarbageCollection) | Fine — supported by MinGW GCC | +| `-fno-unique-section-names` | `_build_flags` | Clang-only branch already; no change | +| `-Wl,--start-group` / `--end-group` | `_binary_build_flags` | Fine — GNU ld branch, taken automatically | +| `-Wl,--whole-archive` / `--no-whole-archive` | `_binary_build_flags` | Fine — same | +| `-Wl,--build-id=none` | `_binary_build_flags` | **Remove for Windows.** ELF-only; MinGW ld errors or warns. Guard the existing `{{ gnuld \|\| gold \|\| lld ? … }}` expression with an OS check, since MinGW ld *does* match `gnuld`. | +| `-Wl,--gc-sections` | `_binary_build_flags` | Fine — MinGW ld supports it | +| `-Wl,--strip-all` | `_binary_build_flags` | Fine | +| `-shared` | `_binary_build_flags` | Fine — produces a DLL | +| `-static` | `cc_binary` | Works, but means "static libgcc/libstdc++" rather than a fully static image. Document the difference; do not silently disable it as Darwin does. | +| `-static-libgcc` | `cc_binary` (gcc branch) | Fine | +| `--coverage`, `-fprofile-dir=.` | `_COVERAGE_FLAGS` | Works with MinGW + `gcov`, but the `cover` command copies `.gcno` from `$GCNO_DIR` and shells out — verify end-to-end in M6 rather than assuming | +| `DefaultLdFlags = -lpthread -ldl` | `.plzconfig` | **Both wrong on MinGW.** `-ldl` does not exist; `-lpthread` is unnecessary (winpthreads is implicit) and may not resolve. Needs a `windows_amd64` override — set `DefaultLdFlags` to empty. | + +The `-fPIC` and `-Wl,--build-id=none` changes both need a way to express "target OS is +windows" inside the `{{ … }}` expression language, or an `if CONFIG.OS == "windows"` in the +Python-side flag assembly. **Prefer the latter** — the expression language identifies +*tools*, not targets, and overloading it with OS knowledge would be a category error. + +## `pkg-config` + +`_build_flags` and `_binary_build_flags` emit backticked `pkg-config --cflags` / +`--libs` invocations. There is no `pkg-config` in busybox and no Windows convention for it. + +**Decision: document as unsupported on Windows.** Leave the codepath intact — it will simply +fail if used — and have users pass `compiler_flags`/`linker_flags` explicitly. Do not ship a +`pkg-config` shim; that is a package-management problem, not a build-system one. + +Emit a clear diagnostic rather than a shell "command not found": add a check in the rules +that raises at parse time when `pkg_config_libs` is set and `CONFIG.OS == "windows"`. + +## What MSVC would later require + +Recorded so the extension point stays visible, not as scheduled work. + +1. **New `cctool` matchers.** `cl.exe /?` prints *"Microsoft (R) C/C++ Optimizing Compiler + Version 19.NN.NNNNN"*; `link.exe` prints *"Microsoft (R) Incremental Linker Version …"*. + Note that `please_cc` currently probes with `-v -Wl,-v`, which MSVC does not understand — + the probe itself needs to become tool-family-aware, which is a deeper change than adding + a regex. +2. **A second flag dialect.** Not a translation layer — a parallel set of flag-assembly + functions selected by toolchain, because the mappings are not one-to-one + (`--whole-archive` → `/WHOLEARCHIVE:lib`, `--gc-sections` → `/OPT:REF`, and + `--start-group` has no equivalent at all because MSVC's linker does not care about + library order). +3. **Separate archiver and linker tools.** MinGW links through the compiler driver + (`$TOOLS_CC`); MSVC needs `lib.exe` and `link.exe` as distinct tools. The plugin config has + no `LdTool` today — it would need one. +4. **`vcvarsall` environment discovery.** `INCLUDE`, `LIB`, `LIBPATH` and SDK version + selection. This conflicts with Please's hermetic-environment model + (`src/core/build_env.go` builds the env from scratch rather than inheriting), so it needs + a deliberate design — most likely a `pass_env` allowlist plus a documented setup step. +5. **`.obj`/`.lib`/`.pdb`** output naming, and `/showIncludes` if header scanning is ever + added (it is not today — see below). + +`clang-cl` is a middle path: one binary, MSVC-compatible flags, already partially matched by +the existing Clang regex. It still needs items 2, 4 and 5. + +## What is pleasantly absent + +**No header-dependency scanning.** There is no `-MD`, `-MF`, `-MMD` or `.d` handling +anywhere in the rules. Please does not scan headers; correctness comes from declared +`hdrs`/`private_hdrs` plus the sandboxed tmp dir. + +This removes a whole class of portability work — no dep-file path munging, no +`/showIncludes` parsing when MSVC eventually lands. + +## Development and upstreaming + +Point `plugins/BUILD` at a fork or branch revision during development: + +```python +plugin_repo( + name = "cc", + plugin = "cc-rules", + revision = "", +) +``` + +Upstream to `please-build/cc-rules` as the final step of M5. Per `CONTRIBUTING.md`, raise the +issue in that repo *before* writing the code. + +Extend the plugin's own CI (`.github/workflows/plugin_test_cc.yaml`) with a MinGW +cross-compile job on `ubuntu-latest` — `apt-get install g++-mingw-w64-x86-64` plus +`plz build --arch windows_amd64 //test/...`. + +## Exit criterion — met + +On a Linux box, in the cc-rules repo with `cc-rules-windows.patch` applied: + +```console +$ plz build --arch windows_amd64 //test/binary:test_binary +plz-out/bin/windows_amd64/test/binary/test_binary.exe +$ file plz-out/bin/windows_amd64/test/binary/test_binary.exe +PE32+ executable (console) x86-64, for MS Windows +``` + +A `cc_library` + `cc_binary` + `cc_shared_object` triple produces `lib.a`, `prog.exe` and +`libshared.dll`; `prog.exe` links against the static library and prints the right answer under +Wine. The same targets still produce `prog` and `libshared.so` on Linux, and all 12 of +cc-rules' own tests pass there. + +### What the experiments changed + +1. **Module-level `CONFIG` does not see the target architecture.** The first attempt defined + `_EXE_SUFFIX` as a module-level constant and it silently had no effect — these build defs + are subincluded, and `CONFIG.OS` at module level reflects the host. It has to be a function + evaluated per call. This is a trap for any future platform-conditional logic here. +2. **A repeatable config key cannot be cleared by assigning empty.** `defaultldflags =` + yields a list containing one empty string rather than an empty list, which + `_escape_linker_flag` turns into a bare `-Wl,` and the linker rejects with + `cannot find : Invalid argument`. Set an actual value instead. +3. **`-lpthread` is fine on MinGW**, so only `-ldl` had to go. The Windows default is + `defaultldflags = -lpthread`. +4. **`-fPIC` and `-Wl,--build-id=none` were passed and neither broke the link.** They remain + worth removing as noise, but they are not blockers, so that is deferred rather than done. +5. **A `cc_shared_object` that sets `out` explicitly keeps whatever extension it was given.** + The rules' *default* is now correct, but a BUILD file hardcoding `out = "libfoo.so"` — as + cc-rules' own `//test/so:libdolphin` does — will still produce a `.so` on Windows. That is + arguably right, since `out` is an explicit instruction, but it is a portability trap worth + documenting for users. + +### Still open + +- **`please_cc` has no `windows_amd64` release.** `tools/BUILD` fetches it as a prebuilt + binary per platform with a pinned hash, so upstreaming needs a Windows build published + alongside the others. It did not block this work because tools are built for the *host*, + which is Linux under Axis 2 — but a native Windows `plz` will need it. +- ~~**`UnitTest++` does not compile for Windows** as packaged: it needs its `Win32/` platform + sources, which the plugin's target does not include.~~ **Wrong, and it cost time.** The + plugin's `unittest.build` has selected `Win32` on Windows since before this port started. + What actually blocked `cc_test` was that the sources were selected correctly and then + compiled by `/usr/bin/c++`: the UnitTest++ test main is a `cc_library` *inside the plugin*, + and a target inside a plugin sees the `PluginConfig` defaults rather than the using repo's + `[Plugin "cc"]` values. The toolchain the user configures applies to their code and not to + the plugin's. Fixed by defaulting the tools per platform in the build defs, which is where + the rest of the platform handling already lives. `//test/windows:cc_test_test` guards it. +- `SUPPORTED_ARCHITECTURES` still lacks `windows_amd64`; it gates the plugin's own release + rather than its use. diff --git a/docs/design/windows/04-release-and-ci.md b/docs/design/windows/04-release-and-ci.md new file mode 100644 index 0000000000..4ad467d512 --- /dev/null +++ b/docs/design/windows/04-release-and-ci.md @@ -0,0 +1,275 @@ +# Release Pipeline and CI + +Status: **Draft** · Milestone: M4 · Last updated: 2026-09-10 + +How `windows_amd64` artifacts get built on Linux, signed and published. This is Axis 1 from +`00-overview.md`. + +## The template: FreeBSD + +Please already cross-compiles a platform it does not test natively on CircleCI. The +`build-freebsd` job runs **on Linux**, in a Docker image, using a prebuilt Linux `plz`: + +```yaml +build-freebsd: + docker: + - image: ghcr.io/thought-machine/please_freebsd_builder:20260318 + steps: + - checkout + - attach_workspace: { at: /tmp/workspace } + - run: + name: Extract plz + command: tar -xzf /tmp/workspace/linux_amd64/please_*.tar.gz + - run: + name: Cross-compile + command: ./please/please build -p -v2 --profile ci --arch freebsd_amd64 //package:release_files + - persist_to_workspace: + root: plz-out/pkg + paths: [ freebsd_amd64/* ] +``` + +`build-windows` is a near-verbatim copy. **Copy it; do not invent a new shape.** + +Note the dependency: `build-freebsd` `requires: [build-alpine]`, because `build-alpine` +produces the canonical `linux_amd64` release tarball that every cross job extracts and runs. +`build-windows` takes the same dependency. + +The builder image is equally simple — `tools/images/freebsd_builder/Dockerfile` is 13 lines: + +```dockerfile +FROM ubuntu:noble +RUN apt-get update && apt-get install -y curl git gcc xz-utils && apt-get clean +RUN curl -fsSL https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz | tar -xzC /usr/local +RUN ln -s /usr/local/go/bin/go /usr/local/bin/go && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt +RUN GOOS=freebsd go install std +``` + +`tools/images/windows_builder/Dockerfile` is the same with `GOOS=windows go install std`, +plus `g++-mingw-w64-x86-64` in the apt line for Axis 2. Add the image name to the list in +`tools/images/build.sh`; images are tagged by date and pushed to +`ghcr.io/thought-machine/please_`. + +## Prerequisites + +Four things must land before the cross-build can even start. + +### 1. Go toolchain hash + +`third_party/go/BUILD` pins per-platform SHA-256s for the Go distribution: + +```python +go_toolchain( + name = "toolchain", + hashes = [ + "…", # go1.27.0.darwin-amd64.tar.gz + … + ], + version = "1.27.0", +) +``` + +Add `go1.27.0.windows-amd64.zip`. Note Windows Go distributions are `.zip`, not `.tar.gz` — +confirm the go plugin's `go_toolchain` rule handles that, or the hash is useless. + +### 2. arcat — the real gate + +**What arcat is.** A small standalone Go binary (`github.com/please-build/arcat`, 6 source +files) that Please downloads as a prebuilt release keyed by `HOSTOS_HOSTARCH`. It is the +built-in archive toolkit, so rules never depend on the host having `tar`, `zip`, `ar` or +`unzip`, or on those behaving consistently. Three uses: + +| Invocation | Used by | +|---|---| +| `arcat x` — extract zip/tar | `remote_file(extract=True)`, `http_archive`, **`plugin_repo`** | +| `arcat tar` — create tarballs | `tarball()` | +| `arcat ar -r` / `--combine` — create/merge `.a` static libraries | **`cc_library`** | + +**Why it is more severe than "a hash to add".** `plugin_repo` extracts the plugin zip with +it, and *every language plugin is delivered that way*. So without arcat, Windows cannot load +the cc rules at all — and `cc_library` then needs it again to build `.a` archives. It is on +the critical path for any real build. Measured: `plz.exe` under Wine fails with + +``` +failed to generate internal package: arcat tool not supported for platform: windows_amd64 +``` + +as soon as a plugin is involved. Simple genrules and parsing work without it, which is why +the earlier assessment understated this. + +**Why it is nevertheless easy.** arcat is pure Go with **no** `syscall`, `x/sys/unix` or cgo +usage anywhere. Verified: it cross-compiles to a PE32+ binary, and under Wine both +critical paths work — `arcat x` extracts a zip correctly, and `arcat ar -r` produces a `.a` +that MinGW links into a working `.exe`. + +One snag, and it is not a Windows one: arcat's `go.mod` still says `go 1.17` while the code +uses generics, so it fails to build on *any* platform with a modern toolchain +(`implicit function instantiation requires go1.18 or later`). Bumping the directive is a +one-line fix that should go upstream regardless. + +**So the work is:** publish a `windows_amd64` arcat release alongside the others, then add its +hash to the switch in `src/parse/internal_package.go`. The switch is exhaustive and hard-fails +by default, which is what produces the error above. + +### 3. `.plzconfig_windows_amd64` + +Per-arch config, layered by `state.ForArch` (`src/core/state.go`). Compare +`.plzconfig_freebsd_amd64`, which is two lines. + +```ini +[Plugin "cc"] +cctool = x86_64-w64-mingw32-gcc +cpptool = x86_64-w64-mingw32-g++ +artool = x86_64-w64-mingw32-ar +defaultldflags = ; -lpthread -ldl are both wrong on MinGW + +[build] +xattrs = false + +[sandbox] +build = false +test = false +``` + +### 4. `//package:installed_files` must stop pulling in the Linux sandbox + +```python +filegroup( + name = "tools", + srcs = [ + "//tools/build_langserver", + "//tools/sandbox:please_sandbox", # <- C binary of Linux-namespace code + ], +) +``` + +`please_sandbox` is a `c_binary` whose source is `#ifdef __linux__` throughout, with a no-op +fallback. It has no meaning on Windows and building it requires a C toolchain for the target. +Gate it with `is_platform(os = "linux")` (`rules/misc_rules.build_defs`; see `src/BUILD.plz` +for the usage pattern). + +**This is already latently wrong for the FreeBSD cross build**, which gets away with it +because `.plzconfig_freebsd_amd64` points `cctool` at the host Linux `cc`. Fixing it properly +benefits both platforms. + +## Packaging + +### Add a `.zip` alongside the tarballs + +`package/BUILD` produces `please_.tar.gz`, `.tar.xz` and a tools tarball. Windows +has no guaranteed `tar -xJ`; ship a zip. + +Note from `02-shell-and-build-actions.md`: busybox has `unzip` but not `zip`, and its `xz` is +decompress-only. Both are irrelevant here because release artifacts are *produced* on Linux — +but it does mean the xz tarball rule should be gated to Linux rather than attempted on +Windows. + +### Contents of the Windows release + +| File | Source | +|---|---| +| `please.exe` | `//src:please` | +| `busybox.exe` | vendored `remote_file`, pinned hash — see `02-shell-and-build-actions.md` | +| `build_langserver.exe` | `//tools/build_langserver` | +| — | **no** `please_sandbox` | + +`//package:installed_files` sets `entry_points = {"please": "please"}` — verify this resolves +with the `.exe` suffix, or add a Windows-conditional entry point. + +### The `plz` alias + +`install.sh` does `ln -sf please plz`. On Windows, symlinks need Developer Mode. Ship a +`plz.cmd` one-liner (`@"%~dp0please.exe" %*`) instead — a file copy is also acceptable but +doubles the download size. + +## Bootstrap and self-update + +### `pleasew` + +`pleasew` is POSIX `sh` and has an explicit OS whitelist: + +```sh +Linux|Darwin|FreeBSD) ;; +*) echo "Please does not support the %s operating system"; exit 1 ;; +``` + +**Do not make it polyglot.** Add a sibling `pleasew.ps1` (PowerShell) implementing the same +flow: find repo root, read `.plzconfig`/`.plzconfig__` for the version, download +`${URL_BASE}/windows_amd64/${VERSION}/please_${VERSION}.zip`, extract, exec. + +`pleasew` is embedded into the binary via `src/assets/BUILD` (`plz init` writes it out), so +`pleasew.ps1` needs adding there and to the root `BUILD` filegroup too. + +### `src/update/update.go` + +The download URL is already built from `runtime.GOOS`/`runtime.GOARCH`: + +```go +url = fmt.Sprintf("%s/%s_%s/%s/please_%s%s", DownloadLocation, GOOS, GOARCH, Version, Version, ext) +``` + +so it works as soon as the bucket has a `windows_amd64/` folder. Three things around it do +not: + +- `syscall.Exec(newPlease, …)` to hand over to the new binary → `process.ExecReplace` + (`01-os-abstraction.md`). +- `writeTarFile` recreates `tar.TypeSymlink` members → needs the M2 copy-fallback. +- The binary is opened with mode `0555` and `fileMode()` returns `0664`/`0775` → harmless on + Windows, but the symlink at the end (`please` → version dir) is not. + +Also: **Windows will not let you overwrite a running executable.** The self-updater must +rename the running `please.exe` aside before writing the new one, or update into a +version-stamped directory and switch a `.cmd` shim. The version-directory layout Please +already uses (`~/.please//`) makes the second option natural. + +### `tools/please_shim` + +Same exec-replace problem, plus `filepath.Join(Location, "please")` needs `.exe`. Covered in +`01-os-abstraction.md`. + +## CI wiring + +`.circleci/config.yml`: + +1. New `build-windows` job (copy `build-freebsd`), `requires: [build-alpine]`. +2. Add it to the workflow `jobs:` list. +3. Add it to `release-gs`'s `requires:` list alongside `build-freebsd`. + +`.circleci/release.sh`: + +```sh +release_folder /tmp/workspace/windows_amd64 windows_amd64/$VERSION +``` + +The signing globs above it are `{*_amd64,*_arm64}`, which **already match** `windows_amd64` — +so signing needs no change, but note that means an unreleased `windows_amd64` folder in the +workspace would be signed and then silently dropped. Add the `release_folder` line in the +same commit as the CI job, not later. + +The idempotency guard at the top of `release.sh` checks whether +`gs://get.please.build/linux_arm64/$VERSION/` exists. Leave it — adding Windows to it would +make the first Windows release re-upload everything. + +`tools/misc/gen_release.py` — its `_arch()` helper defaults anything non-darwin/non-freebsd +to `linux_*`. Add a windows branch, or the GitHub release assets get mislabelled. + +## Non-blocking guardrail (M0) + +Before any of the above, add a **non-blocking** job that runs: + +```bash +plz build --arch windows_amd64 //src:please +``` + +It will fail. That is the point: it makes the compile-error count visible and +monotonically decreasing, and it catches regressions from contributors who are not thinking +about Windows. Record the initial output in `appendix-baseline-errors.md`. + +## Exit criterion + +```bash +plz build --arch windows_amd64 //package:release_files +ls plz-out/pkg/windows_amd64/ +# please_.zip, please_.tar.gz, please_, please_shim_ +``` + +on a Linux CI box, with the artifacts signed by the existing `release_signer` step. diff --git a/docs/design/windows/05-testing-strategy.md b/docs/design/windows/05-testing-strategy.md new file mode 100644 index 0000000000..1960fa1a63 --- /dev/null +++ b/docs/design/windows/05-testing-strategy.md @@ -0,0 +1,291 @@ +# Testing Strategy + +Status: **Draft** · Milestone: M6 (with M9 as the follow-up) · Last updated: 2026-09-10 + +The programme constraint is that development and CI stay on Linux, with real Windows testing +deferred. This document is how that is made to work rather than merely asserted. + +## Four test loops + +| Loop | Runs | Tests | Available from | +|---|---|---|---| +| **A — compile gate** | Linux, natively | Does `plz.exe` build for `GOOS=windows`? | M0 | +| **B — C++ cross-build** | Linux, natively | Do the cc rules produce correct PE32+ artifacts? | M5 | +| **C — Wine** | Linux, under Wine | Does `plz.exe` actually *run*? | M1 onwards | +| **D — the documentation** | Windows, natively | Do the published codelabs work as written? | M10 | + +Loops A and B need no emulation at all. Loop C is where the leverage is, and it is why M6 +should start as soon as M1 produces a binary — the milestone number is a completion point, +not a start date. + +## Loop A — compile gate + +```bash +plz build --arch windows_amd64 //src:please +``` + +Wired into CI as a non-blocking job in M0 (see `04-release-and-ci.md`). Its output is the +burn-down list in `appendix-baseline-errors.md`. + +Cheap, fast, and catches the majority of M1's work. It catches nothing about behaviour. + +## Loop B — C++ cross-build (Axis 2) + +```bash +plz build --arch windows_amd64 //test/cc/... +file plz-out/bin/windows_amd64/test/cc/binary.exe +# PE32+ executable (console) x86-64, for MS Windows +``` + +`plz` runs as a native Linux binary; `x86_64-w64-mingw32-g++` runs as a native Linux binary; +the *output* is Windows. Nothing is emulated. + +**This validates roughly 80% of the C++ work**: flag assembly, output naming, the +`please_cc` tool-identification path, archive combination, transitive label propagation, and +the whole `cc_library` → `cc_binary` graph. All of it before a Windows machine exists. + +What it does not validate: that the same flags are produced when `plz` itself is running on +Windows with the bundled busybox shell. That needs Loop C. + +Assertions worth making beyond `file(1)`: + +- `x86_64-w64-mingw32-objdump -p out.exe` — check the import table names the expected DLLs. +- `x86_64-w64-mingw32-nm` on the archive — check `--whole-archive` actually pulled symbols in + for an `alwayslink` library. +- Run the produced `.exe` under Wine — which is Loop C applied to the *output* rather than to + `plz`. + +## Loop C — Wine + +**Proven, not hypothetical.** Wine 9.0 (the Ubuntu/Pop!_OS package, `apt install wine64`) +already runs `please.exe` end to end: version, `query alltargets //...`, and real builds +including shell pipelines. See `appendix-baseline-errors.md`. + +Wine implements precisely the primitives the port introduces: Job Objects, `LockFileEx`, +`CreateProcess`, console control events, `PATHEXT` resolution. That is not a coincidence — +they are the well-trodden Win32 core, which is what Wine covers best. + +Setup used: + +```bash +export WINEPREFIX=$PWD/.wineprefix WINEDEBUG=-all +wineboot --init +# busybox-w64 as the build shell +cp busybox64.exe wbin/bash.exe +export WINEPATH='Z:\path\to\wbin' +``` + +`WINEDEBUG=-all` suppresses Wine's chatter; a job-local `WINEPREFIX` keeps runs isolated. + +**Clear Please's cache between runs, or you will verify nothing.** `rm -rf plz-out` is not +enough: Please also keeps a directory cache, which under Wine lands in +`$WINEPREFIX/drive_c/users//AppData/Local/please`. A build that appears to succeed may +be replaying cached artifacts from an earlier, differently-built binary — this happened during +M0 and produced a false pass on a binary whose shell handling was in fact broken. Wipe both: + +```bash +rm -rf plz-out "$WINEPREFIX"/drive_c/users/*/AppData/Local/please +``` + +(Incidentally this confirms `os.UserCacheDir()` resolves correctly on Windows.) +Note Wine maps Unix paths to the `Z:` drive, so `$PWD` becomes `Z:\...` inside the binary — +useful to know when reading error messages. + +```bash +wine plz-out/bin/windows_amd64/src/please.exe --version +wine plz-out/bin/windows_amd64/src/please.exe build //test/cc:binary +wine plz-out/bin/windows_amd64/test/cc/binary.exe +``` + +### What to run under it + +1. **Unit tests.** Add a test macro that runs a Go test binary cross-compiled for Windows + under `wine`. The high-value packages are exactly the ones M1 and M2 touch: + `src/core` (`lock_test.go`, config loading), `src/fs`, `src/process`. +2. **The genrule smoke test** from `02-shell-and-build-actions.md` — a `cmd` with a pipe and + a redirect, proving the bundled shell is wired up. +3. **The headline end-to-end test**, combining both axes: `wine plz.exe` building a C++ + project with MinGW. This is the M6 exit criterion and the single most valuable test in the + programme, because it is the first thing that exercises busybox, the generated cc command + lines, the Windows process layer and PE output *together*. + +### CI + +**Implemented.** `test/build_defs/wine.build_defs` has two macros — `wine_go_test` for a +cross-built Go test binary, and `wine_plz_test` for `please.exe` driving a small repo laid out +the way the release is. `//test/windows` uses them; the `test-windows-wine` CircleCI job runs +them and blocks the release. + +They are labelled `wine` and excluded from the other test passes, because building them means +cross-compiling the Go standard library for another platform. `test.sh` runs them as a third +pass where Wine is installed. + +**Never run these binaries by hand in the source tree.** Under `plz test` they run in a +sandboxed temp directory; run directly from the repo root they operate on the repo. Doing that +once during this work deleted the whole of `test/` — `AsyncDeleteDir` re-executes +`os.Executable()`, which for a test binary is the test binary, with arguments it then ignores. +Everything below was found by running them through the macros, which is the only safe way. + +Things learned building it: + +- **Rename the binary to `.exe` first.** Go's `exec` on Windows will not run a file whose name + has no extension in `PATHEXT`, even given its full path, and the go plugin names test + binaries after the rule. Any test whose subject re-execs itself — `TestComplete` in + `src/core` does — fails obscurely otherwise. +- **A `go_test`'s own `data` doesn't come with it** when another rule depends on the binary, so + anything the test reads has to be repeated on the `wine_go_test`. `$DATA` has to be set to + just that, too: a `gentest` would otherwise include the test binary in it, and tests that + read `$DATA` expect only their own data. +- **Wine's hosts file leaves the `localhost` line commented out.** Anything that resolves it + hangs until it gives up; three `remote_file` tests were each burning fifteen seconds. The + macro appends the line to the prefix. +- **`~` resolves inside the shared prefix**, so anything a test writes to its home directory + leaks into the next run — and a read-only file left there cannot be replaced on Windows at + all. Tests that write to `~` set `USERPROFILE` themselves, which is what `os.UserHomeDir` + reads there. + +A Wine job on `ubuntu-latest` (`apt-get install wine64`) is cheap. Make it blocking once M1 +lands — the whole point is to catch Windows regressions from contributors who are not +thinking about Windows. + +Set `WINEDEBUG=-all` to suppress Wine's chatter, and `WINEPREFIX` to a job-local directory so +the prefix is not shared between runs. + +## Loop D — the documentation + +The codelabs at https://please.build/codelabs.html are what a new user follows, and until this +loop nothing had ever executed a line of them, on any platform. Loop D replays them on +`windows-latest` with the release zip, the way a reader would. + +It is built in three parts, and only the last needs Windows: + +- `//test/windows/codelab_script` reduces `docs/codelabs/*.md` to a plan: the files each codelab + says to create and the commands it says to run, in order. It refuses to guess. A block no rule + can classify is an error, not a skipped block, so a codelab edit that introduces one fails + `//test/windows/codelab_script/script:script_test` on Linux, in the default test pass. +- `test/windows/codelab_steps.conf` records what the Markdown cannot say, each stanza with its + reason above it: that a `.plzconfig` block is a fragment to merge rather than a whole file, + that a block is output rather than a command, that a step cannot run on a CI machine at all. + Each stanza pins the text it was decided about, so an edit to that block fails extraction + rather than moving the decision onto something else. +- `test/windows/run_codelabs.ps1` replays the plan, handing each command to `pwsh` exactly as the + codelab writes it. Every step ends PASS, FAIL, KNOWN, SKIPPED or BLOCKED; a known failure that + starts passing fails the job, as in the unit-test job. + +**Wine contributes nothing here, and there is deliberately no Wine target for it.** What this +loop exists to find is PowerShell rejecting bash syntax, Unix tools that are not there, plugin +tools with no Windows release, and GitHub's API refusing anonymous callers. Wine emulates Win32 +and has no PowerShell; it can show none of those. + +What Linux can check before a push is everything except the execution, which covers all of the +bookkeeping that decides whether the job goes red: + +```bash +plz test //test/windows/codelab_script/... +plz build //test/windows:codelab_plan +pwsh ./test/windows/run_codelabs.ps1 -DryRun -Plan plz-out/gen/test/windows/codelab_plan.json \ + -KnownFailures test/windows/codelab_known_failures.txt +``` + +The runner's execution path was exercised on Linux once, against a synthetic plan with a fake +release, to cover every outcome and every rule that fails the job. Its answers about the real +codelabs only come from `windows-latest`. + +## What Wine does not cover + +Be honest about this. Wine passing is evidence, not proof. These are the M9 agenda, and they +should be listed in the M9 issue rather than discovered during it. + +**Measured on 2026-09-11**, by the GitHub Actions `windows-latest` job. The predictions below +were mostly right, and the list found one thing nobody had predicted. + +| Item | What a real Windows machine said | +|---|---| +| `ERROR_SHARING_VIOLATION` | **Materialised, and it broke `plz clean` outright.** Please held its own `plz-out/log/build.log` open and then asked Windows to delete the directory containing it. Both the background rename and the synchronous fallback failed, every time. Fixed by closing the log first; `RemoveAll` now also recognises the case and retries briefly rather than reporting it as a permissions problem | +| `MAX_PATH` | Not reproduced. The runner has long paths enabled, and a build at a 200-character path succeeded. Still untested with long paths off | +| Symlink privileges | `SeCreateSymbolicLinkPrivilege` is **disabled** on the runner, which is the case most users are in. `CopyOrLinkFile` already fell back; `buildLinks` did not, so every `link:` label quietly became a warning. Now falls back too, and the three tests that covered it no longer skip | +| Case-insensitivity | Not yet probed directly, but it caught `plz run` appending a second `PATH`: Windows stores the variable as `Path`, and the name was being compared exactly | +| Antivirus | Defender runs on the job, so every result above is already under a live scanner. No flakiness seen yet | +| Console, Ctrl-C | Still unreachable. A step's stdout is a pipe, so the interactive display never engages. Needs a machine with a real console session | +| **Unpredicted** | **`plz run` could not run anything on Windows.** An `sh_binary` is a `.cmd`, a `.cmd` runs through `cmd.exe`, and `cmd.exe` reads the forward slash in `plz-out/bin/x.cmd` as a switch. Wine's `cmd` parses it happily. Found by deleting a skip, not by adding a test | +| **Unpredicted** | **Any `plz` run outside a repo hung at 100% CPU for ever.** The walk towards the filesystem root never terminated, because trimming the separator off `C:\` leaves `C:` and splitting that returns it unchanged | + +### Filesystem semantics + +- **Case-insensitivity.** Wine on ext4 is case-*sensitive* by default. A BUILD graph with + `Foo.h` and `foo.h` works under Wine and collides on NTFS. Please's glob and hash code has + no case-folding anywhere. +- **`ERROR_SHARING_VIOLATION`.** Windows refuses to delete or rename a file that another + process has open. Wine is more permissive. This is the single most likely source of + real-Windows-only failures, and it hits exactly where Please works hardest: `plz-out/tmp` + teardown, `RemoveAll`, and the self-updater overwriting a running binary. +- **`MAX_PATH`.** 260 characters unless long-path support is enabled *and* the binary has the + manifest opt-in. `plz-out/bin///` nests deeply; a monorepo will hit + this. Wine does not enforce it. +- **Symlink privileges.** `os.Symlink` needs Developer Mode or + `SeCreateSymbolicLinkPrivilege`. This entry predicted Wine would grant it unconditionally, so + that the M2 copy-fallback path would never be exercised. **Measured, and it is worse than + that:** Wine's `os.Symlink` returns no error and produces a link that `os.Lstat` then cannot + find. `TestSymlink` skips on Windows for that reason. So Wine tells us nothing either way + here, and the fallback still needs testing by injecting a failure, not by hoping. + +### Process and console + +- **Real console behaviour.** VT sequence support (`golang.org/x/term`), the interactive + display in `src/output/interactive_display.go`, window resize. Wine's console is not + conhost. +- **Ctrl-C / Ctrl-Break delivery.** Wine's `GenerateConsoleCtrlEvent` is approximate. The + graceful-then-forceful kill path in `01-os-abstraction.md` needs native verification. +- **Antivirus.** Real-time scanning locks freshly written executables, causing intermittent + `ERROR_SHARING_VIOLATION` and slow builds. Invisible under Wine and a genuine user-facing + problem — worth a documented note in the eventual user docs. + +### Toolchain + +- **MSVC**, when it arrives, cannot be tested under Wine at all. + +## Regression protection for the platforms that already work + +Every change in M1–M3 touches shared code paths. Two things must hold for every PR: + +```bash +./bootstrap.sh # full build + unit + e2e on Linux +plz lint # golangci-lint + plz fmt check +``` + +**Hash stability is a hard requirement.** Please's cache is content-hash based over rule +definition, config, sources and secrets. Any change to command generation, environment +variables or config defaults shifts target hashes and invalidates every user's cache. Before +merging M3 in particular: + +```bash +plz hash //src/... # record +git stash -u -- src && plz hash //src/... # record again at HEAD +git stash pop +``` + +**Compare within one working directory.** Hashes are *not* comparable between two checkouts +of the same commit — a `git worktree` at HEAD produces different hashes from the main repo +for reasons unrelated to any change, so a worktree-vs-repo diff reports dozens of false +positives. Stash and unstash in place instead. + +Expect the dependency cone of whatever you edited to change; that is content hashing working. +What matters is that nothing *outside* that cone moves. + +A diff here is not necessarily wrong, but it must be *intended* and called out in the PR +description. + +**The e2e tests in `test/` assert on exact output text** and are documented as brittle. Expect +to update `.txt` golden files. Treat any *unexpected* change as a real regression rather than +noise — that is what they are there for. + +## Exit criterion for M6 + +A single CI job, on Linux, that: + +1. cross-builds `please.exe`, +2. cross-builds a C++ project for Windows using MinGW, +3. runs `please.exe` under Wine to drive that build, +4. runs the resulting `cc_test` binary under Wine and collects its results. + +If that passes, the Windows port is real, and M9 is about hardening rather than discovery. diff --git a/docs/design/windows/06-milestones.md b/docs/design/windows/06-milestones.md new file mode 100644 index 0000000000..b6842ebf73 --- /dev/null +++ b/docs/design/windows/06-milestones.md @@ -0,0 +1,773 @@ +# Milestone Tracker + +Status: **Living document** · Last updated: 2026-09-10 + +> **M0 is done and it changed the plan.** `plz.exe` compiles, links, parses BUILD files and +> executes build actions under Wine after ~290 lines of probe changes. Estimates below are +> revised down accordingly. See `appendix-baseline-errors.md` for evidence. + +The one file in this directory expected to change weekly. Update `Status` and `Notes` as work +lands; keep the exit criteria fixed unless the design genuinely changes, in which case update +the corresponding design doc too. + +Legend: ⬜ not started · 🟡 in progress · ✅ done · ⚠️ blocked + +## Summary + +| # | Milestone | Est. | Status | Owner | Issue | +|---|---|---|---|---|---| +| M0 | Baseline and guardrail | 2d | ✅ | — | — | +| M1 | OS abstraction layer | 1–2w | ✅ | — | — | +| M2 | Paths, environment and the `.exe` model | 1w | ✅ | — | — | +| M3 | Build actions and the bundled shell | 1w | ✅ | — | — | +| M4 | Release pipeline: cross-built Windows artifacts | 1w | 🟡 | — | — | +| M5 | C++ on Windows: cc-rules (workstream B) | 2w | 🟡 | — | — | +| M6 | Linux-hosted verification harness | 1w | ✅ | — | — | +| M7 | Sandboxing parity | 2w | 🟡 | — | — | +| M8 | Remote execution and plugin parity | 3w | 🟡 | — | — | +| M9 | Native Windows CI and GA | 2w | ⬜ | — | — | +| M10 | The codelabs, replayed on Windows | — | 🟡 | — | — | + +Rough total: 14–15 weeks of focused work. M0–M6 (the C++ vertical slice) is 7–8 weeks. + +**M1 was re-estimated from 2–3 weeks to 1–2 weeks.** Making it *compile* turned out to be a +two-day job (5 sites). The remaining time is the part the compiler cannot help with: Job +Objects, `ExecReplace`, and real file locking — all silent runtime failures today. + +## Sequencing + +``` +M0 ─┬─ M1 ─┬─ M2 ─┬─ M3 ─── M4 ─┐ + │ │ │ ├─ M6 ─┬─ M7 ─┬─ M9 + └─ M5 ─┴──────┴─────────────┘ └─ M8 ─┘ + (workstream B, parallel) +``` + +- **M5 can start immediately** and run in parallel with M1–M4. It is a different repo and its + Loop B verification (`05-testing-strategy.md`) needs only a Linux `plz` and MinGW — neither + of which depends on the core port. +- **M6 should start as soon as M1 produces a binary**, not after M5 finishes. The milestone + number is a completion point. +- **M7 is not a blocker for anything.** `sandbox_other.go` already compiles on Windows and + degrades to a plain `exec.Command`. + +## M0 — Baseline and guardrail ✅ + +**Exit:** the non-blocking CI job runs, fails, and its output is recorded in +`appendix-baseline-errors.md`. — *Met, and exceeded: the probe went all the way to a working +build under Wine.* + +- [x] Baseline measured — 5 compile-blocking sites in 4 packages, 4 layers +- [x] `pkg/xattr` verified: ships `xattr_unsupported.go`, no build tag needed +- [x] These design documents +- [x] `probe/m1-skeleton.patch` — verified to apply cleanly and produce a working `please.exe` +- [x] ~~Non-blocking CI job~~ — overtaken by events. M4 added a *blocking* `build-windows` + job that builds the whole release, which is strictly stronger +- [x] ~~`go1.27.0.windows-amd64` hash in `third_party/go/BUILD`~~ — **not needed.** Go + cross-compiles from the host toolchain; there is no Windows distribution to fetch +- [x] `tools/images/windows_builder/Dockerfile`, added to `tools/images/build.sh` — done in M4 + +### Findings that changed the plan + +1. **Errors are layered, not parallel.** `src/process` is a dependency of nearly everything, + so a single `go build ./...` reports 3 errors and leaves 30 of 51 packages unchecked. + Build packages independently and iterate. +2. **`syscall.Exec` is not a compile blocker** — Windows ships a stub returning `EWINDOWS`. + Same for `Chdir` and the signal constants. Silent runtime failures instead. +3. **New: go-flags breaks Please's label syntax on Windows** (D5). `-tags forceposix` fixes + it. Not in the original plan at all. +4. **`src/output/shell_output.go` is an abstraction leak** the source survey missed. +5. **`path.Dir` in `src/cli/logging.go` is a hard startup blocker**, not a cosmetic bug. +6. **busybox-w64 has a `bash` applet** but **rejects `--noprofile`/`--norc`** — contradicting + the Linux busybox result. `ShellArgs` is mandatory. + +## M1 — OS abstraction layer + +**Exit:** `plz build --arch windows_amd64 //src:please` produces `please.exe` via the real +BUILD-file path (not raw `go build`), and `//src/...` unit tests compile. ✅ **Met.** +Full suite: 837 tests, 835 passed, 2 skipped. The cross-built binary parses labels and runs +cold-cache builds under Wine. + +Design: `01-os-abstraction.md`. `probe/m1-skeleton.patch` is a starting shape — but its +`lock_windows.go` and `kill_windows.go` are deliberately wrong and must be replaced, not +adopted. + +**The compile fixes are ~2 days. The rest of the milestone is the runtime work the compiler +gives no signal for.** Do the cheap fixes first to unblock Wine testing, then the real ones. + +- [x] `src/cli/logging.go` `path.Dir` → `filepath.Dir` (hard startup blocker) +- [x] `.plzconfig_windows_amd64` with `[Plugin "go"] BuildTags = forceposix` (D5) — + `go_binary` has no `tags` param, so this is config, not a BUILD edit +- [x] `src/output/shell_output.go` — leak removed; `process.ShareParentProcessGroup` +- [x] `src/core/lock.go` → `lock_other.go` / `lock_windows.go` (`LockFileEx`), real + implementation; all 12 lock tests pass under Wine +- [x] `process.ExecReplace` + 5 call sites (the 6th, `sandbox_linux.go`, is Linux-only). + Verified under Wine: stdout passthrough, exit codes 0 and 3. Also releases the repo + lock before handing over — on Unix the exec did that implicitly via `O_CLOEXEC` +- [x] `src/process/exec_windows.go` — `CREATE_NEW_PROCESS_GROUP`; job objects in + `kill_windows.go` +- [x] `src/process/kill_windows.go` / `kill_other.go` — Ctrl-Break then `TerminateJobObject` +- [x] Narrow `exec_other.go` from `!linux` to `!linux && !windows` +- [x] `src/clean/clean.go` — `ForkExec` → detached `exec.Command` (`DETACHED_PROCESS`) +- [x] `src/cli/process.go` — signal set and exit-code convention now per-platform +- [x] `Build.Xattrs` defaults false on Windows; `pkg/xattr` needed no build tag +- [x] `.exe`/`PATHEXT` via `fs.ExecutableNames`, wired into `core.LookPath`. + Note `isExecutable`'s `0111` check is only reachable on the FreeBSD path, so it needed + nothing — the design doc over-stated this +- [x] `src/run/run_step.go` — `ExitError.ExitCode()`; `syscall.Chdir` → `os.Chdir` + +**Landed early from M3** (M1 is untestable under Wine without it): platform-specific shell +init args, since busybox rejects `--noprofile`/`--norc`. Note this is a property of the shell +being invoked, not the host — remote execution keeps the full flag set via a new +`process.RemoteBashCommand`. The `[build] Shell`/`ShellArgs` *config* is still M3. + +## M2 — Paths, environment and the `.exe` model + +**Exit:** `//src/core/...` and `//src/fs/...` unit tests pass under Wine, including +`lock_test.go` and config loading. + +Design: `01-os-abstraction.md` (the `.exe` model) and `02-shell-and-build-actions.md` (the +path-format rule). + +- [x] Promote `splitPathList` → `fs.SplitPathList` (done in M1, needed by `LookPath`) +- [x] Remaining raw `":"` splits in `src/core/config.go` (6) and `src/remote/action.go`. + The remote one splits locally but still joins with `":"` for the POSIX worker +- [x] `src/fs/home.go` — `os.UserHomeDir()`; `~` regex built from the platform separators +- [x] `MachineConfigFileName` (ProgramData) and `DefaultPath` (empty on Windows — there is + no equivalent of `/usr/bin` holding build tools) +- [x] `USERPROFILE`, `TEMP`, `TMP` — Windows only, so Unix hashes are untouched +- [x] `src/core/build_target.go` — platform-conditional `SandboxDir` +- [x] `src/fs/copy.go` — symlink privilege fallback (copies the target, warns once) +- [x] `src/fs/fs.go` — `RemoveAll` clears the read-only attribute on files too +- [x] **`glob()` returned nothing at all on Windows** — not on the original list, and fatal. + `patternToMatcher` built the pattern with `filepath.Join` while the walk goes through + `io/fs`, whose paths are always slash-separated. Fixed by using `path` throughout +- [x] The raw `"/"` handling in `src/fs/sort.go` and elsewhere in `glob.go` turns out to be + **correct** for the same reason — `io/fs` paths are always `/`. The design doc was + wrong to flag them +- [x] Forward-slash normalisation in `BuildEnvironment` + a test asserting no `\`. + Confirmed by experiment rather than assumption: `echo` and `printf '%s'` round-trip a + backslash path unharmed, but `sed -e "s#x#$TMP_DIR#"` turned `\tmp` into a literal tab + and ate the rest — and the cc rules build their link line with `sed` + +## M3 — Build actions and the bundled shell ✅ + +**Exit:** a `genrule` with `cmd = "cat $SRCS | sort > $OUT"` builds under `plz.exe` on Wine. +**Met**, and this time through config and packaging rather than by hand: nothing was placed on +the PATH and no file was renamed to `bash.exe`. The repo has no `[build] shell` line at all; +`plz.exe` finds the `busybox.exe` sitting next to it and runs the action through it. + +Design: `02-shell-and-build-actions.md`. + +- [x] `[build] Shell` / `ShellArgs` config. Defaults are per-platform, from + `process.DefaultShell` / `DefaultShellArgs`: `bash` with `--noprofile --norc` on Unix, + `busybox` with `bash` (the applet name) on Windows +- [x] `src/process/process.go`, `src/run/run_step.go` — `BashCommand` is now a method on + `Executor`, which carries the shell. `RemoteBashCommand` is untouched and still hardcodes + the full flag set, because the remote worker is a real bash whatever we are running on +- [x] `src/cache/cmd_cache.go` — both `sh -c` sites use the configured shell +- [x] `src/output/shell_output.go` — `plz build --shell` had a third hardcoded shell, not on + the original list. It gets `Executor.InteractiveShellCommand`, which is the same shell + without `-e`/`-u` +- [x] Vendor `busybox.exe` (`remote_file`, pinned hash, GPL-2.0 noted) +- [x] Add to `//package:installed_files` under `is_platform(os = "windows")` +- [x] Applet and flag audit against busybox-w64 — done in M0, see + `02-shell-and-build-actions.md` +- [x] Gate the `xz -zc` tarball rule to Linux (busybox `xz` is decompress-only — re-verified + against busybox-w64 1.38.0, which is also decompress-only despite listing the applet). + `tarball(xzip = True)` now fails at parse time on Windows, and `package/BUILD` defines + the two xz tarballs only where they can be built +- [x] `plz hash //...` unchanged on Linux — verified by hashing the same tree with the old and + new binaries, which agree exactly. The config addition is invisible to the hash because + `Configuration.Hash` covers only `Build.Lang`, `Build.Nonce`, the rejected licences and + the build environment + +**Landed early from M4** (`//package:installed_files` does not build for Windows without it): +`please_sandbox` is gated off Windows. It is built on Linux namespaces, so there was never +anything to ship there, and MinGW rejects `sandbox.c` outright. + +### Findings + +1. **Resolving the shell on `$PATH` alone would have made the bundling pointless.** Nothing + puts Please's install directory on the user's PATH on Windows, so a default of `busybox` + would never have been found. `resolveShell` (`src/core/state.go`) keeps the old behaviour + for a shell that is on the PATH and falls back to the *build* path — which already has + `Please.Location` prepended — only when it is not. Verified both ways under Wine: with + `busybox.exe` beside `plz.exe` the build works with no configuration; with it moved away + the build fails with `exec: "busybox": executable file not found in %PATH%`. +2. **The `busybox bash` applet form behaves exactly like the `bash.exe` copy M0 tested.** + Re-verified under Wine: `-e` stops at the first failure (exit 1), `-u` rejects an unset + variable (exit 2), `-o pipefail` propagates a failure from the left of a pipe (exit 1). + So `ShellArgs = bash` costs nothing over renaming the binary, and avoids installing a file + called `bash.exe` that would shadow a user's real bash. +3. **A repeatable config key still cannot be cleared by assigning it empty** — the M5 trap, + met again. `ShellArgs = ` yields `[""]`, so an empty argument would have been passed + through to the shell. `ShellArgs` drops empty entries, which is what lets a user turn the + shell's arguments off entirely. +4. **Bundling busybox needed a licence-policy change.** `.plzconfig` rejected `GPL-2.0`, so + `//third_party/binary:busybox` failed to build before it was ever run. Accepted with a + comment: Please execs busybox rather than linking it, so they are separately distributed + works, but the release has to carry the licence and a pointer to the source. +5. **`plz run` had the same bare-filename bug as `toolPath`** (M6 finding 2), one layer up: + `run_step.go` decided "does this need looking up on the PATH?" by searching for `/` only, + so an entry-point path built with `filepath.Join` would have been sent to `LookPath`. Now + checks `filepath.Separator` too. + +## M4 — Release pipeline 🟡 + +**Exit:** `plz build --arch windows_amd64 //package:release_files` on Linux CI produces a +signed `windows_amd64/` folder. The command **passes locally**; what is left is the arcat +release it depends on for plugins, and running it on CI for real. + +Design: `04-release-and-ci.md`. + +- [ ] **arcat for `windows_amd64`** — publish the release, then add its hash to + `src/parse/internal_package.go`. **Upgraded in severity:** it blocks `plugin_repo`, and + every language plugin is delivered that way, so no plugin can load on Windows without + it. `cc_library` needs it again for `.a` archives. Simple genrules and parsing work + without it, which is why this was first recorded as minor. + Good news: arcat is pure Go with no syscall/cgo, cross-compiles to PE32+, and both + `arcat x` and `arcat ar -r` verified working under Wine. Its `go.mod` says `go 1.17` + while the code uses generics, so it fails to build on *any* platform with a modern + toolchain — a one-line upstream fix, unrelated to Windows. + **This is the only thing between here and the exit criterion**, and it needs someone with + push access to the arcat repo. Confirmed still true: the v1.3.1 release has assets for + darwin, freebsd and linux only. + + **What has been done instead is to stop it being a wall.** Generating the internal + package used to fail outright on any platform with no published arcat, which stopped + everything rather than only the things that need one. The arcat rule is simply left out + now, so the rest of `//_please` still works, and `plz` warns once at startup that anything + needing arcat — including loading a plugin — will fail unless `[build] arcattool` points + at a build of your own. That is the difference between "Windows cannot parse anything" and + "supply this one binary yourself". +- [x] `.plzconfig_windows_amd64` — landed early in M1 (needed for `forceposix`) +- [x] `package/BUILD` — gate `please_sandbox` on `is_platform(os = "linux")` — done in M3 +- [x] `package/BUILD` — `.zip` release target, built with `arcat zip` on the Linux release + box. The two xz tarballs are replaced by it on Windows rather than added to +- [x] `plz.cmd` shim instead of the `ln -sf please plz` symlink +- [x] `src/update/update.go` — `linkFile` is now per-platform. Windows hard-links instead of + symlinking, and renames a file it cannot replace out of the way to `.stale`, which the + next run sweeps up in `clean()` +- [x] `pleasew.ps1` + `src/assets/BUILD` + root `BUILD`. **Not executed anywhere yet** — + there is no PowerShell on the Linux host, so it has been reviewed but not run +- [x] `.circleci/config.yml` — `build-windows` job, workflow entry, `release-gs` requires +- [x] `.circleci/release.sh` — `release_folder … windows_amd64/$VERSION` +- [x] `tools/misc/gen_release.py` — `_arch()` windows branch, and `.zip` added to the + content-type table +- [x] `tools/images/windows_builder/Dockerfile`, added to `tools/images/build.sh` — the M0 + item. The image tag in `config.yml` is dated `20260910` and **the image has to be built + and pushed before that job can run** + +### What the release turned up + +1. **The `.exe` suffix had to be asked for per target.** `go_binary` names the output after + the rule, so `//src:please` produced a file called `please` for Windows, which cmd will + not run and `LookPath` will not find. `out = "please.exe" if is_platform(...)` fixes it + where it matters today; the general fix belongs in the go plugin (M8). +2. **The whole packaging path is verified end to end under Wine.** Extract the zip as a user + would, run `plz.cmd`, and a genrule builds: the shim finds `please.exe`, which finds + `busybox.exe` beside it with no configuration at all. `query alltargets //...` also works, + which is the `forceposix` smoke test the risk register asked for. +3. **`plz init` now writes `pleasew.ps1` as well as `pleasew`**, on every platform. A repo is + often worked on from more than one, and picking by host would give a Linux developer no way + to set one up for their Windows colleagues. +4. **The release artifact names carry no extension and that is correct.** `please_` + is a URL key, not a filename; the updater downloads it and writes it as `please.exe` + locally. Only the archive members need the suffix. + +## Working on the plugins locally + +All four plugins are separate repos we have no push access to, so they are cloned into +`~/code/-rules` on a `windows` branch each, branched at the tag `plugins/BUILD` pins. + +`plugins/BUILD` takes each checkout path from a `[buildconfig]` key, so switching is a +gitignored `.plzconfig.local` and nothing else: + +```ini +[buildconfig] +go-rules-path = /home/peter/code/go-rules +``` + +Delete that file and you are back on the pinned downloads. Verified both directions. + +Three things learned wiring this up: + +- **`local_repository` is not usable** for this. It omits `plugin = True`, so the subrepo + registers as `plugins/go` rather than `go` and `///go//...` never resolves. Call the + `subrepo()` builtin directly. +- **A subrepo `path` outside the repo root is fine.** It is used verbatim with no containment + check, so no symlink is needed. +- **The local definition has to replace the download, not sit beside it.** The first attempt put + the `subrepo()` calls in a separate untracked package and pointed `[Plugin "go"] Target` at it. + That builds fine until anything parses `plugins/BUILD` as well — `plz test //...` does — and + then dies with *"Found multiple definitions for subrepo 'go'"*. Hence the conditional in the + one file, which costs a few tracked lines but cannot conflict with itself. + +Expect every hash to change when you flip, since the full source path goes into the digest. + +## M5 — C++ on Windows (workstream B) + +**Exit:** `plz build --arch windows_amd64 //test/...` in cc-rules produces PE32+ `.exe` and +`.dll`. + +Design: `03-cc-toolchain.md`. Repo: `please-build/cc-rules`. + +- [x] **D1 confirmed.** Both a WinLibs 16.2.0 and an Ubuntu 13 MinGW match the existing GCC + and GNU ld matchers; the Clang matcher correctly does not. No new matchers needed +- [x] `build_defs/arch.build_defs` — add `windows_amd64` (gates the plugin's own release, + not its use) +- [x] `cc_binary` / `cc_test` → `.exe`; `cc_shared_object` → `.dll`. **Must be a function, + not a module-level constant** — subincluded `CONFIG.OS` reflects the host at module level +- [x] A `cc_library` + `cc_binary` + `cc_shared_object` triple builds and `prog.exe` runs + under Wine, linking the static lib correctly +- [x] Drop `-fPIC` and `-Wl,--build-id=none` for Windows — both were passed and neither + broke the link, so this is noise reduction rather than a blocker +- [x] `DefaultLdFlags` → `-lpthread`. Only `-ldl` was wrong. Note a repeatable config key + **cannot be cleared by assigning empty** — that yields `[""]`, which becomes a bare + `-Wl,` and the linker rejects it. + + **It was set that way anyway**, in *this* repo's `.plzconfig_windows_amd64`, and stayed + broken because nothing here built a C++ target for Windows until one was added. The + platform default now comes from the plugin's build defs, where `CONFIG.OS` is the target; + the plugin's own `.plzconfig_windows_amd64` never applied to anyone using it as a plugin +- [x] **`cc_shared_object` can be linked against.** Windows resolves a DLL's symbols through an + import library rather than through the DLL, so `-l` had nothing to find. The link + now writes one with `--out-implib`, named after the output — `lib.dll.a` for the + default `lib.dll`, which is what `-l` looks for. + + The note in the rule said a second output was impossible, because the shared link command + names its output `$OUT` and that is unset on a multi-output rule. True of `outs`, not of + `optional_outs`, which don't count towards it. `//test/windows:dll_test` builds the pair, + links one against the other and runs it under Wine; taking the DLL away makes it exit 53, + so the linkage is genuinely dynamic. Windows has no rpath, so the DLL has to sit beside + the binary — which is what the test rule's data does +- [ ] `please_cc` `execvp_windows.go` (needed for native Windows, not for Axis 2) +- [x] Parse-time error when `pkg_config_libs` is used on Windows, naming the rule that asked +- [ ] MinGW cross-compile job in `plugin_test_cc.yaml` +- [ ] **`please_cc` needs a `windows_amd64` release.** `tools/BUILD` fetches it as a prebuilt + binary with a pinned hash per platform. Not a blocker under Axis 2, where tools build for + the Linux host, but required for a native Windows plz +- [x] **`cc_test` works on Windows.** Recorded as blocked on `UnitTest++` needing its `Win32/` + sources; that was never true. The sources were selected and then compiled with the host + toolchain, because a target inside a plugin does not see the using repo's plugin config. + One portability fix went with it: the test main called `unsetenv`, which Windows has no + such function for. Guarded by `//test/windows:cc_test_test` + Blocks `cc_test`, not `cc_library`/`cc_binary` +- [ ] Upstream PR; bump `plugins/BUILD` revision + +## M6 — Linux-hosted verification harness + +**Exit:** one CI job builds and runs a C++ `cc_test` for Windows, from Linux, end to end. + +Design: `05-testing-strategy.md`. + +- [x] Wine test macro for cross-compiled Go test binaries — `test/build_defs/wine.build_defs`, + `wine_go_test` and `wine_plz_test` +- [x] Wine CI job — `test-windows-wine`, blocking, and a third pass in `test.sh` where Wine is + installed. Every `//src/...` package whose tests run there at all: 19 targets, 717 tests, + 787 passing and 7 skipped — **every `//src/...` package**. `//src/build`, `//src/exec` + and `//src/process` are the valuable ones: they run real build actions and real + subprocesses, so they cover the layers that are almost entirely separate code on Windows +- [x] The genrule shell smoke test — plus a `query alltargets //...` test, which is the + `forceposix` guard the risk register asked for +- [x] **The headline end-to-end passes.** `wine plz.exe` extracts the cc plugin with + `arcat.exe`, runs build actions through busybox, identifies the toolchain with + `please_cc.exe`, compiles and links with MinGW `g++.exe`, and the resulting `hello.exe` + runs and prints correctly. Every component in that chain is a Windows binary. + + Two environmental caveats, neither a Please defect: Wine here has no working DNS, so + the plugin zip has to be supplied locally rather than downloaded; and `arcat` is pointed + at a cross-built binary via `[build] arcattool`, since there is no published + `windows_amd64` release yet. Both stand in for release infrastructure, not code. + + Getting there surfaced two real bugs — see the M6 findings below. +- [x] Make the Wine job blocking + +### What running the unit tests under Wine surfaced + +Doing this properly for the first time found **four correctness bugs**, all of the same shape +and none visible on Linux: code handling repo-relative or label-derived paths through +`filepath`, whose separator on Windows is a backslash. M2 recorded the inverse of this lesson +(`io/fs` paths are always `/`, so use `path`) and fixed the producers; these are the consumers +it missed. + +1. **`glob()` crossed package boundaries.** `isBuildFile` called `filepath.Base` on a path from + `io/fs`, so on Windows it compared the whole path against `BUILD` and never matched. No + subpackage was ever detected, and a glob in one package would take files belonging to + another. This is the same function M2 fixed the *pattern* side of. +2. **The initial package was wrong from any subdirectory.** `getRepoRoot` returned it with + backslashes, which are illegal in a package name, so validation failed and Please walked up + until something parsed — usually the repo root. `plz build ...` from `src/core` would have + built the wrong thing silently. +3. **Relative labels didn't parse at all.** `path/to:thingy` became `//current_package\path\to`. +4. **`$(location)`, `$(exe)`, `$(worker)` and tool paths expanded with backslashes** into shell + commands, where a backslash is an escape character. M2 normalised the *environment*, which is + a different path. + +**This overturned an M2 decision.** `02-shell-and-build-actions.md` argued for normalising only +at the environment boundary because it was the smaller change. It isn't: the replacements above +are not environment values, and the existing tests already assumed forward slashes throughout. +`plz-out` paths are now built with `path`, so they are slash-separated on every platform. Win32 +accepts either, and it is a no-op on Unix. + +Extending the job past `core` and `fs` found **four more of the same kind**: + +5. **`output_dirs` produced doubled paths.** `copyOutDir` strips the temp directory off a path + to get an output name, comparing a `filepath.Join` result against a slash-separated + `TmpDir()`. Neither prefix matched, the whole path survived as the output name, and + `moveOutputs` then joined the temp directory onto a path that already contained it. +6. **JS coverage file names were never sanitised.** `sanitiseFileNameDir` compared paths from a + coverage file against `filepath.Dir` of a plz-out directory, so coverage was reported + against absolute build paths instead of source files. +7. **Coverage-by-directory keys came out backslashed**, so they neither read correctly nor + matched anything configured. +8. **`file://` URLs could not name a Windows path.** RFC 8089 puts a slash before the drive + letter, so `file:///C:/foo` arrives as `/C:/foo`, which `filepath.IsAbs` rejects. No + `remote_file` with a local URL could work. + +And widening it again, past `build` to the parser, the query layer and the rest, found three +more — including the worst one so far: + +9. **`join_path` in the BUILD language returned backslashes on Windows.** The BUILD language is + the same language on every platform: its paths become labels, sources and command strings, + and every hash that reaches one would have differed. `subinclude`, `subrepo` names and + Bazel-compatibility `load()` had the same problem. +10. **Completion returned backslashed package names**, which are not valid build labels, so + every completion below the top level was unusable. +11. **`plz query changes` matched no package** for a changed file, because it walked up the + directory tree with `filepath.Dir` and looked the result up as a package name. + +And the last packages found a gap in M3's own work: + +12. **The command cache never got the resolved shell.** `resolveShell` lived in + `executorFromConfig`, so only build actions and tests benefited; `[cache] storecommand` and + `retrievecommand` were handed the bare name and could not find the bundled busybox. It is + now `Configuration.Shell()`, resolved once and shared, with a final fallback to the + directory of the running binary — which is where a bundled shell sits, and unlike the build + path does not depend on `Please.Location` having been resolved yet. +13. **`ShellArgs` was empty for any hand-built configuration.** The defaults for a repeatable + key can only be applied after parsing, or gcfg appends to them rather than replacing, so + anything using `DefaultConfiguration()` directly got none. On Unix that quietly dropped + `--noprofile --norc`; on Windows it dropped the applet name, so the shell did not run at + all. `Configuration.ShellArgs()` supplies the platform default when nothing is set. + +### The job objects from M1, finally under test + +`//src/process` used to be excluded because its tests ran `true`, `false` and `sleep` as bare +argv — programs on the PATH on Unix, applets inside the shell on Windows, so only one spelling +works anywhere. Building the argv through the configured shell instead makes them portable, and +that puts the whole process layer under test on Windows for the first time. + +`TestKillsProcessTree` is new, and covers what process groups on Unix and job objects on Windows +both exist for: when a command times out, what it started has to die with it. Nothing tested +that on **any** platform before. It fails on Linux if the signal goes to the process rather than +the group. + +What the Windows side of it actually proves took some establishing, and the answer is not the +obvious one: + +- Disabling `TerminateJobObject` — the test still passes. +- Disabling the Ctrl-Break path as well — the test still passes. +- Disabling `trackProcessTree`, so there is no job object at all — **the run hangs + indefinitely.** + +So it is `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` that does the work, when the handle is closed on +the way out of `ExecWithTimeout`, rather than either explicit kill. And the failure mode without +it is not a surviving grandchild but a hang: the orphans keep Please's pipes open, so it never +finishes rather than failing. That is worth knowing, because a hang is the harder thing to +diagnose in the field. + +Note the grandchild has to be a separate process to test any of this. busybox implements a +subshell as a thread on Windows, so `( ... ) &` would die with its parent and prove nothing. + +A sweep for the same mistake elsewhere, rather than waiting for a test to find it, turned up +four more places where a path Please *prints* came out backslashed: `plz query outputs`, `plz +query graph`, the paths `plz build` reports, and the entries `plz generate` writes into a +`.gitignore` — the last of which would simply not have matched, since git speaks forward slashes +on every platform. None of these has a test that would notice, so they are worth naming. + +A further kind of finding is recorded rather than fixed: **Go's `exec` on Windows will not run a file +whose name has no extension in `PATHEXT`, even given its full path.** The `wine_go_test` macro +copies each test binary to a `.exe` before running it. The same trap is why `//src:please` +needs `out = "please.exe"` (M4), and it will bite `plz run` on any `go_binary` until the go +plugin names Windows outputs properly (M8). + +Two things about the harness itself, both found by tests failing for reasons that were nothing +to do with Please: + +- **Wine's hosts file has the `localhost` line commented out.** Anything resolving it hangs + until it gives up — three `remote_file` tests each burned 15 seconds. The macro appends the + line to the prefix. +- **`~` resolves inside the shared prefix**, so a test writing to its home directory leaks into + the next run. One of these left a read-only file behind, which on Windows the next run cannot + replace. Tests that write to `~` now set `USERPROFILE` themselves, which is what + `os.UserHomeDir` reads there. + +Three tests are honestly unrunnable rather than fixed: + +- **`TestSymlink` skips on Windows.** Creating a symlink needs Developer Mode, and under Wine + `os.Symlink` reports success and produces a link that cannot even be `Lstat`ed. The testing + strategy predicted Wine would grant the privilege unconditionally and so never exercise the + copy fallback; the reality is worse, and belongs on the M9 agenda. +- **`TestSymlinkedOutputs` and `TestCreatePlzOutGo` skip on Windows**, for the same symlink + reason, and the permission assertions in `TestOutputDirDoubleStar` skip because Windows has + no mode to preserve — Go synthesises one from the read-only attribute. + +### What the end-to-end test surfaced + +1. **No package below the top level parsed on Windows.** `buildFileName` joined with + `filepath.Join` and then called `iofs.Stat`, so it looked for a file whose *name* contained + a backslash. Only the root package worked, because `filepath.Join("", "BUILD")` has no + separator to get wrong — which is exactly why every earlier test missed it. This alone + would have made Windows unusable for any real repo. +2. **`toolPath` prepended `./` to absolute paths**, producing + `./Z:/tmp/.../please_cc.exe`, because it decided "is this a bare filename?" by looking for + `/` only. +3. **`.plzconfig` rejects unquoted backslashes** — `unquoted '\' must be followed by new line + or double quote`. Windows paths in config files must use forward slashes or be quoted. + ~~Worth a note in the user docs.~~ **Fixed instead**: the error now names the file it came + from — it did not before — and says to use forward slashes or quote the value. The parser's + own message gives no hint that a path is even involved. +4. **`DefaultPath` being empty on Windows is load-bearing**, not cosmetic: `ar.exe not found + in path` until `[build] path` is configured. That is the intended design, but it means a + Windows user must configure tool locations before anything builds. **The message now says + so** when the only directory searched was Please's own, which is exactly the + nothing-configured state. Note the check counts non-empty entries: clearing a repeatable key + by assigning it empty yields `[""]`, not an empty list — the same trap as `ShellArgs`, met + for the third time. + +## M7 — Sandboxing parity 🟡 + +- [x] Default `Sandbox.Build`/`Sandbox.Test` false on Windows, with a clear log line. The + defaults were already false — the zero value — so the work was the log line, and it + mattered more than it looks. Setting either on Windows previously produced + `Can't find sandbox tool please_sandbox on the path`, which invites you to install + something that does not exist. It now says sandboxing is not implemented on this platform + and that actions will run without isolation, and does not construct a sandboxing executor +- [ ] `sandbox_windows.go` — Job Objects (reuse M1), restricted token, scrubbed environment +- [x] Document the filesystem-isolation gap. There is no mount-namespace analogue on Windows. + A job object can bound processes and a restricted token can drop privileges, but neither + hides a directory, and the only thing that does is a Windows Container — a dependency far + too large to take on for a build tool. So a Windows sandbox could isolate *processes* but + not the *filesystem*, which is the half that matters most for build hermeticity. That is + why refusing to act on the setting, rather than half-implementing it, is the right shape + until someone has a use for the process half on its own + +Note `resolveOut` already guards its sandbox branch on `runtime.GOOS == "linux"`, so `$OUT` +does not change shape on a platform without a sandbox. `target.Sandbox` is still folded into +the target hash, so a Windows user who turns sandboxing on gets different hashes for no +benefit — which is why refusing to act on the setting, rather than quietly ignoring it, is +the right shape. + +## M8 — Remote execution and plugin parity 🟡 + +- [x] ~~`src/remote/action.go` `translateOS` — add `windows`~~ — **nothing to do.** + `reallyTranslateOS` special-cases only `darwin` → `macos` and passes everything else + through, so `windows` already comes out as `windows`. Recorded rather than changed +- [ ] go plugin — `windows_amd64` arch, `.exe` naming. **Now blocking more than it looks:** + Go's `exec` on Windows will not run a file with no `PATHEXT` extension even given its + full path, so `plz run` on any `go_binary` fails until this lands. `//src:please` and + `//tools/build_langserver` work around it per-target (M4). + + **Measured, and the constraint is narrower than it appears.** Windows itself runs such a + file happily — `os.StartProcess` on the bare path works, and prints its output. It is + only Go's `os/exec` that refuses, in `lookExtensions`, and it refuses even when `Cmd.Path` + is set directly, so there is no way to keep `os/exec` and bypass it. Reimplementing + process handling to avoid that is far worse than naming the output correctly, so the fix + stays with the plugin. + + What is fixed here is the message. `executable file not found in %PATH%` for a file that + is plainly there is baffling; `fs.ExplainUnrunnable` adds that the name has no extension + Windows will run, and what it would need to be called +- [x] **go plugin — `.exe` naming done** in the local clone. `go_binary`, `go_test` and + `go_benchmark` append the suffix from a per-call function. Proof it works: the + `out = "please.exe" if is_platform(...)` workarounds in `src/BUILD.plz` and + `//tools/build_langserver` can be deleted and `please.exe` still comes out with the right + name. **They are deliberately still in the tree**, because this repo pins the unfixed + upstream plugin; drop them in the same change that bumps `plugins/BUILD` +- [x] **go plugin — a `windows_amd64` `please_go` release, published from the fork.** Upstream + publishes five platforms and not Windows, so a native Windows plz could not build a Go + target at all. Only that one architecture is redirected to the fork; everything else + still comes from please-build. The download needs an explicit `out`, because the asset + name carries the version and platform and so has no extension in PATHEXT. The same was + done for `please_cc` and `please_pex` +- [ ] go plugin — `windows_amd64` arch for its own release. `tools/please_go:bootstrap` runs + `go build ... && mv please_go $OUT`, which fails where `go build` writes `please_go.exe`, + and hardcodes `TMPDIR=/tmp`. Native-Windows only +- [x] **shell plugin — `sh_test` and `sh_cmd` done.** Windows has no shebang mechanism, so a + `.sh` is not runnable by name however it is written. `sh_test` hands the script to a shell + explicitly and `sh_cmd` takes its interpreter from a new `shell_tool` plugin config rather + than hardcoding `/bin/sh`. Verified under Wine through the bundled busybox + + **Its Windows default was in the wrong place, and did nothing.** It was set in the + plugin's own `.plzconfig_windows_amd64`, and *a plugin's architecture config is never read + when it is used as a plugin*: `readSubrepoConfig` reads only `.plzconfig` from the + subrepo, and the `.plzconfig_` that `state.ForArch` merges belongs to the repo doing + the building. So it worked in the plugin's own tests, where it *is* that repo, and a + Windows user of the plugin silently got `/bin/sh` — a path Windows does not have. + + Reading the subrepo's arch file would not be enough on its own either: a plugin's own + `[Plugin "x"]` values are not consulted for its config, only `[PluginConfig "x"] + DefaultValue`, and merging that appends rather than replaces, so a scalar default cannot + be overridden by a second file. The platform default now lives in the build defs, where + the rest of the platform handling already is. The python plugin does the same for its + run-time interpreters, for the same reason +- [x] **shell plugin — `sh_binary` done** in the local clone. It wrote a shebang, appended the + script, then appended a zip, and relied on the shebang. The payload was never the + problem, since busybox has `unzip`; only the launching was. It **cannot emit a `.cmd` + alongside** because `plz run` requires a single output, so on Windows the single output + *is* the `.cmd`: a four-line batch preamble with the same zip appended after it. cmd.exe + reads a batch file a line at a time and stops at `exit /b`, so it never reaches the + archive. + + The script is left inside the zip rather than inlined the way the Unix version inlines + it. There is no syntax a batch file and a shell script both ignore — a shebang line + works precisely because it is a comment to the shell — so the preamble unpacks the + payload and hands the script to `busybox sh`. It sources it rather than running it, so + that `$0` is the file the user ran, as it is on Unix where the two are one file. Scripts + find their unpacked dependencies relative to `$0`, and would not otherwise. + + **Two things bit on the way, neither of them about batch files.** Build outputs are + read-only and the zip preserved that, so the second run of an `sh_binary` could not + replace what the first one unpacked — Windows forbids it outright — and went on to run + the stale payload, having complained only on stderr. The rule now zips a *copy* of the + build directory with the modes relaxed, because the originals are hardlinked to + `plz-out` and a `chmod` there would quietly make another target's outputs writable. + + And `wine foo.cmd` is not the same thing as running it: what Wine cannot load as a PE it + hands to the host, so the unfixed `.cmd` — still carrying a Unix shebang — ran under + `/bin/sh` and printed exactly what the test wanted. `//test/windows:sh_binary_test` goes + through `cmd.exe` explicitly, runs it twice in the one directory, and fails without + either fix +- [x] **python plugin — `python_binary` and `python_test` done** in the local clone. A pex is a + static ELF preamble with a zip appended, so a Windows cross-build produced an ELF-prefixed + file that was dead on arrival. Four things were needed, and only one of them was the one + we expected: + + 1. **A Windows preamble**, in Go, cross-compiled and embedded in `please_pex` beside the + native C one; `--os`, defaulting to the build environment's `OS`, picks between them. + It reads the same configuration from the same place in the archive. It is a separate + program rather than a port because Windows has no `exec`: it runs the interpreter as a + child and passes the exit status back, which `_execv` cannot do + 2. **`plz.py` built a regex out of `os.sep`** to match distribution metadata inside the + zip. Zip member names are always `/`-separated, so on Windows that was a backslash, + which the regex compiler read as an escape — every pex died on startup, before any of + its own code ran. The `filepath`-for-`path` mistake again, in Python + 3. **`.pex.exe` naming**, for the same reason `go_binary` needs `.exe` + 4. **A run-time interpreter default of `python` then `py`.** The fallback elsewhere is the + interpreter that compiled the sources, which cross-compiling makes the host's, and + `python3` is a spelling a normal Windows install does not have + + **The two-stage plan in the original note was unnecessary.** It assumed the Python inside + would need work too. It does not: Python skips leading non-zip data, so an ELF-prefixed + pex already imported and ran correctly under Wine once the `os.sep` bug was fixed. Only + the launcher was broken, so only the launcher was replaced + + Measured under Wine by `//test/windows:pex_test`, against the embeddable Python from + python.org: `os.name`, importing the test module out of the zip, reading a data file + beside it, and importing third-party code through the meta path hook. Exit-code + propagation is separately guarded by `//test/windows:pex_exit_code_test`, because a + preamble that always returned 0 would make every failing `python_binary` look fine +- [ ] python plugin — a `please_pex` release carrying the Windows preamble. Until there is one + this repo builds the tool from the plugin's source, through `PexTool` in + `.plzconfig_windows_amd64`. Same blocker as `arcat`: no push access, not a porting problem +- [ ] python plugin — `.pyd` extension modules. `SoImport` writes one to a `NamedTemporaryFile` + and loads it while the handle is still open, which Windows does not allow. Only affects + pexes containing native wheels; none of the tests here do +- [x] **A parse deadlock in Please itself**, found building the python plugin's `please_pex` + from source. Resolving a subrepo declared inside another subrepo makes Please look for + the package that would declare it in the host repo first, and fall back to the subrepo + that asked when it isn't there. Not being there is the ordinary case, so that lookup + swallows the error — and it swallowed the parse claim with it, leaving the next caller + waiting on a parse nobody was going to do. Twelve targets hung with no output at all. + `//test/subrepo/nested_subrepo_probe` guards it; two subrepos are needed, because the + first to look is the one that poisons the lookup + +- [x] `src/watch` — **this was a bug, not a documentation task.** `plz watch` compares the + paths it recorded against the ones fsnotify reports. Ours are slash-separated; fsnotify + on Windows reports backslashes. Nothing matched, so every event was discarded as + belonging to a file we weren't watching, and the watch would simply never fire — silently, + since the mismatch looks exactly like an unrelated file changing. Both sides now go + through `watchKey`. `//test/windows:watch_test` guards it, and fails without the fix + +## M9 — Native Windows CI and GA + +- [x] **GitHub Actions `windows-latest` job — done, and blocking.** A Linux job cross-builds + the twenty-three Windows test binaries with their data, plus the release, and hands both + to a `windows-latest` job that runs them. It builds its own artifacts rather than taking + CircleCI's, because a CircleCI workspace is scoped to one CircleCI run and cannot be read + from Actions; the duplicated cross-compile is the price, and in exchange every pull + request produces a downloadable Windows build. + + It was advisory for exactly one run, to produce a failure list without a red check nobody + had read yet. 800 of the Wine suite's tests now run natively, and + `test/windows/run_native_probes.ps1` also builds a repo with the release, cleans and + rebuilds it five times under a live virus scanner, and builds at a long path +- [x] **Work through the Wine-invisible failures — first pass done.** The first native run + found five things, four of them real bugs, and one of those was not on anyone's list: + every `plz` run outside a repo hung at 100% CPU for ever, because the walk towards the + filesystem root never terminates on Windows. `plz clean` also failed every time, exactly + the `ERROR_SHARING_VIOLATION` the risk register predicted, and on Please's own log file. + + `05-testing-strategy.md` now records what a real machine said against each prediction, + with a date. Two items remain out of reach from a CI step: console behaviour, because a + step's stdout is a pipe so the interactive display never engages, and Ctrl-C, which needs + a console the sender is attached to. Both need a machine with a real session +- [x] **`get_plz.ps1`**, served and signed from the same bucket as `get_plz.sh` and run the + same way: `irm https://get.please.build/get_plz.ps1 | iex` +- [x] **`README.md` and `docs/faq.html`.** The FAQ said Windows was not supported natively; + it now says what is supported, and names the two things that behave differently - no + sandbox, and virus scanners holding files open +- [x] **`docs/milestones/18.0.0.html`** +- [x] **`VERSION` 18.0.0 + `ChangeLog` entry** + +## M10 — The codelabs, replayed on Windows 🟡 + +M9 showed that Please runs on Windows. It said nothing about whether the documentation does, and +nothing had ever executed a line of the codelabs, on any platform. + +- [x] **An extractor that refuses to guess.** `//test/windows/codelab_script` reduces + `docs/codelabs/*.md` to a plan of files, commands and directory changes. A block no rule + can classify is an error, not a silently dropped block, and + `//test/windows/codelab_script/script:script_test` runs it against the real codelabs in + the default Linux pass. That is what stops the check drifting from the published pages +- [x] **`test/windows/codelab_steps.conf`**, for what the Markdown cannot say: `.plzconfig` + fragments to merge rather than overwrite, output shown in a `bash` fence, steps that never + exit. Every stanza carries a reason and pins the text it was decided about. It is kept out + of `docs/` so the codelabs still read as documentation +- [x] **`test/windows/run_codelabs.ps1`**, which hands each command to `pwsh` exactly as written, + with a home of its own per codelab. PASS, FAIL, KNOWN, SKIPPED and BLOCKED, and the same + shrink-only known-failures rule as the unit-test job. Its execution path was exercised on + Linux against a synthetic plan; its answers about the real codelabs come only from Windows +- [x] **A blocking `codelabs` job** in `.github/workflows/windows.yml`, beside `test`, fed a plan + the Linux job built and checked +- [x] **The first native runs, and the known-failures list they produced.** Four entries were + listed ahead of the first run from facts checked directly, and it confirmed them and + added two. It also caught the harness: appending `.plzconfig` fragments repeated `GoTool`, + which a plugin section refuses, so the runner now merges key by key. The second run added + the last entry, a failure that is not Windows at all: the Go codelabs' `third_party/go/BUILD` + drops the `go_stdlib` that `plz init plugin go` generates +- [ ] **What to do about the codelabs that cannot work as written.** Deliberately not decided + here, and no codelab has been edited. `test/windows/codelab_known_failures.txt` is the + record that decision should be taken from + +## Risk register + +| Risk | Impact | Mitigation | +|---|---|---| +| MinGW does not match `please_cc`'s existing regexes | Blocks all of M5 | One-command check, first task in M5 | +| ~~busybox-w64 diverges from Linux busybox~~ | **Materialised, resolved.** `--noprofile`/`--norc` rejected | Audit re-run against busybox-w64 in M0; `ShellArgs` promoted from hedge to requirement | +| ~~go-flags `/` option delimiter breaks label syntax~~ | **Found and resolved in M0** | `-tags forceposix` (D5). Must not regress — it is invisible in Please's own source | +| `go_repo` won't generate Windows-only third-party packages on a Linux host | Any unconditional dep on `x/sys/windows` breaks the normal Linux build | Guard such deps with `is_platform(os = "windows")`; `go_library` filters the `_windows.go` srcs to match | +| Assuming `filepath` is always right on Windows | **Materialised twice.** `glob()` matched nothing (M2), then crossed package boundaries and broke relative labels (M6) | Paths from `io/fs`, build labels, and anything going into a shell command are all `/`-separated: use `path`. The inverse of the `logging.go` bug, where `filepath` was the fix. The Wine unit-test job is the guard | +| BUILD files verified only by `go build` | Real breakage invisible until someone runs `plz` | Always verify through `plz build`, not `go build` — this found 3 bugs in one pass | +| Prebuilt per-platform helper binaries with no Windows release | Blocks plugins (arcat) and native cc builds (please_cc) | Both are pure Go and cross-compile cleanly; the work is publishing releases and recording hashes, not porting | +| A dropped `forceposix` tag silently breaks every label | Total CLI breakage, only visible at runtime | ~~Add a Wine smoke test~~ — done: `//test/windows:label_test` | +| Backslash escaping in shell command strings | Intermittent, hard-to-diagnose build failures | The forward-slash rule in `02-shell-and-build-actions.md`, plus an assertion test | +| `.exe` needs to be a core concept after all | Rework of the M2 decision | Verify `plz run` on a `cc_binary` early in M5, before the rest of M5 depends on it | +| Hash drift invalidates every user's cache | Silent, affects all platforms | `plz hash //...` diff on every M1–M3 PR | +| `ERROR_SHARING_VIOLATION` on real Windows | Invisible until M9 | Listed explicitly in the M9 issue; design `RemoveAll` and the updater defensively now | +| arcat platform gate forgotten | `plz.exe` cannot parse anything, discovered late | Called out as a hard gate in M4; it fails at runtime on Windows, not at build time on Linux | +| `plz init plugin` points at upstream plugins with no Windows tools | Every codelab that installs a plugin fails at its first build | Recorded per step in `codelab_known_failures.txt`. The fix is Windows releases upstream, or `plz init plugin` using the forks; that is a docs and release decision, not taken in M10 | +| The codelab replay interprets a block differently from its prose | The check passes or fails for its own reasons rather than the codelab's | Unclassified blocks are fatal; every stanza in `codelab_steps.conf` pins its text with `matches` and carries its reason | diff --git a/docs/design/windows/07-state-of-play.md b/docs/design/windows/07-state-of-play.md new file mode 100644 index 0000000000..aca8864a31 --- /dev/null +++ b/docs/design/windows/07-state-of-play.md @@ -0,0 +1,192 @@ +# State of Play + +Status: **Living document** · Last updated: 2026-09-12 + +Where the Windows port actually is, and what to pick up next. `06-milestones.md` is the +per-milestone tracker with the reasoning; this is the short version for someone starting cold. + +## What works today + +`please.exe` cross-builds from Linux, runs under Wine, ships busybox as its build shell, and +builds a C++ binary end to end through an entirely Windows toolchain, including a DLL and a +binary linked against it. Python works too: a `python_test` and a `python_binary` both build +for Windows and run there, and so does an `sh_binary`, as a `.cmd` with its payload appended. +The release is a `.zip` containing `please.exe`, `busybox.exe`, `build_langserver.exe` and a +`plz.cmd` shim; extracting it and running `plz.cmd` builds a genrule with no configuration at +all. + +Test coverage on Linux is unchanged and green. Coverage *of Windows behaviour* is 34 targets and +873 tests under Wine, run by a blocking CI job and by `./test.sh` as a third pass. Eight of those +targets only exist when a local plugin checkout is configured — see below. + +**And 809 of those tests now run on a real Windows machine, with no Windows-specific skips +left.** The two that still skip there are skipped on every platform and always were. A blocking GitHub Actions job +cross-builds them on Linux and runs them on `windows-latest`, alongside probes that build a repo +with the release zip, clean and rebuild it five times, and build at a long path. That job is the +only thing anywhere that is not taking Wine's word for it. + +**The codelabs are now replayed there as well, and only one can be followed to its end.** Nothing +had ever executed a codelab on any platform. `using_plugins` runs through; `genrule` gets as far +as its custom tool, a `#!/bin/bash` script Windows cannot run; every codelab that builds Go or +Python stops at its first build, and `github_actions` has nothing to run. The causes are upstream +plugin tools and Puku with no Windows release, a Go 1.20 toolchain requested as a `.tar.gz` that +Windows releases never are, Python absent from the empty default build path, and bash syntax. One +cause is not Windows at all: the Go codelabs write a `third_party/go/BUILD` that drops the +`go_stdlib` `plz init plugin go` now generates. Each is in `test/windows/codelab_known_failures.txt` +with the log line behind it. See Loop D in `05-testing-strategy.md`. + +| # | Milestone | State | +|---|---|---| +| M0–M3, M6 | baseline, OS layer, paths, shell, Wine harness | done | +| M4 | release pipeline | done; `arcat` is built from source rather than downloaded | +| M5 | C++ / cc-rules | done, `cc_test` included | +| M7 | sandboxing | decided against, documented | +| M8 | plugins | go, cc, shell, python all done in local clones | +| M9 | native Windows CI and GA | done — 18.0.0 | +| M10 | The codelabs, replayed on Windows | done; findings recorded, docs decision open | + +## The five repos + +| Repo | Branch | Head | +|---|---|---| +| `~/code/please` | `wine` | merged to `master` on the fork | +| `~/code/go-rules` | `windows` | don't double the `.exe` | +| `~/code/cc-rules` | `windows` | emit an import library | +| `~/code/shell-rules` | `windows` | build an `sh_binary` as a `.cmd` | +| `~/code/python-rules` | `windows` | build a `.pex` Windows can run | + +The plugin clones are branched at the tag `plugins/BUILD` used to pin, not at `master`. There is +no push access to any of the *upstream* repos, so nothing is upstreamed, but all five are pushed +to forks at `PeterNeiss/{please,go-rules,cc-rules,shell-rules,python-rules}`, and `plugins/BUILD` +now downloads the four plugins from there, pinned to commit SHAs. The local checkouts are no +longer wired into anything: `.plzconfig.local` is inert and can be deleted. + +Every Wine test is now unconditional. Eight of them used to exist only when a local checkout was +configured — two pex tests, the DLL test, the `cc_test`, the `sh_binary` test — because no +plugin anyone could download carried the fix each one covers. They run in CI now, on every +change, which is where they were always meant to run. + +## Environment + +- Wine prefix: `/tmp/claude-1000/-home-peter-code-please//scratchpad/wineprefix`. It is + session-scoped, so a new session recreates it with `wineboot --init`; the test macros do this + themselves under `plz-out/wineprefix`. +- MinGW is installed (`x86_64-w64-mingw32-g++`), which is what cross-builds C++ for Windows. +- **`plz` on the PATH is not this repo's Please**, and the difference is not cosmetic: the + installed one predates the parse-deadlock fix, so running the Wine tests with it hangs at + the end of the parse with no error. Build `//src:please` and run `plz-out/bin/src/please`, + or `./test.sh`, which does that itself. `plz install` also settles it. +- `go` is not on the default PATH. Use `export PATH="$PWD/plz-out/bin/third_party/go/toolchain/bin:$PATH"` + before `plz lint` or `./test.sh`. +- The `BUILD` files this repo already had are not `plz fmt` clean. Format only the files you + touch, or you will bury your diff. + +## Pick up here + +In rough order of value. + +1. **Decide what to do about the codelabs.** The codelabs job passes only because every failure is + listed in `test/windows/codelab_known_failures.txt` with its evidence, and that file is the input + to the decision. The largest fixes are not in the prose: `plz init plugin` pointing at plugin + releases that exist for Windows, and Go codelabs that do not delete the stdlib it generates. No + codelab has been edited. +2. **`sh_test` cannot take an `sh_binary` as its `src` on Windows.** It copies whatever it is + given to `.sh` and hands that to a shell, and a `.cmd` is not a shell script. The + plugin's own tests are written that way, so they are the thing to fix it against. The + smallest real functional gap left. +3. **Ctrl-Break is delivered but never verified.** `KillProcess` sends one, waits 30ms, then + terminates the job object. `TestKillsProcessTree` passes natively, but it only asserts a + grandchild died, which terminating the job achieves either way — so the graceful path could + be dead code on Windows and no test would notice. + + Harder than it looks, which is why it is still here. The window is 30ms: `KillProcess` sends + the break, waits that long, then terminates the job regardless. A test that asserts the child + shut down gracefully is racing that timer on a CI machine, and a flaky test in a blocking job + is worse than no test. Either call `killProcessTree` directly and wait generously, which + tests the delivery without the timer, or widen the window and say why. +4. **`.pyd` extension modules in a pex.** + `SoImport` writes one to a `NamedTemporaryFile` and + loads it while the handle is still open, which Windows does not allow. Only bites a pex + containing native wheels. +5. **`plz debug` and `plz cover` on a Windows target** are untested. `plz cover` has one + concrete suspicion against it: coverage paths come back from the Python side with + backslashes in them. Both are unknowns rather than known defects, so the native job is + likely to find them faster than guessing will. + +Nothing is blocked on access any more. `arcat` is built from source in this repo, and +`please_go`, `please_cc` and `please_pex` all have `windows_amd64` releases published from the +forks, so a native Windows `plz` has everything it needs to download. What remains is upstream +adoption, which is a matter of someone merging rather than of permission. + +## Things that will bite you again + +Each of these has already cost time once. + +- **`filepath` is wrong for anything that is a build label, a plz-out path, or destined for a + shell command.** Use `path`. This produced roughly a dozen bugs across the port, including + globs crossing package boundaries and `join_path` in the BUILD language returning backslashes, + which would have changed every hash that reached it. It has a Python dialect too: `os.sep` in + the pex bootstrap, matching zip member names, which are always `/`-separated. +- **In a plugin's build defs, a suffix must come from a function, not a module-level constant.** + At module level `CONFIG.OS` is the host, so a constant passes on Linux and misnames everything + when cross-compiling. Make the function idempotent while you are there: appending `.exe` to a + name that already ends in `.exe` produced `please.exe.exe`, which nothing could find. +- **A plugin's own `.plzconfig_` is never read when it is used as a plugin.** Only + `.plzconfig` is read from the subrepo, and the arch file that gets merged belongs to the repo + doing the building. A platform default set that way works in the plugin's own tests and + nowhere else. Put it in the build defs instead. +- **A repeatable config key cannot be cleared by assigning it empty** — that yields a list of one + empty string. Four separate bugs so far. The most recent sat in this repo's own + `.plzconfig_windows_amd64` for weeks, because nothing here built a C++ target for Windows + until a test did. +- **Go's `os/exec` will not run a file with no `PATHEXT` extension**, even given its full path. + Windows itself is fine with it; `os.StartProcess` proves that. Only the lookup refuses. +- **Never run a cross-built test binary by hand in the source tree.** Under `plz test` they get a + sandboxed temp directory; run from the repo root they operate on the repo. Doing this once + deleted the whole of `test/`. +- **A build output is read-only, and on Windows that means it cannot be replaced at all.** + Unpacking an archive of build outputs over a previous unpacking of itself therefore fails, + and tools tend to report it on stderr and carry on with the stale copy. `sh_binary` hit this; + anything else that unpacks build outputs beside themselves will too. +- **A skip hides a bug better than a missing test does.** Deleting two has now found two real + failures that Wine had passed for months. `plz run` handed `cmd.exe` a forward-slashed path, + which it reads as a switch; and every `link:` label silently became a warning. Both were + behind `runtime.GOOS == "windows"` skips that looked reasonable when they were written. +- **`filepath.Split` does not terminate a walk on Windows.** Trimming the separator off `C:\` + leaves `C:`, and splitting that returns it unchanged, so a loop that stops at an empty string + never stops. Compare each step against the previous one instead. This hung every `plz` run + outside a repo, at 100% CPU, and nothing predicted it. +- **A handle this process holds is still a handle.** Windows will not rename or delete a + directory containing a file anything has open, including us. The log file lives under + `plz-out` by default, which is what `plz clean` deletes. +- **`upload-artifact` drops hidden files** unless `include-hidden-files` is set. A dotfile that + exists in `plz-out` is silently not in the artifact, and nothing on the Linux side can catch + it, because the Wine tests never go through one. +- **`plz-out/pkg` is never refreshed once it exists.** The `hlink:` label goes through + `fs.LinkIfNotExists`, and the destination is named after the version, so rebuilding a release + at the same version leaves the previous bytes there, silently. `plz-out/gen//package/` + always has the real artifact. Affects every platform; cost an hour here, twice. +- **`chmod` in a build directory writes through to `plz-out`.** Inputs are hardlinked in, so + relaxing a mode there silently makes another target's outputs writable. Copy first if the + modes need changing. +- **`wine foo.cmd` does not run it as Windows would.** What Wine cannot load as a PE it hands + to the host, so a `.cmd` that still has a Unix shebang on it runs under `/bin/sh` and passes + the test you wrote to catch exactly that. Go through `cmd.exe` explicitly. +- **`plz update` on Windows fetches only the bare binary, not the zip.** Everything else the + release ships - busybox, arcat, and the plz.cmd shim - stays at the + version it was first installed at, silently, getting staler with each update. Nothing has + ever exercised this. +- **Python under Wine needs its output to be a pipe.** Wine's console emulation hands it handles + it rejects at startup otherwise, and the error — `can't initialize sys standard streams` — reads + like a problem with whatever you were testing. It is not. +- **`plz init plugin ` hands a Windows user plugins that cannot build there.** It writes + `owner = "please-build"`, and upstream `please_go`, `please_pex` and `please_cc` publish no + `windows_amd64` asset. This repo's `plugins/BUILD` uses the forks for exactly that reason, and + every codelab that installs a plugin inherits the problem. It is the single largest cause of + codelab failures and will be rediscovered by anyone who follows the docs. +- **`plz init plugin` asks GitHub's API for the latest tag anonymously.** Shared CI addresses hit + the unauthenticated rate limit, and the failure reads as a plugin that cannot be found. A 403 + from `api.github.com` in the codelabs job is that, not a regression. +- **The Go codelabs predate `plz init plugin go` generating a toolchain and a stdlib.** Their + `third_party/go/BUILD` holds only a `go_toolchain`, so following them replaces the generated + `go_stdlib`, and every Go build then fails to find `//third_party/go:std`, on every platform. diff --git a/docs/design/windows/appendix-baseline-errors.md b/docs/design/windows/appendix-baseline-errors.md new file mode 100644 index 0000000000..0c3907676a --- /dev/null +++ b/docs/design/windows/appendix-baseline-errors.md @@ -0,0 +1,207 @@ +# Appendix — Baseline Compile and Runtime Errors + +Status: **Measured** · Milestone: M0 · Last updated: 2026-09-10 + +Real results, not predictions. Measured against Go 1.27.0 (`GOOS=windows GOARCH=amd64`) at +commit `8cddc25` (Release 17.33.0), with a Linux control build as a baseline. + +**Headline: the port is far closer than the source survey suggested.** `please.exe` compiles, +links, parses BUILD files and executes build actions after ~290 lines of change. The hard +work is not making it build — it is making it correct. + +## Method + +```bash +GOOS=windows GOARCH=amd64 go build ./src/... ./tools/... +``` + +**This is misleading on its own.** Go stops at the first failing package in the dependency +graph, and `src/process` is a dependency of nearly everything. The first run reports 3 errors +in 1 package; 30 of 51 packages simply never get type-checked. + +The errors are **layered, not parallel**. You cannot enumerate them up front — each fix +reveals the next layer. Build each package independently and iterate: + +```bash +go list ./src/... ./tools/... > pkgs.txt +while read -r p; do GOOS=windows GOARCH=amd64 go build -o /dev/null "$p"; done < pkgs.txt +``` + +Plan M1 as an iterative loop, not as a checklist derived from a single error dump. + +## Compile blockers — the complete set + +Four layers, five sites, four packages. That is all. + +| Layer | Package | Site | Error | +|---|---|---|---| +| 1 | `src/process` | `exec_other.go:17,18` | `unknown field Setpgid / Foreground in syscall.SysProcAttr` | +| 1 | `src/process` | `process.go:206` | `undefined: syscall.Kill` | +| 2 | `src/core` | `lock.go` ×10 | `undefined: syscall.Flock`, `LOCK_SH/EX/UN/NB` | +| 3 | `src/clean` | `clean.go:96` | `undefined: syscall.ForkExec` | +| 3 | `src/output` | `shell_output.go:467` | `cmd.SysProcAttr.Setpgid undefined` | + +After layer 3, **every package compiles and `./src` links to a valid PE32+ binary.** + +``` +please.exe: PE32+ executable (console) x86-64, for MS Windows, 16 sections +``` + +### Layer 3 contains a site the source survey missed + +`src/output/shell_output.go:467`: + +```go +cmd := state.ProcessExecutor.ExecCommand(...) +// TODO(jpoole): Read the docs. Attaching stdin and out doesn't seem to work with this. +cmd.SysProcAttr.Setpgid = false +``` + +A caller **outside** `src/process` reaching into platform-specific process attributes. This is +an abstraction leak, and the fix is not a build tag here — it is to expose the intent from +`src/process` (e.g. `process.ClearProcessGroup(cmd)`, or a parameter on `ExecCommand`) so the +platform detail stays in one package. Worth auditing for other instances during M1. + +## Corrections to the source survey + +Predictions that were **wrong**, and why. All three arise from the same mistake: assuming +"Unix-only API" means "does not compile on Windows". + +### `syscall.Exec` is not a compile blocker + +Go's `syscall/exec_windows.go` defines: + +```go +func Exec(argv0 string, argv []string, envv []string) (err error) { + return EWINDOWS +} +``` + +It **compiles** and fails at **runtime**. All five call sites (`src/please.go`, +`src/run/run_step.go`, `src/tool/tool.go`, `src/update/update.go`, +`tools/please_shim/main.go`) build cleanly. + +This is *more* dangerous, not less: `plz run`, `plz tool`, `plz update`, `plz op` and the shim +will build, ship, and then fail at runtime with an opaque *"not supported by windows"*. The +`process.ExecReplace` work in `01-os-abstraction.md` is still required — it just cannot be +driven by the compiler. It needs tests. + +`syscall.Chdir` likewise exists on Windows. + +### Signal constants are not a compile blocker + +`syscall.SIGHUP`, `SIGQUIT` and `SIGABRT` are all defined in `syscall/types_windows.go`. +`src/cli/process.go` compiles unchanged. Narrowing the `signal.Notify` set is a *correctness* +change (Windows delivers only `os.Interrupt` and a synthesised `SIGTERM`), not a build fix. + +### `pkg/xattr` needs no build tag + +The module ships `xattr_unsupported.go`. It compiles for Windows and returns `ENOTSUP`. +The M0 open question is resolved: **defaulting `Build.Xattrs = false` on Windows is +sufficient.** No `attr_unix.go`/`attr_windows.go` split needed. + +### `syscall.ForkExec` genuinely is absent + +`src/clean` is the one prediction that held exactly. + +## Runtime findings + +Compiling is not the interesting part. These were found by running the binary under Wine 9.0 +and are ordered as encountered — each one blocks everything after it. + +### R1 — go-flags parses `//pkg:target` as a flag *(new; not in the original plan)* + +```console +$ wine please.exe query alltargets //... +CRITICAL: unknown flag `/...' +``` + +`github.com/thought-machine/go-flags` ships `optstyle_windows.go`: + +```go +// Windows uses a front slash for both short and long options. Also it uses +// a colon for name/argument delimter. +const ( + defaultShortOptDelimiter = '/' + defaultLongOptDelimiter = "/" + defaultNameArgDelimiter = ':' +) +``` + +**This collides with Please's entire label syntax.** `//pkg:target` parses as option `/pkg` +with argument `target`; `//...` is an unknown flag. Every command taking a build label — which +is nearly all of them — is broken. + +**Fix:** the file is guarded `// +build !forceposix`. Build with `-tags forceposix`: + +```bash +go build -tags forceposix ./src +``` + +Verified: label parsing works completely with the tag. + +Note it is **not** a per-target BUILD change — `go_binary` has no `tags` parameter. The go +plugin sources build tags from `CONFIG.GO.BUILD_TAGS`, so it goes in +`.plzconfig_windows_amd64` as `[Plugin "go"] BuildTags = forceposix`, where it covers every +go-flags binary built for Windows without touching other platforms. + +### R2 — `path.Dir` on a filesystem path blocks startup + +```console +CRITICAL: Error opening log file: open Z:\...\plz-out\log\build.log: Path not found. +``` + +`src/cli/logging.go:64` uses `path.Dir(logFile)` instead of `filepath.Dir`. Predicted in +`01-os-abstraction.md` as a "genuine bug"; confirmed here as a **hard startup blocker**, not a +cosmetic issue. `plz` cannot run at all until it is fixed. + +### R3 — no shell + +```console +Error building target //:hello: exec: "bash": executable file not found in %PATH% +``` + +Exactly as designed for in M3. Everything upstream of the shell works. + +## What works, verified end to end + +With the four compile fixes, `-tags forceposix`, the `path.Dir` fix, and busybox-w64 on +`%PATH%` as `bash.exe`: + +```console +$ wine please.exe query alltargets //... +//:hello + +$ wine please.exe build //:hello +plz-out\gen\hello.txt + +$ wine please.exe build //:pipeline //:findpipe +plz-out\gen\sorted.txt +plz-out\gen\found.txt +``` + +where `pipeline` is `cat $SRCS | sort > $OUT` and `findpipe` is +`find . -name '*.o' -or -name '*.a' | sort | tr '\n' ','` — the construct the cc rules depend +on. Both produce correct output. + +**This means the parser (`src/parse/asp`), config loading, the build graph, target hashing, +`plz-out` population and build-action execution all already work on Windows.** + +Note the predicted "hard gate" in `src/parse/internal_package.go` (the exhaustive arcat +platform switch) did **not** trigger for parse or for simple genrules. It is only reached when +the `_please` internal package is actually needed. Still required for M4, but it is not the +early blocker the plan implied. + +## Reference artifact + +The throwaway probe patch is at `probe/m1-skeleton.patch` (287 lines, 17 files). + +**It is not an implementation.** `lock_windows.go` returns `nil` — a no-op lock — and +`kill_windows.go` kills only the direct child, not the tree. It exists to prove the layering +and to give M1 a starting shape. Do not ship it. + +## Progress + +| Date | Compile-blocking sites | Notes | +|---|---|---| +| 2026-09-10 | 5 (4 packages, 4 layers) | Baseline. `please.exe` links, parses and builds after ~290 lines of probe changes. | diff --git a/docs/design/windows/probe/README.md b/docs/design/windows/probe/README.md new file mode 100644 index 0000000000..5a478d0af7 --- /dev/null +++ b/docs/design/windows/probe/README.md @@ -0,0 +1,33 @@ +# Probe artifacts + +Throwaway material from the M0 investigation. **Not implementations — do not ship.** + +- `m1-skeleton.patch` — the minimal set of changes that makes `plz` compile, link, parse and + build under Windows. 17 files, 287 lines. Deliberately incorrect in places: + `lock_windows.go` returns `nil` (a no-op lock, which would corrupt concurrent builds) and + `kill_windows.go` kills only the direct child rather than the process tree. + +Its value is that it proves the layering described in `../appendix-baseline-errors.md` and +gives M1 a starting shape. + +Apply with `git apply docs/design/windows/probe/m1-skeleton.patch` from the repo root, then +build with `-tags forceposix` (see R1 in the appendix). + +## Workstream B (`please-build/cc-rules`) + +- `cc-rules-windows.patch` — the M5 changes to `build_defs/cc.build_defs`, against v0.7.3. + Unlike the M1 skeleton these are real and were verified end to end, but they live in + another repo, so they are recorded here until they are upstreamed. +- `cc-rules.plzconfig_windows_amd64` — the arch config used to test them. The toolchain paths + assume `g++-mingw-w64-x86-64` is installed. + +Reproduce with: + +```bash +git clone --branch v0.7.3 https://github.com/please-build/cc-rules +cd cc-rules +git apply /path/to/cc-rules-windows.patch +cp /path/to/cc-rules.plzconfig_windows_amd64 .plzconfig_windows_amd64 +plz build --arch windows_amd64 //test/binary:test_binary +file plz-out/bin/windows_amd64/test/binary/test_binary.exe # PE32+ executable +``` diff --git a/docs/design/windows/probe/cc-rules-windows.patch b/docs/design/windows/probe/cc-rules-windows.patch new file mode 100644 index 0000000000..5c6b9b1973 --- /dev/null +++ b/docs/design/windows/probe/cc-rules-windows.patch @@ -0,0 +1,56 @@ +diff --git a/build_defs/cc.build_defs b/build_defs/cc.build_defs +index bfdfaa6..92db3cd 100644 +--- a/build_defs/cc.build_defs ++++ b/build_defs/cc.build_defs +@@ -26,6 +26,23 @@ _ACTION_FLAGS = [ + + _COVERAGE_FLAGS = ["--coverage", "-fprofile-dir=."] + ++def _exe_suffix(): ++ """Returns the filename suffix for an executable on the target platform. ++ ++ Windows decides what is executable by extension, and the toolchain appends .exe to a -o ++ name that has none, so the rule has to declare the name the compiler will actually write. ++ ++ N.B. this has to be evaluated per call rather than once at module level: these build defs ++ are subincluded, and CONFIG.OS there reflects the host until a rule is actually being ++ instantiated for the target architecture. ++ """ ++ return ".exe" if CONFIG.OS == "windows" else "" ++ ++ ++def _so_suffix(): ++ """Returns the filename suffix for a shared library on the target platform.""" ++ return ".dll" if CONFIG.OS == "windows" else ".so" ++ + # Clang >= 16 just require C++20 features to be enabled in order to enable modules, while older versions explicitly + # require the -fmodules-ts flag. + _MODULE_FLAGS = ["""'{{ clang && clang < 16 ? "-fmodules-ts" : "-std=c++20" }}'"""] +@@ -436,7 +453,8 @@ def cc_shared_object(name:str, srcs:list=[], hdrs:list=[], out:str='', optional_ + } + cmds, tools = _binary_cmds(_c, linker_flags, pkg_config_libs, strip, shared=True, test=False) + if not out: +- out = f'{name}.so' if name.startswith('lib') else f'lib{name}.so' ++ so = _so_suffix() ++ out = f'{name}{so}' if name.startswith('lib') else f'lib{name}{so}' + return build_rule( + name=name, + srcs={'srcs': srcs, 'hdrs': hdrs}, +@@ -584,7 +602,7 @@ def cc_binary(name:str, srcs:list=[], hdrs:list=[], private_hdrs:list=[], + deps += [lib_rule] + return build_rule( + name=name, +- outs=[name], ++ outs=[name + _exe_suffix()], + deps=deps, + visibility=visibility, + cmd=cmds, +@@ -675,7 +693,7 @@ def cc_test(name:str, srcs:list=[], hdrs:list=[], compiler_flags:list&cflags&cop + + return build_rule( + name=name, +- outs=[name], ++ outs=[name + _exe_suffix()], + deps=deps, + data=data, + visibility=visibility, diff --git a/docs/design/windows/probe/cc-rules.plzconfig_windows_amd64 b/docs/design/windows/probe/cc-rules.plzconfig_windows_amd64 new file mode 100644 index 0000000000..176f0dcbe4 --- /dev/null +++ b/docs/design/windows/probe/cc-rules.plzconfig_windows_amd64 @@ -0,0 +1,5 @@ +[Plugin "cc"] +cctool = x86_64-w64-mingw32-gcc +cpptool = x86_64-w64-mingw32-g++ +artool = x86_64-w64-mingw32-ar +defaultldflags = -lpthread diff --git a/docs/design/windows/probe/m1-skeleton.patch b/docs/design/windows/probe/m1-skeleton.patch new file mode 100644 index 0000000000..536e21a2f5 --- /dev/null +++ b/docs/design/windows/probe/m1-skeleton.patch @@ -0,0 +1,287 @@ +--- a/src/process/exec_other.go ++++ b/src/process/exec_other.go +@@ -1,5 +1,5 @@ +-//go:build !linux +-// +build !linux ++//go:build !linux && !windows ++// +build !linux,!windows + + package process + +--- a/src/process/exec_windows.go ++++ b/src/process/exec_windows.go +@@ -0,0 +1,7 @@ ++package process ++ ++import "os/exec" ++ ++func (e *Executor) ExecCommand(sandbox SandboxConfig, foreground bool, command string, args ...string) *exec.Cmd { ++ return exec.Command(command, args...) ++} +--- a/src/process/kill_unix.go ++++ b/src/process/kill_unix.go +@@ -0,0 +1,12 @@ ++//go:build !windows ++ ++package process ++ ++import ( ++ "os/exec" ++ "syscall" ++) ++ ++func killGroup(cmd *exec.Cmd, sig syscall.Signal) { ++ syscall.Kill(-cmd.Process.Pid, sig) ++} +--- a/src/process/kill_windows.go ++++ b/src/process/kill_windows.go +@@ -0,0 +1,10 @@ ++package process ++ ++import ( ++ "os/exec" ++ "syscall" ++) ++ ++func killGroup(cmd *exec.Cmd, sig syscall.Signal) { ++ _ = cmd.Process.Kill() ++} +--- a/src/process/pgroup_unix.go ++++ b/src/process/pgroup_unix.go +@@ -0,0 +1,7 @@ ++//go:build !windows ++ ++package process ++ ++import "os/exec" ++ ++func ClearProcessGroup(cmd *exec.Cmd) { cmd.SysProcAttr.Setpgid = false } +--- a/src/process/pgroup_windows.go ++++ b/src/process/pgroup_windows.go +@@ -0,0 +1,5 @@ ++package process ++ ++import "os/exec" ++ ++func ClearProcessGroup(cmd *exec.Cmd) {} +--- a/src/process/shellargs_unix.go ++++ b/src/process/shellargs_unix.go +@@ -0,0 +1,5 @@ ++//go:build !windows ++ ++package process ++ ++var shellArgs = []string{"--noprofile", "--norc"} +--- a/src/process/shellargs_windows.go ++++ b/src/process/shellargs_windows.go +@@ -0,0 +1,4 @@ ++package process ++ ++// busybox-w64's bash applet rejects --noprofile/--norc but honours -e/-u/-o pipefail. ++var shellArgs []string +--- a/src/process/process.go ++++ b/src/process/process.go +@@ -203,7 +203,7 @@ + // This is a bit of a fiddle. We want to wait for the process to exit but only for just so + // long (we do not want to get hung up if it ignores our SIGTERM). + log.Debug("Sending signal %s to -%d", sig, cmd.Process.Pid) +- syscall.Kill(-cmd.Process.Pid, sig) // Kill the group - we always set one in ExecCommand. ++ killGroup(cmd, sig) + + select { + case <-ch: +@@ -293,8 +293,10 @@ + + // BashCommand returns the command that we'd use to execute a subprocess in a shell with. + func BashCommand(binary, command string, exitOnError bool) []string { ++ argv := append([]string{binary}, shellArgs...) + if exitOnError { +- return []string{binary, "--noprofile", "--norc", "-e", "-u", "-o", "pipefail", "-c", command} ++ argv = append(argv, "-e") + } +- return []string{binary, "--noprofile", "--norc", "-u", "-o", "pipefail", "-c", command} ++ argv = append(argv, "-u", "-o", "pipefail", "-c", command) ++ return argv + } +--- a/src/core/lock.go ++++ b/src/core/lock.go +@@ -6,7 +6,6 @@ + "fmt" + "os" + "strconv" +- "syscall" + + "github.com/thought-machine/please/src/fs" + ) +@@ -25,7 +24,7 @@ + // AcquireSharedRepoLock acquires a shared lock on the repo lock file. The file descriptor is reused if already opened + // allowing its lock mode to be replaced. Dies if the lock cannot be successfully acquired. + func AcquireSharedRepoLock() { +- if err := acquireRepoLock(syscall.LOCK_SH); err != nil { ++ if err := acquireRepoLock(plzLOCK_SH); err != nil { + log.Fatal(err) + } + } +@@ -33,7 +32,7 @@ + // AcquireExclusiveRepoLock acquires an exclusive lock on the repo lock file. The file descriptor is reused if already opened + // allowing its lock mode to be replaced. Dies if the lock cannot be successfully acquired. + func AcquireExclusiveRepoLock() { +- if err := acquireRepoLock(syscall.LOCK_EX); err != nil { ++ if err := acquireRepoLock(plzLOCK_EX); err != nil { + log.Fatal(err) + } + } +@@ -75,13 +74,13 @@ + + // AcquireExclusiveFileLock opens a file to acquire an exclusive lock. + func AcquireExclusiveFileLock(filePath string) (*os.File, error) { +- return acquireOpenFileLock(filePath, syscall.LOCK_EX) ++ return acquireOpenFileLock(filePath, plzLOCK_EX) + } + + // AcquireSharedFileLock opens a file to acquire a shared lock. + // Multiple of these can be held at once, but not concurrently with an exclusive lock (ala a RWMutex or similar). + func AcquireSharedFileLock(filePath string) (*os.File, error) { +- return acquireOpenFileLock(filePath, syscall.LOCK_SH) ++ return acquireOpenFileLock(filePath, plzLOCK_SH) + } + + // Base function that allows to set up different lock modes and facilitate testing. +@@ -105,7 +104,7 @@ + return + } + +- if err := syscall.Flock(int(file.Fd()), syscall.LOCK_UN); err != nil { ++ if err := plzFlock(int(file.Fd()), plzLOCK_UN); err != nil { + log.Errorf("Failed to release lock for %s: %s", file.Name(), err) // No point making this fatal really + } + if err := file.Close(); err != nil { +@@ -118,7 +117,7 @@ + func acquireFileLock(file *os.File, how int, levelLog logFunc) error { + // Try a non-blocking acquire first so we can warn the user if we're waiting. + log.Debug("Attempting to acquire lock for %s...", file.Name()) +- err := syscall.Flock(int(file.Fd()), how|syscall.LOCK_NB) ++ err := plzFlock(int(file.Fd()), how|plzLOCK_NB) + if err != nil { + pid, err := os.ReadFile(file.Name()) + if err == nil && len(pid) > 0 { +@@ -127,14 +126,14 @@ + levelLog("Looks like another process has already acquired the lock for %s. Waiting for it to finish...", file.Name()) + } + +- if err := syscall.Flock(int(file.Fd()), how); err != nil { ++ if err := plzFlock(int(file.Fd()), how); err != nil { + return fmt.Errorf("Failed to acquire lock for %s: %w", file.Name(), err) + } + } + log.Debug("Acquired lock for %s", file.Name()) + + // Record content, only if we have an exclusive lock. +- if how&syscall.LOCK_EX != 0 { ++ if how&plzLOCK_EX != 0 { + if err := file.Truncate(0); err == nil { + file.WriteAt([]byte(strconv.Itoa(os.Getpid())), 0) + } +--- a/src/core/lock_unix.go ++++ b/src/core/lock_unix.go +@@ -0,0 +1,14 @@ ++//go:build !windows ++ ++package core ++ ++import "syscall" ++ ++const ( ++ plzLOCK_SH = syscall.LOCK_SH ++ plzLOCK_EX = syscall.LOCK_EX ++ plzLOCK_UN = syscall.LOCK_UN ++ plzLOCK_NB = syscall.LOCK_NB ++) ++ ++func plzFlock(fd, how int) error { return syscall.Flock(fd, how) } +--- a/src/core/lock_windows.go ++++ b/src/core/lock_windows.go +@@ -0,0 +1,10 @@ ++package core ++ ++const ( ++ plzLOCK_SH = 1 ++ plzLOCK_EX = 2 ++ plzLOCK_UN = 8 ++ plzLOCK_NB = 4 ++) ++ ++func plzFlock(fd, how int) error { return nil } +--- a/src/clean/clean.go ++++ b/src/clean/clean.go +@@ -8,7 +8,6 @@ + "fmt" + "os" + "path/filepath" +- "syscall" + + "github.com/thought-machine/please/src/build" + "github.com/thought-machine/please/src/cli/logging" +@@ -93,7 +92,7 @@ + } + // Note that we can't fork() directly and continue running Go code, but ForkExec() works okay, + // so we re-execute ourselves with a specific command that will remove this. +- _, err = syscall.ForkExec(exec, []string{exec, "clean", "--rm", newDir}, nil) ++ err = forkExec(exec, []string{exec, "clean", "--rm", newDir}) + return err + } + +--- a/src/clean/forkexec_unix.go ++++ b/src/clean/forkexec_unix.go +@@ -0,0 +1,10 @@ ++//go:build !windows ++ ++package clean ++ ++import "syscall" ++ ++func forkExec(bin string, argv []string) error { ++ _, err := syscall.ForkExec(bin, argv, nil) ++ return err ++} +--- a/src/clean/forkexec_windows.go ++++ b/src/clean/forkexec_windows.go +@@ -0,0 +1,7 @@ ++package clean ++ ++import "os/exec" ++ ++func forkExec(bin string, argv []string) error { ++ return exec.Command(bin, argv[1:]...).Start() ++} +--- a/src/cli/logging.go ++++ b/src/cli/logging.go +@@ -7,7 +7,7 @@ + "container/list" + "fmt" + "os" +- "path" ++ "path/filepath" + "regexp" + "strings" + "sync" +@@ -61,7 +61,7 @@ + // InitFileLogging initialises an optional logging backend to a file. + func InitFileLogging(logFile string, logFileLevel Verbosity, append bool) { + fileLogLevel = logging.Level(logFileLevel) +- if err := os.MkdirAll(path.Dir(logFile), os.ModeDir|0775); err != nil { ++ if err := os.MkdirAll(filepath.Dir(logFile), os.ModeDir|0775); err != nil { + log.Fatalf("Error creating log file directory: %s", err) + } + flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC +--- a/src/output/shell_output.go ++++ b/src/output/shell_output.go +@@ -464,7 +464,7 @@ + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // TODO(jpoole): Read the docs. Attaching stdin and out doesn't seem to work with this. +- cmd.SysProcAttr.Setpgid = false ++ process.ClearProcessGroup(cmd) + cmd.Run() // Ignore errors, it will typically end by the user killing it somehow. + } + } diff --git a/docs/faq.html b/docs/faq.html index ef1e881f2c..dab34b8046 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -37,9 +37,24 @@

    - Windows is unfortunately not supported natively, since it's just too - different from the Unix environment Please is designed for. It is possible - to run it within + Windows is supported natively on amd64. Please ships busybox as its build + shell there, so a fresh install needs no configuration: extract the release + and run 

    plz
    . Building Go, C++ and Python + all work, and the test suite runs on Windows in CI rather than only under + emulation. +

    + +

    + Two differences are worth knowing. There is no build sandbox on Windows, + because it is built on Linux namespaces and there is no equivalent; build + actions and tests see the whole machine. And real-time virus scanning holds + freshly written files open, which shows up as intermittent sharing-violation + errors and slow builds, so excluding your + 

    plz-out
    directories is worth doing. +

    + +

    + Running under rel="noopener" >WSL - though. + also works, and is still the better choice if your build depends on Unix + tooling that has no Windows equivalent.

    diff --git a/docs/milestones/18.0.0.html b/docs/milestones/18.0.0.html new file mode 100644 index 0000000000..1fcbe5cddd --- /dev/null +++ b/docs/milestones/18.0.0.html @@ -0,0 +1,64 @@ +

    Please 18.0.0

    + +

    + This release adds native Windows support. You can find the complete changelog + here. +

    + +
    +

    Windows

    +

    + Please now runs natively on Windows, on amd64. Extract the release and run + plz; there is nothing to configure. Windows has no shell that can run a + build action, so the release bundles busybox and + uses it by default. +

    +

    + Install it with: +

    + 
    irm https://get.please.build/get_plz.ps1 | iex
    +

    + Building Go, C++ and Python all work. C++ builds through MinGW-w64, including shared libraries, + which Windows links through an import library rather than through the DLL itself. A + sh_binary becomes a .cmd, because Windows has no + shebang mechanism and a batch file is the only kind of script it will run by name. +

    +
    + +
    +

    What to expect

    +

    + Two things behave differently, and neither is a bug we intend to fix soon. +

    +

    + There is no build sandbox. Please's sandbox is built on Linux namespaces and Windows has no + equivalent, so build actions and tests see the whole machine. plz will not + pretend otherwise: sandboxing is off in the default Windows configuration rather than silently + ineffective. +

    +

    + Real-time virus scanning holds freshly written files open, which Windows treats as a reason to + refuse deleting or renaming them. That surfaces as intermittent sharing-violation errors and slower + builds. Excluding your plz-out directories from scanning is worth doing. +

    +

    + Symlinks need Developer Mode or SeCreateSymbolicLinkPrivilege, which an + ordinary account has neither of. Where Please would make one it copies instead, so nothing fails; you + will see a single warning saying so. +

    +
    + +
    +

    How it is tested

    +

    + The Windows build is exercised two ways. A Wine job on Linux runs the cross-built binaries on every + change, which is what made the port developable at all. On top of that, a + windows-latest job runs the same test binaries natively, alongside probes + that build a repository with the release itself, clean and rebuild it repeatedly under a live virus + scanner, and build at a long path. +

    +

    + The native job is the one that counts, and it is blocking. Wine passing is evidence, not proof, and + most of the bugs found late in this work were ones Wine had been passing for months. +

    +
    diff --git a/package/BUILD b/package/BUILD index 97d140ff3b..16cdf7b740 100644 --- a/package/BUILD +++ b/package/BUILD @@ -1,11 +1,37 @@ subinclude("//build_defs:version") +# arcat, built for Windows. Please normally downloads a prebuilt arcat keyed by platform and +# there is no Windows one, so the release carries this instead; [build] arcattool defaults to +# it once it is installed beside please.exe. See docs/design/windows/08-offline-release.md. +# +# Renamed because Windows will not run a file whose name has no extension in PATHEXT, and the +# go plugin names a binary after its rule. cp, rather than asking for a different out, because +# the target is not ours to change. +# +# please_go, please_cc and please_pex are not here, and cannot be: cross-compiling a plugin's +# own tool through ///go//tools/please_go and friends collides on subrepo names, because an arch +# subrepo's name does not carry the subrepo that owns it, so the plugin's third_party/go and +# ours both claim third_party/go/github.com_stretchr_testify@windows_amd64. Until those three +# have windows_amd64 releases to download, a native Windows plz can parse and extract plugins +# but cannot build a Go, C++ or Python target. Cross-building from Linux is unaffected, since +# tools resolve to the host. +if is_platform(os = "windows"): + genrule( + name = "arcat", + srcs = ["///third_party/go/github.com_please-build_arcat//:arcat"], + outs = ["arcat.exe"], + binary = True, + cmd = "cp $SRC $OUT", + ) + filegroup( name = "tools", srcs = [ "//tools/build_langserver", + ] + ([ + # The sandbox is built on Linux namespaces, so there is nothing to ship elsewhere. "//tools/sandbox:please_sandbox", - ], + ] if not is_platform(os = "windows") else []), binary = True, visibility = ["//src:tools"], ) @@ -15,23 +41,61 @@ filegroup( srcs = [ ":tools", "//src:please", - ], + ] + ([ + # Windows has no shell that can run a build action, so we ship one. The default + # [build] shell is 'busybox', which resolves to this once it's installed. + "//third_party/binary:busybox", + # install.sh gets the short name with 'ln -sf please plz'. Symlinks need Developer + # Mode on Windows, so a one-line batch file stands in for it. + ":plz_cmd", + # Windows has no installer and no package manager to carry the instructions, so they + # travel in the zip. + ":install_md", + # Extracting a plugin needs arcat, and there is no arcat release for Windows, so a + # fresh install there cannot get a plugin at all without one of its own. + ":arcat", + ] if is_platform(os = "windows") else []), binary = True, entry_points = { - "please": "please", + "please": "please.exe" if is_platform(os = "windows") else "please", }, labels = ["link:plz-out/please"], visibility = ["PUBLIC"], ) -tarball( - name = "please_tarball_xz", - srcs = [":installed_files"], - out = "please_%s.tar.xz" % VERSION, - subdir = "please", - xzip = True, +filegroup( + name = "plz_cmd", + srcs = ["plz.cmd"], + binary = True, +) + +filegroup( + name = "install_md", + srcs = ["Install.md"], ) +# xz only compresses where there is an xz binary to do it, which excludes Windows - the +# busybox we bundle there decompresses only. Windows gets a .zip in place of the two xz +# tarballs; the gzip one is built everywhere. +XZIP = not is_platform(os = "windows") + +if XZIP: + tarball( + name = "please_tarball_xz", + srcs = [":installed_files"], + out = "please_%s.tar.xz" % VERSION, + subdir = "please", + xzip = True, + ) + + tarball( + name = "please_tools_tarball", + srcs = [":tools"], + out = "please_tools_%s.tar.xz" % VERSION, + subdir = "please_tools", + xzip = True, + ) + tarball( name = "please_tarball", srcs = [":installed_files"], @@ -39,13 +103,20 @@ tarball( subdir = "please", ) -tarball( - name = "please_tools_tarball", - srcs = [":tools"], - out = "please_tools_%s.tar.xz" % VERSION, - subdir = "please_tools", - xzip = True, -) +# Windows has no guaranteed tar, so its release is a zip. This is built on the Linux release +# box like everything else, so it uses the host's arcat, not a Windows one. +if is_platform(os = "windows"): + genrule( + name = "please_zip", + srcs = [":installed_files"], + outs = ["please_%s.zip" % VERSION], + # --rename_dir, rather than --prefix, because entries come out under the package + # directory they were built in and the release wants them at the top of a please/. + cmd = "$TOOL zip --dumb --input package --output $OUT --rename_dir package:please", + tools = [CONFIG.ARCAT_TOOL], + # //test/windows runs the artifact itself, rather than a reconstruction of it. + visibility = ["//test/windows:all"], + ) genrule( name = "please", @@ -69,9 +140,10 @@ filegroup( ":please", ":please_shim", ":please_tarball", + ] + ([ ":please_tarball_xz", ":please_tools_tarball", - ], + ] if XZIP else [":please_zip"]), labels = ["hlink:plz-out/pkg/${OS}_${ARCH}"], ) diff --git a/package/Install.md b/package/Install.md new file mode 100755 index 0000000000..a490dc80d8 --- /dev/null +++ b/package/Install.md @@ -0,0 +1,78 @@ +# Installing Please on Windows + +This is the Windows build of Please. Windows support is new, so read the last two sections +before you rely on it. + +## Install + +1. Extract the zip. It contains a single `please` directory; put that wherever you keep tools, + for example `C:\Tools\please`. Nothing writes to the directory afterwards, so Program Files + is fine too. +2. Add that directory to your `PATH`, so that `plz` works from any repository. +3. Check it: + + ``` + plz --version + ``` + +`plz.cmd` is the entry point, and it does nothing but run `please.exe` beside it. Everywhere +else Please installs `plz` as a symlink; Windows needs Developer Mode for those, so a one-line +batch file stands in. + +There is no installer, no registry key and no service. Uninstalling is deleting the directory. + +## What is in here + +| File | What it is | +|---|---| +| `please.exe` | Please itself | +| `plz.cmd` | the short name you type | +| `busybox.exe` | the shell that build actions run in | +| `build_langserver.exe` | the BUILD-file language server, for editor integration | +| `arcat.exe` | Please's archive tool, used by many built-in rules | + +Some builds also carry the language plugins, as `plugin_*.zip` alongside `please_go.exe`, +`please_cc.exe` and `please_pex.exe`. If `plugin_revisions.txt` is here, yours is one of them, +and that file says exactly which build of each plugin you have. + +**Keep these files together.** Please finds the shell, the archive tool and the bundled plugins +by looking beside its own binary. Copying `please.exe` out on its own leaves it unable to run a +build action. + +## Nothing else to configure + +A repository that asks for the go, cc, shell or python plugin the ordinary way works as it is. +Where this build bundles them, they resolve from the install directory rather than being +downloaded, so a machine with no internet access builds the same as one with it. To use the +published plugins instead, delete the `plugin_*.zip` files. + +**Language toolchains are not bundled, and never are on any platform.** Building Go needs Go, +C++ needs a compiler, and Python needs an interpreter, each installed separately and pointed at +from your repository's `.plzconfig` the same way it would be anywhere else. What is bundled is +only what Please itself needs. + +## Known limitations + +- **`plz update` does not refresh everything.** It fetches only the Please binary, so busybox, + the archive tool and any bundled plugins stay at the version you first installed. Re-extract + the zip instead of updating in place. +- **A repository that pins an exact `[please] version`** will try to download that version from + the Please download server, which has no Windows release yet. Use a `>=` constraint, or set + `selfupdate = false`. +- **Build sandboxing is off.** It is built on Linux namespaces and there is no Windows + equivalent yet, so build actions and tests see the whole machine. +- **Real-time antivirus scanning locks files Please has just written**, which shows up as + intermittent sharing-violation errors and slow builds. Excluding your `plz-out` directories + helps. +- **Long paths.** Output paths nest deeply. If your repository lives far from the root of a + drive, turn on Windows long-path support. + +## Where a bundled plugin came from + +If this build carries plugins, `plugin_revisions.txt` names the exact commit of each, and they +are **not** the upstream releases their version numbers suggest: each is that release plus +Windows fixes that are not published anywhere yet. + +They also answer for whatever revision your repository asks for. A repository pinning a +different version of a plugin silently gets the bundled one on Windows. That is what lets an +unmodified repository build with no network at all; deleting the archive opts back out of it. diff --git a/package/plz.cmd b/package/plz.cmd new file mode 100644 index 0000000000..ebb44257c6 --- /dev/null +++ b/package/plz.cmd @@ -0,0 +1,4 @@ +@echo off +rem Please is installed as please.exe; this is the short name people actually type. +rem %~dp0 is the directory this script lives in, with a trailing backslash. +"%~dp0please.exe" %* diff --git a/pleasew.ps1 b/pleasew.ps1 new file mode 100644 index 0000000000..e9d94f03a4 --- /dev/null +++ b/pleasew.ps1 @@ -0,0 +1,140 @@ +# The Windows counterpart of pleasew: find or download the Please version this repo asks for, +# then hand over to it. Kept deliberately parallel to that script rather than clever, so the +# two can be read side by side. + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# This fork publishes GitHub Releases rather than to the bucket upstream uses; see +# tools/misc/get_plz.ps1. Set [please] downloadlocation to the bucket to use that instead - the +# path shape differs, and the base says which one to build. +$DefaultUrlBase = 'https://github.com/PeterNeiss/please/releases/download' + +if ($env:PROCESSOR_ARCHITECTURE -eq 'AMD64' -or $env:PROCESSOR_ARCHITEW6432 -eq 'AMD64') { + $Arch = 'amd64' +} else { + Write-Error "Please does not support the $env:PROCESSOR_ARCHITECTURE architecture on Windows." + exit 1 +} +$Os = 'windows' + +# Check PLZ_CONFIG_PROFILE, or fall back to a --profile argument. +function Get-Profile { + if ($env:PLZ_CONFIG_PROFILE) { return $env:PLZ_CONFIG_PROFILE } + for ($i = 0; $i -lt $args.Count; $i++) { + if ($args[$i] -like '--profile=*') { return $args[$i].Split('=', 2)[1] } + if ($args[$i] -eq '--profile' -and $i + 1 -lt $args.Count) { return $args[$i + 1] } + } + return '' +} + +# Find the repo root by walking up until we see a .plzconfig. +function Find-RepoRoot { + $dir = Get-Location + while ($dir) { + if (Test-Path (Join-Path $dir '.plzconfig')) { return $dir.ToString() } + $parent = Split-Path -Parent $dir + if ($parent -eq $dir -or -not $parent) { return '' } + $dir = $parent + } + return '' +} + +$Profile_ = Get-Profile @args +$RepoRoot = Find-RepoRoot + +# Config files in order of precedence, high to low. +$Configs = @() +if ($RepoRoot) { + $Configs += Join-Path $RepoRoot '.plzconfig.local' + if ($Profile_) { $Configs += Join-Path $RepoRoot ".plzconfig.$Profile_" } + $Configs += Join-Path $RepoRoot ".plzconfig_${Os}_${Arch}" + $Configs += Join-Path $RepoRoot '.plzconfig' +} +$Configs += Join-Path $env:USERPROFILE '.config\please\plzconfig' +$Configs += Join-Path $env:ProgramData 'please\plzconfig' + +# Returns the value of the first key matching the pattern, across the config files in order. +function Read-Config([string] $Pattern) { + foreach ($config in $Configs) { + if (-not (Test-Path $config)) { continue } + $match = Select-String -Path $config -Pattern $Pattern -CaseSensitive:$false | Select-Object -First 1 + if ($match) { + $parts = $match.Line -split '=', 2 + if ($parts.Count -eq 2) { return $parts[1].Trim() } + } + } + return '' +} + +$Location = Read-Config '^\s*location' +if ($Location) { + # It can contain a literal ~, which nothing on Windows expands for us. + $Location = $Location -replace '^~', $env:USERPROFILE +} else { + $Location = Join-Path $env:USERPROFILE '.please' +} + +# If Please is already here at any version, let it handle any update itself. +$Target = Join-Path $Location 'please.exe' +if (Test-Path $Target) { + & $Target @args + exit $LASTEXITCODE +} + +$UrlBase = Read-Config '^\s*downloadlocation' +if (-not $UrlBase) { $UrlBase = $DefaultUrlBase } +$UrlBase = $UrlBase.TrimEnd('/') + +$Version = Read-Config '^\s*version[^a-z]' +$Version = $Version -replace '^>=', '' +if (-not $Version) { + Write-Warning "Can't determine version, will use latest." + if ($UrlBase -like '*github.com*') { + # A GitHub release has no latest_version file; the redirect on /releases/latest names + # the tag. + $Repo = ($UrlBase -replace '/releases/download$', '') + $Version = ((Invoke-WebRequest -UseBasicParsing "$Repo/releases/latest").BaseResponse.RequestMessage.RequestUri.AbsoluteUri -split '/')[-1] -replace '^v', '' + } else { + $Version = (Invoke-WebRequest -UseBasicParsing "$UrlBase/latest_version").Content.Trim() + } +} + +$Dir = Join-Path $Location $Version +$Zip = Join-Path ([System.IO.Path]::GetTempPath()) "please_$Version.zip" + +Write-Host "Downloading Please $Version to $Dir..." -ForegroundColor Green +if (Test-Path $Dir) { Remove-Item -Recurse -Force $Dir } +New-Item -ItemType Directory -Force -Path $Dir | Out-Null +# The two layouts differ: a release keeps everything under one tag with the platform in the +# filename, the bucket keeps a directory per platform and version. +$Url = if ($UrlBase -like '*github.com*') { + "$UrlBase/v$Version/please_${Version}_${Os}_${Arch}.zip" +} else { + "$UrlBase/${Os}_${Arch}/$Version/please_$Version.zip" +} +Invoke-WebRequest -UseBasicParsing $Url -OutFile $Zip + +# The zip holds everything under a please/ directory, which is the layer the tarball strips +# with --strip-components=1. Expand-Archive has no equivalent, so unpack and move up. +$Staging = Join-Path ([System.IO.Path]::GetTempPath()) "please_$Version" +if (Test-Path $Staging) { Remove-Item -Recurse -Force $Staging } +Expand-Archive -Path $Zip -DestinationPath $Staging +Move-Item (Join-Path $Staging 'please\*') $Dir +Remove-Item -Recurse -Force $Staging, $Zip + +# Link it all back up a directory. Symlinks need Developer Mode on Windows, so hard-link +# where we can and copy where we can't; this is the same choice the self-updater makes. +foreach ($file in Get-ChildItem -File $Dir) { + $link = Join-Path $Location $file.Name + if (Test-Path $link) { Remove-Item -Force $link } + try { + New-Item -ItemType HardLink -Path $link -Target $file.FullName -ErrorAction Stop | Out-Null + } catch { + Copy-Item -Force $file.FullName $link + } +} + +Write-Host 'Should be good to go now, running plz...' -ForegroundColor Green +& $Target @args +exit $LASTEXITCODE diff --git a/plugins/BUILD b/plugins/BUILD index 3d8c22f52d..51a142083a 100644 --- a/plugins/BUILD +++ b/plugins/BUILD @@ -1,23 +1,21 @@ -plugin_repo( - name = "go", - plugin = "go-rules", - revision = "v1.31.1", -) +# Each plugin is pinned here as an archive download. +# +# These point at forks rather than at please-build, because the Windows support in all four is +# on a branch that is not merged anywhere. Pinned to commit SHAs rather than to that branch: a +# branch archive changes whenever it is pushed to, which would silently move every build hash +# that reaches it and leave the cache serving something else. When the work lands upstream, the +# owner goes back to please-build and these become ordinary version tags. +PLUGINS = [ + ("go", "go-rules", "abd06f1"), + ("cc", "cc-rules", "90913bb"), + ("shell", "shell-rules", "7ed07be"), + ("python", "python-rules", "2edd835"), +] -plugin_repo( - name = "cc", - plugin = "cc-rules", - revision = "v0.7.3", -) - -plugin_repo( - name = "shell", - plugin = "shell-rules", - revision = "v0.2.1", -) - -plugin_repo( - name = "python", - plugin = "python-rules", - revision = "v2.0.2", -) +for name, plugin, revision in PLUGINS: + plugin_repo( + name = name, + owner = "PeterNeiss", + plugin = plugin, + revision = revision, + ) diff --git a/rules/misc_rules.build_defs b/rules/misc_rules.build_defs index 0aaf7e7510..734b299b0f 100644 --- a/rules/misc_rules.build_defs +++ b/rules/misc_rules.build_defs @@ -650,6 +650,11 @@ def tarball(name:str, srcs:list, out:str=None, deps:list=None, subdir:str=None, tar_out = out or (name + ('.tar.gz' if gzip else '.tar')) cmd = '$TOOL tar ' tar_name = name + if xzip and CONFIG.OS == 'windows': + # The xz applet in the busybox we bundle on Windows only decompresses, and Windows + # has no other xz to fall back on. Callers that need a compressed archive there + # should use gzip, which is built into the tar tool. + fail('tarball(xzip = True) is not supported on Windows; use gzip = True instead') if xzip: tar_out = name + '.tar' tar_name = f'_{name}#tar' diff --git a/src/assets/assets.go b/src/assets/assets.go index b5507d40b7..036a092b5d 100644 --- a/src/assets/assets.go +++ b/src/assets/assets.go @@ -10,6 +10,11 @@ import ( //go:embed pleasew var Pleasew []byte +// PleasewPS1 is the Windows counterpart of the wrapper script +// +//go:embed pleasew.ps1 +var PleasewPS1 []byte + // PlzComplete is the plz completion script // //go:embed plz_complete.sh diff --git a/src/assets/pleasew.ps1 b/src/assets/pleasew.ps1 new file mode 100644 index 0000000000..c247535c1c --- /dev/null +++ b/src/assets/pleasew.ps1 @@ -0,0 +1 @@ +needed for `go build src/please.go` \ No newline at end of file diff --git a/src/build/BUILD b/src/build/BUILD index 5a30845694..0598ec6f8f 100644 --- a/src/build/BUILD +++ b/src/build/BUILD @@ -31,7 +31,9 @@ go_test( "incrementality_test.go", "remote_file_test.go", ], - data = ["test_data"], + data = [":test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":build", "///third_party/go/github.com_stretchr_testify//assert", @@ -54,3 +56,11 @@ go_test( "//src/plz", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/build/build_step.go b/src/build/build_step.go index af13ea2b20..b49b96dcb1 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -671,7 +671,11 @@ func addOutputDirectoryToBuildOutput(target *core.BuildTarget, dir core.OutputDi func copyOutDir(target *core.BuildTarget, from string, to string) ([]string, error) { relativeToTmpdir := func(path string) string { - return strings.TrimPrefix(strings.TrimPrefix(path, target.TmpDir()), "/") + // ToSlash first: the argument was assembled with filepath.Join and so uses the host + // separator, while TmpDir is slash-separated. Without it neither prefix matches on + // Windows, the whole path survives as the output name, and moveOutputs then joins the + // temp directory onto a path that already contains it. + return strings.TrimPrefix(strings.TrimPrefix(filepath.ToSlash(path), target.TmpDir()), "/") } var outs []string @@ -700,6 +704,17 @@ func copyOutDir(target *core.BuildTarget, from string, to string) ([]string, err return outs, os.Rename(from, to) } +// fileURLPath returns the filesystem path a file:// URL refers to. +// The path component of such a URL always begins with a slash, so on Windows the drive letter +// arrives as /C:/foo and the slash has to come off before it is an absolute path at all. +func fileURLPath(url string) string { + path := strings.TrimPrefix(url, "file://") + if filepath.Separator == '\\' && len(path) >= 3 && path[0] == '/' && path[2] == ':' { + return path[1:] + } + return path +} + func moveOutputs(state *core.BuildState, target *core.BuildTarget) ([]string, bool, error) { changed := false tmpDir := target.TmpDir() @@ -1016,11 +1031,14 @@ func checkLicences(state *core.BuildState, target *core.BuildTarget) { // buildLinks builds links from the given target if it's labelled appropriately. // For example, Go targets may link themselves into plz-out/go/src etc. func buildLinks(state *core.BuildState, target *core.BuildTarget) { - buildLinksOfType(state, target, "link:", false, os.Symlink) + // SymlinkOrCopy rather than os.Symlink: Windows refuses to create one without a privilege + // an ordinary user does not have, and a link: label that silently becomes a warning is + // worse than a copy. + buildLinksOfType(state, target, "link:", false, fs.SymlinkOrCopy) buildLinksOfType(state, target, "hlink:", false, os.Link) // Directly link to the path of the label for these (i.e. don't append out to the destination dir) - buildLinksOfType(state, target, "dlink:", true, os.Symlink) + buildLinksOfType(state, target, "dlink:", true, fs.SymlinkOrCopy) buildLinksOfType(state, target, "dhlink:", true, os.Link) if state.Config.ShouldLinkGeneratedSources() && target.HasLabel("codegen") { @@ -1094,10 +1112,10 @@ func fetchOneRemoteFile(state *core.BuildState, target *core.BuildTarget, url st } defer f.Close() if strings.HasPrefix(url, "file://") { - filename := strings.TrimPrefix(url, "file://") + filename := fileURLPath(url) if !filepath.IsAbs(filename) { return fmt.Errorf("URL %s must be an absolute path", url) - } else if strings.HasPrefix(filename, core.RepoRoot) { + } else if core.IsInRepoRoot(filename) { return fmt.Errorf("URL %s is within the repo, you cannot use remote_file for this", url) } fromfile, err := os.Open(filename) diff --git a/src/build/build_step_test.go b/src/build/build_step_test.go index 0e4642ce83..d0a4dc378f 100644 --- a/src/build/build_step_test.go +++ b/src/build/build_step_test.go @@ -14,6 +14,7 @@ import ( iofs "io/fs" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -78,6 +79,7 @@ func TestModifiedBuildTargetStillNeedsRebuilding(t *testing.T) { } func TestSymlinkedOutputs(t *testing.T) { + skipIfNoSymlinks(t) // Test behaviour when the output is a symlink. state, target := newState("//package1:target5") target.AddOutput("file5") @@ -117,6 +119,17 @@ func TestPostBuildFunction(t *testing.T) { assert.Equal(t, []string{"file7"}, target.Outputs()) } +// assertPermissionsPreserved checks that the mode a build action set on a file survived being +// moved into plz-out. Windows has no mode to preserve - Go synthesises one from the read-only +// attribute - so there is nothing to assert there. +func assertPermissionsPreserved(t *testing.T, info os.FileInfo) { + t.Helper() + if runtime.GOOS == "windows" { + return + } + assert.Equal(t, "-rwxrwxrwx", info.Mode().Perm().String()) +} + func TestOutputDir(t *testing.T) { newTarget := func() (*core.BuildState, *core.BuildTarget) { // Test modifying a command in the post-build function. @@ -147,6 +160,19 @@ func TestOutputDir(t *testing.T) { assert.Equal(t, core.Reused, target.State()) } +// skipIfNoSymlinks skips a test that needs working symlinks. Creating one on Windows needs +// Developer Mode, and under Wine os.Symlink reports success while producing a link that cannot +// even be stat'ed - so a failure here says nothing about Please. See docs/design/windows. +func skipIfNoSymlinks(t *testing.T) { + t.Helper() + if fs.IsWine() { + // Only under Wine, where os.Symlink reports success and produces a link that cannot be + // stat'ed. Real Windows either makes the link or falls back to a copy, and both leave + // the content these tests assert on where it should be. + t.Skip("Wine's symlinks are not real enough to assert against") + } +} + func TestOutputDirDoubleStar(t *testing.T) { newTarget := func(withDoubleStar bool) (*core.BuildState, *core.BuildTarget) { // Test modifying a command in the post-build function. @@ -176,7 +202,7 @@ func TestOutputDirDoubleStar(t *testing.T) { info, err := os.Lstat(filepath.Join(target.OutDir(), "foo/file7")) require.NoError(t, err) - assert.Equal(t, info.Mode().Perm().String(), "-rwxrwxrwx") + assertPermissionsPreserved(t, info) state, target = newTarget(true) @@ -186,7 +212,7 @@ func TestOutputDirDoubleStar(t *testing.T) { info, err = os.Lstat(filepath.Join(target.OutDir(), "foo/file7")) require.NoError(t, err) - assert.Equal(t, info.Mode().Perm().String(), "-rwxrwxrwx") + assertPermissionsPreserved(t, info) } func TestCacheRetrieval(t *testing.T) { @@ -272,6 +298,7 @@ func TestGoModCreation(t *testing.T) { } func TestCreatePlzOutGo(t *testing.T) { + skipIfNoSymlinks(t) state, target := newState("//package1:target") target.AddLabel("link:plz-out/go/${PKG}/src") target.AddOutput("file1.go") @@ -399,7 +426,12 @@ func TestHashCheckers(t *testing.T) { func TestFetchLocalRemoteFile(t *testing.T) { state, target := newState("//package4:target1") - target.AddSource(core.URLLabel("file://" + os.Getenv("TMP_DIR") + "/src/build/test_data/local_remote_file.txt")) + // From the working directory, which is the test data directory, rather than $TMP_DIR: + // that names the same place but as the host we started from writes it, which under Wine + // is not an absolute path at all. + wd, err := os.Getwd() + require.NoError(t, err) + target.AddSource(core.URLLabel("file://" + filepath.ToSlash(wd) + "/local_remote_file.txt")) target.AddOutput("local_remote_file.txt") // Temporarily reset the repo root so we can test this locally @@ -409,11 +441,21 @@ func TestFetchLocalRemoteFile(t *testing.T) { core.RepoRoot = oldRoot }() - err := fetchRemoteFile(state, target) - assert.NoError(t, err) + assert.NoError(t, fetchRemoteFile(state, target)) assert.True(t, fs.FileExists(filepath.Join(target.TmpDir(), "local_remote_file.txt"))) } +func TestFileURLPath(t *testing.T) { + // A Unix path round-trips unchanged; a Windows one loses the slash the URL form requires + // before its drive letter, but only where that is what a drive letter means. + assert.Equal(t, "/home/user/file.txt", fileURLPath("file:///home/user/file.txt")) + if filepath.Separator == '\\' { + assert.Equal(t, `C:/foo/bar`, fileURLPath("file:///C:/foo/bar")) + } else { + assert.Equal(t, `/C:/foo/bar`, fileURLPath("file:///C:/foo/bar")) + } +} + func TestFetchLocalRemoteFileCannotBeRelative(t *testing.T) { state, target := newState("//package4:target2") target.AddSource(core.URLLabel("src/build/test_data/local_remote_file.txt")) diff --git a/src/build/remote_file_test.go b/src/build/remote_file_test.go index 6fe86a6705..e1bb003e0d 100644 --- a/src/build/remote_file_test.go +++ b/src/build/remote_file_test.go @@ -25,6 +25,17 @@ func listen(s *http.Server) net.Listener { return lis } +// writeHomeSecret puts the secret the tests read at ~/secret, with the home directory pointed +// somewhere this test owns. Writing to the real one would leave a file behind, and it is left +// read-only, which on Windows means the next run cannot replace it. +func writeHomeSecret(t *testing.T) { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) // What os.UserHomeDir reads on Windows. + require.NoError(t, fs.CopyFile("secret", fs.ExpandHomePath("~/secret"), 0444)) +} + func TestHeader(t *testing.T) { state, target := newState("//pkg:header_test") target.IsRemoteFile = true @@ -53,8 +64,7 @@ func TestSecretHeader(t *testing.T) { target.AddLabel("remote_file:secret_header:foo:~/secret") target.AddLabel("remote_file:secret_header:bar:secret") - err := fs.CopyFile("secret", fs.ExpandHomePath("~/secret"), 0444) - require.NoError(t, err) + writeHomeSecret(t) s, m := Server() m.HandleFunc("/header", func(writer http.ResponseWriter, request *http.Request) { @@ -68,8 +78,7 @@ func TestSecretHeader(t *testing.T) { lis := listen(s) go s.Serve(lis) - err = fetchRemoteFile(state, target) - require.NoError(t, err) + require.NoError(t, fetchRemoteFile(state, target)) } func TestBasicAuth(t *testing.T) { @@ -80,8 +89,7 @@ func TestBasicAuth(t *testing.T) { target.AddLabel("remote_file:username:foo") target.AddLabel("remote_file:password_file:~/secret") - err := fs.CopyFile("secret", fs.ExpandHomePath("~/secret"), 0444) - require.NoError(t, err) + writeHomeSecret(t) s, m := Server() m.HandleFunc("/header", func(writer http.ResponseWriter, request *http.Request) { @@ -94,6 +102,5 @@ func TestBasicAuth(t *testing.T) { lis := listen(s) go s.Serve(lis) - err = fetchRemoteFile(state, target) - require.NoError(t, err) + require.NoError(t, fetchRemoteFile(state, target)) } diff --git a/src/cache/BUILD b/src/cache/BUILD index 72d59791b0..6a4c484828 100644 --- a/src/cache/BUILD +++ b/src/cache/BUILD @@ -15,16 +15,27 @@ go_library( "//src/cli/logging", "//src/core", "//src/fs", + "//src/process", ], ) go_test( name = "cache_test", srcs = glob(["*_test.go"]), - data = ["test_data"], + data = [":cache_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":cache", "///third_party/go/github.com_stretchr_testify//assert", "//src/core", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "cache_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/cache/cmd_cache.go b/src/cache/cmd_cache.go index 650c3fe786..927a2a5567 100644 --- a/src/cache/cmd_cache.go +++ b/src/cache/cmd_cache.go @@ -8,14 +8,18 @@ import ( "io" "os/exec" "path/filepath" + "slices" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/process" ) type cmdCache struct { storeCommand string retrieveCommand string + // The shell the two commands run in, as argv up to but not including the command. + shell []string } func keyToString(key []byte) string { @@ -30,7 +34,8 @@ func (cache *cmdCache) Store(target *core.BuildTarget, key []byte, files []strin ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cmd := exec.CommandContext(ctx, "sh", "-c", cache.storeCommand) + argv := append(cache.shell, cache.storeCommand) + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) cmd.Env = append(cmd.Env, "CACHE_KEY="+strKey) r, w := io.Pipe() @@ -52,7 +57,8 @@ func (cache *cmdCache) Retrieve(target *core.BuildTarget, key []byte, _ []string strKey := keyToString(key) log.Debug("Retrieve %s: %s from custom cache...", target.Label, strKey) - cmd := exec.Command("sh", "-c", cache.retrieveCommand) + argv := append(cache.shell, cache.retrieveCommand) + cmd := exec.Command(argv[0], argv[1:]...) cmd.Env = append(cmd.Env, "CACHE_KEY="+strKey) var cmdOutputBuffer bytes.Buffer @@ -130,8 +136,13 @@ func write(w io.WriteCloser, target *core.BuildTarget, files []string, cancel co } func newCmdCache(config *core.Configuration) *cmdCache { + // These are shell strings like any build action, so they run in the configured shell - + // on Windows there is no 'sh' to fall back on. Clipped so that appending the command to + // it can't write into this slice from two goroutines at once. + shell := process.ShellArgv(config.Shell(), config.ShellArgs()) return &cmdCache{ storeCommand: config.Cache.StoreCommand, retrieveCommand: config.Cache.RetrieveCommand, + shell: slices.Clip(append(shell, "-c")), } } diff --git a/src/clean/BUILD b/src/clean/BUILD index 1780bb6549..89178876fe 100644 --- a/src/clean/BUILD +++ b/src/clean/BUILD @@ -1,6 +1,10 @@ go_library( name = "clean", - srcs = ["clean.go"], + srcs = [ + "clean.go", + "detach_other.go", + "detach_windows.go", + ], pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ @@ -9,12 +13,14 @@ go_library( "//src/core", "//src/fs", "//src/test", - ], + ] + (["///third_party/go/golang.org_x_sys//windows"] if is_platform(os = "windows") else []), ) go_test( name = "clean_test", srcs = ["clean_test.go"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":clean", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/clean/clean.go b/src/clean/clean.go index 79b09d7f6d..de2b35b36e 100644 --- a/src/clean/clean.go +++ b/src/clean/clean.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "syscall" "github.com/thought-machine/please/src/build" "github.com/thought-machine/please/src/cli/logging" @@ -91,10 +90,9 @@ func AsyncDeleteDir(dir string) error { if err != nil { return err } - // Note that we can't fork() directly and continue running Go code, but ForkExec() works okay, - // so we re-execute ourselves with a specific command that will remove this. - _, err = syscall.ForkExec(exec, []string{exec, "clean", "--rm", newDir}, nil) - return err + // Note that we can't fork() directly and continue running Go code, so we re-execute + // ourselves detached, with a specific command that will remove this. + return startDetached(exec, []string{"clean", "--rm", newDir}) } // moveDir moves a directory to a new location and returns that new location. diff --git a/src/clean/detach_other.go b/src/clean/detach_other.go new file mode 100644 index 0000000000..849460cac3 --- /dev/null +++ b/src/clean/detach_other.go @@ -0,0 +1,11 @@ +//go:build !windows +// +build !windows + +package clean + +import "os/exec" + +// startDetached starts a process that will outlive us, and does not wait for it. +func startDetached(bin string, args []string) error { + return exec.Command(bin, args...).Start() +} diff --git a/src/clean/detach_windows.go b/src/clean/detach_windows.go new file mode 100644 index 0000000000..1784c85de1 --- /dev/null +++ b/src/clean/detach_windows.go @@ -0,0 +1,17 @@ +package clean + +import ( + "os/exec" + "syscall" + + "golang.org/x/sys/windows" +) + +// startDetached starts a process that will outlive us, and does not wait for it. +// DETACHED_PROCESS keeps it off our console, so it isn't killed when we exit or when the +// user closes the window. +func startDetached(bin string, args []string) error { + cmd := exec.Command(bin, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.DETACHED_PROCESS} + return cmd.Start() +} diff --git a/src/cli/BUILD b/src/cli/BUILD index 666d64eb77..16ef8abe0d 100644 --- a/src/cli/BUILD +++ b/src/cli/BUILD @@ -7,6 +7,8 @@ go_library( "progress.go", "prompt.go", "replacements.go", + "signals_other.go", + "signals_windows.go", "suggest.go", "winch_other.go", "winch_windows.go", @@ -35,6 +37,8 @@ go_test( "flags_test.go", "logging_test.go", ], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":cli", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/cli/logging.go b/src/cli/logging.go index 41547e0b22..7c4ea43262 100644 --- a/src/cli/logging.go +++ b/src/cli/logging.go @@ -7,7 +7,7 @@ import ( "container/list" "fmt" "os" - "path" + "path/filepath" "regexp" "strings" "sync" @@ -61,7 +61,7 @@ func InitLogging(verbosity Verbosity) { // InitFileLogging initialises an optional logging backend to a file. func InitFileLogging(logFile string, logFileLevel Verbosity, append bool) { fileLogLevel = logging.Level(logFileLevel) - if err := os.MkdirAll(path.Dir(logFile), os.ModeDir|0775); err != nil { + if err := os.MkdirAll(filepath.Dir(logFile), os.ModeDir|0775); err != nil { log.Fatalf("Error creating log file directory: %s", err) } flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC @@ -75,11 +75,28 @@ func InitFileLogging(logFile string, logFileLevel Verbosity, append bool) { fileBackend = logging.NewLogBackend(file, "", 0) fileBackend = logging.NewBackendFormatter(fileBackend, logFormatter(false)) setLogBackend(logging.NewLogBackend(os.Stderr, "", 0)) - AtExit(func() { - fileBackend = nil - setLogBackend(logging.NewLogBackend(os.Stderr, "", 0)) - file.Close() - }) + openLogFile = file + AtExit(CloseFileLogging) +} + +// openLogFile is the open log file, if there is one, so that CloseFileLogging can reach it. +var openLogFile *os.File + +// CloseFileLogging stops logging to a file and closes it, leaving stderr logging in place. +// +// Anything that deletes or renames a directory the log file is in has to call this first. +// Windows refuses to rename or unlink a file another handle has open, and the default log file +// lives at plz-out/log/build.log - which is inside the directory plz clean removes, so a clean +// could not do either. AtExit is not enough on its own: it only runs on a terminating signal, +// never on an ordinary exit. +func CloseFileLogging() { + if openLogFile == nil { + return + } + fileBackend = nil + setLogBackend(logging.NewLogBackend(os.Stderr, "", 0)) + openLogFile.Close() + openLogFile = nil } func logFormatter(coloured bool) logging.Formatter { diff --git a/src/cli/process.go b/src/cli/process.go index 44a0c4cd23..d9ac784d3d 100644 --- a/src/cli/process.go +++ b/src/cli/process.go @@ -3,7 +3,6 @@ package cli import ( "os" "os/signal" - "syscall" ) var atexitHandlers []func() @@ -16,7 +15,7 @@ func init() { // functions previously registered with AtExit, and then exits the process. func handleSignals() { ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGABRT, syscall.SIGTERM) + signal.Notify(ch, terminatingSignals...) sig := <-ch log.Info("Received signal %s", sig) // Allow a second signal to terminate the process regardless @@ -47,8 +46,5 @@ func AtExit(f func()) { // exit kills the process with an exit code suitable for the given signal. func exit(sig os.Signal) { - if s, ok := sig.(syscall.Signal); ok { - os.Exit(128 + int(s)) - } - os.Exit(1) + os.Exit(exitCodeForSignal(sig)) } diff --git a/src/cli/signals_other.go b/src/cli/signals_other.go new file mode 100644 index 0000000000..2ca76a4acc --- /dev/null +++ b/src/cli/signals_other.go @@ -0,0 +1,22 @@ +//go:build !windows +// +build !windows + +package cli + +import ( + "os" + "syscall" +) + +// terminatingSignals are the signals we clean up and exit on. +var terminatingSignals = []os.Signal{ + syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGABRT, syscall.SIGTERM, +} + +// exitCodeForSignal returns the conventional shell exit code for dying to a signal. +func exitCodeForSignal(sig os.Signal) int { + if s, ok := sig.(syscall.Signal); ok { + return 128 + int(s) + } + return 1 +} diff --git a/src/cli/signals_windows.go b/src/cli/signals_windows.go new file mode 100644 index 0000000000..3cebccf77b --- /dev/null +++ b/src/cli/signals_windows.go @@ -0,0 +1,16 @@ +package cli + +import ( + "os" + "syscall" +) + +// terminatingSignals are the signals we clean up and exit on. Windows only ever delivers +// Ctrl-C as os.Interrupt and a synthesised SIGTERM; the others are defined but never sent. +var terminatingSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + +// exitCodeForSignal returns the exit code to use when dying to a signal. The 128+signum +// convention is a shell idiom with no meaning on Windows, so just report a plain failure. +func exitCodeForSignal(sig os.Signal) int { + return 1 +} diff --git a/src/cmap/cmap.go b/src/cmap/cmap.go index f8058ef732..8e197ebf68 100644 --- a/src/cmap/cmap.go +++ b/src/cmap/cmap.go @@ -84,6 +84,13 @@ func (m *Map[K, V]) GetOrWait(key K) (val V, wait <-chan struct{}, first bool) { return m.shards[m.hasher(key)&m.mask].Get(key) } +// Delete removes the given key from the map, returning the value it had and whether there was +// one. A key that only exists because something is waiting on it through GetOrWait is left +// alone: deleting a key is not the same as it arriving, and the waiters would never be woken. +func (m *Map[K, V]) Delete(key K) (V, bool) { + return m.shards[m.hasher(key)&m.mask].Delete(key) +} + // Values returns a slice of all the current values in the map. // No particular consistency guarantees are made. func (m *Map[K, V]) Values() []V { @@ -183,6 +190,19 @@ func (s *shard[K, V]) Get(key K) (val V, wait <-chan struct{}, first bool) { return } +// Delete removes a key that has a value, returning it. It reports false, and does nothing, for +// a key that is absent or is only a placeholder something is waiting on. +func (s *shard[K, V]) Delete(key K) (V, bool) { + s.l.Lock() + defer s.l.Unlock() + if v, present := s.m[key]; present && v.Wait == nil { + delete(s.m, key) + return v.Val, true + } + var zero V + return zero, false +} + // Values returns a copy of all the targets currently in the map. func (s *shard[K, V]) Values() []V { s.l.RLock() diff --git a/src/cmap/cmap_test.go b/src/cmap/cmap_test.go index 2c4394f8a8..03a5f3ba06 100644 --- a/src/cmap/cmap_test.go +++ b/src/cmap/cmap_test.go @@ -56,6 +56,33 @@ func TestReAdd(t *testing.T) { assert.False(t, first) } +func TestDelete(t *testing.T) { + m := New[int, int](DefaultShardCount, hashInts) + assert.True(t, m.Add(5, 7)) + v, deleted := m.Delete(5) + assert.True(t, deleted) + assert.Equal(t, 7, v) + assert.False(t, m.Contains(5)) + // Deleting it again does nothing, and the key is free to be added afresh. + _, deleted = m.Delete(5) + assert.False(t, deleted) + assert.True(t, m.Add(5, 9)) + assert.Equal(t, 9, m.Get(5)) +} + +func TestDeleteLeavesWaitersAlone(t *testing.T) { + // A key that only exists because something is waiting on it has no value to delete, and + // removing it would leave the waiter waiting on a channel nothing can close. + m := New[int, int](DefaultShardCount, hashInts) + _, ch, first := m.GetOrWait(5) + assert.True(t, first) + _, deleted := m.Delete(5) + assert.False(t, deleted) + m.Set(5, 7) + <-ch + assert.Equal(t, 7, m.Get(5)) +} + func TestAddOrGet(t *testing.T) { m := New[int, int](DefaultShardCount, hashInts) x, inserted := m.AddOrGet(5, func() int { return 7 }) diff --git a/src/core/BUILD b/src/core/BUILD index d4b351dd22..e4294844b9 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -29,14 +29,15 @@ go_library( "//src/process", "//src/scm", "//src/version", - ], + ] + (["///third_party/go/golang.org_x_sys//windows"] if is_platform(os = "windows") else []), ) go_test( name = "core_test", srcs = glob(["*_test.go"]), - data = ["test_data"], - filter_srcs = False, # As above + data = [":test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":core", "///third_party/go/github.com_stretchr_testify//assert", @@ -64,3 +65,11 @@ benchmark( srcs = ["graph_benchmark_test.go"], deps = [":core"], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/core/build_env.go b/src/core/build_env.go index b3c0bcfbb8..cc06476eaf 100644 --- a/src/core/build_env.go +++ b/src/core/build_env.go @@ -80,6 +80,7 @@ func BuildEnvironment(state *BuildState, target *BuildTarget, tmpDir string) Bui env["TMPDIR"] = tmpDir env["OUTS"] = strings.Join(outEnv, " ") env["HOME"] = tmpDir + setPlatformTmpEnv(env, tmpDir) // Set a consistent hash seed for Python. Important for build determinism. env["PYTHONHASHSEED"] = "42" @@ -132,6 +133,7 @@ func BuildEnvironment(state *BuildState, target *BuildTarget, tmpDir string) Bui env["BINDIR"] = filepath.Join(RepoRoot, BinDir) } + env.normalisePathSeparators() return withUserProvidedEnv(target, env) } @@ -164,6 +166,7 @@ func TestEnvironment(state *BuildState, target *BuildTarget, testDir string, run env["TMP_DIR"] = testDir env["TMPDIR"] = testDir env["HOME"] = testDir + setPlatformTmpEnv(env, testDir) env["TEST_ARGS"] = strings.Join(state.TestArgs, ",") env["RESULTS_FILE"] = resultsFile // We shouldn't really have specific things like this here, but it really is just easier to set it. @@ -190,6 +193,7 @@ func TestEnvironment(state *BuildState, target *BuildTarget, testDir string, run if len(state.TestArgs) > 0 { env["TESTS"] = strings.Join(state.TestArgs, " ") } + env.normalisePathSeparators() return withUserProvidedEnv(target, env) } @@ -204,6 +208,7 @@ func RunEnvironment(state *BuildState, target *BuildTarget, inTmpDir bool) Build env["OUT"] = resolveOut(outEnv[0], ".", false) } + env.normalisePathSeparators() return withUserProvidedEnv(target, env) } @@ -213,6 +218,7 @@ func ExecEnvironment(state *BuildState, target *BuildTarget, execDir string) Bui env["TMP_DIR"] = execDir env["TMPDIR"] = execDir env["HOME"] = execDir + setPlatformTmpEnv(env, execDir) // This is used by programs that use display terminals for correct handling // of input and output in the terminal where the program is run. env["TERM"] = os.Getenv("TERM") @@ -228,6 +234,7 @@ func ExecEnvironment(state *BuildState, target *BuildTarget, execDir string) Bui } } + env.normalisePathSeparators() return withUserProvidedEnv(target, env) } @@ -350,7 +357,10 @@ func toolPath(state *BuildState, tool BuildInput, abs bool) string { entryPoint = o.Annotation } path := state.Graph.TargetOrDie(label).toolPath(abs, entryPoint) - if !strings.Contains(path, "/") { + // A bare filename is made explicit so the shell runs it rather than searching PATH. + // Check both separators: on Windows the path may still contain backslashes at this + // point, and treating one as a bare name yields nonsense like "./C:\dir\tool.exe". + if !strings.ContainsRune(path, '/') && !strings.ContainsRune(path, os.PathSeparator) { path = "./" + path } return path diff --git a/src/core/build_env_test.go b/src/core/build_env_test.go index ca2b22210f..8302a81e41 100644 --- a/src/core/build_env_test.go +++ b/src/core/build_env_test.go @@ -2,6 +2,7 @@ package core import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -203,3 +204,23 @@ func TestDeduplicateEnvVars(t *testing.T) { env := TestEnvironment(state, target, "/path/to/runtime/dir", 1) assert.Equal(t, env["COVERAGE"], "wibble") } + +// TestBuildEnvironmentUsesForwardSlashes asserts the invariant that the environment Please +// generates never contains a backslash. Build commands are shell strings, and anything that +// interprets its arguments - sed, for one, which the C/C++ rules use to build their link line +// - will silently mangle a Windows path embedded in one. This is trivially true on platforms +// whose separator is already a forward slash; it is the real assertion on Windows. +func TestBuildEnvironmentUsesForwardSlashes(t *testing.T) { + target := NewBuildTarget(NewBuildLabel("pkg", "t")) + target.AddOutput("out_file1") + target.AddSource(FileLabel{File: "src_file1", Package: "pkg"}) + + for name, env := range map[string]BuildEnv{ + "build": BuildEnvironment(NewDefaultBuildState(), target, filepath.Join("path", "to", "tmp")), + "exec": ExecEnvironment(NewDefaultBuildState(), target, filepath.Join("path", "to", "run")), + } { + for k, v := range env { + assert.NotContains(t, v, `\`, "%s environment: %s contains a backslash", name, k) + } + } +} diff --git a/src/core/build_label.go b/src/core/build_label.go index 365e851c8c..a02d095470 100644 --- a/src/core/build_label.go +++ b/src/core/build_label.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "os" - "path/filepath" + "path" "strings" "time" @@ -74,7 +74,7 @@ func (label BuildLabel) ShortString(context BuildLabel) string { return label.String() } else if label.PackageName == context.PackageName { return ":" + label.Name - } else if label.Name == filepath.Base(label.PackageName) { + } else if label.Name == path.Base(label.PackageName) { return "//" + label.PackageName } label.Subrepo = "" @@ -258,7 +258,8 @@ func parseMaybeRelativeBuildLabel(target, subdir string) (BuildLabel, error) { return TryParseBuildLabel(target, subdir, "") } // Presumably it's just underneath this directory (note that if it was absolute we returned above) - return TryParseBuildLabel("//"+filepath.Join(subdir, target), "", "") + // path, not filepath: this is a build label, which is slash-separated on every platform. + return TryParseBuildLabel("//"+path.Join(subdir, target), "", "") } // ParseBuildLabels parses a bunch of build labels from strings. It dies on failure. @@ -352,10 +353,12 @@ func (label BuildLabel) FullPaths(graph *BuildGraph) []string { } // addPathPrefix adds a prefix to all the entries in a slice. +// path, not filepath: these are plz-out paths, which stay slash-separated - they are what +// $(location) and friends expand to inside a shell command. func addPathPrefix(paths []string, prefix string) []string { ret := make([]string, len(paths)) for i, output := range paths { - ret[i] = filepath.Join(prefix, output) + ret[i] = path.Join(prefix, output) } return ret } diff --git a/src/core/build_target.go b/src/core/build_target.go index 90c64943e2..1831ca0a95 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -40,7 +40,8 @@ const SubrepoDir = "plz-out/subrepos" const DefaultBuildingDescription = "Building..." // SandboxDir is the directory that sandboxed actions are run in. -const SandboxDir = "/tmp/plz_sandbox" +// This is platform-specific; see sandboxdir_other.go and sandboxdir_windows.go. +var SandboxDir = sandboxDir // Suffixes for temporary directories const buildDirSuffix = "._build" @@ -418,13 +419,19 @@ func (target *BuildTarget) String() string { return target.Label.String() } +// The directories below are built with path, not filepath, and so are slash-separated on every +// platform. They are assembled from build label components, which are slash-separated by +// definition, and they end up interpolated into shell commands, where a backslash is an escape +// character rather than a separator. Win32 accepts either, so nothing is lost by being +// consistent. See docs/design/windows/02-shell-and-build-actions.md. +// // TmpDir returns the temporary working directory for this target, eg. // //mickey/donald:goofy -> plz-out/tmp/mickey/donald/goofy._build // Note the extra subdirectory to keep rules separate from one another, and the .build suffix // to attempt to keep rules from duplicating the names of sub-packages; obviously that is not // 100% reliable but we don't have a better solution right now. func (target *BuildTarget) TmpDir() string { - return filepath.Join(TmpDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name+buildDirSuffix) + return path.Join(TmpDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name+buildDirSuffix) } // BuildLockFile returns the lock filename for the target's build stage. @@ -436,17 +443,17 @@ func (target *BuildTarget) BuildLockFile() string { // //mickey/donald:goofy -> plz-out/gen/mickey/donald (or plz-out/bin if it's a binary) func (target *BuildTarget) OutDir() string { if target.IsSubrepo { - return filepath.Join(SubrepoDir, target.Label.Subrepo, target.Label.PackageName) + return path.Join(SubrepoDir, target.Label.Subrepo, target.Label.PackageName) } else if target.IsBinary { - return filepath.Join(BinDir, target.Label.Subrepo, target.Label.PackageName) + return path.Join(BinDir, target.Label.Subrepo, target.Label.PackageName) } - return filepath.Join(GenDir, target.Label.Subrepo, target.Label.PackageName) + return path.Join(GenDir, target.Label.Subrepo, target.Label.PackageName) } // ExecDir returns the exec directory for this target, e.g. // //mickey/donald:goofy -> plz-out/exec/mickey/donald/goofy func (target *BuildTarget) ExecDir() string { - return filepath.Join(ExecDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name) + return path.Join(ExecDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name) } // TestDir returns the test directory for this target, eg. @@ -454,7 +461,7 @@ func (target *BuildTarget) ExecDir() string { // This is different to TmpDir so we run tests in a clean environment // and to facilitate containerising tests. func (target *BuildTarget) TestDir(runNumber int) string { - return filepath.Join(target.TestDirs(), fmt.Sprint("run_", runNumber)) + return path.Join(target.TestDirs(), fmt.Sprint("run_", runNumber)) } // TestLockFile returns the lock filename for the target's test stage. @@ -464,7 +471,7 @@ func (target *BuildTarget) TestLockFile(runNumber int) string { // TestDirs contains the parent directory of all the test run directories above func (target *BuildTarget) TestDirs() string { - return filepath.Join(TmpDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name+testDirSuffix) + return path.Join(TmpDir, target.Label.Subrepo, target.Label.PackageName, target.Label.Name+testDirSuffix) } // IsTest returns whether or not the target is a test target i.e. has its Test field populated @@ -483,12 +490,12 @@ func (target *BuildTarget) CompleteRun(state *BuildState) bool { // TestResultsFile returns the output results file for tests for this target. func (target *BuildTarget) TestResultsFile() string { - return filepath.Join(target.OutDir(), ".test_results_"+target.Label.Name) + return path.Join(target.OutDir(), ".test_results_"+target.Label.Name) } // CoverageFile returns the output coverage file for tests for this target. func (target *BuildTarget) CoverageFile() string { - return filepath.Join(target.OutDir(), ".test_coverage_"+target.Label.Name) + return path.Join(target.OutDir(), ".test_coverage_"+target.Label.Name) } // AddTestResults adds results to the target @@ -934,7 +941,7 @@ func (target *BuildTarget) FullOutputs() []string { outs := target.Outputs() outDir := target.OutDir() for i, out := range outs { - outs[i] = filepath.Join(outDir, out) + outs[i] = path.Join(outDir, out) } return outs } @@ -1079,14 +1086,14 @@ func (target *BuildTarget) CheckTargetOwnsBuildOutputs(state *BuildState) error for _, output := range target.Outputs() { targetPackage := target.Label.PackageName - out := filepath.Join(targetPackage, output) + out := path.Join(targetPackage, output) if fs.IsPackage(state.Config.Parse.BuildFileName, out) { return fmt.Errorf("trying to output file %s, but that directory is another package", out) } // If the output is just a file in the package root, we don't need to check anything else. - if filepath.Dir(output) == "." { + if path.Dir(output) == "." { continue } @@ -1861,9 +1868,11 @@ func (target *BuildTarget) toolPath(abs bool, namedOutput string) string { ret := make([]string, len(outputs)) for i, o := range outputs { if abs { - ret[i] = filepath.Join(RepoRoot, target.OutDir(), o) + // ToSlash because RepoRoot is a native path: this whole string is about to be + // interpolated into a shell command. + ret[i] = filepath.ToSlash(filepath.Join(RepoRoot, target.OutDir(), o)) } else { - ret[i] = filepath.Join(target.PackageDir(), o) + ret[i] = path.Join(target.PackageDir(), o) } } return strings.Join(ret, " ") @@ -2073,7 +2082,7 @@ func (target *BuildTarget) HasLinks(state *BuildState) bool { func (target *BuildTarget) PackageDir() string { if target.Subrepo != nil { - return filepath.Join(target.Subrepo.PackageRoot, target.Label.PackageDir()) + return path.Join(target.Subrepo.PackageRoot, target.Label.PackageDir()) } return target.Label.PackageDir() } diff --git a/src/core/build_target_test.go b/src/core/build_target_test.go index 4360be1316..528ba776ee 100644 --- a/src/core/build_target_test.go +++ b/src/core/build_target_test.go @@ -4,10 +4,12 @@ package core import ( "fmt" "os" + "path/filepath" "slices" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTmpDir(t *testing.T) { @@ -395,7 +397,9 @@ func TestToolPath(t *testing.T) { target.AddOutput("file2.go") wd, _ := os.Getwd() RepoRoot = wd - root := wd + "/plz-out/gen/src/core" + // Tool paths are interpolated into shell commands, so they are slash-separated even where + // the working directory we started from isn't. + root := filepath.ToSlash(wd) + "/plz-out/gen/src/core" assert.Equal(t, fmt.Sprintf("%s/file1.go %s/file2.go", root, root), target.toolPath(true, "")) assert.Equal(t, "src/core/file1.go src/core/file2.go", target.toolPath(false, "")) } @@ -407,7 +411,7 @@ func TestToolPathWithEntryPoint(t *testing.T) { target.EntryPoints = map[string]string{"f1": "file1.go"} wd, _ := os.Getwd() RepoRoot = wd - root := wd + "/plz-out/gen/src/core" + root := filepath.ToSlash(wd) + "/plz-out/gen/src/core" assert.Equal(t, root+"/file1.go", target.toolPath(true, "f1")) assert.Equal(t, "src/core/file1.go", target.toolPath(false, "f1")) } @@ -626,7 +630,12 @@ func TestAllURLs(t *testing.T) { func TestCheckSecrets(t *testing.T) { target := makeTarget1("//src/core:target1", "") assert.NoError(t, target.CheckSecrets()) - target.Secrets = append(target.Secrets, "/bin/sh") + // A file that exists, made rather than assumed. This used to be /bin/sh, which Windows + // does not have - and which passed under Wine anyway, because its Z: drive maps the host's + // root, so the test proved nothing there and failed on a real machine. + existing := filepath.Join(t.TempDir(), "a_secret") + require.NoError(t, os.WriteFile(existing, []byte("shhh"), 0644)) + target.Secrets = append(target.Secrets, existing) assert.NoError(t, target.CheckSecrets()) // Checking for files in the home directory is awkward because nothing is really // guaranteed to exist. We just check the directory itself for now. diff --git a/src/core/command_replacements.go b/src/core/command_replacements.go index 07f1b98e67..4d7dc818f2 100644 --- a/src/core/command_replacements.go +++ b/src/core/command_replacements.go @@ -55,6 +55,7 @@ package core import ( "encoding/base64" "fmt" + "path" "path/filepath" "runtime/debug" "strings" @@ -220,12 +221,12 @@ func replaceSequence(state *BuildState, target *BuildTarget, in string, runnable } } if hash { - return base64.RawURLEncoding.EncodeToString(state.PathHasher.MustHash(filepath.Join(target.Label.PackageName, in), target.HashLastModified())) + return base64.RawURLEncoding.EncodeToString(state.PathHasher.MustHash(path.Join(target.Label.PackageName, in), target.HashLastModified())) } if strings.HasPrefix(in, "/") { return in // Absolute path, probably on a tool or system src. } - return quote(filepath.Join(target.Label.PackageName, in)) + return quote(path.Join(target.Label.PackageName, in)) } // replaceWorkerSequence is like replaceSequence but for worker commands, which do not @@ -288,7 +289,9 @@ func checkAndReplaceSequence(state *BuildState, target, dep *BuildTarget, ep, in if err != nil { log.Fatalf("Couldn't calculate relative path: %s", err) } - outputBuilder.WriteString(quote(abs)) + // ToSlash because the absolute part comes from the OS: this is going + // straight into a shell command. + outputBuilder.WriteString(quote(filepath.ToSlash(abs))) } else { outputBuilder.WriteString(quote(fileDestination(target, dep, out, dir, outPrefix, test))) } @@ -327,9 +330,10 @@ func quote(s string) string { } // handleDir chooses either the out dir or the actual output location depending on the 'dir' flag. +// path, not filepath: the result is interpolated into a shell command. func handleDir(outDir, output string, dir bool) string { if dir { return outDir } - return filepath.Join(outDir, output) + return path.Join(outDir, output) } diff --git a/src/core/command_replacements_test.go b/src/core/command_replacements_test.go index 4187278cdf..f57e4b8260 100644 --- a/src/core/command_replacements_test.go +++ b/src/core/command_replacements_test.go @@ -108,7 +108,7 @@ func TestToolReplacement(t *testing.T) { target1.Tools = append(target1.Tools, target2.Label) wd, _ := os.Getwd() - expected := quote(filepath.Join(wd, "plz-out/gen/path/to/target2.py")) + expected := quote(filepath.ToSlash(filepath.Join(wd, "plz-out/gen/path/to/target2.py"))) cmd, _ := ReplaceSequences(state, target1, target1.Command) assert.Equal(t, expected, cmd) } @@ -119,7 +119,7 @@ func TestToolReplacementSubrepo(t *testing.T) { target1.Tools = append(target1.Tools, target2.Label) wd, _ := os.Getwd() - expected := quote(filepath.Join(wd, "plz-out/gen/subrepo/path/to/target2.py")) + expected := quote(filepath.ToSlash(filepath.Join(wd, "plz-out/gen/subrepo/path/to/target2.py"))) cmd, _ := ReplaceSequences(state, target1, target1.Command) assert.Equal(t, expected, cmd) } @@ -151,7 +151,7 @@ func TestToolDirReplacement(t *testing.T) { target1.Tools = append(target1.Tools, target2.Label) wd, _ := os.Getwd() - expected := quote(filepath.Join(wd, "plz-out/gen/path/to")) + expected := quote(filepath.ToSlash(filepath.Join(wd, "plz-out/gen/path/to"))) cmd, _ := ReplaceSequences(state, target1, target1.Command) assert.Equal(t, expected, cmd) } @@ -191,7 +191,7 @@ func TestWorkerReplacement(t *testing.T) { target.Tools = append(target.Tools, tool.Label) worker, remoteArgs, localCmd, err := WorkerCommandAndArgs(state, target) assert.NoError(t, err) - assert.Equal(t, wd+"/plz-out/bin/path/to/target2.py", worker) + assert.Equal(t, filepath.ToSlash(wd)+"/plz-out/bin/path/to/target2.py", worker) assert.Equal(t, "--some_arg", remoteArgs) assert.Equal(t, "", localCmd) } @@ -213,7 +213,7 @@ func TestLocalCommandWorker(t *testing.T) { target.Tools = append(target.Tools, tool.Label) worker, remoteArgs, localCmd, err := WorkerCommandAndArgs(state, target) assert.NoError(t, err) - assert.Equal(t, wd+"/plz-out/bin/path/to/target2.py", worker) + assert.Equal(t, filepath.ToSlash(wd)+"/plz-out/bin/path/to/target2.py", worker) assert.Equal(t, "--some_arg", remoteArgs) assert.Equal(t, "find . | xargs rm && echo hello", localCmd) } diff --git a/src/core/config.go b/src/core/config.go index 7076c3b334..fd4992cdcc 100644 --- a/src/core/config.go +++ b/src/core/config.go @@ -9,6 +9,7 @@ import ( iofs "io/fs" "maps" "os" + "os/exec" "path/filepath" "reflect" "runtime" @@ -28,6 +29,7 @@ import ( "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/metrics" + "github.com/thought-machine/please/src/process" "github.com/thought-machine/please/src/version" ) @@ -50,7 +52,8 @@ const LocalConfigFileName string = ".plzconfig.local" // MachineConfigFileName is the file name for the machine-level config - can use this to override // things for a particular machine (e.g. build machine with different caching behaviour). -const MachineConfigFileName = "/etc/please/plzconfig" +// This is platform-specific; see config_other.go and config_windows.go. +var MachineConfigFileName = machineConfigFileName // UserConfigFileName is the file name for user-specific config (for all their repos). const UserConfigFileName = "~/.config/please/plzconfig" @@ -58,8 +61,9 @@ const UserConfigFileName = "~/.config/please/plzconfig" // DefaultPleaseLocation is the default location where Please is installed. const DefaultPleaseLocation = "~/.please" -// DefaultPath is the default location please looks for programs in -var DefaultPath = []string{"/usr/local/bin", "/usr/bin", "/bin"} +// DefaultPath is the default location please looks for programs in. +// This is platform-specific; see config_other.go and config_windows.go. +var DefaultPath = defaultPath // readConfigFileOnly reads a single config file into the config struct func readConfigFileOnly(fs iofs.FS, config *Configuration, filename string, quiet bool) error { @@ -80,7 +84,7 @@ func readConfigFileOnly(fs iofs.FS, config *Configuration, filename string, quie } if gcfg.FatalOnly(err) != nil { - return err + return configError(filename, err) } if quiet { log.Debug("Error in config file %s: %s", filename, err) @@ -90,6 +94,17 @@ func readConfigFileOnly(fs iofs.FS, config *Configuration, filename string, quie return nil } +// configError names the file a config error came from, and for the one mistake people are most +// likely to make on Windows says what to do about it. A backslash starts an escape sequence in +// this format, so a path written the way Windows writes it fails to parse, with a message that +// gives no hint that a path is even involved. +func configError(filename string, err error) error { + if strings.Contains(err.Error(), `unquoted '\'`) { + return fmt.Errorf("%s: %w\nA backslash begins an escape sequence here; write paths with forward slashes, which Windows accepts too, or put the value in double quotes", filename, err) + } + return fmt.Errorf("%s: %w", filename, err) +} + // readConfigFile reads a single config file into the config struct taking into account // some context like subrepos and plugins. func readConfigFile(fs iofs.FS, config *Configuration, filename string, subrepo bool) error { @@ -154,7 +169,7 @@ func defaultGlobalConfigFiles() []string { } if xdgConfigDirs := os.Getenv("XDG_CONFIG_DIRS"); xdgConfigDirs != "" { - for _, p := range strings.Split(xdgConfigDirs, ":") { + for _, p := range fs.SplitPathList(xdgConfigDirs) { if !filepath.IsAbs(p) { continue } @@ -212,12 +227,15 @@ func ReadConfigFiles(fs iofs.FS, filenames []string, profiles []string) (*Config } } + // Resolve the full path to Please's own location. This has to happen before the plugin + // repo defaults below, which are relative to it. It is idempotent, and the call further + // down is left alone. + config.EnsurePleaseLocation() + // Set default values for slices. These add rather than overwriting so we can't set // them upfront as we would with other config values. - setDefault(&config.Please.PluginRepo, - "https://github.com/{owner}/{plugin}/archive/{revision}.zip", - "https://github.com/{owner}/{plugin}-rules/archive/{revision}.zip", - ) + setDefault(&config.Please.PluginRepo, config.defaultPluginRepos()...) + config.useBundledTools() if usingBazelWorkspace { setDefault(&config.Parse.BuildFileName, "BUILD.bazel", "BUILD", "BUILD.plz") } else { @@ -225,6 +243,7 @@ func ReadConfigFiles(fs iofs.FS, filenames []string, profiles []string) (*Config } setBuildPath(&config.Build.Path, config.Build.PassEnv, config.Build.PassUnsafeEnv) setDefault(&config.Build.HashCheckers, "sha1", "sha256", "blake3") + setDefault(&config.Build.ShellArgs, process.DefaultShellArgs...) setDefault(&config.Build.PassUnsafeEnv) setDefault(&config.Build.PassEnv) setDefault(&config.Cover.FileExtension, ".go", ".py", ".java", ".tsx", ".ts", ".js", ".cc", ".h", ".c", ".rs") @@ -288,9 +307,6 @@ func ReadConfigFiles(fs iofs.FS, filenames []string, profiles []string) (*Config } } - // Resolve the full path to its location. - config.EnsurePleaseLocation() - // If the HTTP proxy config is set and there is no env var overriding it, set it now // so various other libraries will honour it. if config.Build.HTTPProxy != "" { @@ -347,12 +363,12 @@ func setBuildPath(conf *[]string, passEnv []string, passUnsafeEnv []string) { pathVal := DefaultPath for _, i := range passUnsafeEnv { if i == "PATH" { - pathVal = strings.Split(os.Getenv("PATH"), ":") + pathVal = fs.SplitPathList(os.Getenv("PATH")) } } for _, i := range passEnv { if i == "PATH" { - pathVal = strings.Split(os.Getenv("PATH"), ":") + pathVal = fs.SplitPathList(os.Getenv("PATH")) } } setDefault(conf, pathVal...) @@ -372,7 +388,7 @@ func defaultPathIfExists(conf *string, dir, file string) { // DefaultConfiguration returns the default configuration object with no overrides. // N.B. Slice fields are not populated by this (since it interferes with reading them) func DefaultConfiguration() *Configuration { - config := Configuration{buildEnvStored: &storedBuildEnv{}} + config := Configuration{buildEnvStored: &storedBuildEnv{}, shellStored: &storedShell{}} config.Please.SelfUpdate = true config.Please.Autoclean = true config.Please.DownloadLocation = "https://get.please.build" @@ -386,9 +402,10 @@ func DefaultConfiguration() *Configuration { config.Build.Timeout = cli.Duration(10 * time.Minute) config.Build.Config = "opt" // Optimised builds by default config.Build.FallbackConfig = "opt" // Optimised builds as a fallback on any target that doesn't have a matching one set - config.Build.Xattrs = true + config.Build.Xattrs = defaultXattrs config.Build.HashFunction = "sha256" config.Build.ParallelDownloads = 4 + config.Build.Shell = process.DefaultShell config.BuildConfig = map[string]string{} config.BuildEnv = map[string]string{} config.Cache.HTTPWriteable = true @@ -466,7 +483,7 @@ func DefaultConfiguration() *Configuration { config.Python.PexTool = "/////_please:please_pex" config.Java.JavacWorker = "/////_please:javac_worker" config.Java.JarCatTool = "/////_please:arcat" - config.Build.ArcatTool = "/////_please:arcat" + config.Build.ArcatTool = DefaultArcatTool config.Java.JUnitRunner = "/////_please:junit_runner" config.Metrics.Timeout = cli.Duration(2 * time.Second) @@ -528,6 +545,8 @@ type Configuration struct { UpdateGitignore bool `help:"Whether to automatically update the nearest gitignore with generated sources"` ParallelDownloads int `help:"Max number of remote_file downloads to run in parallel."` ArcatTool string `help:"Defines the tool used to concatenate files which we use in various build rules. Defaults to Arcat." var:"ARCAT_TOOL"` + Shell string `help:"The shell that build actions and tests are run in. Defaults to 'bash', which is looked up on Please's PATH; on Windows it defaults to the busybox that Please bundles, since Windows has no system shell that can run a build action." example:"bash | /bin/sh"` + ShellArgs []string `help:"Arguments passed to the shell before the command to run. Defaults to --noprofile and --norc, which stop bash reading the invoking user's startup files. On Windows the default is 'bash', selecting busybox's shell applet; busybox reads no startup files and rejects those two flags. Note that -u, -o pipefail and (where applicable) -e are always passed and are not configurable here."` } `help:"A config section describing general settings related to building targets in Please.\nSince Please is by nature about building things, this only has the most generic properties; most of the more esoteric properties are configured in their own sections."` BuildConfig map[string]string `help:"A section of arbitrary key-value properties that are made available in the BUILD language. These are often useful for writing custom rules that need some configurable property.\n\n[buildconfig]\nandroid-tools-version = 23.0.2\n\nFor example, the above can be accessed as CONFIG.ANDROID_TOOLS_VERSION."` BuildEnv map[string]string `help:"A set of extra environment variables to define for build rules. For example:\n\n[buildenv]\nsecret-passphrase = 12345\n\nThis would become SECRET_PASSPHRASE for any rules. These can be useful for passing secrets into custom rules; any variables containing SECRET or PASSWORD won't be logged.\n\nIt's also useful if you'd like internal tools to honour some external variable."` @@ -691,6 +710,8 @@ type Configuration struct { // buildEnvStored is a cached form of BuildEnv. buildEnvStored *storedBuildEnv + // shellStored is a cached form of Shell(). + shellStored *storedShell FeatureFlags struct { } `help:"Flags controlling preview features for the next release. Typically these config options gate breaking changes and only have a lifetime of one major release."` @@ -742,6 +763,62 @@ type storedBuildEnv struct { Once sync.Once } +type storedShell struct { + Shell string + Once sync.Once +} + +// Shell returns the shell that build actions, tests and the command cache run in. +// +// A bare name is left for the OS to resolve on Please's own PATH, as it always has been. The +// exception is when it isn't there at all: then we look on the build path, which includes +// Please's own install directory. That is how the shell Please bundles on Windows gets found, +// since nothing puts that directory on the user's PATH. +func (config *Configuration) Shell() string { + if config.shellStored == nil { + // A Configuration built by hand rather than through DefaultConfiguration; nothing to + // cache in, so just work it out each time. + return config.resolveShell() + } + config.shellStored.Once.Do(func() { + config.shellStored.Shell = config.resolveShell() + }) + return config.shellStored.Shell +} + +// ShellArgs returns the arguments passed to the shell before the command itself. +// A Configuration built by hand has none set - the defaults for a repeatable key can only be +// applied after parsing, or they would be appended to rather than replaced - so the platform +// default stands in. +func (config *Configuration) ShellArgs() []string { + if len(config.Build.ShellArgs) == 0 { + return process.DefaultShellArgs + } + return config.Build.ShellArgs +} + +func (config *Configuration) resolveShell() string { + shell := config.Build.Shell + if shell == "" { + return process.DefaultShell + } else if filepath.IsAbs(shell) || strings.ContainsRune(shell, filepath.Separator) { + return shell + } else if _, err := exec.LookPath(shell); err == nil { + return shell + } else if path, err := LookPath(shell, config.Path()); err == nil { + return path + } else if exe, err := fs.Executable(); err == nil { + // Last resort: next to the binary that is running. That is where a bundled shell sits + // in an install, and unlike the build path above it doesn't depend on Please.Location + // having been resolved yet. + if path, err := LookPath(shell, []string{filepath.Dir(exe)}); err == nil { + return path + } + } + // Leave it as it is; the exec will fail with a better message than anything we'd write. + return shell +} + // Hash returns a hash of the parts of this configuration that affect building targets in general. // Most parts are considered not to (e.g. cache settings) or affect specific targets (e.g. changing // tool paths which get accounted for on the targets that use them). @@ -771,12 +848,47 @@ func (config *Configuration) GetBuildEnv() BuildEnv { config.buildEnvStored.Once.Do(func() { config.buildEnvStored.Env = config.getBuildEnv(true, true) if path, present := config.buildEnvStored.Env["PATH"]; present { - config.buildEnvStored.Path = strings.Split(path, ":") + config.buildEnvStored.Path = fs.SplitPathList(path) } }) return config.buildEnvStored.Env } +// DefaultArcatTool is the [build] arcattool that means "whichever one Please downloads". +// parse.ArcatUnavailable recognises it, to say something useful on a platform where there is +// nothing to download. +// The literal is parse.InternalPackageName, which core cannot import; parse asserts they agree. +const DefaultArcatTool = "/////_please:arcat" + +// defaultPluginRepos returns the templates a plugin_repo() is resolved against when nothing is +// configured. Setting any [please] pluginrepo replaces the whole list, as it always has. +func (config *Configuration) defaultPluginRepos() []string { + return []string{ + "https://github.com/{owner}/{plugin}/archive/{revision}.zip", + "https://github.com/{owner}/{plugin}-rules/archive/{revision}.zip", + } +} + +// useBundledTools points the config at any helper tool the release bundles beside the binary, +// where nothing else has been configured. +// +// This is only arcat. The plugins' own tools - please_go, please_cc, please_pex - are chosen by +// the plugins' build defs, because a plugin's config is not ours to default. +// +// Only Windows bundles anything. Everywhere else there is a published arcat to download, and +// the internal package rule is the better answer because it is hashed and cached like anything +// else. +func (config *Configuration) useBundledTools() { + if runtime.GOOS != "windows" || config.Build.ArcatTool != DefaultArcatTool { + return + } + if fs.FileExists(filepath.Join(config.Please.Location, "arcat"+fs.ExeSuffix)) { + // A bare name rather than a path: Please.Location is already the head of the build + // PATH, and the lookup adds the .exe. The same route the bundled busybox takes. + config.Build.ArcatTool = "arcat" + } +} + // EnsurePleaseLocation will resolve `config.Please.Location` to a full path location where it is to be found. func (config *Configuration) EnsurePleaseLocation() { defaultPleaseLocation := fs.ExpandHomePath(DefaultPleaseLocation) @@ -823,7 +935,7 @@ func (config *Configuration) getBuildEnv(includePath bool, includeUnsafe bool) B if v, isSet := os.LookupEnv(k); isSet { if k == "PATH" { // plz's install location always needs to be on the path. - v = config.Please.Location + ":" + v + v = config.Please.Location + string(os.PathListSeparator) + v includePath = false // skip this in a bit } env[k] = v @@ -842,7 +954,7 @@ func (config *Configuration) getBuildEnv(includePath bool, includeUnsafe bool) B // but really external environment variables shouldn't affect this. // The only concession is that ~ is expanded as the user's home directory // in PATH entries. - env["PATH"] = strings.Join(append([]string{config.Please.Location}, config.Build.Path...), ":") + env["PATH"] = strings.Join(append([]string{config.Please.Location}, config.Build.Path...), string(os.PathListSeparator)) } return env } diff --git a/src/core/config_other.go b/src/core/config_other.go new file mode 100644 index 0000000000..3637395b39 --- /dev/null +++ b/src/core/config_other.go @@ -0,0 +1,8 @@ +//go:build !windows +// +build !windows + +package core + +const machineConfigFileName = "/etc/please/plzconfig" + +var defaultPath = []string{"/usr/local/bin", "/usr/bin", "/bin"} diff --git a/src/core/config_test.go b/src/core/config_test.go index d4d047ebf6..54c2f205e3 100644 --- a/src/core/config_test.go +++ b/src/core/config_test.go @@ -3,6 +3,7 @@ package core import ( "bytes" "os" + "path/filepath" "reflect" "strings" "testing" @@ -17,7 +18,9 @@ import ( ) func TestPlzConfigWorking(t *testing.T) { - RepoRoot = "/repo/root" + // A genuinely absolute path: on Windows a leading slash isn't one without a drive letter, + // so the location below would be resolved relative to the repo root a second time. + RepoRoot = filepath.Join(t.TempDir(), "repo", "root") config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/working.plzconfig"}, nil) assert.NoError(t, err) @@ -28,7 +31,16 @@ func TestPlzConfigWorking(t *testing.T) { assert.Equal(t, "8", config.Java.SourceLevel) assert.Equal(t, "7", config.Java.TargetLevel) assert.Equal(t, "10", config.Java.ReleaseLevel) - assert.Equal(t, "/repo/root/plz-out/please", config.Please.Location) + assert.Equal(t, filepath.Join(RepoRoot, "plz-out", "please"), config.Please.Location) +} + +func TestPlzConfigBackslash(t *testing.T) { + // The mistake a Windows user makes first. The parser's own message says nothing about + // paths, so check we name the file and say what to do instead. + _, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/backslash.plzconfig"}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "backslash.plzconfig") + assert.Contains(t, err.Error(), "forward slashes") } func TestPlzConfigFailing(t *testing.T) { @@ -157,22 +169,26 @@ func TestConfigOverrideOptions(t *testing.T) { } func TestPleaseRelativeLocationOverride(t *testing.T) { - RepoRoot = "/repo/root" + RepoRoot = filepath.Join(t.TempDir(), "repo", "root") config := DefaultConfiguration() err := config.ApplyOverrides(map[string]string{"please.location": "./plz-out/please"}) assert.NoError(t, err) - assert.Equal(t, "/repo/root/plz-out/please", config.Please.Location) + assert.Equal(t, filepath.Join(RepoRoot, "plz-out", "please"), config.Please.Location) } func TestPleaseTildeLocationOverride(t *testing.T) { - t.Setenv("HOME", "/path/to/home") + // USERPROFILE as well as HOME: os.UserHomeDir reads the former on Windows. + home := filepath.Join(t.TempDir(), "home") + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) config := DefaultConfiguration() err := config.ApplyOverrides(map[string]string{"please.location": "~/please-location"}) assert.NoError(t, err) - assert.Equal(t, "/path/to/home/please-location", config.Please.Location) + // Only the ~ is substituted, so the separator the user wrote survives as they wrote it. + assert.Equal(t, home+"/please-location", config.Please.Location) } func TestReadSemver(t *testing.T) { @@ -258,13 +274,20 @@ func TestUnknownHashChecker(t *testing.T) { assert.Error(t, err) } +// buildPath returns the PATH a config should produce: Please's own location, then the build +// path. Both of those are platform-specific - there is no default build path at all on Windows +// - so it's computed rather than written out. +func buildPath(config *Configuration) string { + return strings.Join(append([]string{config.Please.Location}, config.Build.Path...), string(os.PathListSeparator)) +} + func TestBuildEnvSection(t *testing.T) { config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/buildenv.plzconfig"}, nil) assert.NoError(t, err) expected := BuildEnv{ "BAR_BAR": "first", "FOO_BAR": "second", - "PATH": os.Getenv("TMP_DIR") + ":/usr/local/bin:/usr/bin:/bin", + "PATH": buildPath(config), } assert.EqualValues(t, expected, config.GetBuildEnv()) } @@ -277,7 +300,7 @@ func TestPassEnv(t *testing.T) { expected := BuildEnv{ "BAR": "second", "FOO": "first", - "PATH": os.Getenv("TMP_DIR") + ":" + os.Getenv("PATH"), + "PATH": buildPath(config), } assert.EqualValues(t, expected, config.GetBuildEnv()) } @@ -290,7 +313,7 @@ func TestPassUnsafeEnv(t *testing.T) { expected := BuildEnv{ "BAR": "second", "FOO": "first", - "PATH": os.Getenv("TMP_DIR") + ":" + os.Getenv("PATH"), + "PATH": buildPath(config), } assert.EqualValues(t, expected, config.GetBuildEnv()) } @@ -316,7 +339,7 @@ func TestPassUnsafeEnvExcludedFromHash(t *testing.T) { func TestBuildPathWithPathEnv(t *testing.T) { config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/passenv.plzconfig"}, nil) assert.NoError(t, err) - assert.Equal(t, config.Build.Path, strings.Split(os.Getenv("PATH"), ":")) + assert.Equal(t, config.Build.Path, fs.SplitPathList(os.Getenv("PATH"))) } func TestUpdateArgsWithAliases(t *testing.T) { @@ -415,25 +438,32 @@ func TestGetTags(t *testing.T) { } func TestEnsurePleaseLocation(t *testing.T) { - t.Setenv("HOME", "/path/to/home") + // The home directory is read through os.UserHomeDir, which looks at a different variable + // on Windows, and the paths below have to be genuinely absolute to be recognised as such + // there - a leading slash isn't enough without a drive letter. + home := filepath.Join(t.TempDir(), "home") + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) config := DefaultConfiguration() // Empty please location config resolves to this executable's directory config.Please.Location = "" config.EnsurePleaseLocation() - assert.Equal(t, os.Getenv("PWD"), config.Please.Location) + wd, err := os.Getwd() + require.NoError(t, err) + assert.Equal(t, wd, config.Please.Location) // Expands ~ config.Please.Location = "~" config.EnsurePleaseLocation() - assert.Equal(t, "/path/to/home", config.Please.Location) + assert.Equal(t, home, config.Please.Location) // Resolves relative path to repo root - RepoRoot = "/repo/root" + RepoRoot = filepath.Join(t.TempDir(), "repo", "root") config.Please.Location = "./plz-out/please" config.EnsurePleaseLocation() - assert.Equal(t, "/repo/root/plz-out/please", config.Please.Location) + assert.Equal(t, filepath.Join(RepoRoot, "plz-out", "please"), config.Please.Location) } func TestPluginConfig(t *testing.T) { @@ -441,3 +471,23 @@ func TestPluginConfig(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []string{"fooc"}, config.Plugin["foo"].ExtraValues["fooctool"]) } + +func TestDefaultPluginRepos(t *testing.T) { + // The URL list is hashed into every plugin download's rule hash, so an entry added here + // moves build hashes on every platform at once. It is the same list everywhere, and was + // briefly not: a Windows release used to carry its plugins and point at them with a + // file:// template, which is gone now that they are downloadable like anything else. + repos := DefaultConfiguration().defaultPluginRepos() + assert.Len(t, repos, 2) + for _, repo := range repos { + assert.True(t, strings.HasPrefix(repo, "https://github.com/"), repo) + } +} + +func TestUseBundledToolsLeavesAConfiguredArcatAlone(t *testing.T) { + config := DefaultConfiguration() + config.Please.Location = "/opt/please" + config.Build.ArcatTool = "//my/own:arcat" + config.useBundledTools() + assert.Equal(t, "//my/own:arcat", config.Build.ArcatTool) +} diff --git a/src/core/config_windows.go b/src/core/config_windows.go new file mode 100644 index 0000000000..f35f2837de --- /dev/null +++ b/src/core/config_windows.go @@ -0,0 +1,22 @@ +package core + +import ( + "os" + "path/filepath" +) + +// machineConfigFileName lives under ProgramData, which is the Windows equivalent of /etc for +// machine-wide configuration. If the variable isn't set we fall back to the conventional path. +var machineConfigFileName = filepath.Join(programData(), "please", "plzconfig") + +// defaultPath is deliberately empty. There is no Windows equivalent of /usr/bin holding the +// tools a build might need, and the conventional locations (System32 and friends) hold none +// of them, so there is nothing useful to default to; users configure [build] path instead. +var defaultPath []string + +func programData() string { + if dir := os.Getenv("ProgramData"); dir != "" { + return dir + } + return `C:\ProgramData` +} diff --git a/src/core/lock.go b/src/core/lock.go index 629cb12e5b..ec99e511c3 100644 --- a/src/core/lock.go +++ b/src/core/lock.go @@ -1,4 +1,4 @@ -// The logic below relies heavily on flock (advisory locks). +// The logic below relies heavily on advisory file locking; see lock_other.go and lock_windows.go. package core @@ -6,7 +6,6 @@ import ( "fmt" "os" "strconv" - "syscall" "github.com/thought-machine/please/src/fs" ) @@ -25,7 +24,7 @@ var repoLockFile *os.File // AcquireSharedRepoLock acquires a shared lock on the repo lock file. The file descriptor is reused if already opened // allowing its lock mode to be replaced. Dies if the lock cannot be successfully acquired. func AcquireSharedRepoLock() { - if err := acquireRepoLock(syscall.LOCK_SH); err != nil { + if err := acquireRepoLock(lockShared); err != nil { log.Fatal(err) } } @@ -33,7 +32,7 @@ func AcquireSharedRepoLock() { // AcquireExclusiveRepoLock acquires an exclusive lock on the repo lock file. The file descriptor is reused if already opened // allowing its lock mode to be replaced. Dies if the lock cannot be successfully acquired. func AcquireExclusiveRepoLock() { - if err := acquireRepoLock(syscall.LOCK_EX); err != nil { + if err := acquireRepoLock(lockExclusive); err != nil { log.Fatal(err) } } @@ -75,13 +74,13 @@ func openRepoLockFile() error { // AcquireExclusiveFileLock opens a file to acquire an exclusive lock. func AcquireExclusiveFileLock(filePath string) (*os.File, error) { - return acquireOpenFileLock(filePath, syscall.LOCK_EX) + return acquireOpenFileLock(filePath, lockExclusive) } // AcquireSharedFileLock opens a file to acquire a shared lock. // Multiple of these can be held at once, but not concurrently with an exclusive lock (ala a RWMutex or similar). func AcquireSharedFileLock(filePath string) (*os.File, error) { - return acquireOpenFileLock(filePath, syscall.LOCK_SH) + return acquireOpenFileLock(filePath, lockShared) } // Base function that allows to set up different lock modes and facilitate testing. @@ -105,7 +104,7 @@ func ReleaseFileLock(file *os.File) { return } - if err := syscall.Flock(int(file.Fd()), syscall.LOCK_UN); err != nil { + if err := flock(file, lockUnlock); err != nil { log.Errorf("Failed to release lock for %s: %s", file.Name(), err) // No point making this fatal really } if err := file.Close(); err != nil { @@ -118,7 +117,7 @@ type logFunc func(format string, args ...interface{}) func acquireFileLock(file *os.File, how int, levelLog logFunc) error { // Try a non-blocking acquire first so we can warn the user if we're waiting. log.Debug("Attempting to acquire lock for %s...", file.Name()) - err := syscall.Flock(int(file.Fd()), how|syscall.LOCK_NB) + err := flock(file, how|lockNonBlocking) if err != nil { pid, err := os.ReadFile(file.Name()) if err == nil && len(pid) > 0 { @@ -127,14 +126,14 @@ func acquireFileLock(file *os.File, how int, levelLog logFunc) error { levelLog("Looks like another process has already acquired the lock for %s. Waiting for it to finish...", file.Name()) } - if err := syscall.Flock(int(file.Fd()), how); err != nil { + if err := flock(file, how); err != nil { return fmt.Errorf("Failed to acquire lock for %s: %w", file.Name(), err) } } log.Debug("Acquired lock for %s", file.Name()) // Record content, only if we have an exclusive lock. - if how&syscall.LOCK_EX != 0 { + if how&lockExclusive != 0 { if err := file.Truncate(0); err == nil { file.WriteAt([]byte(strconv.Itoa(os.Getpid())), 0) } diff --git a/src/core/lock_other.go b/src/core/lock_other.go new file mode 100644 index 0000000000..a5200c7389 --- /dev/null +++ b/src/core/lock_other.go @@ -0,0 +1,21 @@ +//go:build !windows +// +build !windows + +package core + +import ( + "os" + "syscall" +) + +const ( + lockShared = syscall.LOCK_SH + lockExclusive = syscall.LOCK_EX + lockUnlock = syscall.LOCK_UN + lockNonBlocking = syscall.LOCK_NB +) + +// flock applies or releases an advisory lock on an open file. +func flock(file *os.File, how int) error { + return syscall.Flock(int(file.Fd()), how) +} diff --git a/src/core/lock_test.go b/src/core/lock_test.go index 5940baeae7..38cecea2c2 100644 --- a/src/core/lock_test.go +++ b/src/core/lock_test.go @@ -3,7 +3,6 @@ package core import ( "os" "strconv" - "syscall" "testing" "github.com/stretchr/testify/assert" @@ -60,11 +59,11 @@ func TestAcquireExclusiveRepoRoot(t *testing.T) { } func TestAcquireRepoRootOverride(t *testing.T) { - err := acquireRepoLock(syscall.LOCK_SH | syscall.LOCK_NB) + err := acquireRepoLock(lockShared | lockNonBlocking) assert.NoError(t, err) // It is able to immediately override the lock mode since it uses the same file descriptor. - err = acquireRepoLock(syscall.LOCK_EX | syscall.LOCK_NB) + err = acquireRepoLock(lockExclusive | lockNonBlocking) assert.NoError(t, err) ReleaseRepoLock() @@ -73,7 +72,7 @@ func TestAcquireRepoRootOverride(t *testing.T) { // This attempts to mimic how 2 plz processes acquire a shared repo lock. func TestAcquireSharedRepoRootTwice(t *testing.T) { // 1st process. - err := acquireRepoLock(syscall.LOCK_SH | syscall.LOCK_NB) + err := acquireRepoLock(lockShared | lockNonBlocking) assert.NoError(t, err) // Keep file descriptor reference alive. @@ -83,7 +82,7 @@ func TestAcquireSharedRepoRootTwice(t *testing.T) { // 2nd process. repoLockFile = nil // Reset. // It is able to immediately acquire another shared lock via a different file descriptor. - err = acquireRepoLock(syscall.LOCK_SH | syscall.LOCK_NB) + err = acquireRepoLock(lockShared | lockNonBlocking) assert.NoError(t, err) ReleaseRepoLock() @@ -92,7 +91,7 @@ func TestAcquireSharedRepoRootTwice(t *testing.T) { // This attempts to mimic how 1 plz process acquires a shared repo lock and another tries to acquire an exclusive one. func TestAcquireSharedAndExclusiveRepoRoot(t *testing.T) { // 1st process. - err := acquireRepoLock(syscall.LOCK_SH | syscall.LOCK_NB) + err := acquireRepoLock(lockShared | lockNonBlocking) assert.NoError(t, err) // Keep file descriptor reference alive. @@ -102,7 +101,7 @@ func TestAcquireSharedAndExclusiveRepoRoot(t *testing.T) { // 2nd process. repoLockFile = nil // Reset. // It errors immediately trying to acquire an exclusive lock as a shared one already exists from process 1. - err = acquireRepoLock(syscall.LOCK_EX | syscall.LOCK_NB) + err = acquireRepoLock(lockExclusive | lockNonBlocking) assert.Error(t, err) ReleaseRepoLock() @@ -111,7 +110,7 @@ func TestAcquireSharedAndExclusiveRepoRoot(t *testing.T) { // This attempts to mimic how 1 plz process acquires an exclusive repo lock and another tries to acquire a shared one. func TestAcquireExclusiveAndSharedRepoRoot(t *testing.T) { // 1st process. - err := acquireRepoLock(syscall.LOCK_EX | syscall.LOCK_NB) + err := acquireRepoLock(lockExclusive | lockNonBlocking) assert.NoError(t, err) // Keep file descriptor reference alive. @@ -121,7 +120,7 @@ func TestAcquireExclusiveAndSharedRepoRoot(t *testing.T) { // 2nd process. repoLockFile = nil // Reset. // It errors immediately trying to acquire a shared lock as an exclusive one already exists from process 1. - err = acquireRepoLock(syscall.LOCK_SH | syscall.LOCK_NB) + err = acquireRepoLock(lockShared | lockNonBlocking) assert.Error(t, err) ReleaseRepoLock() @@ -149,7 +148,7 @@ func TestAcquireExclusiveFileLock(t *testing.T) { // This attempts to mimic how 1 plz process acquires an exclusive file lock and another tries to do the same thing to the same file. func TestAcquireExclusiveFileLockTwice(t *testing.T) { // 1st process. - fd1, err := acquireOpenFileLock("path/to/file", syscall.LOCK_EX|syscall.LOCK_NB) + fd1, err := acquireOpenFileLock("path/to/file", lockExclusive|lockNonBlocking) assert.NoError(t, err) // Keep file descriptor reference alive. @@ -158,7 +157,7 @@ func TestAcquireExclusiveFileLockTwice(t *testing.T) { // 2nd process. // It errors immediately trying to acquire an exclusive lock as the same lock mode was already placed by process 1. - fd2, err := acquireOpenFileLock("path/to/file", syscall.LOCK_EX|syscall.LOCK_NB) + fd2, err := acquireOpenFileLock("path/to/file", lockExclusive|lockNonBlocking) assert.Error(t, err) ReleaseFileLock(fd2) diff --git a/src/core/lock_windows.go b/src/core/lock_windows.go new file mode 100644 index 0000000000..bf30fa6626 --- /dev/null +++ b/src/core/lock_windows.go @@ -0,0 +1,75 @@ +package core + +import ( + "os" + "sync" + + "golang.org/x/sys/windows" +) + +// These mirror the flock(2) constants; their values are arbitrary since Windows doesn't +// define them, but they must remain distinct bits because callers combine and test them. +const ( + lockShared = 0x1 + lockExclusive = 0x2 + lockUnlock = 0x8 + lockNonBlocking = 0x4 +) + +// LockFileEx locks a byte range rather than a whole file. We lock a single byte far past any +// plausible content so that the PID written into the lock file stays readable by other +// processes, which is what produces the "process N has already acquired the lock" message. +const ( + lockOffsetLow = 0 + lockOffsetHigh = 0x40000000 +) + +// Windows has no equivalent of flock's atomic conversion between shared and exclusive on a +// single handle, so we have to release before re-acquiring. Track what each handle holds in +// order to do that only when it's actually needed. +var ( + locksMux sync.Mutex + locks = map[*os.File]bool{} +) + +// flock applies or releases an advisory lock on an open file. +// +// N.B. unlike flock(2), changing mode on a handle that already holds a lock is not atomic: +// the lock is dropped and re-taken, so another process can take it in between. Please only +// changes mode at startup (shared -> exclusive in acquireRepoLock), so in practice this +// window is not contended. +func flock(file *os.File, how int) error { + handle := windows.Handle(file.Fd()) + overlapped := &windows.Overlapped{Offset: lockOffsetLow, OffsetHigh: lockOffsetHigh} + + locksMux.Lock() + defer locksMux.Unlock() + + if how&lockUnlock != 0 { + if !locks[file] { + return nil + } + delete(locks, file) + return windows.UnlockFileEx(handle, 0, 1, 0, overlapped) + } + + if locks[file] { + if err := windows.UnlockFileEx(handle, 0, 1, 0, overlapped); err != nil { + return err + } + delete(locks, file) + } + + var flags uint32 + if how&lockExclusive != 0 { + flags |= windows.LOCKFILE_EXCLUSIVE_LOCK + } + if how&lockNonBlocking != 0 { + flags |= windows.LOCKFILE_FAIL_IMMEDIATELY + } + if err := windows.LockFileEx(handle, flags, 0, 1, 0, overlapped); err != nil { + return err + } + locks[file] = true + return nil +} diff --git a/src/core/package.go b/src/core/package.go index 2bca9a0160..4ab074c113 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -3,6 +3,7 @@ package core import ( "fmt" "maps" + "path" "path/filepath" "slices" "sort" @@ -267,12 +268,14 @@ func FindOwningPackages(state *BuildState, files []string) []BuildLabel { // FindOwningPackage returns a build label identifying the package that owns a given file. func FindOwningPackage(state *BuildState, file string) BuildLabel { - f := filepath.Dir(file) + // path, not filepath: the result becomes a build label package name, and filepath.Dir on + // Windows would hand back backslashes, which aren't legal in one. + f := path.Dir(filepath.ToSlash(file)) for f != "." { if fs.IsPackage(state.Config.Parse.BuildFileName, f) { return BuildLabel{PackageName: f, Name: "all"} } - f = filepath.Dir(f) + f = path.Dir(f) } return BuildLabel{PackageName: "", Name: "all"} } diff --git a/src/core/pathsep_other.go b/src/core/pathsep_other.go new file mode 100644 index 0000000000..114cdf2cf4 --- /dev/null +++ b/src/core/pathsep_other.go @@ -0,0 +1,7 @@ +//go:build !windows +// +build !windows + +package core + +// normalisePathSeparators is a no-op where the path separator is already a forward slash. +func (env BuildEnv) normalisePathSeparators() {} diff --git a/src/core/pathsep_windows.go b/src/core/pathsep_windows.go new file mode 100644 index 0000000000..cfdfe51f6b --- /dev/null +++ b/src/core/pathsep_windows.go @@ -0,0 +1,21 @@ +package core + +import "strings" + +// normalisePathSeparators rewrites the paths Please generates to use forward slashes. +// +// Build commands are shell strings, and a backslash is an escape character to much of what +// runs in them. Expanding a variable is safe, but passing one to anything that interprets its +// arguments is not: `sed -e "s#x#$TMP_DIR#"` silently turns `\tmp` into a tab, and the C/C++ +// rules build their link line with sed. Win32, MinGW and busybox all accept forward slashes, +// so we use those throughout. +// +// This deliberately runs before withUserProvidedEnv: values the user wrote themselves are +// left exactly as written, since they may not be paths at all. +func (env BuildEnv) normalisePathSeparators() { + for k, v := range env { + if strings.ContainsRune(v, '\\') { + env[k] = strings.ReplaceAll(v, `\`, `/`) + } + } +} diff --git a/src/core/platform_env_other.go b/src/core/platform_env_other.go new file mode 100644 index 0000000000..52d5cc2131 --- /dev/null +++ b/src/core/platform_env_other.go @@ -0,0 +1,8 @@ +//go:build !windows +// +build !windows + +package core + +// setPlatformTmpEnv sets any platform-specific environment variables pointing at a build +// action's temporary directory. Unix tools use HOME and TMPDIR, which are set already. +func setPlatformTmpEnv(env BuildEnv, dir string) {} diff --git a/src/core/platform_env_windows.go b/src/core/platform_env_windows.go new file mode 100644 index 0000000000..cdf158ab98 --- /dev/null +++ b/src/core/platform_env_windows.go @@ -0,0 +1,11 @@ +package core + +// setPlatformTmpEnv sets any platform-specific environment variables pointing at a build +// action's temporary directory. Windows-native tools look at USERPROFILE rather than HOME, +// and at TEMP/TMP rather than TMPDIR, so they need the same redirection for the build +// environment to stay hermetic. +func setPlatformTmpEnv(env BuildEnv, dir string) { + env["USERPROFILE"] = dir + env["TEMP"] = dir + env["TMP"] = dir +} diff --git a/src/core/sandboxdir_other.go b/src/core/sandboxdir_other.go new file mode 100644 index 0000000000..064af19bce --- /dev/null +++ b/src/core/sandboxdir_other.go @@ -0,0 +1,6 @@ +//go:build !windows +// +build !windows + +package core + +const sandboxDir = "/tmp/plz_sandbox" diff --git a/src/core/sandboxdir_windows.go b/src/core/sandboxdir_windows.go new file mode 100644 index 0000000000..21a340ad4a --- /dev/null +++ b/src/core/sandboxdir_windows.go @@ -0,0 +1,5 @@ +package core + +// sandboxDir is unused for now - there is no sandbox on Windows (see M7 in +// docs/design/windows) - but it still has to be a path that never collides with a repo. +const sandboxDir = `C:\plz_sandbox` diff --git a/src/core/sandboxsupport_other.go b/src/core/sandboxsupport_other.go new file mode 100644 index 0000000000..c1f465bd5c --- /dev/null +++ b/src/core/sandboxsupport_other.go @@ -0,0 +1,9 @@ +//go:build !windows +// +build !windows + +package core + +// sandboxSupported reports whether this platform can isolate a build action at all. +// Everywhere but Windows there is at least a sandbox tool to hand the action to, even if what +// it does varies; on Linux it does the whole job. +func sandboxSupported() bool { return true } diff --git a/src/core/sandboxsupport_windows.go b/src/core/sandboxsupport_windows.go new file mode 100644 index 0000000000..209585b5fd --- /dev/null +++ b/src/core/sandboxsupport_windows.go @@ -0,0 +1,8 @@ +package core + +// sandboxSupported reports whether this platform can isolate a build action at all. +// +// Nothing on Windows does yet. The pieces exist - job objects, restricted tokens - but there +// is no analogue of a mount namespace, so filesystem isolation would need Windows Containers, +// which is far too large a dependency to take on. See docs/design/windows. +func sandboxSupported() bool { return false } diff --git a/src/core/state.go b/src/core/state.go index 685c6184a7..089d36838e 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -12,6 +12,7 @@ import ( iofs "io/fs" "iter" "path/filepath" + "runtime" "runtime/pprof" "sort" "strings" @@ -886,6 +887,19 @@ func (state *BuildState) SyncParsePackage(label BuildLabel) *Package { return state.Graph.PackageByLabel(label) // Important to check again; it's possible to race against this whole lot. } +// ReleasePendingParse gives back the claim on parsing a package that SyncParsePackage granted, +// for a package that has not been parsed and is not going to be. +// +// It exists for callers that parse a package speculatively - to find out whether it exists at +// all - and swallow the error when it doesn't. Such a caller still took the claim, and if it +// keeps it every later caller asking about the same package waits forever for a parse nobody is +// going to do. That shows up as a hang with no output rather than an error. +func (state *BuildState) ReleasePendingParse(label BuildLabel) { + if ch, present := state.progress.pendingPackages.Delete(label.packageKey()); present { + close(ch) // Anything already waiting goes back to trying for itself. + } +} + func waitOnChan[T any](ch chan T, message string, args ...any) { start := time.Now() t := time.NewTimer(10 * time.Second) @@ -1479,11 +1493,18 @@ func newXXHash() hash.Hash { } func executorFromConfig(config *Configuration) *process.Executor { + wantsSandbox := config.Sandbox.Build || config.Sandbox.Test + if wantsSandbox && !sandboxSupported() { + // Saying the tool is missing would be misleading here - there is nothing to install. + log.Warningf("Sandboxing is not implemented on %s; build actions and tests will run without isolation.", runtime.GOOS) + return process.NewSandboxingExecutor(false, process.NamespaceNever, "", config.Shell(), config.ShellArgs()) + } + tool := config.Sandbox.Tool if !filepath.IsAbs(tool) { var err error tool, err = LookBuildPath(tool, config) - if err != nil && (config.Sandbox.Build || config.Sandbox.Test) { + if err != nil && wantsSandbox { log.Warningf("Can't find sandbox tool %v on the path: %v", config.Sandbox.Tool, err) } } else if !fs.FileExists(tool) { @@ -1491,9 +1512,11 @@ func executorFromConfig(config *Configuration) *process.Executor { } return process.NewSandboxingExecutor( - config.Sandbox.Tool == "" && (config.Sandbox.Build || config.Sandbox.Test), + config.Sandbox.Tool == "" && wantsSandbox, process.NamespacingPolicy(config.Sandbox.Namespace), tool, + config.Shell(), + config.ShellArgs(), ) } diff --git a/src/core/subrepo.go b/src/core/subrepo.go index 100d4e0ab8..2713477a2d 100644 --- a/src/core/subrepo.go +++ b/src/core/subrepo.go @@ -4,7 +4,7 @@ import ( "fmt" iofs "io/fs" "os" - "path/filepath" + "path" "strings" "sync" @@ -110,8 +110,9 @@ func LabelToArch(label BuildLabel, arch cli.Arch) BuildLabel { } // Dir returns the directory for a package of this name. +// path, not filepath: subrepo roots are plz-out paths, which stay slash-separated. func (s *Subrepo) Dir(dir string) string { - return filepath.Join(s.Root, dir) + return path.Join(s.Root, dir) } func readSubrepoConfig(repoConfig *Configuration, subrepo *Subrepo) error { diff --git a/src/core/test_data/backslash.plzconfig b/src/core/test_data/backslash.plzconfig new file mode 100644 index 0000000000..9530a96298 --- /dev/null +++ b/src/core/test_data/backslash.plzconfig @@ -0,0 +1,2 @@ +[build] +path = C:\tools\bin diff --git a/src/core/test_results.go b/src/core/test_results.go index 727d9c533b..5756925f3a 100644 --- a/src/core/test_results.go +++ b/src/core/test_results.go @@ -3,7 +3,6 @@ package core import ( "bytes" "fmt" - "strings" "time" "github.com/thought-machine/please/src/fs" @@ -304,9 +303,7 @@ func MergeCoverageLines(existing, coverage []LineCoverage) []LineCoverage { func (coverage *TestCoverage) OrderedFiles() []string { files := make([]string, 0, len(coverage.Files)) for file := range coverage.Files { - if strings.HasPrefix(file, RepoRoot) { - file = strings.TrimLeft(file[len(RepoRoot):], "/") - } + file = TrimRepoRoot(file) files = append(files, file) } fs.SortPaths(files) diff --git a/src/core/utils.go b/src/core/utils.go index 3c63f53516..d429e4b659 100644 --- a/src/core/utils.go +++ b/src/core/utils.go @@ -6,6 +6,7 @@ import ( "fmt" "iter" "os" + "path" "path/filepath" "strings" @@ -20,7 +21,8 @@ var RepoRoot string var InitialWorkingDir string // InitialPackagePath is the initial subdir of the working directory, ie. what package did we start in. -// This is similar but not identical to InitialWorkingDir. +// This is similar but not identical to InitialWorkingDir. It is a build label package name, so it +// is always slash-separated, even on Windows. var InitialPackagePath string // usingBazelWorkspace is true if we detected a Bazel WORKSPACE file to find our repo root. @@ -78,7 +80,8 @@ func InitialPackage() []BuildLabel { label.Name = "..." return []BuildLabel{label} } - dir = filepath.Dir(dir) + // path, not filepath: this is a package name, which is slash-separated everywhere. + dir = path.Dir(dir) } return WholeGraph } @@ -89,18 +92,74 @@ func getRepoRoot(filename string) (string, string) { if err != nil { log.Fatalf("Couldn't determine working directory: %s", err) } - // Walk up directories looking for a .plzconfig file, which we use to identify the root. + return findRepoRootFrom(dir, filename) +} + +// findRepoRootFrom walks up from the given directory looking for the file that marks a repo +// root, and returns that directory and the package the walk started in. +func findRepoRootFrom(dir, filename string) (string, string) { initial := dir for dir != "" { if PathExists(filepath.Join(dir, filename)) { - return dir, strings.TrimLeft(initial[len(dir):], "/") + // The second return is a package name, so it has to come back slash-separated + // whatever the OS gave us - anything else fails build label validation, and the + // initial package silently becomes the whole repo. + return dir, strings.Trim(filepath.ToSlash(initial[len(dir):]), "/") } - dir, _ = filepath.Split(dir) - dir = strings.TrimRight(dir, "/") + // Stop when the walk stops going anywhere, rather than when it reaches an empty + // string. On Windows it never reaches one: trimming the separator off "C:\" leaves + // "C:", and splitting that returns it unchanged, because the volume name is the whole + // path. Before this, any plz run outside a repo spun here for ever, one stat per + // iteration, instead of reporting that it couldn't find a root. + parent, _ := filepath.Split(dir) + parent = strings.TrimRight(parent, fs.PathSeparators) + if parent == dir { + break + } + dir = parent } return "", "" } +// IsInRepoRoot returns true if the given path is inside the repo. +// +// It exists because comparing against RepoRoot directly is wrong on Windows. RepoRoot is in the +// OS's own separator, so it is backslashed there, while paths that arrive from outside - a +// file:// URL, a coverage report from another tool - are usually slash-separated. A plain +// HasPrefix then never matches, and a guard written that way silently stops guarding. +// +// It also only matches at a path boundary, so that /repo/elsewhere is not inside /repo/else. +func IsInRepoRoot(path string) bool { + _, ok := trimRepoRoot(path) + return ok +} + +// TrimRepoRoot returns the given path relative to the repo root, or unchanged if it is not +// inside it. The result keeps whatever separators it arrived with. +func TrimRepoRoot(path string) string { + if trimmed, ok := trimRepoRoot(path); ok { + return trimmed + } + return path +} + +func trimRepoRoot(path string) (string, bool) { + root := filepath.ToSlash(RepoRoot) + normalised := filepath.ToSlash(path) + if root == "" || !strings.HasPrefix(normalised, root) { + return path, false + } + rest := path[len(root):] + if rest == "" { + return "", true + } + // Only a match at a boundary; "/repo" is not a prefix of "/repository". + if !strings.ContainsRune(fs.PathSeparators, rune(rest[0])) && !strings.HasSuffix(root, "/") { + return path, false + } + return strings.TrimLeft(rest, fs.PathSeparators), true +} + // StartedAtRepoRoot returns true if the build was initiated from the repo root. // Used to provide slightly nicer output in some places. func StartedAtRepoRoot() bool { @@ -125,7 +184,9 @@ func IterSources(state *BuildState, graph *BuildGraph, target *BuildTarget, incl for input := range IterInputs(state, graph, target, includeTools, false) { fullPaths := input.FullPaths(graph) for i, sourcePath := range input.Paths(graph) { - if tmpPath := filepath.Join(tmpDir, sourcePath); !done[tmpPath] { + // path, not filepath: these are plz-out paths, and they reach build actions + // through the environment as $SRCS. + if tmpPath := path.Join(tmpDir, sourcePath); !done[tmpPath] { if !yield(fullPaths[i], tmpPath) { return } @@ -511,15 +572,29 @@ func CollapseHash(key []byte) []byte { // The main difference is that it looks based on our config which isn't necessarily the same // as the external environment variable. func LookPath(filename string, paths []string) (string, error) { + names := fs.ExecutableNames(filename) + dirs := 0 for _, p := range paths { - for _, p2 := range strings.Split(p, ":") { - p3 := filepath.Join(p2, filename) - if _, err := os.Stat(p3); err == nil { - return p3, nil + for _, p2 := range fs.SplitPathList(p) { + if p2 != "" { + dirs++ + } + for _, name := range names { + p3 := filepath.Join(p2, name) + if _, err := os.Stat(p3); err == nil { + return p3, nil + } } } } - return "", fmt.Errorf("%s not found in path %s", filename, strings.Join(paths, ":")) + err := fmt.Errorf("%s not found in path %s", filename, strings.Join(paths, string(os.PathListSeparator))) + if dirs <= 1 { + // Only Please's own directory was searched, which means no build path is configured. + // There is no default one on Windows - nothing there corresponds to /usr/bin - so this + // is the first thing a new user hits, and the message above doesn't hint at the answer. + return "", fmt.Errorf("%w\nNo [build] path is configured; set one to the directories your tools live in", err) + } + return "", err } // LookBuildPath is like LookPath but takes the config's build path into account. diff --git a/src/core/utils_test.go b/src/core/utils_test.go index adfa07dd63..3e8107c078 100644 --- a/src/core/utils_test.go +++ b/src/core/utils_test.go @@ -4,9 +4,15 @@ import ( "crypto/sha1" "encoding/base64" "os" + "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/thought-machine/please/src/fs" ) func TestCollapseHash(t *testing.T) { @@ -118,29 +124,49 @@ func TestInitialPackageUpToRoot(t *testing.T) { assert.Equal(t, []BuildLabel{{PackageName: "", Name: "..."}}, p) } +// writeFakeTool creates an executable named tool, plus whatever extension the platform needs +// to consider it one, in a new directory, and returns the directory and the full path. +func writeFakeTool(t *testing.T, tool string) (string, string) { + t.Helper() + dir := t.TempDir() + file := filepath.Join(dir, tool+fs.ExeSuffix) + require.NoError(t, os.WriteFile(file, nil, 0o755)) + return dir, file +} + func TestLookPath(t *testing.T) { - // Assume this will be present on the path somewhere (you've really got to have bash for plz) - path, err := LookPath("bash", []string{"/usr/local/bin", "/usr/bin", "/bin"}) - assert.NoError(t, err) - assert.Contains(t, []string{"/usr/local/bin/bash", "/usr/bin/bash", "/bin/bash"}, path) - info, err := os.Stat(path) - assert.NoError(t, err) - assert.Equal(t, "bash", info.Name()) + // A tool we put there ourselves, rather than something the host is assumed to have: the + // directories Please looks in by default differ per platform, and on Windows there are none. + dir, file := writeFakeTool(t, "plz_look_path_test") + found, err := LookPath("plz_look_path_test", []string{filepath.Join(dir, "nonexistent"), dir}) + require.NoError(t, err) + assert.Equal(t, file, found) } func TestLookPathColons(t *testing.T) { - // We support having colons inside the path elements because people might find that more natural. - path, err := LookPath("bash", []string{"/usr/local/bin:/usr/bin:/bin"}) - assert.NoError(t, err) - assert.Contains(t, []string{"/usr/local/bin/bash", "/usr/bin/bash", "/bin/bash"}, path) - info, err := os.Stat(path) - assert.NoError(t, err) - assert.Equal(t, "bash", info.Name()) + // We support having the list separator inside the path elements because people might find + // that more natural. + dir, file := writeFakeTool(t, "plz_look_path_test") + joined := strings.Join([]string{filepath.Join(dir, "nonexistent"), dir}, string(os.PathListSeparator)) + found, err := LookPath("plz_look_path_test", []string{joined}) + require.NoError(t, err) + assert.Equal(t, file, found) } func TestLookPathDoesntExist(t *testing.T) { - _, err := LookPath("wibblewobbleflibble", []string{"/usr/local/bin", "/usr/bin", "/bin"}) + dir, _ := writeFakeTool(t, "plz_look_path_test") + _, err := LookPath("wibblewobbleflibble", []string{dir, t.TempDir()}) assert.Error(t, err) + assert.NotContains(t, err.Error(), "No [build] path", "shouldn't advise configuring a path that is configured") +} + +func TestLookPathWithNothingConfigured(t *testing.T) { + // Only Please's own directory to search, which is what a Windows user gets before they set + // [build] path - there is no default one there. Say so rather than just naming the one + // directory we looked in. + _, err := LookPath("wibblewobbleflibble", []string{t.TempDir()}) + require.Error(t, err) + assert.Contains(t, err.Error(), "No [build] path is configured") } // buildGraph builds a test graph which we use to test IterSources etc. @@ -181,3 +207,72 @@ func makeTarget4(graph *BuildGraph, label string, deps ...string) *BuildTarget { target.AddOutput(target.Label.Name + ".a") return target } + +func TestFindRepoRootFromTerminatesAtTheRoot(t *testing.T) { + // A walk that reaches the top of the filesystem without finding a marker has to stop. + // On Windows it used not to: trimming the separator off "C:\" leaves "C:", and splitting + // that returns it unchanged, so this spun for ever at one stat per iteration and every plz + // run outside a repo hung instead of reporting that it could not find a root. + wd, err := os.Getwd() + require.NoError(t, err) + root := filepath.VolumeName(wd) + string(filepath.Separator) + + done := make(chan struct{}) + go func() { + defer close(done) + dir, pkg := findRepoRootFrom(root, "a_file_that_is_not_there_"+t.Name()) + assert.Empty(t, dir) + assert.Empty(t, pkg) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + // Failing rather than hanging the whole package, which is what this used to do. + t.Fatal("findRepoRootFrom did not terminate at the filesystem root") + } +} + +func TestFindRepoRootFromFindsTheMarker(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "some", "package") + require.NoError(t, os.MkdirAll(nested, os.ModeDir|0755)) + marker := "marker_" + t.Name() + require.NoError(t, os.WriteFile(filepath.Join(root, marker), nil, 0644)) + + dir, pkg := findRepoRootFrom(nested, marker) + assert.Equal(t, root, dir) + // Slash-separated whatever the OS gave us, because it is a package name. + assert.Equal(t, "some/package", pkg) +} + +func TestIsInRepoRoot(t *testing.T) { + // The comparison this replaces was a plain HasPrefix against RepoRoot, which is in the + // OS's own separator. Paths that arrive from outside - a file:// URL, a coverage report + // from another tool - are slash-separated, so on Windows it never matched and the guard + // that uses it silently stopped guarding. + old := RepoRoot + defer func() { RepoRoot = old }() + RepoRoot = filepath.Join(string(filepath.Separator)+"home", "user", "repo") + slashed := filepath.ToSlash(RepoRoot) + + assert.True(t, IsInRepoRoot(RepoRoot)) + assert.True(t, IsInRepoRoot(slashed), "a slash-separated path inside the repo is inside it") + assert.True(t, IsInRepoRoot(slashed+"/src/core/utils.go")) + assert.True(t, IsInRepoRoot(filepath.Join(RepoRoot, "src", "core"))) + + assert.False(t, IsInRepoRoot(slashed+"sitory/src"), "only matches at a path boundary") + assert.False(t, IsInRepoRoot("/somewhere/else")) + assert.False(t, IsInRepoRoot("")) +} + +func TestTrimRepoRoot(t *testing.T) { + old := RepoRoot + defer func() { RepoRoot = old }() + RepoRoot = filepath.Join(string(filepath.Separator)+"home", "user", "repo") + slashed := filepath.ToSlash(RepoRoot) + + assert.Equal(t, "src/core", TrimRepoRoot(slashed+"/src/core")) + assert.Equal(t, filepath.Join("src", "core"), TrimRepoRoot(filepath.Join(RepoRoot, "src", "core"))) + // Left alone rather than mangled when it isn't ours. + assert.Equal(t, "/somewhere/else", TrimRepoRoot("/somewhere/else")) +} diff --git a/src/core/xattrs_other.go b/src/core/xattrs_other.go new file mode 100644 index 0000000000..ed8ec07779 --- /dev/null +++ b/src/core/xattrs_other.go @@ -0,0 +1,7 @@ +//go:build !windows +// +build !windows + +package core + +// defaultXattrs is whether we try to record file metadata in extended attributes by default. +const defaultXattrs = true diff --git a/src/core/xattrs_windows.go b/src/core/xattrs_windows.go new file mode 100644 index 0000000000..6267d1d98a --- /dev/null +++ b/src/core/xattrs_windows.go @@ -0,0 +1,5 @@ +package core + +// defaultXattrs is whether we try to record file metadata in extended attributes by default. +// Windows has no equivalent, so we always fall back to writing separate files. +const defaultXattrs = false diff --git a/src/exec/BUILD b/src/exec/BUILD index 727071ff7f..3f74d37d13 100644 --- a/src/exec/BUILD +++ b/src/exec/BUILD @@ -14,6 +14,8 @@ go_library( go_test( name = "exec_test", srcs = ["exec_test.go"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":exec", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/export/BUILD b/src/export/BUILD index 5069f9dc85..8f36a767bf 100644 --- a/src/export/BUILD +++ b/src/export/BUILD @@ -20,7 +20,9 @@ go_library( go_test( name = "export_test", srcs = ["export_test.go"], - data = ["test_data"], + data = [":export_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":export", "///third_party/go/github.com_stretchr_testify//assert", @@ -28,3 +30,11 @@ go_test( "//src/parse/asp", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "export_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/format/BUILD b/src/format/BUILD index 1a6fbed019..67474e7cee 100644 --- a/src/format/BUILD +++ b/src/format/BUILD @@ -16,7 +16,9 @@ go_library( go_test( name = "format_test", srcs = ["fmt_test.go"], - data = ["test_data"], + data = [":format_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":format", "///third_party/go/github.com_stretchr_testify//assert", @@ -24,3 +26,11 @@ go_test( "//src/core", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "format_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/fs/BUILD b/src/fs/BUILD index 6636b1f795..95c21236db 100644 --- a/src/fs/BUILD +++ b/src/fs/BUILD @@ -11,7 +11,7 @@ go_library( "///third_party/go/github.com_peterebden_go-deferred-regex//:go-deferred-regex", "///third_party/go/github.com_pkg_xattr//:xattr", "//src/cli/logging", - ], + ] + (["///third_party/go/golang.org_x_sys//windows"] if is_platform(os = "windows") else []), ) go_test( @@ -23,7 +23,9 @@ go_test( "*_benchmark_test.go", ], ), - data = ["test_data"], + data = [":test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":fs", "///third_party/go/github.com_stretchr_testify//assert", @@ -50,3 +52,11 @@ go_benchmark( "///third_party/go/github.com_zeebo_blake3//:blake3", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/fs/copy.go b/src/fs/copy.go index a3d8345d73..eabdccd17b 100644 --- a/src/fs/copy.go +++ b/src/fs/copy.go @@ -4,8 +4,12 @@ import ( "fmt" "os" "path/filepath" + "sync" ) +// warnSymlinkFallback ensures we only mention the symlink degradation once. +var warnSymlinkFallback sync.Once + // CopyOrLinkFile either copies or hardlinks a file based on the link argument. // Falls back to a copy if link fails and fallback is true. func CopyOrLinkFile(from, to string, fromMode, toMode os.FileMode, link, fallback bool) error { @@ -17,7 +21,15 @@ func CopyOrLinkFile(from, to string, fromMode, toMode os.FileMode, link, fallbac if err != nil { return err } - return os.Symlink(dest, to) + if err := os.Symlink(dest, to); err == nil || !isSymlinkPrivilegeError(err) { + return err + } + // Windows won't create a symlink without Developer Mode or + // SeCreateSymbolicLinkPrivilege. Copy what it points at instead; for populating + // plz-out the content is what matters, not that the link is reproduced. + warnSymlinkFallback.Do(func() { + log.Warning("Cannot create symlinks; copying instead. Enable Developer Mode to avoid this.") + }) } if err := os.Link(from, to); err == nil || !fallback { return err @@ -83,6 +95,28 @@ func copySymlink(name, dest string) error { type LinkFunc func(string, string) error +// SymlinkOrCopy creates dest as a symlink to src, copying instead where the OS will not make +// one. +// +// Windows needs Developer Mode or SeCreateSymbolicLinkPrivilege to create a symlink at all, and +// an ordinary user has neither, so every link: label a build declares was quietly turning into +// a warning there. For populating plz-out the content is what matters, not that the link is +// reproduced - the same trade CopyOrLinkFile already makes. +func SymlinkOrCopy(src, dest string) error { + err := os.Symlink(src, dest) + if err == nil || !isSymlinkPrivilegeError(err) { + return err + } + warnSymlinkFallback.Do(func() { + log.Warning("Cannot create symlinks; copying instead. Enable Developer Mode to avoid this.") + }) + info, lerr := os.Lstat(src) + if lerr != nil { + return lerr + } + return CopyFile(src, dest, info.Mode()) +} + // LinkIfNotExists creates dest as a link to src if it doesn't already exist. func LinkIfNotExists(src, dest string, f LinkFunc) { if PathExists(dest) { diff --git a/src/fs/copy_test.go b/src/fs/copy_test.go index 2f053e0857..d618c546d8 100644 --- a/src/fs/copy_test.go +++ b/src/fs/copy_test.go @@ -67,6 +67,13 @@ func TestLink(t *testing.T) { } func TestSymlink(t *testing.T) { + if IsWine() { + // Only under Wine, where os.Symlink reports success and produces a link os.Lstat then + // cannot find - so this asserts nothing there. Real Windows is the case worth testing: + // it refuses without Developer Mode or SeCreateSymbolicLinkPrivilege, which is what + // SymlinkOrCopy's fallback exists for, and what the CI runner actually has. + t.Skip("Wine's symlinks are not real enough to assert against; see docs/design/windows") + } var tests = []struct { description string srcExists bool diff --git a/src/fs/executable.go b/src/fs/executable.go index e076e1f871..687ecb95ce 100644 --- a/src/fs/executable.go +++ b/src/fs/executable.go @@ -49,7 +49,7 @@ func executable() (string, error) { return exePath, nil } // Search for executable in $PATH. - for _, dir := range splitPathList(os.Getenv("PATH")) { + for _, dir := range SplitPathList(os.Getenv("PATH")) { if len(dir) == 0 { dir = "." } @@ -86,9 +86,9 @@ func isExecutable(path string) error { return nil } -// splitPathList splits a path list. +// SplitPathList splits a PATH-style list on the platform's list separator. // This is based on genSplit from strings/strings.go -func splitPathList(pathList string) []string { +func SplitPathList(pathList string) []string { if pathList == "" { return nil } diff --git a/src/fs/exename_other.go b/src/fs/exename_other.go new file mode 100644 index 0000000000..3b9a76fffc --- /dev/null +++ b/src/fs/exename_other.go @@ -0,0 +1,18 @@ +//go:build !windows +// +build !windows + +package fs + +// PathSeparators are the characters that separate elements of a path. +const PathSeparators = "/" + +// ExeSuffix is what an executable's filename ends in. Unix decides by the executable bit +// rather than the name, so there is nothing to add. +const ExeSuffix = "" + +// ExecutableNames returns the filenames to try when searching the path for an executable +// called name. On Unix an executable is just a file with the executable bit set, so there is +// only ever one candidate. +func ExecutableNames(name string) []string { + return []string{name} +} diff --git a/src/fs/exename_windows.go b/src/fs/exename_windows.go new file mode 100644 index 0000000000..8ca8ca1dd8 --- /dev/null +++ b/src/fs/exename_windows.go @@ -0,0 +1,37 @@ +package fs + +import ( + "os" + "strings" +) + +// PathSeparators are the characters that separate elements of a path. Windows accepts either, +// and both turn up in practice: its own APIs return backslashes, but plenty of paths reaching +// us were written with forward slashes. +const PathSeparators = `/\` + +// ExeSuffix is what an executable's filename ends in. Windows will not run a file without +// it, whatever the file actually contains. +const ExeSuffix = ".exe" + +// defaultPathExt is used when PATHEXT isn't set in the environment; it matches what Windows +// itself defaults to. +const defaultPathExt = ".COM;.EXE;.BAT;.CMD" + +// ExecutableNames returns the filenames to try when searching the path for an executable +// called name. Windows decides what is executable by extension, so a bare name like "bash" +// has to be tried as "bash.exe", "bash.cmd" and so on. The bare name is returned first, since +// callers may already have passed a full filename. +func ExecutableNames(name string) []string { + pathExt := os.Getenv("PATHEXT") + if pathExt == "" { + pathExt = defaultPathExt + } + names := []string{name} + for _, ext := range strings.Split(pathExt, ";") { + if ext = strings.TrimSpace(ext); ext != "" { + names = append(names, name+strings.ToLower(ext)) + } + } + return names +} diff --git a/src/fs/fs.go b/src/fs/fs.go index 73bb3dfd90..231f766719 100644 --- a/src/fs/fs.go +++ b/src/fs/fs.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "time" "github.com/thought-machine/please/src/cli/logging" ) @@ -180,14 +181,36 @@ func copyFile(from, to string) (err error) { // RemoveAll will try and remove the path with `os.RemoveAll`; if that fails with a permission error, // it will attempt to adjust permissions to make things writable, then remove them. +// +// On Windows it also retries briefly where the failure was something else holding the file open, +// which real-time virus scanning causes routinely and which is usually over in a moment. A handle +// that is genuinely held outlives the retries and is reported, since no amount of waiting will +// help - see docs/design/windows/05-testing-strategy.md. func RemoveAll(path string) error { + err := removeAll(path) + for i := 1; i < removeRetries && isTransientRemoveError(err); i++ { + time.Sleep(removeRetryDelay) + err = removeAll(path) + } + if isTransientRemoveError(err) { + return fmt.Errorf("%w; something else has a file in %s open. On Windows a file cannot be "+ + "deleted while any process holds it open, virus scanners included", err, path) + } + return err +} + +func removeAll(path string) error { if err := os.RemoveAll(path); err == nil || !errors.Is(err, os.ErrPermission) { return err + } else if isTransientRemoveError(err) { + // Not a permissions problem however much it looks like one: the chmod walk below would + // make every file in the tree writable and then fail again for the same reason. + return err } else if err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { const writable = 0o220 if err != nil { return err - } else if d.IsDir() && d.Type()&writable != writable { + } else if (d.IsDir() || removeNeedsWritableFiles) && d.Type()&writable != writable { if info, err := d.Info(); err != nil { return fmt.Errorf("could not read info for %s: %w", path, err) } else if err := os.Chmod(path, info.Mode()|writable); err != nil { diff --git a/src/fs/glob.go b/src/fs/glob.go index 073e29cc43..d1553a4115 100644 --- a/src/fs/glob.go +++ b/src/fs/glob.go @@ -3,6 +3,7 @@ package fs import ( "fmt" iofs "io/fs" + "path" "path/filepath" "regexp" "strings" @@ -15,7 +16,9 @@ type matcher interface { type builtInGlob string func (p builtInGlob) Match(name string) (bool, error) { - matched, err := filepath.Match(string(p), name) + // path.Match, not filepath.Match: the names come from io/fs and are slash-separated, and + // on Windows filepath would treat the separator as a backslash and let * cross directories. + matched, err := path.Match(string(p), name) if err != nil { return false, fmt.Errorf("failed to glob, invalid patern: %v, %w", string(p), err) } @@ -33,7 +36,9 @@ func (r regexGlob) Match(name string) (bool, error) { // This converts the string pattern into a matcher. A matcher can either be one of our homebrew compiled regexs that // support ** or a matcher that uses the built in filesystem.Match functionality. func patternToMatcher(root, pattern string) (matcher, error) { - fullPattern := filepath.Join(root, pattern) + // These patterns are matched against paths from io/fs, which are always slash-separated + // whatever the host OS, so they have to be built with path rather than filepath. + fullPattern := path.Join(root, pattern) // Use the built in filesystem.Match globs when not using double star as it's far more efficient if !strings.Contains(pattern, "**") { @@ -168,10 +173,13 @@ func (globber *Globber) walkDir(rootPath string) (walkedDir, error) { return dir, nil } dir := walkedDir{} - err := iofs.WalkDir(globber.fs, rootPath, func(path string, d iofs.DirEntry, err error) error { + err := iofs.WalkDir(globber.fs, rootPath, func(name string, d iofs.DirEntry, err error) error { typeMode := mode(d.Type()) - if isBuildFile(globber.buildFileNames, path) { - packageName := filepath.Dir(path) + if isBuildFile(globber.buildFileNames, name) { + // path, not filepath: this comes from io/fs and is slash-separated whatever the + // host OS. filepath.Dir on Windows splits on backslashes only, so it would return + // the whole string here and no subpackage would ever be found. + packageName := path.Dir(name) if packageName != rootPath { dir.subPackages = append(dir.subPackages, packageName) return filepath.SkipDir @@ -182,9 +190,9 @@ func (globber *Globber) walkDir(rootPath string) (walkedDir, error) { return filepath.SkipDir } if typeMode.IsSymlink() { - dir.symlinks = append(dir.symlinks, path) + dir.symlinks = append(dir.symlinks, name) } else { - dir.fileNames = append(dir.fileNames, path) + dir.fileNames = append(dir.fileNames, name) } return nil }) @@ -207,7 +215,8 @@ func isBathPathOf(path string, base string) bool { } rest := strings.TrimPrefix(path, base) - return rest == "" || rest[0] == filepath.Separator + // Always '/', not os.PathSeparator: these paths come from io/fs. + return rest == "" || rest[0] == '/' } // shouldExcludeMatch checks if the match also matches any of the exclude patterns. If the exclude pattern is a relative @@ -221,14 +230,14 @@ func shouldExcludeMatch(root, match string, excludes []string) (bool, error) { rootPath := root m := match - if isBathPathOf(match, filepath.Join(root, excl)) { + if isBathPathOf(match, path.Join(root, excl)) { return true, nil } // If the exclude pattern doesn't contain any slashes and the match does, we only match against the base of the // match path. if strings.ContainsRune(match, '/') && !strings.ContainsRune(excl, '/') { - m = filepath.Base(match) + m = path.Base(match) rootPath = "" } @@ -250,7 +259,7 @@ func shouldExcludeMatch(root, match string, excludes []string) (bool, error) { // isBuildFile checks if the filename is considered a build filename func isBuildFile(buildFileNames []string, name string) bool { - fileName := filepath.Base(name) + fileName := path.Base(name) for _, buildFileName := range buildFileNames { if fileName == buildFileName { return true @@ -271,6 +280,6 @@ func isInDirectories(name string, directories []string) bool { // isHidden checks if the file is a hidden file i.e. starts with . or, starts and ends with #. func isHidden(name string) bool { - file := filepath.Base(name) + file := path.Base(name) return strings.HasPrefix(file, ".") || (strings.HasPrefix(file, "#") && strings.HasSuffix(file, "#")) } diff --git a/src/fs/home.go b/src/fs/home.go index 60c3ae8d45..58ca7fb5d8 100644 --- a/src/fs/home.go +++ b/src/fs/home.go @@ -2,16 +2,34 @@ package fs import ( "os" + "regexp" "strings" "github.com/peterebden/go-deferred-regex" ) -var homeRex = deferredregex.DeferredRegex{Re: "(?:^|:)(~(?:[/:]|$))"} +var homeRex = deferredregex.DeferredRegex{Re: homePathRegex()} -// ExpandHomePath expands all prefixes of ~ without a user specifier to $HOME. +// homePathRegex returns the pattern matching a bare ~ at the start of a path, or at the start +// of an entry within a PATH-style list. Both the list separator and the path separators are +// platform-specific, and Windows accepts either slash. +func homePathRegex() string { + listSep := regexp.QuoteMeta(string(os.PathListSeparator)) + pathSeps := "/" + if os.PathSeparator == '\\' { + pathSeps = `/\\` + } + return `(?:^|` + listSep + `)(~(?:[` + pathSeps + listSep + `]|$))` +} + +// ExpandHomePath expands all prefixes of ~ without a user specifier to the user's home directory. func ExpandHomePath(path string) string { - return ExpandHomePathTo(path, os.Getenv("HOME")) + home, err := os.UserHomeDir() + if err != nil { + // Same as the old behaviour of reading $HOME directly: if we can't tell, expand to nothing. + home = "" + } + return ExpandHomePathTo(path, home) } // ExpandHomePathTo expands all prefixes of ~ without a user specifier to the given string. diff --git a/src/fs/home_test.go b/src/fs/home_test.go index 9c8f2e5c6d..07116524d8 100644 --- a/src/fs/home_test.go +++ b/src/fs/home_test.go @@ -5,19 +5,24 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestExpandHomePath(t *testing.T) { - HOME := os.Getenv("HOME") + home, err := os.UserHomeDir() + require.NoError(t, err) + // Both the home directory and the separator between entries in a PATH-style list are + // platform-specific, so the expectations are built rather than written out. + sep := string(os.PathListSeparator) cases := []struct { in, want string }{ {"", ""}, - {"~", HOME}, + {"~", home}, {"~username", "~username"}, - {"~:/bin/~:/usr/local", HOME + ":/bin/~:/usr/local"}, - {"/bin:~/bin:~/script:/usr/local/bin", - "/bin:" + HOME + "/bin:" + HOME + "/script:/usr/local/bin"}, + {"~" + sep + "/bin/~" + sep + "/usr/local", home + sep + "/bin/~" + sep + "/usr/local"}, + {"/bin" + sep + "~/bin" + sep + "~/script" + sep + "/usr/local/bin", + "/bin" + sep + home + "/bin" + sep + home + "/script" + sep + "/usr/local/bin"}, } for _, c := range cases { assert.Equal(t, c.want, ExpandHomePath(c.in)) diff --git a/src/fs/removeall_other.go b/src/fs/removeall_other.go new file mode 100644 index 0000000000..0615daf2c4 --- /dev/null +++ b/src/fs/removeall_other.go @@ -0,0 +1,19 @@ +//go:build !windows +// +build !windows + +package fs + +import "time" + +// removeNeedsWritableFiles is whether a file has to be writable for its parent directory to be +// removable. On Unix only the directory's own permissions matter. +const removeNeedsWritableFiles = false + +// removeRetries and removeRetryDelay are the Windows retry loop's settings; there is nothing to +// retry here, because Unix is happy to unlink a file that is still open. +const removeRetries = 1 +const removeRetryDelay = time.Duration(0) + +// isTransientRemoveError reports whether a removal failed for a reason that may not still be +// true in a moment. Nothing on Unix qualifies. +func isTransientRemoveError(error) bool { return false } diff --git a/src/fs/removeall_windows.go b/src/fs/removeall_windows.go new file mode 100644 index 0000000000..975eccbed2 --- /dev/null +++ b/src/fs/removeall_windows.go @@ -0,0 +1,31 @@ +package fs + +import ( + "errors" + "time" + + "golang.org/x/sys/windows" +) + +// removeNeedsWritableFiles is whether a file has to be writable for its parent directory to be +// removable. Windows refuses to delete a file carrying FILE_ATTRIBUTE_READONLY - which is what +// os.Chmod manipulates there - and the read-only attribute on a directory means something else +// entirely, so the files themselves have to be cleared. +const removeNeedsWritableFiles = true + +// removeRetries is how many times to retry a removal that failed because something else had the +// file open, and how long to wait between attempts. +// +// Windows will not unlink or rename a file another handle has open, and real-time virus scanning +// opens files Please has just written, for as long as it takes to scan them. That makes this a +// transient failure rather than a permanent one, unlike every other error here. A handle that is +// genuinely held - by this process, or by something the user is running - outlives the retries +// and still fails, which is what we want. +const removeRetries = 10 +const removeRetryDelay = 100 * time.Millisecond + +// isTransientRemoveError reports whether a removal failed for a reason that may not still be +// true in a moment. +func isTransientRemoveError(err error) bool { + return errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_LOCK_VIOLATION) +} diff --git a/src/fs/runnable_other.go b/src/fs/runnable_other.go new file mode 100644 index 0000000000..fab3c1a8c8 --- /dev/null +++ b/src/fs/runnable_other.go @@ -0,0 +1,9 @@ +//go:build !windows +// +build !windows + +package fs + +// ExplainUnrunnable returns extra context for a file that could not be executed, or an empty +// string if there is nothing useful to add. There never is on Unix, where the executable bit +// decides and the error already says so. +func ExplainUnrunnable(string) string { return "" } diff --git a/src/fs/runnable_test.go b/src/fs/runnable_test.go new file mode 100644 index 0000000000..43040a463d --- /dev/null +++ b/src/fs/runnable_test.go @@ -0,0 +1,35 @@ +package fs + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExplainUnrunnableSaysNothingAboutMissingFiles(t *testing.T) { + assert.Empty(t, ExplainUnrunnable(filepath.Join(t.TempDir(), "nothing-here"))) + assert.Empty(t, ExplainUnrunnable("")) + assert.Empty(t, ExplainUnrunnable(t.TempDir()), "a directory isn't a binary that failed to run") +} + +func TestExplainUnrunnableSaysNothingAboutProperlyNamedFiles(t *testing.T) { + file := filepath.Join(t.TempDir(), "tool"+ExeSuffix) + require.NoError(t, os.WriteFile(file, nil, 0o755)) + assert.Empty(t, ExplainUnrunnable(file), "this one is named the way the platform wants") +} + +func TestExplainUnrunnableNamesTheSuffix(t *testing.T) { + file := filepath.Join(t.TempDir(), "tool") + require.NoError(t, os.WriteFile(file, nil, 0o755)) + explanation := ExplainUnrunnable(file) + if runtime.GOOS != "windows" { + // The executable bit decides on Unix, and the error already says so. + assert.Empty(t, explanation) + return + } + assert.Contains(t, explanation, "tool.exe") +} diff --git a/src/fs/runnable_windows.go b/src/fs/runnable_windows.go new file mode 100644 index 0000000000..2974afe9f0 --- /dev/null +++ b/src/fs/runnable_windows.go @@ -0,0 +1,31 @@ +package fs + +import ( + "os" + "strings" +) + +// ExplainUnrunnable returns extra context for a file that could not be executed, or an empty +// string if there is nothing useful to add. +// +// Windows decides what is runnable by extension, and Go's exec package enforces that: a file +// whose name has no extension in PATHEXT will not run even when handed its full path, and the +// error says it was "not found in %PATH%" - which is baffling when the file is plainly there. +// Windows itself is happy to execute it; only the lookup refuses. +// +// The usual cause is a build rule that named its output after the rule, as most language +// plugins do, without adding the suffix Windows needs. +func ExplainUnrunnable(path string) string { + if path == "" || !PathExists(path) { + return "" + } + for _, name := range ExecutableNames("") { + if name != "" && strings.HasSuffix(strings.ToLower(path), name) { + return "" + } + } + if info, err := os.Stat(path); err != nil || info.IsDir() { + return "" + } + return "\n" + path + " exists, but its name has no extension Windows will run; something has to produce it as " + path + ExeSuffix + " instead" +} diff --git a/src/fs/symlink_other.go b/src/fs/symlink_other.go new file mode 100644 index 0000000000..ba6bcfcf40 --- /dev/null +++ b/src/fs/symlink_other.go @@ -0,0 +1,8 @@ +//go:build !windows +// +build !windows + +package fs + +// isSymlinkPrivilegeError reports whether an error from os.Symlink means the OS refused for +// want of a privilege. Only Windows does that. +func isSymlinkPrivilegeError(error) bool { return false } diff --git a/src/fs/symlink_windows.go b/src/fs/symlink_windows.go new file mode 100644 index 0000000000..6733a01720 --- /dev/null +++ b/src/fs/symlink_windows.go @@ -0,0 +1,14 @@ +package fs + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// isSymlinkPrivilegeError reports whether an error from os.Symlink means the OS refused for +// want of a privilege. Creating a symlink on Windows needs either Developer Mode or +// SeCreateSymbolicLinkPrivilege, neither of which an ordinary user has by default. +func isSymlinkPrivilegeError(err error) bool { + return errors.Is(err, windows.ERROR_PRIVILEGE_NOT_HELD) +} diff --git a/src/fs/wine_other.go b/src/fs/wine_other.go new file mode 100644 index 0000000000..830783c3ec --- /dev/null +++ b/src/fs/wine_other.go @@ -0,0 +1,8 @@ +//go:build !windows +// +build !windows + +package fs + +// IsWine reports whether this process is running under Wine rather than on Windows. Nothing +// that is not a Windows binary is. +func IsWine() bool { return false } diff --git a/src/fs/wine_windows.go b/src/fs/wine_windows.go new file mode 100644 index 0000000000..29dde4e5f8 --- /dev/null +++ b/src/fs/wine_windows.go @@ -0,0 +1,17 @@ +package fs + +import ( + "golang.org/x/sys/windows" +) + +// IsWine reports whether this process is running under Wine rather than on Windows. +// +// It exists so that a test can skip where Wine is known to lie, rather than skipping on Windows +// wholesale and telling us nothing about the platform we actually care about. Wine's symlinks +// are the case that forced it: os.Symlink reports success and produces a link os.Lstat cannot +// find, so a test written against real behaviour fails there for a reason that is not a bug. +// +// Detected by a function only Wine exports. Wine documents this as the supported way to tell. +func IsWine() bool { + return windows.NewLazySystemDLL("ntdll.dll").NewProc("wine_get_version").Find() == nil +} diff --git a/src/gc/BUILD b/src/gc/BUILD index a7ef3099d1..f574fb3ac0 100644 --- a/src/gc/BUILD +++ b/src/gc/BUILD @@ -18,9 +18,9 @@ go_test( "gc_test.go", "rewrite_test.go", ], - data = [ - "test_data", - ], + data = [":gc_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":gc", "///third_party/go/github.com_stretchr_testify//assert", @@ -28,3 +28,11 @@ go_test( "//src/fs", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "gc_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/generate/generate.go b/src/generate/generate.go index 33a6582953..9017bf2ec8 100644 --- a/src/generate/generate.go +++ b/src/generate/generate.go @@ -1,6 +1,7 @@ package generate import ( + "path" "path/filepath" "strings" @@ -33,10 +34,12 @@ func UpdateGitignore(graph *core.BuildGraph, labels []core.BuildLabel, gitignore } relativePkg = strings.TrimPrefix(strings.TrimPrefix(t.Label.PackageName, pkg), "/") } - if vcs.AreIgnored(filepath.Join(t.Label.PackageName, out)) { + // path, not filepath: these are matched against .gitignore patterns and then + // written into one, and git speaks forward slashes on every platform. + if vcs.AreIgnored(path.Join(t.Label.PackageName, out)) { continue } - files = append(files, filepath.Join(relativePkg, out)) + files = append(files, path.Join(relativePkg, out)) } } return vcs.IgnoreFiles(gitignore, files) diff --git a/src/hashes/BUILD b/src/hashes/BUILD index 9f2b7f4096..e749145d26 100644 --- a/src/hashes/BUILD +++ b/src/hashes/BUILD @@ -13,9 +13,9 @@ go_library( go_test( name = "hash_rewriter_test", srcs = ["hash_rewriter_test.go"], - data = [ - "test_data", - ], + data = [":hash_rewriter_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":hashes", "///third_party/go/github.com_stretchr_testify//assert", @@ -23,3 +23,11 @@ go_test( "//src/fs", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "hash_rewriter_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/output/BUILD b/src/output/BUILD index 8b89a1b785..3ea9d698c8 100644 --- a/src/output/BUILD +++ b/src/output/BUILD @@ -26,6 +26,8 @@ go_test( "interactive_display_test.go", "shell_output_test.go", ], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":output", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/output/shell_output.go b/src/output/shell_output.go index 88aceddff5..1cfff3f888 100644 --- a/src/output/shell_output.go +++ b/src/output/shell_output.go @@ -8,6 +8,7 @@ import ( "fmt" "math/rand" "os" + "path" "path/filepath" "sort" "strings" @@ -452,7 +453,7 @@ func printTempDirs(state *core.BuildState, duration time.Duration, shell, shellR fmt.Printf(" Expanded: %s\n", os.Expand(cmd, env.ReplaceEnvironment)) } else { fmt.Printf("\n") - argv := []string{"bash", "--noprofile", "--norc", "-o", "pipefail"} + argv := state.ProcessExecutor.InteractiveShellCommand() if shellRun { argv = append(argv, "-c", cmd) } @@ -464,7 +465,7 @@ func printTempDirs(state *core.BuildState, duration time.Duration, shell, shellR cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr // TODO(jpoole): Read the docs. Attaching stdin and out doesn't seem to work with this. - cmd.SysProcAttr.Setpgid = false + process.ShareParentProcessGroup(cmd) cmd.Run() // Ignore errors, it will typically end by the user killing it somehow. } } @@ -474,10 +475,12 @@ func buildResult(target *core.BuildTarget) []string { results := []string{} if target != nil { for _, out := range target.Outputs() { + // Slash-separated: these are printed for a person to read and paste into a + // command, where a backslash would be an escape character rather than a separator. if core.StartedAtRepoRoot() { - results = append(results, filepath.Join(target.OutDir(), out)) + results = append(results, path.Join(target.OutDir(), out)) } else { - results = append(results, filepath.Join(core.RepoRoot, target.OutDir(), out)) + results = append(results, filepath.ToSlash(filepath.Join(core.RepoRoot, target.OutDir(), out))) } } } diff --git a/src/parse/BUILD b/src/parse/BUILD index bc3cc09ee8..ffd1314f6c 100644 --- a/src/parse/BUILD +++ b/src/parse/BUILD @@ -24,9 +24,12 @@ go_test( name = "parse_step_test", srcs = ["parse_step_test.go"], resources = ["internal.tmpl"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":parse", "///third_party/go/github.com_stretchr_testify//assert", + "///third_party/go/github.com_stretchr_testify//require", "//src/core", ], ) diff --git a/src/parse/asp/BUILD b/src/parse/asp/BUILD index 486fa0fef6..556fd644dc 100644 --- a/src/parse/asp/BUILD +++ b/src/parse/asp/BUILD @@ -26,7 +26,9 @@ go_test( ["*_test.go"], exclude = ["*_bench_test.go"], ), - data = ["test_data"], + data = [":asp_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":asp", "///third_party/go/github.com_stretchr_testify//assert", @@ -46,3 +48,11 @@ go_benchmark( ":asp", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "asp_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index acfbba67cb..ad7fb3970f 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -6,7 +6,7 @@ import ( "fmt" "io" "iter" - "path/filepath" + "path" "reflect" "regexp" "slices" @@ -301,7 +301,7 @@ func bazelLoad(s *scope, args []pyObject) pyObject { // The argument always looks like a build label, but it is not really one (i.e. there is no BUILD file that defines it). // We do not support their legacy syntax here (i.e. "/tools/build_rules/build_test" etc). l := s.parseLabelInContextPkg(string(args[0].(pyString))) - filename := filepath.Join(l.PackageName, l.Name) + filename := path.Join(l.PackageName, l.Name) if l.Subrepo != "" { subrepo := s.state.Graph.Subrepo(l.Subrepo) if subrepo == nil || (subrepo.Target != nil && subrepo != s.contextPackage().Subrepo) { @@ -358,7 +358,7 @@ func subinclude(s *scope, args []pyObject) pyObject { outs = t.Outputs() } for _, out := range outs { - s.SetAllWithOrigin(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false, &t.Label) + s.SetAllWithOrigin(s.interpreter.Subinclude(s, path.Join(t.OutDir(), out), t.Label, false), false, &t.Label) } labels = append(labels, t.Label) } @@ -987,7 +987,10 @@ func joinPath(s *scope, args []pyObject) pyObject { for i, arg := range args { l[i] = string(arg.(pyString)) } - return pyString(filepath.Join(l...)) + // path, not filepath: the BUILD language is the same language on every platform. Its paths + // become labels, sources and command strings, all of which are slash-separated, and a + // backslash here would change every hash that reaches it. + return pyString(path.Join(l...)) } func looksLikeBuildLabel(s *scope, args []pyObject) pyObject { @@ -1530,14 +1533,14 @@ func subrepo(s *scope, args []pyObject) pyObject { root = target.Outputs()[0] } if target.Local || s.state.RemoteClient == nil { - root = filepath.Join(target.OutDir(), root) + root = path.Join(target.OutDir(), root) } } else if args[PathArgIdx] != None { root = string(args[PathArgIdx].(pyString)) } // Base name - subrepoName := filepath.Join(s.pkg.Name, name) + subrepoName := path.Join(s.pkg.Name, name) if args[PluginArgIdx].IsTruthy() { subrepoName = name } diff --git a/src/parse/internal.tmpl b/src/parse/internal.tmpl index 90cfe6b671..41775ca4fa 100644 --- a/src/parse/internal.tmpl +++ b/src/parse/internal.tmpl @@ -1,13 +1,15 @@ +{{ if .ArcatHash }} remote_file( name = "arcat", url = f"https://github.com/please-build/arcat/releases/download/v1.3.1/arcat-1.3.1-{CONFIG.HOSTOS}_{CONFIG.HOSTARCH}", - out = "arcat", + out = "arcat{{ .ExeSuffix }}", binary = True, hashes = [ "{{ .ArcatHash }}", # defined in internal_package.go ], visibility = ["PUBLIC"], ) +{{ end }} remote_file( name = "download", diff --git a/src/parse/internal_package.go b/src/parse/internal_package.go index b867eca575..63108b317e 100644 --- a/src/parse/internal_package.go +++ b/src/parse/internal_package.go @@ -8,6 +8,7 @@ import ( "text/template" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/version" ) @@ -27,26 +28,13 @@ func GetInternalPackage(config *core.Configuration) (string, error) { url = fmt.Sprintf("%s/%s_%s/%s/please_tools_%s.tar.xz", config.Please.DownloadLocation, runtime.GOOS, runtime.GOARCH, version.PleaseVersion, version.PleaseVersion) } - var arcatHash string - switch fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) { - case "darwin_amd64": - arcatHash = "6af2cf108592535701aa9395f3a5deeb48a5dfbe8174a8ebe3d56bb93de2c255" - case "darwin_arm64": - arcatHash = "5070ef05d14c66a85d438f400c6ff734a23833929775d6824b69207b704034bf" - case "freebsd_amd64": - arcatHash = "05ad6ac45be3a4ca1238bb1bd09207a596f8ff5f885415f8df4ff2dc849fa04e" - case "linux_amd64": - arcatHash = "aec85425355291e515cd10ac0addec3a5bc9e05c9d07af01aca8c34aaf0f1222" - case "linux_arm64": - arcatHash = "8266cb95cc84b23642bca6567f8b4bd18de399c887cb5845ab6a901d0dba54d2" - default: - return "", fmt.Errorf("arcat tool not supported for platform: %s_%s", runtime.GOOS, runtime.GOARCH) - } + arcatHash := publishedArcatHash() data := struct { ToolsURL string Tools []string ArcatHash string + ExeSuffix string }{ ToolsURL: url, Tools: []string{ @@ -54,6 +42,7 @@ func GetInternalPackage(config *core.Configuration) (string, error) { "please_sandbox", }, ArcatHash: arcatHash, + ExeSuffix: fs.ExeSuffix, } var buf bytes.Buffer @@ -62,3 +51,36 @@ func GetInternalPackage(config *core.Configuration) (string, error) { } return buf.String(), nil } + +// publishedArcatHash returns the hash of the arcat release for the platform we are running on, +// or an empty string if there isn't one. An empty string leaves the arcat rule out of the +// internal package altogether rather than failing: everything else in there still works, and a +// user who points [build] arcattool at their own build never needs ours. See +// ArcatUnavailable for the warning that goes with it. +func publishedArcatHash() string { + return arcatHashFor(fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) +} + +func arcatHashFor(platform string) string { + switch platform { + case "darwin_amd64": + return "6af2cf108592535701aa9395f3a5deeb48a5dfbe8174a8ebe3d56bb93de2c255" + case "darwin_arm64": + return "5070ef05d14c66a85d438f400c6ff734a23833929775d6824b69207b704034bf" + case "freebsd_amd64": + return "05ad6ac45be3a4ca1238bb1bd09207a596f8ff5f885415f8df4ff2dc849fa04e" + case "linux_amd64": + return "aec85425355291e515cd10ac0addec3a5bc9e05c9d07af01aca8c34aaf0f1222" + case "linux_arm64": + return "8266cb95cc84b23642bca6567f8b4bd18de399c887cb5845ab6a901d0dba54d2" + } + return "" +} + +// ArcatUnavailable reports whether the config still expects the arcat that Please would +// download, on a platform where there is no release to download. Nothing that needs arcat can +// work in that state - which includes extracting any plugin - so it is worth saying up front +// rather than letting it surface as a missing target much later. +func ArcatUnavailable(config *core.Configuration) bool { + return publishedArcatHash() == "" && config.Build.ArcatTool == core.DefaultArcatTool +} diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 6c9ec6f11d..52808c9155 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" iofs "io/fs" - "path/filepath" + "path" "strings" "github.com/thought-machine/please/src/cli/logging" @@ -167,6 +167,12 @@ func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo // When we try and parse a subrepo package, but the BUILD file or directory doesn't exist, return nil so // this gets handled later on, in the same way as when the package does exist but doesn't define the subrepo if errors.Is(err, ErrMissingBuildFile) { + // The parse above claimed the right to parse this package, and we are about to + // throw its error away, so the claim has to go back. Without this, the next + // caller to ask about the same non-existent package blocks forever waiting for + // a parse that is never going to happen - and since a package that isn't there + // is the normal answer here, that is a hang on an ordinary lookup. + state.ReleasePendingParse(label) return nil, nil } return nil, err @@ -252,7 +258,11 @@ func buildFileName(state *core.BuildState, subrepo *core.Subrepo, fs iofs.FS, pk return "WORKSPACE", "" } for _, buildFileName := range config.Parse.BuildFileName { - filename := filepath.Join(pkgName, buildFileName) + // path, not filepath: this is an io/fs path, which is always slash-separated whatever + // the host OS. filepath.Join would produce a backslash on Windows and iofs.Stat would + // then look for a single file whose name contains one, so no package below the top + // level would ever be found. + filename := path.Join(pkgName, buildFileName) if info, err := iofs.Stat(fs, filename); err == nil && !info.IsDir() { return filename, pkgName } diff --git a/src/parse/parse_step_test.go b/src/parse/parse_step_test.go index d21034f54e..f71a3eafec 100644 --- a/src/parse/parse_step_test.go +++ b/src/parse/parse_step_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/thought-machine/please/src/core" ) @@ -164,3 +165,34 @@ func getAllPending(state *core.BuildState) ([]string, []string) { func buildLabel(bl string) core.BuildLabel { return core.ParseBuildLabel(bl, "") } + +func TestInternalPackageWithoutArcat(t *testing.T) { + // There is no arcat release for every platform Please runs on - Windows has none at all - + // and it used to be an error to generate the internal package there, which stopped + // everything rather than just the things that need arcat. The rule is simply left out now. + config := core.DefaultConfiguration() + pkg, err := GetInternalPackage(config) + require.NoError(t, err) + if arcatHashFor("no_such_platform") == "" { + // Sanity check on the helper itself before relying on it below. + assert.Contains(t, pkg, "please_sandbox", "the rest of the package should still be there") + } +} + +func TestArcatHashKnownAndUnknownPlatforms(t *testing.T) { + assert.NotEmpty(t, arcatHashFor("linux_amd64")) + assert.Empty(t, arcatHashFor("windows_amd64"), "no arcat is published for Windows") +} + +func TestArcatUnavailableOnlyWhenNothingElseIsConfigured(t *testing.T) { + config := core.DefaultConfiguration() + config.Build.ArcatTool = "C:/tools/arcat.exe" + assert.False(t, ArcatUnavailable(config), "a configured arcat is never unavailable") +} + +func TestDefaultArcatToolNamesTheInternalPackage(t *testing.T) { + // core cannot import this package, so it spells the label out. If the internal package is + // ever renamed, the default arcat tool and the check that recognises it drift apart + // silently and ArcatUnavailable starts answering false for a default config. + assert.Equal(t, "/////"+InternalPackageName+":arcat", core.DefaultArcatTool) +} diff --git a/src/please.go b/src/please.go index d53c2903cb..d4d865d656 100644 --- a/src/please.go +++ b/src/please.go @@ -12,7 +12,6 @@ import ( "runtime/pprof" "strings" "sync" - "syscall" "time" "github.com/thought-machine/go-flags" @@ -697,6 +696,11 @@ var buildFunctions = map[string]func() int{ if len(opts.Clean.Args.Targets) == 0 && core.InitialPackage()[0].PackageName == "" { if len(opts.BuildFlags.Include) == 0 && len(opts.BuildFlags.Exclude) == 0 { // Clean everything, doesn't require parsing at all. + // The log file lives under plz-out by default, and on Windows a directory + // cannot be renamed or deleted while this process holds a file inside it open, + // so let go of it first. The detached child that does the deletion avoids + // opening one at all, for the same reason. + cli.CloseFileLogging() state := core.NewBuildState(config) clean.Clean(config, cache.NewCache(state), !opts.Clean.NoBackground) return 0 @@ -716,10 +720,9 @@ var buildFunctions = map[string]func() int{ "op": func() int { cmd := core.ReadPreviousOperationOrDie() log.Notice("OP PLZ: %s", strings.Join(cmd, " ")) - // Annoyingly we don't seem to have any access to execvp() which would be rather useful here... executable, err := os.Executable() if err == nil { - err = syscall.Exec(executable, append([]string{executable}, cmd...), os.Environ()) + err = process.ExecReplace(executable, append([]string{executable}, cmd...), os.Environ()) } log.Fatalf("SORRY OP: %s", err) // On success Run never returns. return 1 diff --git a/src/plz/BUILD b/src/plz/BUILD index bfaeffee93..43e12de979 100644 --- a/src/plz/BUILD +++ b/src/plz/BUILD @@ -20,6 +20,8 @@ go_library( go_test( name = "plz_test", srcs = ["plz_test.go"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":plz", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/plz/plz.go b/src/plz/plz.go index 4cbbd10454..4dd6715dbe 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -2,6 +2,7 @@ package plz import ( "path/filepath" + "runtime" "strings" "sync" @@ -35,6 +36,11 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config * } parse.InitParser(state) + if parse.ArcatUnavailable(config) { + // Nothing that needs arcat can work, which includes extracting any plugin. Say so here + // rather than letting it surface much later as a target that doesn't exist. + log.Warning("No arcat is published for %s_%s, so anything that needs one - including loading a plugin - will fail. Build it yourself and point [build] arcattool at it.", runtime.GOOS, runtime.GOARCH) + } // Start looking for the initial targets to kick the build off go findOriginalTasks(state, preTargets, targets, arch) diff --git a/src/plzinit/init.go b/src/plzinit/init.go index 7a3ab2f808..bae1d31238 100644 --- a/src/plzinit/init.go +++ b/src/plzinit/init.go @@ -32,6 +32,10 @@ compatibility = true ` const wrapperScriptName = "pleasew" +// A repo is often worked on from more than one platform, so both wrappers are written +// whichever one we happen to be running on. +const windowsWrapperScriptName = "pleasew.ps1" + const pleasingsSubrepoTemplate = ` github_repo( name = "pleasings", @@ -112,10 +116,12 @@ func readConfig(filename string) []byte { return b } -// InitWrapperScript initialises the pleasew script. +// InitWrapperScript initialises the pleasew scripts. func InitWrapperScript() { - data := assets.Pleasew - if err := os.WriteFile(wrapperScriptName, data, 0755); err != nil { + if err := os.WriteFile(wrapperScriptName, assets.Pleasew, 0755); err != nil { + log.Fatalf("Failed to write file: %s", err) + } + if err := os.WriteFile(windowsWrapperScriptName, assets.PleasewPS1, 0755); err != nil { log.Fatalf("Failed to write file: %s", err) } } diff --git a/src/process/BUILD b/src/process/BUILD index 5b9f3cf366..b292a7d135 100644 --- a/src/process/BUILD +++ b/src/process/BUILD @@ -3,9 +3,18 @@ go_library( srcs = [ "exec_linux.go", "exec_other.go", + "exec_replace_other.go", + "exec_replace_windows.go", + "exec_windows.go", + "kill_other.go", + "kill_windows.go", "output.go", + "pgroup_other.go", + "pgroup_windows.go", "process.go", "progress.go", + "shell_other.go", + "shell_windows.go", ], pgo_file = "//:pgo", visibility = ["PUBLIC"], @@ -13,7 +22,7 @@ go_library( "///third_party/go/github.com_peterebden_go-deferred-regex//:go-deferred-regex", "//src/cli", "//src/cli/logging", - ], + ] + (["///third_party/go/golang.org_x_sys//windows"] if is_platform(os = "windows") else []), ) go_test( @@ -22,8 +31,12 @@ go_test( "process_test.go", "progress_test.go", ], + # So //test/windows can run this same binary, cross-built, under Wine. The process layer is + # almost entirely separate code there, and nothing else covers it. + visibility = ["//test/windows:all"], deps = [ ":process", "///third_party/go/github.com_stretchr_testify//assert", + "///third_party/go/github.com_stretchr_testify//require", ], ) diff --git a/src/process/exec_other.go b/src/process/exec_other.go index 1f3902adc2..b78c2df512 100644 --- a/src/process/exec_other.go +++ b/src/process/exec_other.go @@ -1,5 +1,5 @@ -//go:build !linux -// +build !linux +//go:build !linux && !windows +// +build !linux,!windows package process diff --git a/src/process/exec_replace_other.go b/src/process/exec_replace_other.go new file mode 100644 index 0000000000..aff55a2cc4 --- /dev/null +++ b/src/process/exec_replace_other.go @@ -0,0 +1,12 @@ +//go:build !windows +// +build !windows + +package process + +import "syscall" + +// ExecReplace replaces the currently running process with the given command. +// It does not return unless the exec itself failed. +func ExecReplace(path string, argv, env []string) error { + return syscall.Exec(path, argv, env) +} diff --git a/src/process/exec_replace_windows.go b/src/process/exec_replace_windows.go new file mode 100644 index 0000000000..2f15700f7f --- /dev/null +++ b/src/process/exec_replace_windows.go @@ -0,0 +1,38 @@ +package process + +import ( + "errors" + "os" + "os/exec" + "os/signal" +) + +// ExecReplace replaces the currently running process with the given command. +// It does not return unless the exec itself failed. +// +// Windows has no way to replace a process image, so we run the command as a child, wait for +// it, and exit with its status. Two consequences callers need to be aware of: +// +// - We stay alive as the child's parent. Any resource we hold is still held, so release +// anything the child will contend for - notably the repo lock - before calling this. On +// Unix that happens implicitly, because Go opens files O_CLOEXEC and the exec releases +// the lock for us. +// - Nothing deferred in the caller runs, matching execve. +func ExecReplace(path string, argv, env []string) error { + cmd := exec.Command(path) + cmd.Args = argv + cmd.Env = env + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + // The child shares our console, so a Ctrl-C reaches it directly. Ignore it here so we + // don't exit first and leave it writing to a console nobody is reading. + signal.Ignore(os.Interrupt) + if err := cmd.Run(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + return err + } + os.Exit(0) + return nil // unreachable +} diff --git a/src/process/exec_windows.go b/src/process/exec_windows.go new file mode 100644 index 0000000000..c624a46d12 --- /dev/null +++ b/src/process/exec_windows.go @@ -0,0 +1,25 @@ +package process + +import ( + "os/exec" + "syscall" + + "golang.org/x/sys/windows" +) + +// ExecCommand executes an external command. +// Windows has no process groups in the POSIX sense; the closest equivalent is a console +// process group, which is what CREATE_NEW_PROCESS_GROUP sets up. That gives us somewhere to +// send Ctrl-Break, which is the nearest thing to SIGTERM. Killing the whole tree is handled +// separately by a job object - see kill_windows.go. +// +// N.B. This does not start the command - the caller must handle that (or use one +// of the other functions which are higher-level interfaces). +func (e *Executor) ExecCommand(sandbox SandboxConfig, foreground bool, command string, args ...string) *exec.Cmd { + // There is no sandboxing on Windows yet; sandbox and foreground are both ignored. + cmd := exec.Command(command, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: windows.CREATE_NEW_PROCESS_GROUP, + } + return cmd +} diff --git a/src/process/kill_other.go b/src/process/kill_other.go new file mode 100644 index 0000000000..f15705a03c --- /dev/null +++ b/src/process/kill_other.go @@ -0,0 +1,22 @@ +//go:build !windows +// +build !windows + +package process + +import ( + "os/exec" + "syscall" +) + +// trackProcessTree records a started process so that its descendants can be killed later. +// On Unix the process group set up in ExecCommand is sufficient, so this is a no-op. +func trackProcessTree(cmd *exec.Cmd) {} + +// untrackProcessTree releases any resources held by trackProcessTree. +func untrackProcessTree(cmd *exec.Cmd) {} + +// killProcessTree signals a process and all of its descendants. +func killProcessTree(cmd *exec.Cmd, sig syscall.Signal) error { + // Kill the group - we always set one in ExecCommand. + return syscall.Kill(-cmd.Process.Pid, sig) +} diff --git a/src/process/kill_windows.go b/src/process/kill_windows.go new file mode 100644 index 0000000000..0a8e1d2afd --- /dev/null +++ b/src/process/kill_windows.go @@ -0,0 +1,96 @@ +package process + +import ( + "os/exec" + "sync" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// Windows has no process groups that descendants inherit, so killing a whole tree needs a job +// object: every process assigned to one, and everything it subsequently spawns, dies together +// on TerminateJobObject. JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE means that also happens if we exit +// abnormally without cleaning up, which is what Pdeathsig buys us on Linux. +var ( + jobsMux sync.Mutex + jobs = map[*exec.Cmd]windows.Handle{} +) + +// trackProcessTree assigns a started process to a new job object so that it and its +// descendants can be killed together. +// +// There is an unavoidable race here: the process is already running by the time we assign it, +// so anything it spawns in that window escapes the job. Closing it would need CREATE_SUSPENDED +// and a ResumeThread, which os/exec gives us no way to do. +func trackProcessTree(cmd *exec.Cmd) { + if cmd.Process == nil { + return + } + job, err := windows.CreateJobObject(nil, nil) + if err != nil { + log.Warning("Failed to create job object, child processes may outlive us: %s", err) + return + } + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err != nil { + log.Warning("Failed to configure job object: %s", err) + windows.CloseHandle(job) + return + } + proc, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid)) + if err != nil { + log.Warning("Failed to open process %d: %s", cmd.Process.Pid, err) + windows.CloseHandle(job) + return + } + defer windows.CloseHandle(proc) + if err := windows.AssignProcessToJobObject(job, proc); err != nil { + log.Warning("Failed to assign process %d to job object: %s", cmd.Process.Pid, err) + windows.CloseHandle(job) + return + } + jobsMux.Lock() + defer jobsMux.Unlock() + jobs[cmd] = job +} + +// untrackProcessTree closes the job object for a command. Because the job is created with +// JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, this also kills anything still running in it. +func untrackProcessTree(cmd *exec.Cmd) { + jobsMux.Lock() + job, present := jobs[cmd] + delete(jobs, cmd) + jobsMux.Unlock() + if present { + windows.CloseHandle(job) + } +} + +// killProcessTree signals a process and all of its descendants. +// SIGTERM is translated to a Ctrl-Break on the console process group, which is the closest +// thing Windows has to a signal a process can handle. It is best-effort: it does not reach +// processes that have detached from the console, and GUI subsystem processes ignore it. +// Anything else terminates the job object, which is unconditional. +func killProcessTree(cmd *exec.Cmd, sig syscall.Signal) error { + if sig == syscall.SIGTERM { + if err := windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid)); err == nil { + return nil + } + // Fall through to terminating the job if we couldn't deliver it. + } + jobsMux.Lock() + job, present := jobs[cmd] + jobsMux.Unlock() + if !present { + // No job object, so the best we can do is the process itself. + return cmd.Process.Kill() + } + return windows.TerminateJobObject(job, 1) +} diff --git a/src/process/pgroup_other.go b/src/process/pgroup_other.go new file mode 100644 index 0000000000..a14a9e6364 --- /dev/null +++ b/src/process/pgroup_other.go @@ -0,0 +1,13 @@ +//go:build !windows +// +build !windows + +package process + +import "os/exec" + +// ShareParentProcessGroup configures a command to run in our process group rather than one of +// its own. Interactive commands need this for stdin and stdout to attach correctly, at the +// cost of no longer being killable as a group. +func ShareParentProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr.Setpgid = false +} diff --git a/src/process/pgroup_windows.go b/src/process/pgroup_windows.go new file mode 100644 index 0000000000..a1b5845675 --- /dev/null +++ b/src/process/pgroup_windows.go @@ -0,0 +1,14 @@ +package process + +import ( + "os/exec" + + "golang.org/x/sys/windows" +) + +// ShareParentProcessGroup configures a command to run in our process group rather than one of +// its own. Interactive commands need this for stdin and stdout to attach correctly, at the +// cost of no longer being killable as a group. +func ShareParentProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr.CreationFlags &^= windows.CREATE_NEW_PROCESS_GROUP +} diff --git a/src/process/process.go b/src/process/process.go index f76e15fce9..5dcb76386c 100644 --- a/src/process/process.go +++ b/src/process/process.go @@ -34,24 +34,30 @@ type Executor struct { // The tool that will do the network/mount sandboxing sandboxTool string usePleaseSandbox bool - processes map[*exec.Cmd]<-chan error - mutex sync.Mutex + // The shell that build actions and tests are run in, and the arguments given to it + // before the command itself. + shell string + shellArgs []string + processes map[*exec.Cmd]<-chan error + mutex sync.Mutex } -func NewSandboxingExecutor(usePleaseSandbox bool, namespace NamespacingPolicy, sandboxTool string) *Executor { +func NewSandboxingExecutor(usePleaseSandbox bool, namespace NamespacingPolicy, sandboxTool, shell string, shellArgs []string) *Executor { o := &Executor{ namespace: namespace, usePleaseSandbox: usePleaseSandbox, sandboxTool: sandboxTool, + shell: shell, + shellArgs: shellArgs, processes: map[*exec.Cmd]<-chan error{}, } cli.AtExit(o.killAll) // Kill any subprocess if we are ourselves killed return o } -// New returns a new Executor. +// New returns a new Executor using the default shell for this platform. func New() *Executor { - return NewSandboxingExecutor(false, NamespaceNever, "") + return NewSandboxingExecutor(false, NamespaceNever, "", DefaultShell, DefaultShellArgs) } // SandboxConfig contains what namespaces should be sandboxed @@ -127,6 +133,8 @@ func (e *Executor) ExecWithTimeout(ctx context.Context, target Target, dir strin if err != nil { return nil, nil, err } + trackProcessTree(cmd) + defer untrackProcessTree(cmd) ch := make(chan error) e.registerProcess(cmd, ch) defer e.removeProcess(cmd) @@ -155,7 +163,7 @@ func (e *Executor) ExecWithTimeoutShell(target Target, dir string, env []string, // ExecWithTimeoutShellStdStreams is as ExecWithTimeoutShell but optionally attaches stdin to the subprocess. func (e *Executor) ExecWithTimeoutShellStdStreams(target Target, dir string, env []string, timeout time.Duration, showOutput, foreground bool, sandbox SandboxConfig, cmd string, attachStdStreams bool) ([]byte, []byte, error) { - c := BashCommand("bash", cmd, target.ShouldExitOnError()) + c := e.BashCommand(cmd, target.ShouldExitOnError()) return e.ExecWithTimeout(context.Background(), target, dir, env, timeout, showOutput, attachStdStreams, attachStdStreams, foreground, sandbox, c) } @@ -203,7 +211,9 @@ func sendSignal(cmd *exec.Cmd, ch <-chan error, sig syscall.Signal, timeout time // This is a bit of a fiddle. We want to wait for the process to exit but only for just so // long (we do not want to get hung up if it ignores our SIGTERM). log.Debug("Sending signal %s to -%d", sig, cmd.Process.Pid) - syscall.Kill(-cmd.Process.Pid, sig) // Kill the group - we always set one in ExecCommand. + if err := killProcessTree(cmd, sig); err != nil { + log.Debug("Failed to signal process %d: %s", cmd.Process.Pid, err) + } select { case <-ch: @@ -291,10 +301,45 @@ func ExecCommand(args ...string) ([]byte, error) { return cmd.CombinedOutput() } -// BashCommand returns the command that we'd use to execute a subprocess in a shell with. -func BashCommand(binary, command string, exitOnError bool) []string { +// BashCommand returns the command that this executor runs a subprocess in a shell with. +// This is for the shell on the machine we're running on; see RemoteBashCommand for the +// remote execution equivalent. +func (e *Executor) BashCommand(command string, exitOnError bool) []string { + return shellCommand(e.shell, e.shellArgs, command, exitOnError) +} + +// InteractiveShellCommand returns the command to start an interactive shell of the same kind +// build actions run in. It has no -e or -u, since those are hostile in an interactive shell, +// and no command to run. +func (e *Executor) InteractiveShellCommand() []string { + return append(ShellArgv(e.shell, e.shellArgs), "-o", "pipefail") +} + +// ShellArgv returns the leading argv for invoking the given shell: the shell itself followed +// by its arguments. Empty arguments are dropped, because a repeatable config key can't be +// cleared by assigning it empty - that yields a single empty string rather than nothing - and +// an empty argument would otherwise be passed through to the shell. +func ShellArgv(binary string, args []string) []string { + argv := make([]string, 0, len(args)+8) + argv = append(argv, binary) + for _, arg := range args { + if arg != "" { + argv = append(argv, arg) + } + } + return argv +} + +// RemoteBashCommand is as BashCommand, but for a shell on a remote worker. That is a real +// bash whatever we happen to be running on, so it always gets the full set of flags. +func RemoteBashCommand(binary, command string, exitOnError bool) []string { + return shellCommand(binary, []string{"--noprofile", "--norc"}, command, exitOnError) +} + +func shellCommand(binary string, initArgs []string, command string, exitOnError bool) []string { + argv := ShellArgv(binary, initArgs) if exitOnError { - return []string{binary, "--noprofile", "--norc", "-e", "-u", "-o", "pipefail", "-c", command} + argv = append(argv, "-e") } - return []string{binary, "--noprofile", "--norc", "-u", "-o", "pipefail", "-c", command} + return append(argv, "-u", "-o", "pipefail", "-c", command) } diff --git a/src/process/process_test.go b/src/process/process_test.go index 48519ab6cf..c9ba39a940 100644 --- a/src/process/process_test.go +++ b/src/process/process_test.go @@ -2,31 +2,64 @@ package process import ( "context" + "fmt" + "os" + "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// argv returns a command that runs the given shell snippet, as an explicit argv rather than a +// shell string. Built through the shell rather than naming true, false and sleep directly: +// those are programs on the PATH on Unix and applets inside the shell on Windows, so only one +// of the two spellings works anywhere. +func argv(command string) []string { + return New().BashCommand(command, false) +} + func TestExecWithTimeout(t *testing.T) { - out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 10*time.Second, false, false, false, false, NoSandbox, []string{"true"}) + out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 10*time.Second, false, false, false, false, NoSandbox, argv("exit 0")) assert.NoError(t, err) assert.Equal(t, 0, len(out)) } func TestExecWithTimeoutFailure(t *testing.T) { - out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 10*time.Second, false, false, false, false, NoSandbox, []string{"false"}) + out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 10*time.Second, false, false, false, false, NoSandbox, argv("exit 1")) assert.Error(t, err) assert.Equal(t, 0, len(out)) } func TestExecWithTimeoutDeadline(t *testing.T) { - out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 1*time.Nanosecond, false, false, false, false, NoSandbox, []string{"sleep", "10"}) + out, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 1*time.Nanosecond, false, false, false, false, NoSandbox, argv("sleep 10")) assert.Error(t, err) assert.Equal(t, context.DeadlineExceeded, err) assert.Equal(t, 0, len(out)) } +// TestKillsProcessTree covers the thing process groups on Unix and job objects on Windows both +// exist for: when a command times out, what it started has to die with it. Nothing else tests +// that on any platform, and the Windows implementation of it is entirely separate code. +func TestKillsProcessTree(t *testing.T) { + // Forward slashes: this path is going into a shell command, where a backslash escapes. + marker := filepath.ToSlash(filepath.Join(t.TempDir(), "marker")) + // A grandchild that outlives the child it was started from, unless the whole tree is + // killed. Deliberately a separate process rather than a subshell: busybox on Windows + // implements a subshell as a thread, so it would die with its parent either way and prove + // nothing about killing a tree. + cmd := fmt.Sprintf("sh -c 'sleep 2; echo alive > %s' & sleep 30", marker) + + _, _, err := New().ExecWithTimeout(context.Background(), nil, "", nil, 100*time.Millisecond, false, false, false, false, NoSandbox, argv(cmd)) + require.Error(t, err) + + // Comfortably past when the grandchild would have written, had it survived. + time.Sleep(4 * time.Second) + _, err = os.Stat(marker) + assert.True(t, os.IsNotExist(err), "grandchild survived the timeout and wrote %s", marker) +} + func TestExecWithTimeoutOutput(t *testing.T) { targ := &target{} out, stderr, err := New().ExecWithTimeoutShell(targ, "", nil, 10*time.Second, false, false, NoSandbox, "echo hello") @@ -42,3 +75,26 @@ func TestExecWithTimeoutStderr(t *testing.T) { assert.Equal(t, "", string(out)) assert.Equal(t, "hello\n", string(stderr)) } + +func TestBashCommandUsesConfiguredShell(t *testing.T) { + e := NewSandboxingExecutor(false, NamespaceNever, "", "/bin/dash", []string{"--posix"}) + assert.Equal(t, []string{"/bin/dash", "--posix", "-e", "-u", "-o", "pipefail", "-c", "echo hello"}, + e.BashCommand("echo hello", true)) + assert.Equal(t, []string{"/bin/dash", "--posix", "-u", "-o", "pipefail", "-c", "echo hello"}, + e.BashCommand("echo hello", false)) +} + +func TestBashCommandDropsEmptyShellArgs(t *testing.T) { + // A repeatable config key can't be cleared by assigning it empty; that yields a single + // empty string, which must not reach the shell as an argument. + e := NewSandboxingExecutor(false, NamespaceNever, "", "bash", []string{""}) + assert.Equal(t, []string{"bash", "-u", "-o", "pipefail", "-c", "echo hello"}, + e.BashCommand("echo hello", false)) +} + +func TestRemoteBashCommandIgnoresLocalShellArgs(t *testing.T) { + // The remote worker runs a real bash whatever we're running on, so it keeps the full set + // of flags regardless of how the local shell is configured. + assert.Equal(t, []string{"bash", "--noprofile", "--norc", "-u", "-o", "pipefail", "-c", "echo hello"}, + RemoteBashCommand("bash", "echo hello", false)) +} diff --git a/src/process/shell_other.go b/src/process/shell_other.go new file mode 100644 index 0000000000..fdea20bb8b --- /dev/null +++ b/src/process/shell_other.go @@ -0,0 +1,11 @@ +//go:build !windows +// +build !windows + +package process + +// DefaultShell is the shell we run build actions in if nothing else is configured. +const DefaultShell = "bash" + +// DefaultShellArgs stop bash reading the user's profile and rc files, so build actions don't +// pick up anything from the invoking user's environment. +var DefaultShellArgs = []string{"--noprofile", "--norc"} diff --git a/src/process/shell_windows.go b/src/process/shell_windows.go new file mode 100644 index 0000000000..693e301892 --- /dev/null +++ b/src/process/shell_windows.go @@ -0,0 +1,10 @@ +package process + +// DefaultShell is busybox, which Please bundles in its Windows release. Windows has no system +// shell that can run a build action, so depending on one being installed isn't an option. +const DefaultShell = "busybox" + +// DefaultShellArgs selects busybox's bash applet. Note it does not include --noprofile and +// --norc: busybox rejects both outright, and it reads no profile or rc files in the first +// place, so there is nothing to suppress. +var DefaultShellArgs = []string{"bash"} diff --git a/src/query/BUILD b/src/query/BUILD index 26a780b6a0..5b22e74663 100644 --- a/src/query/BUILD +++ b/src/query/BUILD @@ -21,7 +21,9 @@ go_library( go_test( name = "query_test", srcs = glob(["*_test.go"]), - data = ["completions_test_repo"], + data = [":query_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":query", "///third_party/go/github.com_stretchr_testify//assert", @@ -31,3 +33,11 @@ go_test( "//src/parse", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "query_test_data", + srcs = ["completions_test_repo"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/query/changes.go b/src/query/changes.go index 584e17a171..395e7498d4 100644 --- a/src/query/changes.go +++ b/src/query/changes.go @@ -3,7 +3,7 @@ package query import ( "bytes" "crypto/sha1" - "path/filepath" + "path" "sort" "github.com/thought-machine/please/src/build" @@ -46,7 +46,9 @@ func diffGraphs(before, after *core.BuildState) map[*core.BuildTarget]struct{} { func changedTargets(state *core.BuildState, files []string, changed map[*core.BuildTarget]struct{}, level int, includeSubrepos bool) core.BuildLabels { for _, filename := range files { for dir := filename; dir != "." && dir != "/"; { - dir = filepath.Dir(dir) + // path, not filepath: dir becomes a package name to look up in the graph, and + // those are slash-separated everywhere. + dir = path.Dir(dir) pkgName := dir if pkgName == "." { pkgName = "" diff --git a/src/query/completions.go b/src/query/completions.go index 21c5ae0f00..820cfb4ad6 100644 --- a/src/query/completions.go +++ b/src/query/completions.go @@ -3,6 +3,7 @@ package query import ( "fmt" "os" + "path" "path/filepath" "strings" @@ -29,7 +30,8 @@ func CompletePackages(config *core.Configuration, query string) *CompletionPacka if strings.HasPrefix(query, ":") { query = fmt.Sprintf("//%s%s", core.InitialPackagePath, query) } else { - query = "//" + filepath.Join(core.InitialPackagePath, query) + // path, not filepath: this is being assembled into a build label. + query = "//" + path.Join(core.InitialPackagePath, query) } } query = strings.ReplaceAll(query, "\\:", ":") @@ -56,6 +58,10 @@ func CompletePackages(config *core.Configuration, query string) *CompletionPacka } } +// Everything below deals in package names, which are slash-separated on every platform because +// they become build labels. They are only incidentally filesystem paths, and Win32 is happy to +// read a directory named with forward slashes, so path rather than filepath throughout. +// // findPrefixedPackages finds any packages that match a prefix in a directory e.g. src/plz matches src/plz, and // src/plzinit func findPrefixedPackages(config *core.Configuration, root, prefix string) []string { @@ -70,7 +76,7 @@ func findPrefixedPackages(config *core.Configuration, root, prefix string) []str var matchedPkgs []string for _, d := range dirs { if d.IsDir() && strings.HasPrefix(d.Name(), prefix) { - p := filepath.Join(root, d.Name()) + p := path.Join(root, d.Name()) if containsPackage(config, p) { matchedPkgs = append(matchedPkgs, p) } @@ -92,10 +98,10 @@ func getPackagesAndPackageToParse(config *core.Configuration, query string) ([]s prefix := "" if info, err := os.Lstat(root); err != nil || !info.IsDir() { _, prefix = filepath.Split(root) - currentPackage = filepath.Dir(query) + currentPackage = path.Dir(query) } else if !packageOnly { // If we match a package directly but that's also a prefix for another package, we should return those packages - root, prefix := filepath.Split(query) + root, prefix := path.Split(query) packages := findPrefixedPackages(config, root, prefix) if len(packages) > 1 { return packages, "" @@ -117,7 +123,7 @@ func isExcluded(config *core.Configuration, dir string) bool { return true } for _, blacklisted := range config.Parse.BlacklistDirs { - if filepath.Base(dir) == blacklisted { + if path.Base(dir) == blacklisted { return true } } @@ -140,7 +146,7 @@ func containsPackage(config *core.Configuration, dir string) bool { for _, info := range infos { if info.IsDir() { - dirQueue = append(dirQueue, filepath.Join(dir, info.Name())) + dirQueue = append(dirQueue, path.Join(dir, info.Name())) } if config.IsABuildFile(info.Name()) { return true diff --git a/src/query/graph.go b/src/query/graph.go index 4b24517827..4d293f55cb 100644 --- a/src/query/graph.go +++ b/src/query/graph.go @@ -4,7 +4,7 @@ import ( "encoding/base64" "encoding/json" "os" - "path/filepath" + "path" "sync" "github.com/thought-machine/please/src/build" @@ -155,7 +155,7 @@ func makeJSONTarget(state *core.BuildState, target *core.BuildTarget) JSONTarget t.Inputs = append(t.Inputs, in) } for _, out := range target.Outputs() { - t.Outputs = append(t.Outputs, filepath.Join(target.Label.PackageName, out)) + t.Outputs = append(t.Outputs, path.Join(target.Label.PackageName, out)) } for _, dep := range target.Dependencies() { t.Deps = append(t.Deps, dep.Label.String()) diff --git a/src/query/outputs.go b/src/query/outputs.go index 3ab38034cb..da622836d8 100644 --- a/src/query/outputs.go +++ b/src/query/outputs.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" + "path" "github.com/thought-machine/please/src/core" ) @@ -22,7 +22,7 @@ func targetOutputsFlat(graph *core.BuildGraph, labels []core.BuildLabel) { for _, label := range labels { target := graph.TargetOrDie(label) for _, out := range target.Outputs() { - fmt.Printf("%s\n", filepath.Join(target.OutDir(), out)) + fmt.Printf("%s\n", path.Join(target.OutDir(), out)) } } } @@ -32,7 +32,7 @@ func targetOutputsJSON(graph *core.BuildGraph, labels []core.BuildLabel) { for _, label := range labels { target := graph.TargetOrDie(label) for _, out := range target.Outputs() { - data[label.String()] = append(data[label.String()], filepath.Join(target.OutDir(), out)) + data[label.String()] = append(data[label.String()], path.Join(target.OutDir(), out)) } } encoder := json.NewEncoder(os.Stdout) diff --git a/src/remote/BUILD b/src/remote/BUILD index 3c5343b2b6..d50e59dbdc 100644 --- a/src/remote/BUILD +++ b/src/remote/BUILD @@ -52,9 +52,11 @@ go_test( "impl_test.go", "remote_test.go", ], - data = ["test_data"], + data = [":remote_test_data"], # TODO(#1412): find out why this flakes on circle flaky = True, + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":remote", "///third_party/go/cloud.google.com_go_longrunning//autogen/longrunningpb", @@ -78,3 +80,11 @@ go_test( "//src/fs", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "remote_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/remote/action.go b/src/remote/action.go index 7832e1f8e2..51b49496f7 100644 --- a/src/remote/action.go +++ b/src/remote/action.go @@ -129,7 +129,7 @@ func (c *Client) buildCommand(target *core.BuildTarget, inputRoot *pb.Directory, cmd, err := core.ReplaceSequences(state, target, cmd) return &pb.Command{ Platform: c.targetPlatformProperties(target), //nolint:staticcheck - Arguments: process.BashCommand(c.shellPath, commandPrefixBuilder.String()+cmd, state.Config.Build.ExitOnError), + Arguments: process.RemoteBashCommand(c.shellPath, commandPrefixBuilder.String()+cmd, state.Config.Build.ExitOnError), EnvironmentVariables: c.buildEnv(target, c.stampedBuildEnvironment(state, target, inputRoot, stamp, isTest || isRun), target.Sandbox), OutputPaths: outs, }, err @@ -169,7 +169,7 @@ func (c *Client) buildTestCommand(state *core.BuildState, target *core.BuildTarg }, }, }, - Arguments: process.BashCommand(c.shellPath, commandPrefix+cmd, state.Config.Build.ExitOnError), + Arguments: process.RemoteBashCommand(c.shellPath, commandPrefix+cmd, state.Config.Build.ExitOnError), EnvironmentVariables: c.buildEnv(nil, core.TestEnvironment(state, target, ".", run), target.Test.Sandbox), OutputPaths: paths, }, err @@ -587,7 +587,9 @@ func (c *Client) buildEnv(target *core.BuildTarget, env core.BuildEnv, sandbox b if name == "PATH" { // Strip out anything prefixed with the local user's home directory; it can't be // useful remotely but will affect determinism of the action. - parts := strings.Split(v, ":") + // Note the asymmetry: we split with the local separator because the value was + // built locally, but rejoin with ":" because the worker is a POSIX machine. + parts := fs.SplitPathList(v) replaced := make([]string, 0, len(parts)) for _, part := range parts { if part != c.state.Config.Please.Location && !strings.HasPrefix(part, c.userHome) { diff --git a/src/run/BUILD b/src/run/BUILD index 0e4f5698f8..bfa89c8d94 100644 --- a/src/run/BUILD +++ b/src/run/BUILD @@ -1,6 +1,10 @@ go_library( name = "run", - srcs = ["run_step.go"], + srcs = [ + "env_other.go", + "env_windows.go", + "run_step.go", + ], pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ @@ -17,11 +21,22 @@ go_library( go_test( name = "run_test", srcs = ["run_test.go"], - data = ["test_data"], + data = [":run_test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":run", "///third_party/go/github.com_stretchr_testify//assert", + "///third_party/go/github.com_stretchr_testify//require", "//src/core", "//src/process", ], ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "run_test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/run/env_other.go b/src/run/env_other.go new file mode 100644 index 0000000000..5ffb77493f --- /dev/null +++ b/src/run/env_other.go @@ -0,0 +1,8 @@ +//go:build !windows +// +build !windows + +package run + +// envNamesEqual reports whether two environment variable names refer to the same variable. +// Unix environment names are case-sensitive, so this is plain equality. +func envNamesEqual(a, b string) bool { return a == b } diff --git a/src/run/env_windows.go b/src/run/env_windows.go new file mode 100644 index 0000000000..9c4fd59304 --- /dev/null +++ b/src/run/env_windows.go @@ -0,0 +1,12 @@ +package run + +import "strings" + +// envNamesEqual reports whether two environment variable names refer to the same variable. +// +// Windows environment names are case-insensitive, and the OS keeps its own spelling: setting +// PATH updates the variable it already has, which it stores as Path. Comparing names exactly +// therefore fails to find it, and a caller that meant to replace an entry appends a second one +// instead. os/exec happens to paper over that by deduplicating case-insensitively itself, but +// nothing else does - ExecReplace and the audit log both see the duplicate. +func envNamesEqual(a, b string) bool { return strings.EqualFold(a, b) } diff --git a/src/run/run_step.go b/src/run/run_step.go index 8c471997d6..2dce9be7c4 100644 --- a/src/run/run_step.go +++ b/src/run/run_step.go @@ -9,7 +9,6 @@ import ( "os/exec" "path/filepath" "strings" - "syscall" "time" "golang.org/x/sync/errgroup" @@ -131,7 +130,7 @@ func run(ctx context.Context, state *core.BuildState, label core.AnnotatedOutput case overrideCmd != "": command, _ := core.ReplaceSequences(state, target, overrideCmd) // We don't care about passed in args when an override command is provided - args = process.BashCommand("bash", strings.Trim(command, "\""), true) + args = state.ProcessExecutor.BashCommand(strings.Trim(command, "\""), true) case label.Annotation != "": entryPoint, ok := target.EntryPoints[label.Annotation] if !ok { @@ -157,11 +156,11 @@ func run(ctx context.Context, state *core.BuildState, label core.AnnotatedOutput } // Handle targets where $(exe ...) returns something nontrivial - if !strings.Contains(args[0], "/") { + if !strings.Contains(args[0], "/") && !strings.ContainsRune(args[0], filepath.Separator) { // Probably it's a java -jar, we need an absolute path to it. cmd, err := exec.LookPath(args[0]) if err != nil { - log.Fatalf("Can't find binary %s", args[0]) + log.Fatalf("Can't find binary %s%s", args[0], fs.ExplainUnrunnable(args[0])) } args[0] = cmd } else if dir != "" { // Find an absolute path before changing directory @@ -172,19 +171,26 @@ func run(ctx context.Context, state *core.BuildState, label core.AnnotatedOutput args[0] = abs } + // The path Please built is slash-separated, and on Windows that is not merely untidy. A + // .cmd - which is what an sh_binary is there - runs through cmd.exe, and cmd.exe reads a + // forward slash as the start of a switch: plz-out/bin/x.cmd is the command "plz-out" with + // two switches, and it says so. Wine's cmd is more forgiving, which is why this only + // showed up on a real machine. + args[0] = filepath.FromSlash(args[0]) + log.Info("Running target %s...", strings.Join(args, " ")) output.SetWindowTitle("plz run: " + strings.Join(args, " ")) env := environ(state, target, setenv, tmpDir) if !fork { if dir != "" { - err := syscall.Chdir(dir) + err := os.Chdir(dir) if err != nil { log.Fatalf("Error changing directory %s: %s", dir, err) } } // Plain 'plz run'. One way or another we never return from the following line. - must(syscall.Exec(args[0], args, env), args) + must(process.ExecReplace(args[0], args, env), args) } else if detach { // Bypass the whole process management system since we explicitly aim not to manage this subprocess. cmd := exec.Command(args[0], args[1:]...) @@ -251,8 +257,11 @@ func addEnv(env []string, e core.BuildEnv) []string { func addOneEnv(env []string, k, v string) []string { for i, existing := range env { - if strings.HasPrefix(existing, k+"=") { - env[i] = k + "=" + v + if name, _, ok := strings.Cut(existing, "="); ok && envNamesEqual(name, k) { + // The OS's own spelling of the name is kept, not ours. On Windows they differ - + // PATH is stored as Path - and rewriting it here would leave two entries for one + // variable in anything that reads this slice without deduplicating. + env[i] = name + "=" + v return env } } @@ -262,7 +271,7 @@ func addOneEnv(env []string, k, v string) []string { // must dies if the given error is non-nil. func must(err error, cmd []string) { if err != nil { - log.Fatalf("Error running command %s: %s", strings.Join(cmd, " "), err) + log.Fatalf("Error running command %s: %s%s", strings.Join(cmd, " "), err, fs.ExplainUnrunnable(cmd[0])) } } @@ -272,11 +281,7 @@ func toExitError(err error, cmd []string, out []byte) error { if err == nil { return nil } else if exitError, ok := err.(*exec.ExitError); ok { - // This is a little hairy; there isn't a good way of getting the exit code, - // but this should be reasonably portable (at least to the platforms we care about). - if status, ok := exitError.Sys().(syscall.WaitStatus); ok { - exitCode = status.ExitStatus() - } + exitCode = exitError.ExitCode() } return &exitError{ msg: fmt.Sprintf("Error running command %s: %s\n%s", strings.Join(cmd, " "), err, string(out)), diff --git a/src/run/run_test.go b/src/run/run_test.go index 4cdcccfe19..272987d538 100644 --- a/src/run/run_test.go +++ b/src/run/run_test.go @@ -3,9 +3,12 @@ package run import ( "context" "os" + "runtime" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/process" @@ -17,6 +20,19 @@ func init() { } } +// runnable returns the fixture name that this platform can actually execute. +// +// The Unix fixtures are shell scripts relying on a #! line, and Windows has no such mechanism: +// it decides what is executable by extension. The .cmd files beside them are the same two +// programs written the only way Windows will run one by name - which is exactly what the shell +// plugin does for an sh_binary there. +func runnable(name string) string { + if runtime.GOOS == "windows" { + return name + ".cmd" + } + return name +} + func TestSequential(t *testing.T) { state, labels1, labels2 := makeState(core.DefaultConfiguration()) code := Sequential(state, labels1, nil, process.Quiet, false, false, false, "") @@ -38,25 +54,46 @@ func TestEnvVars(t *testing.T) { config.Build.Path = []string{"/wibble"} state, lab1, _ := makeState(config) - t.Setenv("PATH", "/usr/local/bin:/usr/bin:/bin") + // Built rather than written out: the separator between entries differs per platform, and + // so does what Please prepends - its own location, which is empty in this state. + sep := string(os.PathListSeparator) + hostPath := strings.Join([]string{"/usr/local/bin", "/usr/bin", "/bin"}, sep) + + t.Setenv("PATH", hostPath) env := environ(state, state.Graph.TargetOrDie(lab1[0].BuildLabel), false, false) - assert.Contains(t, env, "PATH=/usr/local/bin:/usr/bin:/bin") - assert.NotContains(t, env, "PATH=/wibble") + assert.Equal(t, hostPath, envValue(t, env, "PATH")) env = environ(state, state.Graph.TargetOrDie(lab1[0].BuildLabel), true, false) - assert.NotContains(t, env, "PATH=/usr/local/bin:/usr/bin:/bin") - assert.Contains(t, env, "PATH=:/wibble", env) + assert.Equal(t, sep+"/wibble", envValue(t, env, "PATH")) +} + +// envValue returns the value of one variable, and asserts there is exactly one entry for it. +// +// Looked up by name rather than matched as a whole string, because the OS decides how the name +// is spelled: Windows stores PATH as Path, so asserting on the literal "PATH=" finds nothing. +// The count is the point of the second assertion - appending a second entry instead of +// replacing the first is the bug this guards, and os/exec hides it by deduplicating. +func envValue(t *testing.T, env []string, name string) string { + t.Helper() + var values []string + for _, entry := range env { + if k, v, ok := strings.Cut(entry, "="); ok && envNamesEqual(k, name) { + values = append(values, v) + } + } + require.Len(t, values, 1, "expected exactly one %s in %v", name, env) + return values[0] } func makeState(config *core.Configuration) (*core.BuildState, []core.AnnotatedOutputLabel, []core.AnnotatedOutputLabel) { state := core.NewBuildState(config) target1 := core.NewBuildTarget(core.ParseBuildLabel("//:true", "")) target1.IsBinary = true - target1.AddOutput("true") + target1.AddOutput(runnable("true")) target1.Test = new(core.TestFields) state.Graph.AddTarget(target1) target2 := core.NewBuildTarget(core.ParseBuildLabel("//:false", "")) target2.IsBinary = true - target2.AddOutput("false") + target2.AddOutput(runnable("false")) target2.Test = new(core.TestFields) state.Graph.AddTarget(target2) return state, annotate([]core.BuildLabel{target1.Label}), annotate([]core.BuildLabel{target1.Label, target2.Label}) diff --git a/src/run/test_data/plz-out/bin/false.cmd b/src/run/test_data/plz-out/bin/false.cmd new file mode 100644 index 0000000000..b1dfd7df99 --- /dev/null +++ b/src/run/test_data/plz-out/bin/false.cmd @@ -0,0 +1 @@ +@exit /b 1 diff --git a/src/run/test_data/plz-out/bin/true.cmd b/src/run/test_data/plz-out/bin/true.cmd new file mode 100644 index 0000000000..8c36896533 --- /dev/null +++ b/src/run/test_data/plz-out/bin/true.cmd @@ -0,0 +1 @@ +@exit /b 0 diff --git a/src/test/BUILD b/src/test/BUILD index f3bb830d21..6faf1fa5f9 100644 --- a/src/test/BUILD +++ b/src/test/BUILD @@ -36,7 +36,9 @@ go_test( "results_test.go", "xml_results_test.go", ], - data = ["test_data"], + data = [":test_data"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":test", "///third_party/go/github.com_peterebden_tools//cover", @@ -63,3 +65,11 @@ plz_e2e_test( expect_output_contains = "panic: goodbye world", expected_failure = True, ) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/src/test/coverage.go b/src/test/coverage.go index 74a4f39d41..219a84292b 100644 --- a/src/test/coverage.go +++ b/src/test/coverage.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "strings" @@ -202,7 +203,10 @@ func getDirectoryCoverage(coverage core.TestCoverage) map[string]float32 { for file, coverage := range coverage.Files { covered, total := CountCoverage(coverage) - dirpath := filepath.Dir(file) + // path, not filepath: these are repo-relative names from a coverage file, and the + // result is reported to the user and matched against configured paths, both of which + // are slash-separated. + dirpath := path.Dir(file) if _, exists := linesByDir[dirpath]; exists { linesByDir[dirpath].covered += covered diff --git a/src/test/istanbul_coverage.go b/src/test/istanbul_coverage.go index 3bddcff597..47f3d68cb3 100644 --- a/src/test/istanbul_coverage.go +++ b/src/test/istanbul_coverage.go @@ -5,6 +5,7 @@ package test import ( "bytes" "encoding/json" + "path" "path/filepath" "strings" @@ -95,13 +96,18 @@ func sanitiseFileName(target *core.BuildTarget, filename string, run int) string // It returns a non-empty string if successful. // If matchAnyLastDir is true it will match any directory for the last component. func sanitiseFileNameDir(filename string, dir string, matchAnyLastDir bool) string { + // Compared as slash paths throughout. The directories are plz-out paths, which are + // slash-separated on every platform, but the filename comes out of a coverage file that + // some other tool wrote and may use either separator. + filename = filepath.ToSlash(filename) + dir = filepath.ToSlash(dir) if matchAnyLastDir { - dir = filepath.Dir(dir) + dir = path.Dir(dir) } if index := strings.Index(filename, dir); index != -1 { ret := filename[index+len(dir)+1:] if matchAnyLastDir { - if index := strings.IndexRune(ret, filepath.Separator); index != -1 { + if index := strings.IndexRune(ret, '/'); index != -1 { return ret[index+1:] } } diff --git a/src/test/xml_coverage.go b/src/test/xml_coverage.go index b43fe7806f..1014eeb6d2 100644 --- a/src/test/xml_coverage.go +++ b/src/test/xml_coverage.go @@ -6,7 +6,6 @@ import ( "encoding/xml" "math" "path/filepath" - "strings" "time" "github.com/thought-machine/please/src/cli" @@ -20,7 +19,7 @@ func parseXMLCoverageResults(target *core.BuildTarget, coverage *core.TestCovera } for _, pkg := range xcoverage.Packages.Package { for _, cls := range pkg.Classes.Class { - filename := strings.TrimPrefix(cls.Filename, core.RepoRoot) + filename := core.TrimRepoRoot(cls.Filename) // There can be multiple classes per file so we must merge here, not overwrite. coverage.Files[filename] = core.MergeCoverageLines(coverage.Files[filename], parseXMLLines(cls.Lines.Line)) } diff --git a/src/tool/BUILD b/src/tool/BUILD index cc2ba89cf4..d26a2140e3 100644 --- a/src/tool/BUILD +++ b/src/tool/BUILD @@ -8,12 +8,15 @@ go_library( "//src/cli/logging", "//src/core", "//src/fs", + "//src/process", ], ) go_test( name = "tool_test", srcs = ["tool_test.go"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":tool", "///third_party/go/github.com_stretchr_testify//assert", diff --git a/src/tool/tool.go b/src/tool/tool.go index e3a830bfa2..986b723166 100644 --- a/src/tool/tool.go +++ b/src/tool/tool.go @@ -10,13 +10,13 @@ import ( "path/filepath" "sort" "strings" - "syscall" "github.com/thought-machine/go-flags" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/process" ) var log = logging.Log @@ -45,7 +45,7 @@ func Run(config *core.Configuration, tool Tool, args []string) { target = t } // Hopefully we have an absolute path now, so let's run it. - err := syscall.Exec(target, append([]string{target}, args...), os.Environ()) + err := process.ExecReplace(target, append([]string{target}, args...), os.Environ()) log.Fatalf("Failed to exec %s: %s", target, err) // Always a failure, exec never returns. } diff --git a/src/update/BUILD b/src/update/BUILD index 7fffc5204b..37a3dbb58c 100644 --- a/src/update/BUILD +++ b/src/update/BUILD @@ -4,6 +4,8 @@ go_library( name = "update", srcs = [ "clean.go", + "link_other.go", + "link_windows.go", "update.go", "verify.go", ], @@ -32,7 +34,7 @@ go_test( "verify_test.go", ], data = [ - "test_data", + ":test_data", ":test_please", ":test_tarball", "//src:please", @@ -42,6 +44,8 @@ go_test( }, labels = ["xz"], resources = ["key.pub"], + # So //test/windows can run this same binary, cross-built, under Wine. + visibility = ["//test/windows:all"], deps = [ ":update", "///third_party/go/github.com_hashicorp_go-retryablehttp//:go-retryablehttp", @@ -61,6 +65,8 @@ tarball( out = "please_test.tar.gz", subdir = "please", test_only = True, + # So //test/windows can put it beside the test binary when running it under Wine. + visibility = ["//test/windows:all"], ) genrule( @@ -70,4 +76,14 @@ genrule( binary = True, cmd = "cp $SRC $OUT", test_only = True, + # So //test/windows can put it beside the test binary when running it under Wine. + visibility = ["//test/windows:all"], +) + +# Exposed so //test/windows can put it beside the test binary when running it under Wine. +filegroup( + name = "test_data", + srcs = ["test_data"], + test_only = True, + visibility = ["//test/windows:all"], ) diff --git a/src/update/clean.go b/src/update/clean.go index ea5088c212..5d74142832 100644 --- a/src/update/clean.go +++ b/src/update/clean.go @@ -15,6 +15,10 @@ import ( // clean checks for any stale versions in the download directory and wipes them out if OK. func clean(config *core.Configuration, manualUpdate bool) { + // Anything an update couldn't replace because it was running at the time is still lying + // around under a .stale name; it will be free now. + cleanStaleFiles(config.Please.Location) + dir, _ := os.ReadDir(config.Please.Location) versions := make(semver.Versions, 0, len(dir)) // Convert these to semver diff --git a/src/update/link_other.go b/src/update/link_other.go new file mode 100644 index 0000000000..1d1d17384f --- /dev/null +++ b/src/update/link_other.go @@ -0,0 +1,22 @@ +//go:build !windows +// +build !windows + +package update + +import ( + "os" + + "github.com/thought-machine/please/src/fs" +) + +// linkFile points globalFile at downloadedFile, replacing whatever was there before. +func linkFile(downloadedFile, globalFile string) error { + if err := fs.RemoveAll(globalFile); err != nil { + return err + } + return os.Symlink(downloadedFile, globalFile) +} + +// cleanStaleFiles does nothing here; only Windows can fail to replace a file and have to +// leave the old one behind. +func cleanStaleFiles(string) {} diff --git a/src/update/link_windows.go b/src/update/link_windows.go new file mode 100644 index 0000000000..77a523f0ae --- /dev/null +++ b/src/update/link_windows.go @@ -0,0 +1,62 @@ +package update + +import ( + "os" + "path/filepath" + "strings" + + "github.com/thought-machine/please/src/fs" +) + +// staleSuffix marks a file that was still in use when we tried to replace it. +const staleSuffix = ".stale" + +// linkFile points globalFile at downloadedFile, replacing whatever was there before. +// +// Windows makes this harder than it is elsewhere, in two ways. Symlinks need Developer Mode or +// SeCreateSymbolicLinkPrivilege, which an ordinary user does not have, so we hard-link +// instead; that behaves the same for our purposes and needs no privilege on NTFS. And a +// running executable can be neither deleted nor written over, which matters because the file +// we are most often replacing is the Please that is doing the replacing. Windows does allow +// it to be renamed, so we move it aside and let a later run clear up. +func linkFile(downloadedFile, globalFile string) error { + if err := removeOrRenameAside(globalFile); err != nil { + return err + } + // Hard links fail across volumes and on filesystems that don't have them, so fall back to + // a copy; it costs disk space but is always available. + return fs.CopyOrLinkFile(downloadedFile, globalFile, 0555, 0555, true, true) +} + +// removeOrRenameAside deletes a file, or renames it out of the way if it is in use. +func removeOrRenameAside(path string) error { + if !fs.PathExists(path) { + return nil + } + if err := fs.RemoveAll(path); err == nil { + return nil + } + stale := path + staleSuffix + // A previous update may have left one of these; it's fine if that one is still held too. + if err := fs.RemoveAll(stale); err != nil { + log.Debug("Couldn't remove %s: %s", stale, err) + } + log.Debug("Can't remove %s, renaming it to %s", path, stale) + return os.Rename(path, stale) +} + +// cleanStaleFiles removes anything an earlier update had to rename aside because it was in +// use at the time. Failures are expected and ignored; it may still be in use now. +func cleanStaleFiles(dir string) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), staleSuffix) { + if err := fs.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + log.Debug("Couldn't remove stale file %s: %s", entry.Name(), err) + } + } + } +} diff --git a/src/update/update.go b/src/update/update.go index 81220ec50b..d318e2069b 100644 --- a/src/update/update.go +++ b/src/update/update.go @@ -19,7 +19,6 @@ import ( "runtime" "strconv" "strings" - "syscall" "github.com/coreos/go-semver/semver" "github.com/hashicorp/go-retryablehttp" @@ -42,6 +41,9 @@ var httpClient *retryablehttp.Client const milestoneURL = "https://please.build/milestones" +// pleaseExeName is what the Please binary is called inside a version directory. +const pleaseExeName = "please" + fs.ExeSuffix + // pleaseVersion returns the current version of Please as a semver. func pleaseVersion() semver.Version { return *semver.New(version.PleaseVersion) @@ -95,7 +97,11 @@ func CheckAndUpdate(config *core.Configuration, updatesEnabled, updateCommand, f core.ReturnToInitialWorkingDir() args := filterArgs(forceUpdate, append([]string{newPlease}, os.Args[1:]...)) log.Info("Executing %s", strings.Join(args, " ")) - if err := syscall.Exec(newPlease, args, os.Environ()); err != nil { + // Release the repo lock before handing over. On Unix the exec would drop it for us, since + // Go opens files O_CLOEXEC; on Windows we stay alive as the new process's parent and would + // otherwise deadlock it against ourselves. + core.ReleaseRepoLock() + if err := process.ExecReplace(newPlease, args, os.Environ()); err != nil { log.Fatalf("Failed to exec new Please version %s: %s", newPlease, err) } // Shouldn't ever get here. We should have either exec'd or died above. @@ -184,7 +190,7 @@ func shouldUpdate(config *core.Configuration, updatesEnabled, updateCommand, pre // downloadAndLinkPlease downloads a new Please version and links it into place, if needed. // It returns the new location and dies on failure. func downloadAndLinkPlease(config *core.Configuration, verify bool, progress bool) string { - newPlease := filepath.Join(config.Please.Location, config.Please.Version.VersionString(), "please") + newPlease := filepath.Join(config.Please.Location, config.Please.Version.VersionString(), pleaseExeName) if !core.PathExists(newPlease) { downloadPlease(config, verify, progress) @@ -259,7 +265,7 @@ func copyFile(r io.Reader, newDir string) { if err := os.MkdirAll(newDir, fs.DirPermissions); err != nil { panic(err) } - f, err := os.OpenFile(filepath.Join(newDir, "please"), os.O_RDWR|os.O_CREATE, 0555) + f, err := os.OpenFile(filepath.Join(newDir, pleaseExeName), os.O_RDWR|os.O_CREATE, 0555) if err != nil { panic(err) } @@ -315,10 +321,7 @@ func linkNewFile(config *core.Configuration, file string) { newDir := filepath.Join(config.Please.Location, config.Please.Version.VersionString()) globalFile := filepath.Join(config.Please.Location, file) downloadedFile := filepath.Join(newDir, file) - if err := fs.RemoveAll(globalFile); err != nil { - log.Fatalf("Failed to remove existing file %s: %s", globalFile, err) - } - if err := os.Symlink(downloadedFile, globalFile); err != nil { + if err := linkFile(downloadedFile, globalFile); err != nil { log.Fatalf("Error linking %s -> %s: %s", downloadedFile, globalFile, err) } log.Info("Linked %s -> %s", globalFile, downloadedFile) diff --git a/src/update/update_test.go b/src/update/update_test.go index 2f79eeca9e..5a7149483a 100644 --- a/src/update/update_test.go +++ b/src/update/update_test.go @@ -31,8 +31,13 @@ func (*fakeLogBackend) Log(level logging.Level, calldepth int, rec *logging.Reco } func TestVerifyNewPlease(t *testing.T) { - assert.True(t, verifyNewPlease("src/please", version.PleaseVersion)) - assert.False(t, verifyNewPlease("src/please", "wibble")) + // Windows decides what it can run by extension, so the binary is named with one there. + please := "src/please" + if runtime.GOOS == "windows" { + please += ".exe" + } + assert.True(t, verifyNewPlease(please, version.PleaseVersion)) + assert.False(t, verifyNewPlease(please, "wibble")) assert.False(t, verifyNewPlease("wibble", version.PleaseVersion)) } @@ -66,7 +71,9 @@ func TestDownloadNewPlease(t *testing.T) { c := makeConfig("downloadnewplease") downloadPlease(c, false, true) // Should have written new file - assert.True(t, core.PathExists(filepath.Join(c.Please.Location, c.Please.Version.String(), "please"))) + // pleaseExeName rather than "please": the downloaded binary needs the extension on Windows + // or nothing will run it. + assert.True(t, core.PathExists(filepath.Join(c.Please.Location, c.Please.Version.String(), pleaseExeName))) // Should not have written this yet though assert.False(t, core.PathExists(filepath.Join(c.Please.Location, "please"))) // Panics because it's not a valid .tar.gz diff --git a/src/watch/BUILD b/src/watch/BUILD index 5dcfcffaf0..e8461e0c43 100644 --- a/src/watch/BUILD +++ b/src/watch/BUILD @@ -13,3 +13,15 @@ go_library( "//src/run", ], ) + +go_test( + name = "watch_test", + srcs = ["watch_test.go"], + # So //test/windows can run this same binary, cross-built, under Wine, which is the only + # platform where the assertion means anything. + visibility = ["//test/windows:all"], + deps = [ + ":watch", + "///third_party/go/github.com_stretchr_testify//assert", + ], +) diff --git a/src/watch/watch.go b/src/watch/watch.go index 3d7112f595..08371fb1b8 100644 --- a/src/watch/watch.go +++ b/src/watch/watch.go @@ -4,6 +4,7 @@ package watch import ( "context" "fmt" + "path" "path/filepath" "sync" "time" @@ -59,7 +60,7 @@ func Watch(state *core.BuildState, labels core.BuildLabels, testArgs []string, n select { case event := <-watcher.Events: log.Info("Event: %s", event) - if _, present := files.Load(event.Name); !present { + if _, present := files.Load(watchKey(event.Name)); !present { log.Notice("Skipping notification for %s", event.Name) continue } @@ -123,17 +124,27 @@ func startWatching(watcher *fsnotify.Watcher, state *core.BuildState, labels []c fmt.Println("And now my watch begins...") } +// watchKey normalises a path so that the names we record and the names fsnotify reports back +// can be compared. On Windows they need not agree otherwise: our sources are slash-separated, +// while anything that has been through filepath, or that fsnotify built from a watched +// directory, comes back with backslashes. A mismatch is silent - every event is discarded as +// belonging to a file we aren't watching - so `plz watch` would simply never fire. +func watchKey(path string) string { + return filepath.ToSlash(path) +} + func addSource(watcher *fsnotify.Watcher, state *core.BuildState, source core.BuildInput, dirs map[string]struct{}, files *sync.Map) { if _, ok := source.Label(); !ok { for _, src := range source.Paths(state.Graph) { if err := fs.Walk(src, func(src string, isDir bool) error { + src = watchKey(src) files.Store(src, struct{}{}) if !filepath.IsAbs(src) { files.Store("./"+src, struct{}{}) } dir := src if !isDir { - dir = filepath.Dir(src) + dir = path.Dir(src) } if _, present := dirs[dir]; !present { log.Notice("Adding watch on %s", dir) diff --git a/src/watch/watch_test.go b/src/watch/watch_test.go new file mode 100644 index 0000000000..f7c9c2a839 --- /dev/null +++ b/src/watch/watch_test.go @@ -0,0 +1,20 @@ +package watch + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWatchKeyMatchesNativeSeparators(t *testing.T) { + // The two sides of the comparison come from different places - our own slash-separated + // source paths, and whatever fsnotify reports, which on Windows uses backslashes - so + // they have to normalise to the same thing whichever separator each arrived with. + assert.Equal(t, watchKey("src/core/foo.go"), watchKey(filepath.Join("src", "core", "foo.go"))) +} + +func TestWatchKeyIsIdempotent(t *testing.T) { + once := watchKey(filepath.Join("src", "core", "foo.go")) + assert.Equal(t, once, watchKey(once)) +} diff --git a/test.sh b/test.sh index feed5274ee..c2ce0f2aef 100755 --- a/test.sh +++ b/test.sh @@ -25,6 +25,11 @@ check_path_for_excludes() { echo $EXCLUDES } +# has_wine reports whether the Windows tests can run here. +has_wine() { + hash wine 2>/dev/null +} + # Run the tests to make sure they still work notice "Running tests..." @@ -34,9 +39,19 @@ eval `go env` # repo that are optional and exercise specific rules, and require extra dependencies. EXCLUDES=$(check_path_for_excludes) -plz-out/bin/src/please -p -v2 $PLZ_ARGS ${PLZ_COVER:-test} $EXCLUDES --exclude=e2e --log_file plz-out/log/test_build.log --log_file_level 4 --trace_file plz-out/log/trace.json $@ +plz-out/bin/src/please -p -v2 $PLZ_ARGS ${PLZ_COVER:-test} $EXCLUDES --exclude=e2e --exclude=wine --log_file plz-out/log/test_build.log --log_file_level 4 --trace_file plz-out/log/trace.json $@ # We run the end-to-end tests separately to ensure things don't fight with one another; they are # finicky about some things due to running plz recursively and disabling the lock. notice "Running end-to-end tests..." -plz-out/bin/src/please -p -v2 $PLZ_ARGS ${PLZ_COVER:-test} $EXCLUDES --include=e2e --log_file plz-out/log/e2e_build.log --log_file_level 4 $@ +plz-out/bin/src/please -p -v2 $PLZ_ARGS ${PLZ_COVER:-test} $EXCLUDES --include=e2e --exclude=wine --log_file plz-out/log/e2e_build.log --log_file_level 4 $@ + +# The Windows tests cross-compile for windows_amd64 and run the result under Wine. They are a +# third pass because they are the only thing that builds the Go standard library for another +# platform, which is slow and pointless for someone who just wants the unit tests. +if has_wine; then + notice "Running Windows tests under Wine..." + plz-out/bin/src/please -p -v2 $PLZ_ARGS ${PLZ_COVER:-test} $EXCLUDES --include=wine --log_file plz-out/log/wine_build.log --log_file_level 4 $@ +else + warn "wine not found, skipping the Windows tests" +fi diff --git a/test/build_defs/BUILD b/test/build_defs/BUILD index 46e36c7918..debcfd34b6 100644 --- a/test/build_defs/BUILD +++ b/test/build_defs/BUILD @@ -17,6 +17,18 @@ filegroup( ], ) +filegroup( + name = "wine", + srcs = ["wine.build_defs"], + visibility = ["//test/..."], +) + +filegroup( + name = "windows_bundle", + srcs = ["windows_bundle.build_defs"], + visibility = ["//test/..."], +) + filegroup( name = "base_config", srcs = [ diff --git a/test/build_defs/windows_bundle.build_defs b/test/build_defs/windows_bundle.build_defs new file mode 100644 index 0000000000..d88ef21131 --- /dev/null +++ b/test/build_defs/windows_bundle.build_defs @@ -0,0 +1,94 @@ +# Packages the cross-built Windows test binaries so a real Windows machine can run them. +# +# The port's whole problem is that everything it claims rests on Wine. These are the same +# binaries //test/windows already runs under Wine; shipping them to a windows-latest runner +# turns that into evidence from Windows for almost no new test code. See +# docs/design/windows/05-testing-strategy.md for the list of things Wine cannot show. +# +# Everything here is labelled wine as well as windows, not because any of it runs under Wine +# but because building it means cross-compiling for Windows, which is what that label gates. + +def _windows_test_dir(name:str, test:str, data:list, needs_shell:bool): + """Stages one cross-built Go test binary with its data where the test expects to find it. + + The data lands correctly for free. An architecture subrepo has an empty root and package + root, so ///windows_amd64//src/fs:test_data stages at src/fs/test_data - which is the + literal path the test opens - and cp --parents relocates that subtree intact. + """ + srcs = {"BINARY": [test]} + cmds = [ + 'mkdir -p "$OUT"', + # Renamed for the same reason wine_go_test renames it: Go's exec will not run a file + # whose name has no extension in PATHEXT, even when handed its full path. + 'cp "$SRCS_BINARY" "$OUT/test.exe"', + ] + if data: + srcs["DATA"] = data + cmds.append('cp -r --parents $SRCS_DATA "$OUT/"') + + # What the runner must set $DATA to. Not knowable at parse time - these are the staged + # paths of another rule's outputs - so it is recorded here, where they are known. + cmds.append('echo "$SRCS_DATA" > "$OUT/DATA.txt"') + if needs_shell: + cmds.append('touch "$OUT/NEEDS_SHELL"') + return genrule( + name = f"_{name}#bundle", + srcs = srcs, + outs = [f"tests/{name}"], + cmd = " && ".join(cmds), + labels = ["wine", "windows"], + test_only = True, + ) + +def windows_test_bundle(name:str, tests:list, visibility:list=None): + """Packages the cross-built Windows test binaries for a real Windows machine to run. + + Every entry becomes tests//, holding test.exe, that test's data at the relative paths + it expects, and marker files saying what environment it needs. + + A directory per test rather than one shared tree, for three reasons. The data is a couple + of hundred megabytes, so copying it per test on the runner to isolate them would be most of + a gigabyte of IO. The tests have to be isolated: running these binaries in a shared tree + once deleted the whole of test/. And several of them want the harness directory itself + rather than only their data. + + The result is a directory rather than an archive because actions/upload-artifact zips + whatever it is handed. + + Args: + name (str): Name of the rule. + tests (list): (name, test target, data, needs_shell) tuples, the same list //test/windows + hands to wine_go_test. + visibility (list): Visibility declaration of the rule. + """ + dirs = [_windows_test_dir(n, t, d, s) for n, t, d, s in tests] + + # One test name per line, in the order this file declares them. Parse-time facts only; the + # build-time ones are the marker files beside each binary. A test silently disappearing + # from the list is then something the runner can notice. + manifest = "\n".join([n for n, _, _, _ in tests]) + return genrule( + name = name, + srcs = { + "DIRS": dirs, + "SHELL": ["///windows_amd64//third_party/binary:busybox"], + }, + outs = [name], + cmd = " && ".join([ + 'mkdir -p "$OUT/shell"', + # -l because these are already hardlinks into plz-out, and this is a few hundred + # megabytes that does not need copying twice. + 'cp -rl test/windows/tests "$OUT/tests"', + 'cp "$SRCS_SHELL" "$OUT/shell/busybox.exe"', + # A repo root above every test directory. Several of these tests call + # MustFindRepoRoot, and under Wine they get one by accident, because the harness + # runs them inside this repo's own plz-out. Natively the bundle sits wherever the + # runner unpacked it, with nothing above it, so without this they fail on a missing + # root rather than on whatever they are about. + 'echo "; Marks the bundle as a repo, for the tests that expect to be in one." > "$OUT/.plzconfig"', + f'printf "%s\\n" "{manifest}" > "$OUT/manifest.txt"', + ]), + labels = ["wine", "windows"], + test_only = True, + visibility = visibility, + ) diff --git a/test/build_defs/wine.build_defs b/test/build_defs/wine.build_defs new file mode 100644 index 0000000000..6dd3ed81ae --- /dev/null +++ b/test/build_defs/wine.build_defs @@ -0,0 +1,315 @@ +# Test macros that run Windows binaries under Wine. +# +# The port is developed and built on Linux, so these are how anything about its *runtime* +# behaviour gets checked at all before a Windows machine exists. See +# docs/design/windows/05-testing-strategy.md, which is honest about what Wine does and does +# not cover. +# +# Everything here is labelled 'wine' and excluded from the default test run, because building +# it means cross-compiling the Go standard library for Windows, which is a lot of work to +# impose on someone who only wanted to run the unit tests. + +# Wine's first run initialises a prefix, which is slow and produces a few hundred MB. One +# shared prefix for all these tests is enough - wineserver serialises access to it. $TMP_DIR +# is /plz-out/tmp/..., so trimming from the last /plz-out/ gets back to the root. +# This has to be built in the command rather than passed in env, which isn't expanded. +WINEPREFIX = "${TMP_DIR%/plz-out/*}/plz-out/wineprefix" + +# WINEDEBUG=-all silences Wine's own chatter, which would otherwise be interleaved with the +# test output we are trying to parse. +WINE_ENV = {"WINEDEBUG": "-all"} + +def _wine_setup_cmd(): + """Returns a command that prepares the shared Wine prefix and isolates this test in it.""" + + hosts = '"$WINEPREFIX/drive_c/windows/system32/drivers/etc/hosts"' + return " && ".join([ + f'export WINEPREFIX="{WINEPREFIX}"', + # wineboot is idempotent but not free, so only run it if the prefix isn't there. Two + # tests racing to create it is fine; wineserver serialises access. + 'if [ ! -d "$WINEPREFIX" ]; then wineboot --init >/dev/null 2>&1 || true; fi', + # Wine ships a hosts file with the localhost line commented out, so anything that + # resolves it - a test's own HTTP server, say - hangs until it gives up. + f'grep -q "^127.0.0.1 localhost" {hosts} || echo "127.0.0.1 localhost" >> {hosts}', + # Point the Windows home directory at this test's own temp dir. Otherwise every test + # shares the one inside the prefix and anything written to ~ leaks between runs; a + # read-only file left there can't even be replaced on Windows. + 'export USERPROFILE="$(winepath -w "$TMP_DIR")"', + ]) + +def wine_go_test( + name:str, + test:str, + data:list=[], + needs_shell:bool=False, + labels:list=[], + timeout:int=600, + size:str=None): + """Runs a Go test binary that was cross-compiled for Windows, under Wine. + + Args: + name (str): Name of the rule. + test (str): The go_test target to run, which must be in the windows_amd64 architecture - + i.e. a label of the form ///windows_amd64//src/core:core_test. + data (list): Runtime data the test needs. A go_test's own data doesn't come along when + another rule depends on it, so anything the test reads has to be repeated + here; it lands at the same path it would have under go_test. + needs_shell (bool): True if the test runs build actions, which need a shell. Puts the + bundled busybox next to the test binary, which is where Please looks for it + when it isn't on the PATH - the same arrangement as an install. + labels (list): Extra labels for the rule. + timeout (int): Test timeout in seconds. Wine is slower than native, and these binaries + are being run cold. + size (str): Test size. + """ + + # The binary has to be renamed. The go plugin names its output after the rule, so the + # cross-built binary is called e.g. 'core_test' with no extension - and Go's exec package + # on Windows will not run a file whose name has no extension in PATHEXT, even when handed + # its full path. Any test whose subject re-execs itself fails obscurely without this. + cmds = [ + _wine_setup_cmd(), + 'cp "$DATA_TEST_BINARY" "$TMP_DIR/test.exe"', + # A go_test's $DATA names only its data; here it would also name the test binary, which + # is not something the test put there. Tests that read $DATA expect the former. + # :- because the shell runs with -u and this is unset when the test has no data. + 'export DATA="${DATA_FILES:-}"', + ] + if needs_shell: + # In a directory of its own, on the Windows PATH, rather than in the working directory: + # Go's exec refuses to run something it found relative to the current directory, and + # this is how an install has it anyway. WINEPATH is what Wine adds to the Windows PATH. + cmds.append('mkdir -p "$TMP_DIR/shell" && cp "$DATA_BUSYBOX" "$TMP_DIR/shell/busybox.exe"') + cmds.append('export WINEPATH="$(winepath -w "$TMP_DIR/shell")"') + + # Go test binaries print in the format Please parses when it isn't given JUnit XML. + cmds.append('wine "$TMP_DIR/test.exe" -test.v 2>&1 | tee "$TMP_DIR/test.results"') + test_cmd = " && ".join(cmds) + + test_data = {"TEST_BINARY": [test], "FILES": data} + if needs_shell: + test_data["BUSYBOX"] = ["///windows_amd64//third_party/binary:busybox"] + return gentest( + name = name, + size = size, + timeout = timeout, + data = test_data, + env = WINE_ENV, + labels = labels + ["wine", "windows"], + local = True, + # Wine needs a real filesystem it can put a prefix on, and talks to a wineserver that + # outlives the process; neither survives the sandbox. + sandbox = False, + test_cmd = test_cmd, + ) + +def _wine_run_cmd(binary:str, args:str, exit_code:int, batch:bool=False): + """Returns a command that runs a Windows binary under Wine and checks how it finished. + + The status is captured rather than left to the shell: build actions run with -e, so an exit + code we are expecting would otherwise end the command before we could check it. + + The pipe is not only for the log. Wine's console emulation gives a child process handles + that some programs - Python among them - reject at startup unless the output is a pipe, + which has nothing to do with whatever is being tested. + + A batch file is handed to cmd.exe explicitly, and it matters that it is. Handing one to + `wine` looks like it works and is not the same thing: what Wine cannot load as a PE it + passes to the host, so a file that still has a Unix shebang on it runs under /bin/sh and + produces exactly the output the test was hoping for. The whole point of the wrapper is + that Windows has no shebang mechanism, so a test that leaves this to Wine tests nothing. + """ + run = f'wine "{binary}" {args}' + if batch: + run = f'wine cmd /c "$(winepath -w "{binary}")" {args}' + return " && ".join([ + "code=0", + f'{run} 2>&1 | tee "$TMP_DIR/output" || code=$?', + f'[ "$code" = "{exit_code}" ]', + ]) + +def wine_binary_test( + name:str, + binary:str, + args:str="", + data:list=[], + expected_output:str="", + exit_code:int=0, + batch:bool=False, + needs_shell:bool=False, + runs:int=1, + labels:list=[], + timeout:int=600, + size:str=None): + """Runs a Windows binary under Wine and checks what it printed and how it exited. + + For things that are not tests themselves - a cc_binary, say - where the point is that the + program runs at all and produces the right answer. + + Args: + name (str): Name of the rule. + binary (str): The target to run, which must be in the windows_amd64 architecture - i.e. a + label of the form ///windows_amd64//pkg:it. + args (str): Arguments to pass to it. + data (list): Runtime data it needs. It lands beside the binary, which on Windows is also + how a DLL is found: there is no rpath, so it has to be in the same directory. + expected_output (str): Text its output should contain. Empty to check nothing. + exit_code (int): The exit code it should finish with. + batch (bool): True if it is a .cmd rather than a .exe, so that it is run through cmd.exe + rather than left to Wine, which would fall back to the host shell. + needs_shell (bool): True if running it involves a shell - an sh_binary's wrapper does. + Puts the bundled busybox on the Windows PATH, which is where such a wrapper + expects to find it. + runs (int): How many times to run it, all in the same directory. More than one is worth + asking for when it writes something it will meet again on the next run: a + read-only file left behind cannot be replaced on Windows at all. + labels (list): Extra labels for the rule. + timeout (int): Test timeout in seconds. + size (str): Test size. + """ + + cmds = [_wine_setup_cmd()] + test_data = {"BINARY": [binary], "FILES": data} + if needs_shell: + # Its own directory rather than the working one: this is how an install has it, and Go's + # exec refuses to run something found relative to the current directory anyway. + test_data["BUSYBOX"] = ["///windows_amd64//third_party/binary:busybox"] + cmds.append('mkdir -p "$TMP_DIR/shell" && cp "$DATA_BUSYBOX" "$TMP_DIR/shell/busybox.exe"') + cmds.append('export WINEPATH="$(winepath -w "$TMP_DIR/shell")"') + for _ in range(runs): + cmds.append(_wine_run_cmd("$DATA_BINARY", args, exit_code, batch = batch)) + if expected_output: + cmds.append(f'grep -q "{expected_output}" "$TMP_DIR/output"') + + return gentest( + name = name, + size = size, + timeout = timeout, + data = test_data, + env = WINE_ENV, + labels = labels + ["wine", "windows"], + local = True, + no_test_output = True, + sandbox = False, + test_cmd = " && ".join(cmds), + ) + +def wine_pex_test( + name:str, + pex:str, + args:str="", + data:list=[], + exit_code:int=0, + test_output:bool=True, + labels:list=[], + timeout:int=600, + size:str=None): + """Runs a .pex that was cross-compiled for Windows, under Wine. + + A .pex is not a script with a shebang, it is a zip with an executable stub in front of it, + and that stub has to be a Windows one. Running it here is therefore a real test of the + preamble please_pex prepended, not only of the Python inside. + + The interpreter is the embeddable Python from python.org, put on the Windows PATH rather + than installed - which is also what the preamble's default search for "python" expects to + find. + + Args: + name (str): Name of the rule. + pex (str): The python_test or python_binary target to run, which must be in the + windows_amd64 architecture - i.e. a label of the form ///windows_amd64//pkg:it. + args (str): Arguments to pass to it. + data (list): Runtime data it needs, at the path it expects to read it from. A pex's own + data doesn't come along when another rule depends on it. + exit_code (int): The exit code it should finish with. + test_output (bool): True if it is a python_test, and so writes JUnit XML that Please + should read. False for a python_binary, which writes none. + labels (list): Extra labels for the rule. + timeout (int): Test timeout in seconds. + size (str): Test size. + """ + + # Run it where it lands, under its own name. Unlike a go_test there is nothing to rename: + # the .pex is already called .pex.exe, which is the whole point. + return gentest( + name = name, + size = size, + timeout = timeout, + data = { + "PEX": [pex], + # WINEPATH is what Wine adds to the Windows PATH. python.exe is looked up on it by + # name, which is what a .pex does when nothing has configured an interpreter path. + "PYTHON": ["//third_party/binary:python-windows"], + "FILES": data, + }, + env = WINE_ENV, + # The test runner writes JUnit XML into a directory rather than a single file. + labels = labels + ["wine", "windows"] + (["test_results_dir"] if test_output else []), + local = True, + no_test_output = not test_output, + sandbox = False, + test_cmd = " && ".join([ + _wine_setup_cmd(), + 'export WINEPATH="$(winepath -w "$DATA_PYTHON")"', + _wine_run_cmd("$DATA_PEX", args, exit_code), + ]), + ) + +def wine_plz_test( + name:str, + repo:str, + cmd:str, + expected_output:dict={}, + labels:list=[], + expected_failure:bool=False, + timeout:int=600): + """Runs the cross-built please.exe under Wine against a small test repo. + + This is the counterpart of please_repo_e2e_test for Windows: it checks that Please + actually *runs* there, rather than merely that it compiles. + + Args: + name (str): Name of the rule. + repo (str): A directory containing a small Please repo to run in. + cmd (str): Arguments to pass to please.exe, e.g. 'build //:target'. + expected_output (dict): Maps a file the build should produce, relative to the repo root, + to a file in the repo holding the content it should have. + labels (list): Extra labels for the rule. + expected_failure (bool): True if the command is expected to exit non-zero. + timeout (int): Test timeout in seconds. + """ + + # The Windows release layout: please.exe with busybox.exe beside it, which is where the + # default [build] shell of 'busybox' gets resolved from. Nothing is put on the PATH, so + # this also covers the resolution the bundling depends on. + setup = [ + _wine_setup_cmd(), + 'mkdir -p "$TMP_DIR/plzdir"', + 'cp "$DATA_PLEASE" "$TMP_DIR/plzdir/please.exe"', + 'cp "$DATA_BUSYBOX" "$TMP_DIR/plzdir/busybox.exe"', + 'cp -r "$DATA_REPO" "$TMP_DIR/repo"', + 'cd "$TMP_DIR/repo"', + ] + run = f'wine "$TMP_DIR/plzdir/please.exe" {cmd} 2>&1 | tee "$TMP_DIR/output"' + if expected_failure: + # Please exits non-zero, so check that rather than letting the pipeline fail us. + run = f"if {run}; then exit 1; fi" + test_cmd = " && ".join(setup + [run] + [ + f'diff -u "{expected}" "{out}"' + for out, expected in expected_output.items() + ]) + return gentest( + name = name, + timeout = timeout, + data = { + "PLEASE": ["///windows_amd64//src:please"], + "BUSYBOX": ["///windows_amd64//third_party/binary:busybox"], + "REPO": [repo], + }, + env = WINE_ENV, + labels = labels + ["wine", "windows"], + local = True, + no_test_output = True, + sandbox = False, + test_cmd = test_cmd, + ) diff --git a/test/subrepo/nested_subrepo_probe/BUILD b/test/subrepo/nested_subrepo_probe/BUILD new file mode 100644 index 0000000000..836ed25988 --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/BUILD @@ -0,0 +1,19 @@ +subinclude("//test/build_defs") + +# Resolving a subrepo that is declared inside another subrepo makes Please parse the package it +# would be declared in in the host repo first, to see if it is there. When it isn't - which is the +# normal case, not an error - that lookup used to claim the right to parse the package and never +# give it back, so the next thing to ask about the same package waited on a parse nobody was going +# to do. The build hung with no output at all rather than failing. +# +# Two subrepos are needed to see it: the first one to look poisons the lookup, the second one +# waits on it. timeout is what makes this a failure rather than a hang. +please_repo_e2e_test( + name = "nested_subrepo_probe_test", + expected_output = { + "plz-out/gen/suba/a.txt": "one", + "plz-out/gen/subb/b.txt": "two", + }, + plz_command = "timeout 180 plz build //:both", + repo = "test_repo", +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/.plzconfig b/test/subrepo/nested_subrepo_probe/test_repo/.plzconfig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/subrepo/nested_subrepo_probe/test_repo/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/BUILD_FILE new file mode 100644 index 0000000000..cfd97b2ea3 --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/BUILD_FILE @@ -0,0 +1,21 @@ +# Two subrepos, each of which reaches for a subrepo of its own that is declared in a package +# called "nested". This repo deliberately has no "nested" directory: resolving ///nested/one and +# ///nested/two makes Please look here first, find nothing, and fall back to the subrepo that +# asked. Whichever of the two looks here first has to leave the lookup usable for the other. +local_repository( + name = "suba", + path = "suba", +) + +local_repository( + name = "subb", + path = "subb", +) + +filegroup( + name = "both", + srcs = [ + "///suba//:t", + "///subb//:t", + ], +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/suba/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/suba/BUILD_FILE new file mode 100644 index 0000000000..b96828501d --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/suba/BUILD_FILE @@ -0,0 +1,7 @@ +genrule( + name = "t", + srcs = ["///nested/one//:f"], + outs = ["a.txt"], + cmd = "cp $SRCS $OUT", + visibility = ["PUBLIC"], +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/BUILD_FILE new file mode 100644 index 0000000000..450d24e6ec --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/BUILD_FILE @@ -0,0 +1,8 @@ +# Declared in a package called "nested", so the subrepo is registered as nested/one - which is +# what makes resolving it look for a package called "nested" in the host repo first. +# +# N.B. path is relative to the repo root rather than to suba, which is where this is declared. +local_repository( + name = "one", + path = "suba/nested/one", +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/one/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/one/BUILD_FILE new file mode 100644 index 0000000000..34cf706a48 --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/suba/nested/one/BUILD_FILE @@ -0,0 +1,6 @@ +genrule( + name = "f", + outs = ["f.txt"], + cmd = "echo one > $OUT", + visibility = ["PUBLIC"], +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/subb/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/subb/BUILD_FILE new file mode 100644 index 0000000000..00ecaff73b --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/subb/BUILD_FILE @@ -0,0 +1,7 @@ +genrule( + name = "t", + srcs = ["///nested/two//:f"], + outs = ["b.txt"], + cmd = "cp $SRCS $OUT", + visibility = ["PUBLIC"], +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/BUILD_FILE new file mode 100644 index 0000000000..0d431bc8a1 --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/BUILD_FILE @@ -0,0 +1,8 @@ +# Declared in a package called "nested", so the subrepo is registered as nested/two - which is +# what makes resolving it look for a package called "nested" in the host repo first. +# +# N.B. path is relative to the repo root rather than to subb, which is where this is declared. +local_repository( + name = "two", + path = "subb/nested/two", +) diff --git a/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/two/BUILD_FILE b/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/two/BUILD_FILE new file mode 100644 index 0000000000..efbe6bdd48 --- /dev/null +++ b/test/subrepo/nested_subrepo_probe/test_repo/subb/nested/two/BUILD_FILE @@ -0,0 +1,6 @@ +genrule( + name = "f", + outs = ["f.txt"], + cmd = "echo two > $OUT", + visibility = ["PUBLIC"], +) diff --git a/test/windows/BUILD b/test/windows/BUILD new file mode 100644 index 0000000000..a026c4e7d7 --- /dev/null +++ b/test/windows/BUILD @@ -0,0 +1,284 @@ +subinclude("//test/build_defs:wine", "//test/build_defs:windows_bundle") + +# The unit tests worth running on Windows are the ones the port actually changed: process +# management and locking, the filesystem layer, and config and path handling. Each entry is +# (name, cross-built test target, runtime data, whether it needs a shell). +# +# This list has one definition because it is consumed twice: wine_go_test runs each binary +# under Wine here, and windows_test_bundle packages the same binaries and the same data for a +# real Windows machine to run natively. See docs/design/windows/05-testing-strategy.md for what +# each of those is worth. +WINDOWS_GO_TESTS = [ + ( + "fs_test", + "///windows_amd64//src/fs:fs_test", + ["///windows_amd64//src/fs:test_data"], + False, + ), + ( + "core_test", + "///windows_amd64//src/core:core_test", + ["///windows_amd64//src/core:test_data"], + False, + ), + # Runs real build actions, so it exercises the process layer and the bundled shell as well + # as whatever it is nominally about. + ( + "build_test", + "///windows_amd64//src/build:build_test", + ["///windows_amd64//src/build:test_data"], + True, + ), + # Coverage parsing and the command-line layer: neither runs a build action, but both handle + # paths that came from somewhere else. + ( + "test_test", + "///windows_amd64//src/test:test_test", + ["///windows_amd64//src/test:test_data"], + False, + ), + ("cli_test", "///windows_amd64//src/cli:cli_test", [], False), + # Parsing is the valuable one: the BUILD language interpreter handles paths from every + # direction. + ("parse_test", "///windows_amd64//src/parse:parse_step_test", [], False), + ( + "asp_test", + "///windows_amd64//src/parse/asp:asp_test", + ["///windows_amd64//src/parse/asp:asp_test_data"], + False, + ), + ( + "query_test", + "///windows_amd64//src/query:query_test", + ["///windows_amd64//src/query:query_test_data"], + False, + ), + ( + "format_test", + "///windows_amd64//src/format:format_test", + ["///windows_amd64//src/format:format_test_data"], + False, + ), + ( + "export_test", + "///windows_amd64//src/export:export_test", + ["///windows_amd64//src/export:export_test_data"], + False, + ), + ("output_test", "///windows_amd64//src/output:output_test", [], False), + ( + "hashes_test", + "///windows_amd64//src/hashes:hash_rewriter_test", + ["///windows_amd64//src/hashes:hash_rewriter_test_data"], + False, + ), + ( + "gc_test", + "///windows_amd64//src/gc:gc_test", + ["///windows_amd64//src/gc:gc_test_data"], + False, + ), + ("tool_test", "///windows_amd64//src/tool:tool_test", [], False), + ("plz_test", "///windows_amd64//src/plz:plz_test", [], False), + ("clean_test", "///windows_amd64//src/clean:clean_test", [], False), + ( + "cache_test", + "///windows_amd64//src/cache:cache_test", + ["///windows_amd64//src/cache:cache_test_data"], + True, + ), + ("exec_test", "///windows_amd64//src/exec:exec_test", [], True), + ("process_test", "///windows_amd64//src/process:process_test", [], True), + ( + "update_test", + "///windows_amd64//src/update:update_test", + [ + "///windows_amd64//src/update:test_data", + "///windows_amd64//src/update:test_please", + "///windows_amd64//src/update:test_tarball", + "///windows_amd64//src:please", + ], + False, + ), + ( + "remote_test", + "///windows_amd64//src/remote:remote_test", + ["///windows_amd64//src/remote:remote_test_data"], + False, + ), + ( + "run_test", + "///windows_amd64//src/run:run_test", + ["///windows_amd64//src/run:run_test_data"], + False, + ), + # plz watch compares the paths it recorded against the ones fsnotify reports, which use + # different separators on Windows. A mismatch is silent - every event looks like it belongs + # to a file we aren't watching - so this only means anything when run on one. + ("watch_test", "///windows_amd64//src/watch:watch_test", [], False), +] + +[ + wine_go_test( + name = name, + data = data, + needs_shell = shell, + test = target, + ) + for name, target, data, shell in WINDOWS_GO_TESTS +] + +# The same binaries and the same data, packaged for a windows-latest runner to run natively. +# This is what turns "passes under Wine" into evidence from Windows. +windows_test_bundle( + name = "native_test_bundle", + tests = WINDOWS_GO_TESTS, +) + +# The shell smoke test: a real build action with a pipe and a redirect, run by the busybox +# that ships in the Windows release, found the way a user's install would find it. +wine_plz_test( + name = "shell_test", + cmd = "build //:pipeline", + expected_output = {"plz-out/gen/sorted.txt": "expected_sorted.txt"}, + repo = "smoke_repo", +) + +# A dropped forceposix build tag breaks every build label and is invisible in Please's own +# source, so assert label parsing works at all. +wine_plz_test( + name = "label_test", + cmd = "query alltargets //...", + repo = "smoke_repo", +) + +# A .pex is a zip with an executable stub in front of it, so on Windows it needs a Windows stub; +# the one please_pex prepends everywhere else is an ELF binary and will not run at all. This +# checks the whole chain - the preamble runs, finds an interpreter, and Python imports the test +# out of the zip - which is the only way any of it is exercised before a Windows machine exists. +wine_pex_test( + name = "pex_test", + data = ["//test/windows/python:data"], + pex = "///windows_amd64//test/windows/python:pex_test", +) + +# The preamble runs the interpreter as a child rather than replacing itself with it, because +# Windows has nothing to replace itself with. That makes passing the exit code back its job. +wine_pex_test( + name = "pex_exit_code_test", + args = "7", + exit_code = 7, + pex = "///windows_amd64//test/windows/python:exit_code", + test_output = False, +) + +# Windows resolves a DLL's symbols through an import library rather than through the DLL, so +# linking against a cc_shared_object needs one to exist. This builds the pair, links one against +# the other, and runs it - which also covers the DLL being found at run time, where Windows has +# no rpath and looks beside the binary instead. +# cc_test was recorded as blocked on Windows because UnitTest++ supposedly needs Win32 +# sources the plugin does not include. It does include them. What actually blocked it was +# that the plugin's own targets - the UnitTest++ test main is a cc_library here - compiled +# with the host toolchain whatever the using repo configured, so the Win32 sources were +# handed to a compiler with no windows.h. +# +# The runtime DLLs travel beside it; see //test/windows/cc:mingw_runtime for why they +# cannot simply be linked in. +wine_binary_test( + name = "cc_test_test", + binary = "///windows_amd64//test/windows/cc:greeting_test", + data = [ + "///windows_amd64//test/windows/cc:greeting", + "///windows_amd64//test/windows/cc:mingw_runtime", + ], + # Exit code rather than output: UnitTest++ writes its results as XML to test.results + # and prints nothing when everything passes. It returns the number of failures, so zero + # is the assertion - and a binary that could not start at all exits 53, not 0. +) + +wine_binary_test( + name = "dll_test", + binary = "///windows_amd64//test/windows/cc:hello", + data = ["///windows_amd64//test/windows/cc:greeting"], + expected_output = "hello from a dll", +) + +# Windows has no shebang mechanism, so an sh_binary comes out as a .cmd there: a batch preamble +# that unpacks the zip appended to it and hands the script to the bundled busybox. This runs one +# the way a user would, and covers all three things the preamble has to get right - the payload +# is unpacked, arguments reach the script, and its exit status comes back out. +wine_binary_test( + name = "sh_binary_test", + args = "world 3", + batch = True, + binary = "///windows_amd64//test/windows/shell:greet", + exit_code = 3, + expected_output = "hello from world", + needs_shell = True, + # Twice, in the one directory. The payload it unpacks is a set of build outputs, which + # are read-only, and a read-only file on Windows cannot be replaced at all - so the + # second run is where a stale payload would go unnoticed. + runs = 2, +) + +# Everything the release zip should contain, and nothing else. None of these names needs +# quoting in a shell word, which is what lets them go straight into the printf below. +RELEASE_MEMBERS = [ + "please", + "please/Install.md", + "please/arcat.exe", + "please/build_langserver.exe", + "please/busybox.exe", + "please/please.exe", + "please/plz.cmd", +] + +# The shape of the artifact, checked on Linux without Wine. It catches the things that would +# otherwise only show up on a real Windows machine: a tool that lost its .exe and so cannot be +# run at all, or a file quietly dropped from the release. +# +# Labelled wine like the rest of this package, not because anything here runs under Wine but +# because building it means cross-compiling for Windows, which is what that label gates. +gentest( + name = "release_shape_test", + data = {"ZIP": ["///windows_amd64//package:please_zip"]}, + labels = [ + "wine", + "windows", + ], + no_test_output = True, + test_cmd = " && ".join([ + # $DATA_ZIP is relative to the test directory, and unzip is about to leave it. + 'zip="$PWD/$DATA_ZIP"', + 'unzip -Z1 "$zip" | sed "s|/$||" | sort > "$TMP_DIR/got"', + # Both sides sorted by the same sort, so the list above can stay in a readable order + # rather than whatever collation this machine happens to use. + 'printf "%s\\n" ' + " ".join(RELEASE_MEMBERS) + ' | sort > "$TMP_DIR/want"', + 'diff -u "$TMP_DIR/want" "$TMP_DIR/got"', + ]), +) + +# The decisions about what the codelabs mean, and what is known to fail when they run on Windows. +# Read by //test/windows/codelab_script's tests on Linux, and by run_codelabs.ps1 on Windows. +filegroup( + name = "codelab_metadata", + srcs = [ + "codelab_known_failures.txt", + "codelab_steps.conf", + ], + visibility = ["//test/windows/..."], +) + +# The plan run_codelabs.ps1 replays. Built on Linux and handed to the Windows job as an artifact, +# so the Windows machine consumes something these tests have already checked. Deliberately not +# labelled wine: building it cross-compiles nothing. +genrule( + name = "codelab_plan", + srcs = { + "CODELABS": ["//docs/codelabs:codelab_md"], + "SIDECAR": ["codelab_steps.conf"], + }, + outs = ["codelab_plan.json"], + cmd = "$TOOLS --sidecar $SRCS_SIDECAR --out $OUT $SRCS_CODELABS", + tools = ["//test/windows/codelab_script"], +) diff --git a/test/windows/cc/BUILD b/test/windows/cc/BUILD new file mode 100644 index 0000000000..f1179c8ebf --- /dev/null +++ b/test/windows/cc/BUILD @@ -0,0 +1,68 @@ +# The subject of //test/windows:dll_test. Built for windows_amd64 and run under Wine; nothing +# builds these here, hence the manual labels. +subinclude("///cc//build_defs:cc") + +cc_shared_object( + name = "greeting", + srcs = ["greeting.cpp"], + hdrs = ["greeting.h"], + labels = ["manual"], + visibility = ["//test/windows:all"], +) + +# -lgreeting resolves against the import library the shared object emits alongside the DLL, +# which is the part that did not exist before. The DLL itself is a run-time dependency: Windows +# has no rpath, so it has to be beside the binary, which is what the test rule arranges. +cc_binary( + name = "hello", + srcs = ["main.cpp"], + hdrs = ["greeting.h"], + labels = ["manual"], + linker_flags = [ + "-L" + package_name(), + "-lgreeting", + ], + visibility = ["//test/windows:all"], + deps = [":greeting"], +) + +# The subject of //test/windows:cc_test_test. cc_test was recorded as blocked on Windows, +# because UnitTest++ supposedly needs Win32 sources the plugin does not include - which is not +# true, upstream selects them. This is here to find out what, if anything, is actually wrong. +cc_test( + name = "greeting_test", + srcs = ["greeting_test.cpp"], + hdrs = ["greeting.h"], + labels = ["manual"], + linker_flags = [ + "-L" + package_name(), + "-lgreeting", + ], + visibility = ["//test/windows:all"], + deps = [":greeting"], +) + +# The MinGW C++ runtime, which a cross-built binary needs beside it unless it was linked +# statically. The cc plugin wraps linker_flags in -Wl, so -static-libstdc++ cannot be passed +# through it - those are driver flags, and ld rejects them - and target compiler_flags reach the +# compile step but not the link. So the DLLs travel with the test, the same way the greeting DLL +# does. A cc_binary doing less C++ gets away without them, which is why the DLL test never +# needed this. +# +# Taken from the toolchain rather than pinned: they have to match the compiler that built the +# binary, and -print-file-name is how the compiler says where its own runtime is. +genrule( + name = "mingw_runtime", + outs = [ + "libgcc_s_seh-1.dll", + "libstdc++-6.dll", + ], + binary = True, + cmd = 'for dll in $OUTS; do cp "$($TOOL -print-file-name=$(basename $dll))" "$dll"; done', + labels = ["manual"], + test_only = True, + # The plugin decides this per platform now and leaves the config key unset, so name the + # cross compiler directly, falling back to whatever the repo configured if it did. + tools = [CONFIG.CC.CPP_TOOL or "x86_64-w64-mingw32-g++"], + visibility = ["//test/windows:all"], +) diff --git a/test/windows/cc/greeting.cpp b/test/windows/cc/greeting.cpp new file mode 100644 index 0000000000..5206622208 --- /dev/null +++ b/test/windows/cc/greeting.cpp @@ -0,0 +1,5 @@ +#include "test/windows/cc/greeting.h" + +const char *greeting() { + return "hello from a dll"; +} diff --git a/test/windows/cc/greeting.h b/test/windows/cc/greeting.h new file mode 100644 index 0000000000..64b9ed65dd --- /dev/null +++ b/test/windows/cc/greeting.h @@ -0,0 +1,4 @@ +// Deliberately not marked with __declspec(dllexport): MinGW exports every symbol from a DLL +// that declares none explicitly, and a rule that only works with source annotations would be a +// worse test of the rule. +const char *greeting(); diff --git a/test/windows/cc/greeting_test.cpp b/test/windows/cc/greeting_test.cpp new file mode 100644 index 0000000000..6e997934cb --- /dev/null +++ b/test/windows/cc/greeting_test.cpp @@ -0,0 +1,15 @@ +// The subject of //test/windows:cc_test_test. Built for windows_amd64 and run under Wine; +// nothing builds this here. +// +// It exists to find out whether cc_test works on Windows at all. The recorded blocker was that +// UnitTest++ needs its Win32 sources and the plugin does not include them, which is not true - +// upstream has selected them on Windows since before this port started. +#include + +#include + +#include "test/windows/cc/greeting.h" + +TEST(GreetingIsWhatTheLibrarySays) { + CHECK(std::strcmp(greeting(), "hello from a dll") == 0); +} diff --git a/test/windows/cc/main.cpp b/test/windows/cc/main.cpp new file mode 100644 index 0000000000..753c4e9354 --- /dev/null +++ b/test/windows/cc/main.cpp @@ -0,0 +1,8 @@ +#include + +#include "test/windows/cc/greeting.h" + +int main() { + printf("%s\n", greeting()); + return 0; +} diff --git a/test/windows/codelab_known_failures.txt b/test/windows/codelab_known_failures.txt new file mode 100644 index 0000000000..1c3f3449ae --- /dev/null +++ b/test/windows/codelab_known_failures.txt @@ -0,0 +1,57 @@ +# Codelab steps known to fail on Windows. One "codelab_id" or "codelab_id::step-key" per line, +# with a comment above saying why. Step keys are printed by run_codelabs.ps1 and by +# //test/windows:codelab_plan. +# +# A step listed here that starts passing fails the job too, so this list only ever shrinks. An +# entry naming a step that no longer exists fails on Linux, in +# //test/windows/codelab_script/script:script_test. +# +# This file records what the codelabs do on Windows. It is not a list of things to fix in them: +# no codelab has been edited to make anything here pass. +# +# Every entry rests either on a fact checked directly, or on what a native run printed, quoted in +# its reason. Nothing is guessed at: a wrong guess fails the job exactly as a missing entry does. + +# plz init plugin go writes plugin_repo(owner = "please-build") (src/plzinit/plugins.go), and the +# upstream go-rules releases publish please_go for darwin, freebsd and linux only, with no +# windows_amd64 asset. This repo's own plugins/BUILD pins the PeterNeiss forks for exactly that +# reason. Following the codelab as written, the first build of a Go target cannot succeed on +# Windows, and everything after it in the codelab is blocked behind it. The codelab's toolchain +# step also pins go_toolchain to 1.20, for which the plugin requests go1.20.windows-amd64.tar.gz; +# golang.org publishes Windows releases only as zips, and the first native run got a 404 for it. +# Ahead of both, and on every platform: the codelab's third_party/go/BUILD holds only that +# go_toolchain, so writing it replaces the go_stdlib plz init plugin go generated, and the second +# native run reported //third_party/go:std missing. +go_intro::hello-world/b3.1 + +# The same for Python: plz init plugin python gets upstream python-rules, whose please_pex has no +# windows_amd64 release. On top of that, the default [build] path on Windows is empty, and the +# codelab's only advice for adding an interpreter to it is a colon-separated Unix template. +python_intro::hello-world/b3.1 + +# An inline environment prefix, GODEBUG="installgoroot=all" go install std, is bash syntax. +# PowerShell reads the assignment as the name of a command. The codelab gives no Windows form. +puku::initialising-your-project-and-running-puku-with-please/b12.1 + +# plz puku runs //third_party/binary:puku, a remote_file of puku--_, and puku +# publishes no windows_amd64 release. Every later step of the codelab depends on it. +puku::adding-and-updating-modules/b3.1 + +# plz run //tools:wc runs tools/wc.sh, a filegroup with binary = True holding a #!/bin/bash script. +# Windows runs nothing by its shebang. The first native run failed with "%1 is not a valid Win32 +# application", and Please's own message says the file needs an extension Windows will run. The +# rest of the codelab builds on this tool, so it is blocked behind it. +genrule::managing-tools/b4.1 + +# The codelab clones thought-machine/please-codelabs, whose getting_started_go uses the upstream Go +# plugin. On the first native run its tools/BUILD had no please_go target, there being no +# windows_amd64 release, and its go_toolchain requested go1.20.windows-amd64.tar.gz, a 404. The +# first plz query cannot resolve the graph, and the rest of the codelab is blocked behind it. +plz_query::finding-dependencies-of-a-target/b1.1 + +# Not a Windows failure. plz init plugin go generates third_party/go/BUILD with a go_toolchain and a +# go_stdlib, and points STDLib at the stdlib. The codelab's "Add a go toolchain" block holds only a +# go_toolchain: written as the file it replaces the go_stdlib, and appended it would define a second +# target named toolchain. The second native run stopped here with //third_party/go:std missing. On +# Windows the upstream please_go and the Go 1.20 .tar.gz would fail next, as they do in go_intro. +k8s::using-our-base-image/b3.1 diff --git a/test/windows/codelab_script/BUILD b/test/windows/codelab_script/BUILD new file mode 100644 index 0000000000..0307c53507 --- /dev/null +++ b/test/windows/codelab_script/BUILD @@ -0,0 +1,11 @@ +# Reduces the published codelabs to a plan that test/windows/run_codelabs.ps1 replays on a real +# Windows machine. See script/parse.go for why it refuses to guess. +go_binary( + name = "codelab_script", + srcs = ["main.go"], + visibility = ["//test/windows/..."], + deps = [ + "///third_party/go/github.com_peterebden_go-cli-init_v5//flags", + "//test/windows/codelab_script/script", + ], +) diff --git a/test/windows/codelab_script/main.go b/test/windows/codelab_script/main.go new file mode 100644 index 0000000000..4dd911fea5 --- /dev/null +++ b/test/windows/codelab_script/main.go @@ -0,0 +1,73 @@ +// Command codelab_script reduces the codelabs to a plan test/windows/run_codelabs.ps1 can replay. +// See the script package for why it is built the way it is. +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/peterebden/go-cli-init/v5/flags" + + "github.com/thought-machine/please/test/windows/codelab_script/script" +) + +var opts = struct { + Sidecar string `long:"sidecar" required:"true" description:"codelab_steps.conf"` + Format string `long:"format" default:"plan" choice:"plan" choice:"summary" description:"plan emits the JSON the runner reads; summary prints one line per block, for authoring the sidecar"` + Out string `short:"o" long:"out" description:"File to write to; defaults to stdout"` + Args struct { + Codelabs []string `positional-arg-name:"codelabs" required:"true" description:"The codelab .md files"` + } `positional-args:"true" required:"true"` +}{} + +func main() { + flags.ParseFlagsOrDie("Codelab script", &opts, nil) + + b, err := os.ReadFile(opts.Sidecar) + if err != nil { + die("%s", err) + } + side, err := script.ParseSidecar(opts.Sidecar, string(b)) + if err != nil { + die("%s", err) + } + var codelabs []script.Codelab + for _, filename := range opts.Args.Codelabs { + b, err := os.ReadFile(filename) + if err != nil { + die("%s", err) + } + codelabs = append(codelabs, script.ParseCodelab(filename, string(b))) + } + + var out []byte + if opts.Format == "summary" { + // Deliberately tolerant: this is how the sidecar gets written, so it has to print + // the blocks nothing has decided yet instead of stopping at the first one. + out = []byte(script.Census(codelabs, side)) + } else { + plan, errs := script.BuildPlan(codelabs, side) + if len(errs) > 0 { + for _, err := range errs { + fmt.Fprintf(os.Stderr, "%s\n\n", err) + } + die("%d problem(s) extracting the codelabs; nothing was written", len(errs)) + } + if out, err = json.MarshalIndent(plan, "", " "); err != nil { + die("%s", err) + } + out = append(out, '\n') + } + + if opts.Out == "" { + os.Stdout.Write(out) + } else if err := os.WriteFile(opts.Out, out, 0644); err != nil { + die("%s", err) + } +} + +func die(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/test/windows/codelab_script/script/BUILD b/test/windows/codelab_script/script/BUILD new file mode 100644 index 0000000000..9b4c2c89e4 --- /dev/null +++ b/test/windows/codelab_script/script/BUILD @@ -0,0 +1,27 @@ +go_library( + name = "script", + srcs = [ + "classify.go", + "parse.go", + "plan.go", + "sidecar.go", + ], + visibility = ["//test/windows/..."], +) + +# The drift guard: runs against the real codelabs and the real sidecar, so a codelab edit that +# leaves a block undecided fails here, on Linux, in the default test pass. +go_test( + name = "script_test", + srcs = ["script_test.go"], + data = [ + "test_data", + "//docs/codelabs:codelab_md", + "//test/windows:codelab_metadata", + ], + deps = [ + ":script", + "///third_party/go/github.com_stretchr_testify//assert", + "///third_party/go/github.com_stretchr_testify//require", + ], +) diff --git a/test/windows/codelab_script/script/classify.go b/test/windows/codelab_script/script/classify.go new file mode 100644 index 0000000000..37ee9da0ac --- /dev/null +++ b/test/windows/codelab_script/script/classify.go @@ -0,0 +1,222 @@ +package script + +import ( + "fmt" + "regexp" + "strings" +) + +// Kind is what a fenced block turned out to be. +type Kind string + +const ( + // A file to write, with a path and contents. + KindFile Kind = "file" + // Commands to run. + KindCommand Kind = "command" + // Commands shown with the output they produced. Runnable, but the output is what the + // codelab saw on its author's machine and is advisory here; see plan.go. + KindTranscript Kind = "transcript" + // Shown for reference and never run: a `tree -a` listing, an expected build result, the + // YAML of a CI workflow, a walk-through of an interactive session. + KindIllustration Kind = "illustration" + // The sidecar says to leave this block alone entirely, with a reason. + KindIgnore Kind = "ignore" + // No rule decided. Always an error; see Classify. + KindUnclassified Kind = "unclassified" +) + +// A heading whose entire text is a backticked path, e.g. "### `src/BUILD`". The dominant +// convention: genrule, go_intro, python_intro and using_plugins introduce every file this way. +var fileHeadingRe = regexp.MustCompile("^#{2,6}\\s+`([^`]+)`\\s*$") + +// The same thing said in prose and ending in a colon, which is how puku and k8s do it throughout: +// "Create a file `hello_service/service.go`:". Those two use no file headings at all. +var fileProseRe = regexp.MustCompile("`([^`]+)`[^`]*:\\s*$") + +// The name half of an inline environment assignment. +var envPrefixRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*=`) + +// A transcript line: the command follows the prompt, and the rest of the block is its output. +var promptRe = regexp.MustCompile(`^\$\s+(.*)$`) + +// An introducing line promising output rather than asking for anything to be run. Several codelabs +// tag such a block ```bash anyway - using_plugins shows two `tree -a` listings that way, k8s a +// job-control trace - and running those lines would fail on every platform, Linux included. +var outputIntroRe = regexp.MustCompile(`(?i)\b(output|should look like|should see|will see|prints|looks like|similar to)\b`) + +// Fence languages that mean "this is the content of a file", given a path can be found for it. +var fileLangs = map[string]bool{ + "python": true, "go": true, "golang": true, "yaml": true, + "ini": true, "shell script": true, "dockerfile": true, +} + +// Fence languages that might hold commands. +var commandLangs = map[string]bool{"bash": true, "sh": true, "shell": true, "text": true, "": true} + +// The first word of a line that is plausibly a command. Deliberately a list rather than a pattern: +// the codelabs' output blocks are full of lines that look like commands to a pattern, and a wrong +// guess here becomes a step that fails on every platform for reasons that have nothing to do with +// Windows. Anything not listed is unclassified, which asks rather than guesses. +var commandVerbs = map[string]bool{ + "plz": true, "./pleasew": true, "pleasew": true, "./plz": true, + "go": true, "git": true, "puku": true, "pip": true, "pip3": true, + "python": true, "python3": true, "docker": true, "kubectl": true, "minikube": true, + "mkdir": true, "cd": true, "echo": true, "cat": true, "ls": true, "rm": true, + "touch": true, "cp": true, "mv": true, "curl": true, "wget": true, "tree": true, + "which": true, "pkill": true, "eval": true, "export": true, "source": true, + "printf": true, "wc": true, "sort": true, "sed": true, "grep": true, "chmod": true, +} + +// Classify works out what a block is, consulting the sidecar first. +// +// The order matters and each step earns its place: +// +// 1. The sidecar, which overrides everything and always carries a reason. +// 2. A heading immediately above whose whole text is a backticked path: a file. +// 3. A command-shaped fence, checked before the prose rule because a sentence naming a file is +// as often followed by the command that creates it as by its contents. +// 4. A file-shaped fence with a path findable in the prose before or after it. +// 5. Nothing: KindUnclassified, which the caller must treat as fatal. +// +// Returning KindUnclassified rather than quietly bucketing into "other" is the whole design. It is +// what makes a codelab edit that introduces an unreadable block go red on Linux, in an ordinary +// unit test, instead of silently shrinking what the Windows job checks. +func Classify(c Codelab, b Block, key string, side *Sidecar) (Kind, string, error) { + if entry, ok := side.Lookup(key); ok { + kind, path, err := entry.Kind(b) + if err != nil { + return KindUnclassified, "", fmt.Errorf("%s:%d: %s: %w", c.Source, b.Line, key, err) + } + if kind != "" { + return kind, path, nil + } + } + + if m := fileHeadingRe.FindStringSubmatch(b.Intro); m != nil { + return KindFile, m[1], nil + } + + if commandLangs[b.Lang] { + if hasPrompt(b.Body) { + return KindTranscript, "", nil + } + if outputIntroRe.MatchString(b.Intro) { + return KindIllustration, "", nil + } + if isCommandish(b.Body) { + return KindCommand, "", nil + } + } + + // Only the prose before a block is read for a path. The line after was tried and named the + // wrong file on its first outing: puku's "Add a filegroup for go.mod at `BUILD`:" is + // followed by "Update your `.plzconfig`:", which introduces the next block, not this one. + if fileLangs[b.Lang] || commandLangs[b.Lang] { + if path, ok := proseFilePath(b.Intro); ok { + return KindFile, path, nil + } + } + + return KindUnclassified, "", nil +} + +// Key is the identifier a step is known by, in codelab_steps.conf, in +// codelab_known_failures.txt and in the runner's report. +// +// "::/b", with the command's position appended for a block holding +// several. Readable rather than hashed, because the failures file is a findings record someone +// has to read; an edit in one section does not renumber another, which a whole-file ordinal could +// not promise, and codelab_steps.conf pins the text of what it names so an edit within a section +// cannot silently move a decision onto a different block. +func Key(c Codelab, b Block) string { + return fmt.Sprintf("%s::%s/b%d", c.ID, b.SectionSlug, b.Ordinal) +} + +func hasPrompt(body []string) bool { + for _, line := range body { + if promptRe.MatchString(line) { + return true + } + } + return false +} + +// isCommandish says whether the first non-blank line of a block starts with a word we recognise as +// a command, or with an inline environment assignment such as GODEBUG="installgoroot=all". +func isCommandish(body []string) bool { + for _, line := range body { + line = strings.TrimSpace(line) + if line == "" { + continue + } + first, _, _ := strings.Cut(line, " ") + if commandVerbs[first] { + return true + } + // An inline environment prefix: VAR=value cmd. Bash syntax, so these are exactly the + // steps most likely to fail on Windows - but they are commands, and saying so is what + // lets them run and be reported rather than sit unreadable. The value may be quoted, + // as in GODEBUG="installgoroot=all", so only the name is checked. + return envPrefixRe.MatchString(first) + } + return false +} + +// Commands splits a block into the commands to run and, for each, the output shown after it. +// +// In a transcript the output belongs to the command above it, not to the block. genrule shows +// `$ plz build` with its build summary and then `$ cat` with a word count; attaching both to the +// last command made the first native run report that none of the cat's lines appeared. +func Commands(b Block) (commands []string, expect [][]string) { + prompted := hasPrompt(b.Body) + for _, line := range b.Body { + if strings.TrimSpace(line) == "" { + continue + } + if !prompted { + commands = append(commands, line) + expect = append(expect, nil) + continue + } + if m := promptRe.FindStringSubmatch(line); m != nil { + commands = append(commands, m[1]) + expect = append(expect, nil) + } else if len(expect) > 0 { + expect[len(expect)-1] = append(expect[len(expect)-1], line) + } + } + return commands, expect +} + +// proseFilePath pulls a file path out of a sentence such as +// "Add the following to `common/docker/BUILD`:". +func proseFilePath(line string) (string, bool) { + m := fileProseRe.FindStringSubmatch(line) + if m == nil || !isRepoPath(m[1]) { + return "", false + } + return m[1], true +} + +// isRepoPath says whether a backticked token is plausibly a file in the repo being built. +// +// The exclusions are not hypothetical. Each one is a line in the codelabs as they stand that would +// otherwise be read as a file to create: +// - a space or a URL scheme: prose, or a link +// - a leading slash: a [build] path value, such as `/usr/local/go/bin/go` in puku +// - a dot in the first segment: a module path, such as `github.com/stretchr/testify` in go_intro +func isRepoPath(s string) bool { + if s == "" || strings.ContainsAny(s, " \t") || strings.Contains(s, "://") { + return false + } + if strings.HasPrefix(s, "/") { + return false + } + first, _, _ := strings.Cut(s, "/") + if strings.Contains(first, ".") && !strings.HasPrefix(first, ".") { + return false + } + // A bare BUILD is the one file name without a slash or a dot that the codelabs create. + return s == "BUILD" || strings.Contains(s, "/") || strings.Contains(s, ".") +} diff --git a/test/windows/codelab_script/script/parse.go b/test/windows/codelab_script/script/parse.go new file mode 100644 index 0000000000..29d9331268 --- /dev/null +++ b/test/windows/codelab_script/script/parse.go @@ -0,0 +1,142 @@ +// Package script turns the published codelabs into something a machine can replay. +// +// The codelabs at https://please.build/codelabs.html are the front door for new users, and nothing +// anywhere has ever executed a line of them. This package reads docs/codelabs/*.md and produces an +// ordered plan of the files each one tells you to create and the commands it tells you to run; +// test/windows/run_codelabs.ps1 replays that plan on a real Windows machine. +// +// The plan is derived from the Markdown rather than transcribed into fixtures, so that the check +// and the published page cannot drift apart. That means living with conventions the codelabs were +// never written to satisfy. Where a convention runs out, the answer is never to guess: an +// unclassified block is a hard error, and the decision gets written down in codelab_steps.conf +// with a reason. A tolerant parser that silently ignored what it could not read would emit a +// three-step plan for a thirty-step codelab, pass, and tell nobody anything. +// +// Parsed line by line with no Markdown library, like docs/codelabs/codelab_template.go, which +// reads the same front matter for the index page. A line scanner gives exact line numbers, which +// is what the error messages here are made of. +package script + +import ( + "regexp" + "strings" +) + +// Codelab is one .md file: its front matter and every fenced block in it. +type Codelab struct { + ID string + Title string + Status string + // Path as given, so a failure can be traced back to a file. + Source string + Blocks []Block +} + +// Block is one fenced block, with the context needed to work out what it is. +type Block struct { + // The fence's language tag, empty for a bare ```. + Lang string + Body []string + // 1-based line of the opening fence. + Line int + // Nearest non-blank line above the fence: a heading or a sentence that says what the block + // is, as in "Add the following to `common/docker/BUILD`:". + Intro string + // Text of the enclosing "##" heading, and its slug. Blocks are keyed by section rather than + // by position in the file so that an edit in one section does not renumber another. + Section string + SectionSlug string + // Ordinal of this block within its section, from 1. + Ordinal int +} + +var ( + headingRe = regexp.MustCompile(`^(#{1,6})\s+(.*?)\s*$`) + nonSlugRe = regexp.MustCompile(`[^a-z0-9]+`) + frontKeyRe = regexp.MustCompile(`^([A-Za-z ]+):\s*(.*)$`) +) + +// ParseCodelab reads one codelab into its blocks. It does not classify them; see Classify. +func ParseCodelab(filename, content string) Codelab { + lines := strings.Split(content, "\n") + codelab := Codelab{Source: filename} + readFrontMatter(&codelab, lines) + + section, slug := "", "" + ordinal := 0 + + for i := 0; i < len(lines); i++ { + if m := headingRe.FindStringSubmatch(lines[i]); m != nil { + // Only "##" starts a new section. The codelabs use "###" for sub-steps and for + // file headings, both of which belong to the section around them. + if len(m[1]) == 2 { + section = m[2] + slug = slugify(m[2]) + ordinal = 0 + } + continue + } + if !strings.HasPrefix(lines[i], "```") { + continue + } + // An unterminated fence takes the rest of the file. The codelabs have none, but a + // half-written one should say so rather than silently swallowing every block after it. + end := i + 1 + for end < len(lines) && !strings.HasPrefix(lines[end], "```") { + end++ + } + ordinal++ + codelab.Blocks = append(codelab.Blocks, Block{ + Lang: strings.TrimSpace(strings.TrimPrefix(lines[i], "```")), + Body: lines[i+1 : end], + Line: i + 1, + Intro: nearestNonBlank(lines, i, -1), + Section: section, + SectionSlug: slug, + Ordinal: ordinal, + }) + i = end + } + return codelab +} + +// readFrontMatter reads the "key: value" header the codelabs open with, which runs until the first +// blank line. The same shape codelab_template.go reads, and only the fields this needs. +func readFrontMatter(codelab *Codelab, lines []string) { + for _, line := range lines { + if strings.TrimSpace(line) == "" { + return + } + m := frontKeyRe.FindStringSubmatch(line) + if m == nil { + continue + } + switch strings.ToLower(strings.TrimSpace(m[1])) { + case "id": + codelab.ID = strings.TrimSpace(m[2]) + case "summary": + codelab.Title = strings.TrimSpace(m[2]) + case "status": + codelab.Status = strings.TrimSpace(m[2]) + } + } +} + +// nearestNonBlank walks from i in the given direction and returns the first non-blank line, or "" +// if there is none. Blank lines are skipped and nothing else is. +func nearestNonBlank(lines []string, i, step int) string { + for j := i + step; j >= 0 && j < len(lines); j += step { + if strings.TrimSpace(lines[j]) != "" { + return lines[j] + } + } + return "" +} + +// slugify turns a heading into the section part of a step key: lowercase, words joined by +// hyphens. Readable, because these keys end up in codelab_known_failures.txt, which is a findings +// record someone has to read. +func slugify(s string) string { + s = nonSlugRe.ReplaceAllString(strings.ToLower(s), "-") + return strings.Trim(s, "-") +} diff --git a/test/windows/codelab_script/script/plan.go b/test/windows/codelab_script/script/plan.go new file mode 100644 index 0000000000..d465d10da5 --- /dev/null +++ b/test/windows/codelab_script/script/plan.go @@ -0,0 +1,355 @@ +package script + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Plan is what the runner consumes: every codelab reduced to an ordered list of steps. +type Plan struct { + Codelabs []PlannedCodelab `json:"codelabs"` +} + +// PlannedCodelab is one codelab's steps, plus a census of what its blocks turned out to be. +type PlannedCodelab struct { + ID string `json:"id"` + Title string `json:"title"` + Source string `json:"source"` + // Set when the codelab has nothing anyone runs locally. github_actions teaches CI + // configuration and is entirely YAML. + NotRunnable string `json:"not_runnable,omitempty"` + // How many blocks there were and what they were. The report reconciles against this: a + // summary that counts only what it ran cannot tell you it ran almost nothing. + Blocks map[string]int `json:"blocks"` + Steps []Step `json:"steps"` +} + +// Step is one thing for the runner to do. +type Step struct { + Key string `json:"key"` + // "run", "file", "chdir" or "skip". + Kind string `json:"kind"` + Line int `json:"line"` + Section string `json:"section,omitempty"` + + // kind=run. + Command string `json:"command,omitempty"` + // What the codelab shows this command printing. Advisory: the corpus is full of timings, + // incrementality percentages and a randomly chosen greeting, so asserting on it would + // produce flakes that discredit the whole check. Promoted to a requirement only by the + // sidecar's assert directive. + ExpectedOutput []string `json:"expected_output,omitempty"` + Assert string `json:"assert,omitempty"` + // Tools this step needs, checked on the machine at run time. + Needs []string `json:"needs,omitempty"` + Timeout int `json:"timeout,omitempty"` + // A failure here does not mark the rest of the codelab blocked. True for a command that + // only displays something, and wherever the sidecar says so. + NonBlocking bool `json:"non_blocking,omitempty"` + + // kind=file. + Path string `json:"path,omitempty"` + Mode string `json:"mode,omitempty"` + Content string `json:"content,omitempty"` + + // kind=chdir. + Dir string `json:"dir,omitempty"` + + // kind=skip. + Reason string `json:"reason,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// A command line that only changes directory. The runner owns the working directory across steps, +// because a `cd` in a child process is lost the moment it exits - and the codelabs open with +// "mkdir getting_started_go && cd getting_started_go", with every later step depending on it. +var chdirRe = regexp.MustCompile(`^cd\s+([^\s;|&]+)\s*$`) + +// BuildPlan classifies every block of every codelab and returns the plan. +// +// It fails rather than guessing. An unclassified block, a sidecar stanza whose text no longer +// matches the block it names, and a stanza that names nothing at all are all errors, reported +// together so that one pass over the output fixes all of them. +func BuildPlan(codelabs []Codelab, side *Sidecar) (*Plan, []error) { + plan := &Plan{Codelabs: []PlannedCodelab{}} + var errs []error + + for _, c := range codelabs { + planned := PlannedCodelab{ + ID: c.ID, + Title: c.Title, + Source: c.Source, + Blocks: map[string]int{}, + Steps: []Step{}, + } + if entry, ok := side.Lookup(c.ID); ok { + planned.NotRunnable = entry.NotRunnable + } + + for _, b := range c.Blocks { + key := Key(c, b) + kind, path, err := decide(c, b, key, side, planned.NotRunnable != "") + if err != nil { + errs = append(errs, err) + continue + } + planned.Blocks["total"]++ + planned.Blocks[string(kind)]++ + + entry, hasEntry := side.Lookup(key) + if hasEntry { + if err := checkMatches(c, b, entry, kind, path); err != nil { + errs = append(errs, err) + continue + } + } + + switch kind { + case KindUnclassified: + errs = append(errs, unclassifiedError(c, b, key)) + case KindIllustration, KindIgnore: + // Carried in the census and nowhere else. Not a step, so it never + // appears in the pass, fail or skip tallies. + case KindFile: + planned.Steps = append(planned.Steps, fileStep(b, key, path, entry)) + case KindCommand, KindTranscript: + if hasEntry && entry.Skip != "" { + planned.Steps = append(planned.Steps, Step{ + Key: key, Kind: "skip", Line: b.Line, Section: b.Section, + Reason: entry.Skip, Detail: entry.Reason, + }) + continue + } + steps, stepErrs := commandSteps(c, b, key, entry, side) + planned.Steps = append(planned.Steps, steps...) + errs = append(errs, stepErrs...) + } + } + plan.Codelabs = append(plan.Codelabs, planned) + } + + for _, key := range side.Unresolved() { + errs = append(errs, fmt.Errorf("codelab_steps.conf: %s names no block in any codelab; the codelab it refers to has been edited, so the decision recorded there needs revisiting rather than dropping", key)) + } + return plan, errs +} + +// decide classifies a block the way the plan and the census both need it, so that the two cannot +// disagree about what a block is. A codelab declared not runnable has its reason recorded once, at +// the top, rather than a stanza per block restating it, so what no rule decides there is shown. +func decide(c Codelab, b Block, key string, side *Sidecar, notRunnable bool) (Kind, string, error) { + kind, path, err := Classify(c, b, key, side) + if err == nil && kind == KindUnclassified && notRunnable { + kind = KindIllustration + } + return kind, path, err +} + +func fileStep(b Block, key, path string, entry *Entry) Step { + mode := "write" + if entry != nil && entry.Mode != "" { + mode = entry.Mode + } + return Step{ + Key: key, Kind: "file", Line: b.Line, Section: b.Section, + Path: path, Mode: mode, Content: strings.Join(b.Body, "\n"), + } +} + +// Commands that only display something. Nothing later in a codelab can depend on one succeeding, +// so a failure - `tree -a` has no Windows counterpart that takes that flag - is recorded without +// marking every later step blocked. +var displayVerbs = map[string]bool{"tree": true, "cat": true, "which": true, "ls": true, "printenv": true} + +// commandSteps turns a block into one step per command, splitting "a && b" so that a `cd` can +// become a chdir the runner applies to itself. +// +// A stanza can name a single command as well as a whole block, as ".", for the cases where +// one line of a block needs a decision the others do not: python_intro builds a pex and then runs +// it in the same block, and only the second of those depends on a shebang. +func commandSteps(c Codelab, b Block, key string, entry *Entry, side *Sidecar) ([]Step, []error) { + commands, expects := Commands(b) + var steps []Step + var errs []error + n := 0 + for i, command := range commands { + first := len(steps) + for _, part := range splitChain(command) { + n++ + stepKey := fmt.Sprintf("%s.%d", key, n) + if m := chdirRe.FindStringSubmatch(part); m != nil { + steps = append(steps, Step{ + Key: stepKey, Kind: "chdir", Line: b.Line, + Section: b.Section, Dir: m[1], + }) + continue + } + step := Step{ + Key: stepKey, Kind: "run", Line: b.Line, + Section: b.Section, Command: part, + } + verb, _, _ := strings.Cut(part, " ") + step.NonBlocking = displayVerbs[verb] + applyEntry(&step, entry) + if own, ok := side.Lookup(stepKey); ok { + if own.Matches != "" && own.Matches != part { + errs = append(errs, fmt.Errorf("%s:%d: %s: codelab_steps.conf expects %q here, but the command now says %q. The decision recorded there was made about different text, so re-read it before updating the stanza", + c.Source, b.Line, stepKey, own.Matches, part)) + continue + } + if own.Skip != "" { + steps = append(steps, Step{ + Key: stepKey, Kind: "skip", Line: b.Line, Section: b.Section, + Reason: own.Skip, Detail: own.Reason, + }) + continue + } + applyEntry(&step, own) + } + steps = append(steps, step) + } + // Output shown after a command belongs to the last step that command produced: for + // "mkdir x && plz build", the build. + if len(expects[i]) > 0 && len(steps) > first { + steps[len(steps)-1].ExpectedOutput = expects[i] + } + } + return steps, errs +} + +// applyEntry copies a stanza's run-time directives onto a step. Directives a stanza leaves unset +// leave the step's own values alone, so a command-level stanza refines a block-level one. +func applyEntry(step *Step, e *Entry) { + if e == nil { + return + } + if len(e.Needs) > 0 { + step.Needs = e.Needs + } + if e.Assert != "" { + step.Assert = e.Assert + } + if e.Timeout != 0 { + step.Timeout = e.Timeout + } + if e.NonBlocking { + step.NonBlocking = true + } +} + +// splitChain splits "mkdir x && cd x" into its parts. Only "&&" is split: the codelabs use "|" to +// build real pipelines, which have to reach the shell intact. +func splitChain(command string) []string { + var out []string + for _, part := range strings.Split(command, "&&") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + if len(out) == 0 { + return []string{command} + } + return out +} + +// checkMatches enforces the sidecar's drift guard: the stanza says what it expects to find, and +// extraction fails if the block no longer says it. +func checkMatches(c Codelab, b Block, entry *Entry, kind Kind, path string) error { + if entry.Matches == "" { + return nil + } + var got []string + switch kind { + case KindFile: + // The path alone pins little when the stanza is what chose the path, so the first + // line of the contents is accepted too: `[Alias "puku"]` says which block is meant. + got = []string{path, firstLine(b.Body)} + case KindCommand, KindTranscript: + commands, _ := Commands(b) + got = commands + default: + // An illustration or an ignored block is pinned by its first line, which is what the + // census prints and so what a person writing the stanza has in front of them. + got = []string{firstLine(b.Body)} + } + for _, g := range got { + if strings.TrimSpace(g) == entry.Matches { + return nil + } + } + return fmt.Errorf("%s:%d: %s: codelab_steps.conf expects %q here, but the block now says %q. The decision recorded there was made about different text, so re-read it before updating the stanza", + c.Source, b.Line, entry.Key, entry.Matches, strings.Join(got, " / ")) +} + +// unclassifiedError is the authoring experience for codelab_steps.conf, so it says what to write. +func unclassifiedError(c Codelab, b Block, key string) error { + first := "" + for _, line := range b.Body { + if strings.TrimSpace(line) != "" { + first = strings.TrimSpace(line) + break + } + } + if len(first) > 60 { + first = first[:60] + "..." + } + return fmt.Errorf("%s:%d: cannot tell what this block is (fence %q, introduced by %q, starting %q).\n"+ + "Decide in test/windows/codelab_steps.conf, with the reason above it:\n\n"+ + "; why this block is what it is\n[%s]\nmatches = %s\nkind = illustration", + c.Source, b.Line, b.Lang, truncate(b.Intro, 60), first, key, first) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} + +// Census renders one line per block: the key, what it was decided to be, and its first line. This +// is how codelab_steps.conf is authored from nothing, and the golden file the unit test diffs - +// a compact one-line-per-block record stays reviewable in a way a golden JSON plan would not. +func Census(codelabs []Codelab, side *Sidecar) string { + var b strings.Builder + for _, c := range codelabs { + entry, ok := side.Entries[c.ID] + notRunnable := ok && entry.NotRunnable != "" + for _, block := range c.Blocks { + key := Key(c, block) + kind, path, err := decide(c, block, key, side, notRunnable) + detail := path + if err != nil { + kind, detail = KindUnclassified, err.Error() + } + if detail == "" { + detail = firstLine(block.Body) + } + fmt.Fprintf(&b, "%-56s %-13s %s\n", key, kind, truncate(detail, 60)) + } + } + return b.String() +} + +func firstLine(body []string) string { + for _, line := range body { + if strings.TrimSpace(line) != "" { + return strings.TrimSpace(line) + } + } + return "" +} + +// StepKeys lists every step key in the plan, sorted. Used to check that nothing in +// codelab_known_failures.txt names a step that no longer exists. +func (p *Plan) StepKeys() []string { + var keys []string + for _, c := range p.Codelabs { + for _, s := range c.Steps { + keys = append(keys, s.Key) + } + } + sort.Strings(keys) + return keys +} diff --git a/test/windows/codelab_script/script/script_test.go b/test/windows/codelab_script/script/script_test.go new file mode 100644 index 0000000000..17d3e1c288 --- /dev/null +++ b/test/windows/codelab_script/script/script_test.go @@ -0,0 +1,244 @@ +package script + +import ( + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + sidecarPath = "test/windows/codelab_steps.conf" + knownFailuresPath = "test/windows/codelab_known_failures.txt" + censusPath = "test/windows/codelab_script/script/test_data/census.txt" +) + +// The real codelabs and the real sidecar, not samples that could quietly stop resembling them. +func loadReal(t *testing.T) ([]Codelab, *Sidecar) { + t.Helper() + files, err := filepath.Glob("docs/codelabs/*.md") + require.NoError(t, err) + require.Len(t, files, 8, "the codelabs as published at https://please.build/codelabs.html") + sort.Strings(files) + + var codelabs []Codelab + for _, f := range files { + b, err := os.ReadFile(f) + require.NoError(t, err) + codelabs = append(codelabs, ParseCodelab(f, string(b))) + } + b, err := os.ReadFile(sidecarPath) + require.NoError(t, err) + side, err := ParseSidecar(sidecarPath, string(b)) + require.NoError(t, err) + return codelabs, side +} + +// The drift guard. A codelab edit that introduces a block nothing can classify, or that changes +// a block the sidecar made a decision about, fails here on Linux rather than silently changing +// what the Windows job checks. +func TestEveryBlockIsDecided(t *testing.T) { + codelabs, side := loadReal(t) + _, errs := BuildPlan(codelabs, side) + for _, err := range errs { + t.Errorf("%s", err) + } +} + +func TestCensusMatchesGolden(t *testing.T) { + codelabs, side := loadReal(t) + want, err := os.ReadFile(censusPath) + require.NoError(t, err) + assert.Equal(t, string(want), Census(codelabs, side), + "the census has changed; if the codelab edit that changed it is intended, regenerate %s with --format summary and review the diff", censusPath) +} + +// The Linux half of the shrink-only rule: an entry naming a step that no longer exists fails +// here. The Windows half, an entry whose step passed, is in run_codelabs.ps1. +func TestKnownFailuresNameRealSteps(t *testing.T) { + codelabs, side := loadReal(t) + plan, errs := BuildPlan(codelabs, side) + require.Empty(t, errs) + + names := map[string]bool{} + runnable := map[string]bool{} + for _, c := range plan.Codelabs { + names[c.ID] = true + for _, s := range c.Steps { + names[s.Key] = true + runnable[s.Key] = s.Kind == "run" + } + } + b, err := os.ReadFile(knownFailuresPath) + require.NoError(t, err) + for i, line := range strings.Split(string(b), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if !names[line] { + t.Errorf("%s:%d: %s names no codelab or step", knownFailuresPath, i+1, line) + } else if strings.Contains(line, "::") && !runnable[line] { + t.Errorf("%s:%d: %s is not a step that runs, so it can neither fail nor pass", knownFailuresPath, i+1, line) + } + } +} + +func parseOne(t *testing.T, md string) Codelab { + t.Helper() + return ParseCodelab("test.md", "id: test\nsummary: Test\nstatus: Published\n\n"+md) +} + +func mustSidecar(t *testing.T, conf string) *Sidecar { + t.Helper() + side, err := ParseSidecar("test.conf", conf) + require.NoError(t, err) + return side +} + +func TestFrontMatterAndKeys(t *testing.T) { + c := parseOne(t, "## Hello, world!\n```bash\nplz init\n```\n\n```bash\nplz build\n```\n## Next step\n```bash\nplz test\n```\n") + assert.Equal(t, "test", c.ID) + assert.Equal(t, "Test", c.Title) + require.Len(t, c.Blocks, 3) + assert.Equal(t, "test::hello-world/b1", Key(c, c.Blocks[0])) + assert.Equal(t, "test::hello-world/b2", Key(c, c.Blocks[1])) + // A new section restarts the ordinal, so an edit in one section renumbers no other. + assert.Equal(t, "test::next-step/b1", Key(c, c.Blocks[2])) +} + +func TestClassify(t *testing.T) { + cases := []struct { + name, md string + kind Kind + path string + }{ + {"file heading", "### `src/BUILD`\n```python\ngo_binary()\n```", KindFile, "src/BUILD"}, + {"file in prose", "Create a file `hello_service/service.go`:\n\n```golang\npackage main\n```", KindFile, "hello_service/service.go"}, + {"bare BUILD in prose", "Add a filegroup at `BUILD` in repo root:\n```python\nfilegroup()\n```", KindFile, "BUILD"}, + {"commands", "Run:\n```bash\nplz init\n```", KindCommand, ""}, + {"inline env prefix", "Like so:\n```bash\nGODEBUG=\"installgoroot=all\" go install std\n```", KindCommand, ""}, + {"transcript", "```\n$ plz build //:x\nBuild finished\n```", KindTranscript, ""}, + {"output in a bash fence", "The output should look like this:\n```bash\n.\n├── pleasew\n```", KindIllustration, ""}, + // Each of these is a line in the codelabs that would otherwise be a file to create. + {"absolute path", "if Go is at `/opt/homebrew/bin/go`:\n```ini\n[Build]\n```", KindUnclassified, ""}, + {"module path", "Let's add `github.com/stretchr/testify`:\n```text\ngo_repo()\n```", KindUnclassified, ""}, + {"no rule", "By default:\n```\n/usr/local/bin:/usr/bin:/bin\n```", KindUnclassified, ""}, + // A sentence naming a file is often followed by the command that makes it. + {"command beats prose", "Sync the changes to `third_party/go/BUILD`:\n```bash\nplz puku sync -w\n```", KindCommand, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := parseOne(t, "## S\n"+tc.md+"\n") + require.Len(t, c.Blocks, 1) + kind, path, err := Classify(c, c.Blocks[0], Key(c, c.Blocks[0]), mustSidecar(t, "")) + require.NoError(t, err) + assert.Equal(t, tc.kind, kind) + assert.Equal(t, tc.path, path) + }) + } +} + +func TestUnclassifiedIsAnError(t *testing.T) { + c := parseOne(t, "## S\nBy default:\n```\n/usr/local/bin\n```\n") + _, errs := BuildPlan([]Codelab{c}, mustSidecar(t, "")) + require.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), "test.md:7") + assert.Contains(t, errs[0].Error(), "[test::s/b1]") +} + +func TestCommandsSplitChainsIntoChdir(t *testing.T) { + c := parseOne(t, "## S\n```bash\nmkdir x && cd x\nplz init\ntree -a\n```\n") + plan, errs := BuildPlan([]Codelab{c}, mustSidecar(t, "")) + require.Empty(t, errs) + steps := plan.Codelabs[0].Steps + require.Len(t, steps, 4) + assert.Equal(t, Step{Key: "test::s/b1.1", Kind: "run", Line: 6, Section: "S", Command: "mkdir x"}, steps[0]) + assert.Equal(t, "chdir", steps[1].Kind) + assert.Equal(t, "x", steps[1].Dir) + assert.False(t, steps[2].NonBlocking) + // Only displays something, so its failure blocks nothing after it. + assert.True(t, steps[3].NonBlocking) +} + +func TestTranscriptOutputBelongsToItsCommand(t *testing.T) { + c := parseOne(t, "## S\n```\n$ plz build //:x\nBuild finished\n\n$ cat plz-out/gen/x\nhello\n```\n") + plan, errs := BuildPlan([]Codelab{c}, mustSidecar(t, "")) + require.Empty(t, errs) + steps := plan.Codelabs[0].Steps + require.Len(t, steps, 2) + assert.Equal(t, "plz build //:x", steps[0].Command) + assert.Equal(t, []string{"Build finished"}, steps[0].ExpectedOutput) + assert.Equal(t, []string{"hello"}, steps[1].ExpectedOutput) + // Advisory: nothing promoted it to an assertion. + assert.Empty(t, steps[1].Assert) +} + +func TestSidecarRequiresAReason(t *testing.T) { + _, err := ParseSidecar("test.conf", "[test::s/b1]\nkind = illustration\n") + require.Error(t, err) + assert.Contains(t, err.Error(), "no reason") + + // A blank line between the reason and the stanza detaches it. + _, err = ParseSidecar("test.conf", "; why\n\n[test::s/b1]\nkind = illustration\n") + assert.Error(t, err) +} + +func TestSidecarDecisions(t *testing.T) { + c := parseOne(t, "## S\n### `.plzconfig`\n```\n[Plugin \"go\"]\n```\n\n```bash\nplz build\nplz-out/bin/main.pex\n```\n\n```bash\neval $(minikube docker-env)\n```\n") + side := mustSidecar(t, `; a fragment +[test::s/b1] +matches = .plzconfig +mode = merge + +; nothing after depends on it +[test::s/b2.2] +matches = plz-out/bin/main.pex +blocking = false + +; bash only +[test::s/b3] +matches = eval $(minikube docker-env) +skip = unix-shell +`) + plan, errs := BuildPlan([]Codelab{c}, side) + require.Empty(t, errs) + steps := plan.Codelabs[0].Steps + require.Len(t, steps, 4) + assert.Equal(t, "merge", steps[0].Mode) + assert.False(t, steps[1].NonBlocking) + assert.True(t, steps[2].NonBlocking) + assert.Equal(t, "skip", steps[3].Kind) + assert.Equal(t, "unix-shell", steps[3].Reason) + assert.Equal(t, "bash only", steps[3].Detail) +} + +func TestSidecarMatchesIsEnforced(t *testing.T) { + c := parseOne(t, "## S\n```bash\nplz build //:new\n```\n") + side := mustSidecar(t, "; decided about the old command\n[test::s/b1]\nmatches = plz build //:old\nskip = placeholder\n") + _, errs := BuildPlan([]Codelab{c}, side) + require.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), `"plz build //:old"`) +} + +func TestSidecarStanzaNamingNothingIsAnError(t *testing.T) { + c := parseOne(t, "## S\n```bash\nplz build\n```\n") + side := mustSidecar(t, "; about a block since deleted\n[test::s/b9]\nkind = illustration\n") + _, errs := BuildPlan([]Codelab{c}, side) + require.Len(t, errs, 1) + assert.Contains(t, errs[0].Error(), "test::s/b9 names no block") +} + +func TestNotRunnableCodelab(t *testing.T) { + c := parseOne(t, "## S\n```yaml\nname: CI\n```\n") + side := mustSidecar(t, "; all YAML\n[test]\nnot-runnable = no local commands\n") + plan, errs := BuildPlan([]Codelab{c}, side) + require.Empty(t, errs) + assert.Equal(t, "no local commands", plan.Codelabs[0].NotRunnable) + assert.Empty(t, plan.Codelabs[0].Steps) + assert.Equal(t, 1, plan.Codelabs[0].Blocks["illustration"]) +} diff --git a/test/windows/codelab_script/script/sidecar.go b/test/windows/codelab_script/script/sidecar.go new file mode 100644 index 0000000000..87fd05a66b --- /dev/null +++ b/test/windows/codelab_script/script/sidecar.go @@ -0,0 +1,197 @@ +package script + +import ( + "fmt" + "strconv" + "strings" +) + +// Sidecar is test/windows/codelab_steps.conf: the decisions about what the codelabs mean that the +// Markdown cannot express, kept out of the prose because the codelabs are documentation and have +// to read as documentation. +// +// Every stanza carries a reason, enforced rather than encouraged: a stanza with no comment above it +// is a parse error. This file is read later by whoever decides what to do about the codelabs that +// cannot work on Windows, and a bare directive would tell them nothing. +type Sidecar struct { + Entries map[string]*Entry + // Order the stanzas appeared in, for stable reporting. + Order []string +} + +// Entry is one stanza. +type Entry struct { + Key string + // The comment above the stanza. Required. + Reason string + // The command or path this stanza expects to find at Key. Extraction fails if the block + // there no longer says this, so an edit to a codelab cannot silently move a decision onto + // a different block. This is the drift guard; the keys themselves are readable, not hashed. + Matches string + // "file:", "command", "transcript", "illustration" or "ignore". Overrides the + // heuristics outright. + KindDirective string + // "write" (the default) or "merge", for a file. Several codelabs show a .plzconfig under a + // heading that names the whole file when what they mean is a fragment to add to what + // plz init already wrote. Writing those verbatim drops the earlier keys and manufactures a + // failure that has nothing to do with Windows. + Mode string + // A reason class saying this step cannot run here at all, e.g. "unix-shell". + Skip string + // Tools the step needs, checked on the machine at run time: docker, kubectl, minikube, + // network, github-api, interactive. + Needs []string + // A substring that must appear in the output, promoting one line from advisory to required. + Assert string + // Seconds; 0 means the runner's default. + Timeout int + // Codelab-level: this codelab has nothing to run, with this as the reason. + NotRunnable string + // Set by "blocking = false": a failure here does not mark the rest of the codelab blocked. + // For a step nothing later depends on, so that a bash-only line does not hide every + // finding after it. + NonBlocking bool + // Set when something resolved this stanza against a real block, so ParseSidecar's caller can + // report the ones that matched nothing. + Resolved bool +} + +// Kind returns the kind this stanza forces, if any, and the path for a file. +func (e *Entry) Kind(b Block) (Kind, string, error) { + if e.KindDirective == "" { + return "", "", nil + } + directive, path, hasPath := strings.Cut(e.KindDirective, ":") + switch Kind(directive) { + case KindFile: + if !hasPath || path == "" { + return "", "", fmt.Errorf(`kind = file needs a path, as "file:src/BUILD"`) + } + return KindFile, path, nil + case KindCommand, KindTranscript, KindIllustration, KindIgnore: + if hasPath { + return "", "", fmt.Errorf("kind = %s takes no path", directive) + } + return Kind(directive), "", nil + } + return "", "", fmt.Errorf("unknown kind %q", e.KindDirective) +} + +// Lookup finds the stanza for a key and marks it resolved. +func (s *Sidecar) Lookup(key string) (*Entry, bool) { + e, ok := s.Entries[key] + if ok { + e.Resolved = true + } + return e, ok +} + +// Unresolved lists stanzas that matched no block, in file order. A stanza that names nothing is +// a decision about a codelab that has since been edited, and is reported rather than ignored. +func (s *Sidecar) Unresolved() []string { + var out []string + for _, key := range s.Order { + if !s.Entries[key].Resolved { + out = append(out, key) + } + } + return out +} + +// ParseSidecar reads codelab_steps.conf. +// +// .plzconfig-flavoured: ";" comments, "[stanza]" headers, "key = value" directives. That is this +// repo's idiom for a file a person maintains by hand, and it puts the reason on the line above the +// decision where it belongs. +func ParseSidecar(filename, content string) (*Sidecar, error) { + side := &Sidecar{Entries: map[string]*Entry{}} + var reason []string + var current *Entry + + for i, line := range strings.Split(content, "\n") { + lineno := i + 1 + trimmed := strings.TrimSpace(line) + + if trimmed == "" { + // A blank line separates the file's own header from the first stanza, and one + // stanza from the next. It also discards a comment, so that a reason cannot + // drift away from what it explains. + reason = nil + continue + } + if strings.HasPrefix(trimmed, ";") || strings.HasPrefix(trimmed, "#") { + reason = append(reason, strings.TrimSpace(strings.TrimLeft(trimmed, ";# "))) + continue + } + if strings.HasPrefix(trimmed, "[") && strings.HasSuffix(trimmed, "]") { + key := strings.TrimSpace(trimmed[1 : len(trimmed)-1]) + if key == "" { + return nil, fmt.Errorf("%s:%d: empty stanza name", filename, lineno) + } + if _, dup := side.Entries[key]; dup { + return nil, fmt.Errorf("%s:%d: %s appears twice", filename, lineno, key) + } + if len(reason) == 0 { + return nil, fmt.Errorf("%s:%d: %s has no reason above it; every stanza here needs one, because this file is what the decision about the codelabs will be taken from", filename, lineno, key) + } + current = &Entry{Key: key, Reason: strings.Join(reason, " ")} + side.Entries[key] = current + side.Order = append(side.Order, key) + reason = nil + continue + } + + name, value, ok := strings.Cut(trimmed, "=") + if !ok { + return nil, fmt.Errorf("%s:%d: expected a stanza, a comment, or 'name = value'", filename, lineno) + } + if current == nil { + return nil, fmt.Errorf("%s:%d: %s appears before any stanza", filename, lineno, strings.TrimSpace(name)) + } + if err := current.set(strings.TrimSpace(name), strings.TrimSpace(value)); err != nil { + return nil, fmt.Errorf("%s:%d: %s: %w", filename, lineno, current.Key, err) + } + } + return side, nil +} + +func (e *Entry) set(name, value string) error { + switch strings.ToLower(name) { + case "matches": + e.Matches = value + case "kind": + e.KindDirective = value + case "mode": + if value != "write" && value != "merge" { + return fmt.Errorf("mode is write or merge, not %q", value) + } + e.Mode = value + case "skip": + e.Skip = value + case "needs": + for _, need := range strings.Split(value, ",") { + if need = strings.TrimSpace(need); need != "" { + e.Needs = append(e.Needs, need) + } + } + case "assert": + e.Assert = value + case "timeout": + n, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("timeout is a number of seconds: %w", err) + } + e.Timeout = n + case "not-runnable": + e.NotRunnable = value + case "blocking": + b, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("blocking is true or false: %w", err) + } + e.NonBlocking = !b + default: + return fmt.Errorf("unknown directive %q", name) + } + return nil +} diff --git a/test/windows/codelab_script/script/test_data/census.txt b/test/windows/codelab_script/script/test_data/census.txt new file mode 100644 index 0000000000..c14e150b38 --- /dev/null +++ b/test/windows/codelab_script/script/test_data/census.txt @@ -0,0 +1,167 @@ +genrule::genrule/b1 command mkdir custom_rules && cd custom_rules +genrule::genrule/b2 file BUILD +genrule::genrule/b3 command echo "the quick brown fox jumped over the lazy dog" > file.t... +genrule::genrule/b4 transcript $ plz build //:word_count +genrule::the-build-directory/b1 illustration $ plz build --shell :word_count +genrule::the-build-directory/b2 illustration bash-4.4$ printenv +genrule::build-definitions/b1 file build_defs/word_count.build_defs +genrule::build-definitions/b2 file build_defs/BUILD +genrule::build-definitions/b3 file BUILD +genrule::build-definitions/b4 command plz build //:word_count +genrule::build-definitions/b5 illustration Build finished; total time 30ms, incrementality 100.0%. Outp... +genrule::managing-tools/b1 file build_defs/word_count.build_defs +genrule::managing-tools/b2 file tools/wc.sh +genrule::managing-tools/b3 file tools/BUILD +genrule::managing-tools/b4 transcript $ plz run //tools:wc -- file.txt +genrule::managing-tools/b5 file BUILD +genrule::managing-tools/b6 transcript $ plz build //:lines_words_and_chars //:just_words +genrule::configuration/b1 file .plzconfig +genrule::configuration/b2 file build_defs/word_count.build_defs +genrule::configuration/b3 file BUILD +genrule::configuration/b4 transcript $ plz build //:lines_words_and_chars //:just_words +github_actions::github-actions/b1 illustration name: CI +github_actions::please-build/b1 illustration name: CI +github_actions::setup-please-action/b1 illustration - name: Test +github_actions::setup-please-action/b2 illustration - name: Checkout code +github_actions::setup-please-action/b3 illustration name: CI +go_intro::initialising-your-project/b1 command mkdir getting_started_go && cd getting_started_go +go_intro::initialising-your-project/b2 transcript $ tree -a +go_intro::initialising-your-project/b3 illustration [parse] +go_intro::setting-up-our-import-path/b1 file .plzconfig +go_intro::setting-up-your-toolchain/b1 file third_party/go/BUILD +go_intro::setting-up-your-toolchain/b2 file .plzconfig +go_intro::setting-up-your-toolchain/b3 illustration /usr/local/bin:/usr/bin:/bin +go_intro::setting-up-your-toolchain/b4 ignore [Build] +go_intro::setting-up-your-toolchain/b5 command GODEBUG="installgoroot=all" go install std +go_intro::hello-world/b1 file src/main.go +go_intro::hello-world/b2 file src/BUILD +go_intro::hello-world/b3 command plz run //src:main +go_intro::hello-world/b4 illustration Hello, world! +go_intro::adding-packages/b1 file src/greetings/greetings.go +go_intro::adding-packages/b2 file src/greetings/BUILD +go_intro::adding-packages/b3 command plz build //src/greetings +go_intro::adding-packages/b4 illustration Build finished; total time 290ms, incrementality 50.0%. Outp... +go_intro::using-our-new-package/b1 file src/BUILD +go_intro::using-our-new-package/b2 file src/main.go +go_intro::using-our-new-package/b3 transcript $ plz run //src:main +go_intro::testing-our-code/b1 file src/greetings/greetings_test.go +go_intro::testing-our-code/b2 file src/greetings/BUILD +go_intro::testing-our-code/b3 transcript $ plz test //src/... +go_intro::testing-our-code/b4 file src/greetings/greetings_test.go +go_intro::testing-our-code/b5 file src/greetings/BUILD +go_intro::testing-our-code/b6 transcript $ plz test //src/... +go_intro::third-party-dependencies/b1 transcript $ plz run ///go//tools:please_go -- get github.com/stretchr/... +go_intro::third-party-dependencies/b2 file third_party/go/BUILD +go_intro::third-party-dependencies/b3 file src/greetings/greetings_test.go +go_intro::third-party-dependencies/b4 file src/greetings/BUILD +go_intro::third-party-dependencies/b5 transcript $ plz test +k8s::creating-a-service/b1 command plz init +k8s::creating-a-service/b2 file third_party/go/BUILD +k8s::creating-a-service/b3 file .plzconfig +k8s::creating-a-service/b4 file hello_service/service.go +k8s::creating-a-service/b5 file hello_service/BUILD +k8s::creating-a-service/b6 command plz run //hello_service:hello_service && curl localhost:8000 +k8s::creating-a-service/b7 illustration [1] 28694 +k8s::building-a-docker-image/b1 file common/docker/Dockerfile-base +k8s::building-a-docker-image/b2 command plz init plugin shell && plz init plugin docker +k8s::building-a-docker-image/b3 file common/docker/BUILD +k8s::building-a-docker-image/b4 transcript $ plz build //common/docker:base +k8s::building-a-docker-image/b5 transcript $ cat plz-out/bin/common/docker/base.sh +k8s::building-a-docker-image/b6 command plz run //common/docker:base +k8s::using-our-base-image/b1 file hello_service/k8s/Dockerfile +k8s::using-our-base-image/b2 file hello_service/k8s/BUILD +k8s::using-our-base-image/b3 transcript $ plz build //hello_service/k8s:image +k8s::creating-a-kubernetes-deployment/b1 file hello_service/k8s/deployment.yaml +k8s::creating-a-kubernetes-deployment/b2 file hello_service/k8s/service.yaml +k8s::creating-a-kubernetes-deployment/b3 file hello_service/k8s/BUILD +k8s::creating-a-kubernetes-deployment/b4 transcript $ plz build //hello_service/k8s +k8s::creating-a-kubernetes-deployment/b5 transcript $ plz build //hello_service/k8s:k8s_push +k8s::local-testing-with-minikube/b1 file third_party/binary/BUILD +k8s::local-testing-with-minikube/b2 command plz run //third_party/binary:minikube -- start +k8s::local-testing-with-minikube/b3 command eval $(plz run //third_party/binary:minikube -- docker-env) +k8s::local-testing-with-minikube/b4 command plz run //hello_service/k8s:image_load && plz run //hello_se... +k8s::local-testing-with-minikube/b5 transcript $ kubectl port-forward service/hello-svc 8000:8000 && curl l... +k8s::please-deploy/b1 command plz run sequential --include docker-build --include k8s-push... +k8s::please-deploy/b2 file .plzconfig +k8s::please-deploy/b3 command plz deploy //hello_service/... +k8s::docker-build-and-build-systems/b1 illustration docker_image( +plz_query::setting-up/b1 command git clone https://github.com/thought-machine/please-codelabs +plz_query::setting-up/b2 illustration Cloning into 'please-codelabs'... +plz_query::setting-up/b3 command cd please-codelabs/getting_started_go +plz_query::finding-dependencies-of-a-target/b1 transcript $ plz query deps //src/greetings:greetings_test +plz_query::finding-dependencies-of-a-target/b2 command plz query print ///third_party/go/github.com_stretchr_testif... +plz_query::finding-dependencies-of-a-target/b3 transcript $ cat plz-out/subrepos/third_party/go/github.com_stretchr_te... +plz_query::finding-dependencies-of-a-target/b4 transcript $ plz query deps //src/greetings:greetings --hidden +plz_query::finding-dependencies-of-a-target/b5 transcript $ plz query print //src/greetings:_greetings#srcs +plz_query::reverse-dependencies/b1 transcript $ plz query revdeps ///third_party/go/github.com_stretchr_te... +plz_query::reverse-dependencies/b2 transcript $ plz query revdeps ///third_party/go/github.com_stretchr_te... +plz_query::composing-plz-commands/b1 transcript $ plz query revdeps ///third_party/go/github.com_stretchr_te... +plz_query::including-and-excluding-targets/b1 transcript $ plz query revdeps --exclude //src/greetings:greetings_test... +plz_query::including-and-excluding-targets/b2 transcript $ plz query revdeps --level=-1 ///third_party/go/github.com_... +plz_query::including-and-excluding-targets/b3 command plz build --include go --exclude //third_party/go/... +plz_query::including-and-excluding-targets/b4 file src/greetings/BUILD +plz_query::including-and-excluding-targets/b5 transcript $ plz query alltargets --include=my_label +plz_query::including-and-excluding-targets/b6 command plz test --exclude my_label +puku::initialising-your-project-and-running-puku-with-please/b1 command mkdir puku_sync && cd puku_sync +puku::initialising-your-project-and-running-puku-with-please/b2 file .plzconfig +puku::initialising-your-project-and-running-puku-with-please/b3 ignore [please] +puku::initialising-your-project-and-running-puku-with-please/b4 file .plzconfig +puku::initialising-your-project-and-running-puku-with-please/b5 file third_party/binary/BUILD +puku::initialising-your-project-and-running-puku-with-please/b6 file BUILD +puku::initialising-your-project-and-running-puku-with-please/b7 file .plzconfig +puku::initialising-your-project-and-running-puku-with-please/b8 illustration /usr/local/bin:/usr/bin:/bin +puku::initialising-your-project-and-running-puku-with-please/b9 command which go +puku::initialising-your-project-and-running-puku-with-please/b10 ignore [Build] +puku::initialising-your-project-and-running-puku-with-please/b11 ignore [Build] +puku::initialising-your-project-and-running-puku-with-please/b12 command GODEBUG="installgoroot=all" go install std +puku::adding-and-updating-modules/b1 file src/hello/hello.go +puku::adding-and-updating-modules/b2 command go get github.com/google/uuid +puku::adding-and-updating-modules/b3 command plz puku sync -w +puku::adding-and-updating-modules/b4 file src/hello/BUILD +puku::adding-and-updating-modules/b5 command plz puku fmt //src/hello +puku::adding-and-updating-modules/b6 command plz run //src/hello +puku::adding-and-updating-modules/b7 command GOTOOLCHAIN=local go get github.com/google/uuid@v1.6.0 +puku::adding-and-updating-modules/b8 command GOTOOLCHAIN=local go get -u github.com/google/uuid +puku::adding-and-updating-modules/b9 command go get +puku::adding-and-updating-modules/b10 command go get +puku::stop-a-module-from-updating/b1 command go mod edit -exclude github.com/example/module@v2.0.0 +puku::stop-a-module-from-updating/b2 command go mod edit -dropexclude github.com/example/module@v2.0.0 +puku::stop-a-module-from-updating/b3 command go mod edit -replace github.com/example/module=github.com/ex... +puku::stop-a-module-from-updating/b4 command go mod edit -dropreplace github.com/example/module +puku::stop-a-module-from-updating/b5 command go mod edit -replace github.com/google/uuid=github.com/googl... +puku::removing-modules/b1 command plz query revdeps //third_party/go:module_name --level=-1 | ... +puku::removing-modules/b2 command go mod edit -droprequire github.com/example/module +puku::removing-modules/b3 command plz puku sync -w +puku::using-new-modules/b1 illustration go_library( +puku::using-new-modules/b2 illustration go_repo( +puku::using-new-modules/b3 illustration go_library( +puku::using-new-modules/b4 command plz puku watch //src/... +python_intro::initialising-your-project/b1 command mkdir getting_started_python && cd getting_started_python +python_intro::initialising-your-project/b2 ignore [build] +python_intro::initialising-your-project/b3 transcript $ tree -a +python_intro::initialising-your-project/b4 illustration [parse] +python_intro::hello-world/b1 file src/main.py +python_intro::hello-world/b2 file src/BUILD +python_intro::hello-world/b3 transcript $ plz run //src:main +python_intro::adding-modules/b1 file src/greetings/greetings.py +python_intro::adding-modules/b2 file src/greetings/BUILD +python_intro::adding-modules/b3 transcript $ plz build //src/greetings +python_intro::adding-modules/b4 transcript $ plz build //src:main +python_intro::using-our-new-module/b1 file src/BUILD +python_intro::using-our-new-module/b2 file src/main.py +python_intro::using-our-new-module/b3 transcript $ plz run //src:main +python_intro::testing-our-code/b1 file src/greetings/greetings_test.py +python_intro::testing-our-code/b2 file src/greetings/BUILD +python_intro::testing-our-code/b3 transcript $ plz test //src/... +python_intro::third-party-dependencies/b1 file third_party/python/BUILD +python_intro::third-party-dependencies/b2 file .plzconfig +python_intro::third-party-dependencies/b3 ignore [plugin "python"] +python_intro::third-party-dependencies/b4 file src/greetings/greetings.py +python_intro::third-party-dependencies/b5 file src/greetings/BUILD +python_intro::third-party-dependencies/b6 transcript $ plz run //src:main +using_plugins::initialising-your-please-repo/b1 command plz init +using_plugins::initialising-your-please-repo/b2 illustration . +using_plugins::how-to-install-a-plugin/b1 command plz init plugin go +using_plugins::how-to-install-a-plugin/b2 illustration . +using_plugins::how-to-install-a-plugin/b3 illustration [parse] +using_plugins::how-to-install-a-plugin/b4 illustration plugin_repo( diff --git a/test/windows/codelab_steps.conf b/test/windows/codelab_steps.conf new file mode 100644 index 0000000000..39d33dda41 --- /dev/null +++ b/test/windows/codelab_steps.conf @@ -0,0 +1,322 @@ +; Decisions about what the codelabs mean, for //test/windows/codelab_script, which reduces +; docs/codelabs/*.md to a plan that run_codelabs.ps1 replays on a real Windows machine. +; +; Kept here rather than in the prose, because the codelabs are documentation and have to read as +; documentation. Nothing in this file edits a codelab or makes one pass: it says what a block is +; where the Markdown cannot, and which steps cannot run on a CI machine at all. +; +; Every stanza needs a comment directly above it saying why; the parser refuses one without. +; "matches" pins the text the decision was made about, so an edit to a codelab fails extraction +; on Linux instead of quietly moving a decision onto a different block. Keys are printed by: +; +; plz run //test/windows/codelab_script -- --sidecar test/windows/codelab_steps.conf \ +; --format summary docs/codelabs/*.md +; +; Directives: matches, kind (file: | command | transcript | illustration | ignore), +; mode (write | merge), skip (a reason class), needs (docker, kubectl, minikube, ...), +; blocking (false: a failure here does not block the rest), assert, timeout, not-runnable. +; +; Merging a .plzconfig fragment edits the file the way a reader would: a key the section already +; has is replaced, a new key is added to its section, and a new section is appended. Appending the +; fragment verbatim was tried first and manufactured a failure on the first native run, because +; plz init plugin go already writes GoTool and a plugin section refuses a repeated key. + +; ---- github_actions ----------------------------------------------------------------------- + +; Teaches CI configuration, and every block is the YAML of a workflow. There is nothing in it +; anyone runs locally, on any OS. +[github_actions] +not-runnable = no local commands; every block is GitHub Actions YAML + +; ---- genrule ------------------------------------------------------------------------------ + +; A walk-through of an interactive `plz build --shell` session, bash-4.4$ prompts included. It +; shows what the reader will see after typing, not something a script can replay. +[genrule::the-build-directory/b1] +matches = $ plz build --shell :word_count +kind = illustration + +; The printenv output from inside that same interactive shell. +[genrule::the-build-directory/b2] +matches = bash-4.4$ printenv +kind = illustration + +; A [buildconfig] section to add to the .plzconfig plz init wrote, not the whole file. Written +; verbatim it would drop everything plz init put there. +[genrule::configuration/b1] +matches = .plzconfig +mode = merge + +; ---- go_intro ----------------------------------------------------------------------------- + +; Shows what plz init plugin go has already written ("Please will have initialised this ... for +; us"). Writing it again would replace the generated file with a copy that differs from it. +[go_intro::initialising-your-project/b3] +matches = [parse] +kind = illustration + +; Adds ImportPath to the plugin section plz init plugin go wrote. +[go_intro::setting-up-our-import-path/b1] +matches = .plzconfig +mode = merge + +; Adds GoTool to the same section, for the managed toolchain the codelab recommends. +[go_intro::setting-up-your-toolchain/b2] +matches = .plzconfig +mode = merge + +; The default [build] path, shown for information. +[go_intro::setting-up-your-toolchain/b3] +matches = /usr/local/bin:/usr/bin:/bin +kind = illustration + +; The alternative to the managed toolchain: Go from the system PATH. The codelab offers the two +; as a choice and recommends the other, which is the one followed here. Its example is a +; colon-separated Unix path with no Windows form given, which is worth knowing when the docs +; decision is taken; on Windows the default [build] path is empty. +[go_intro::setting-up-your-toolchain/b4] +matches = [Build] +kind = ignore + +; Part of the same system-PATH alternative, not the recommended route. Also bash syntax: an +; inline environment prefix is a command name to PowerShell. +[go_intro::setting-up-your-toolchain/b5] +matches = GODEBUG="installgoroot=all" go install std +skip = alternative-route + +; ---- python_intro ------------------------------------------------------------------------- + +; Conditional ("If Python isn't in this path") and a template rather than a value: +; $YOUR_PYTHON_INSTALL_HERE followed by a colon-separated Unix path. There is no Windows form to +; follow. On Windows the default [build] path is empty, so whatever this costs shows up in the +; steps that need an interpreter, and is recorded there. +[python_intro::initialising-your-project/b2] +matches = [build] +kind = ignore + +; Shows what plz init plugin python has already written. +[python_intro::initialising-your-project/b4] +matches = [parse] +kind = illustration + +; Runs the built .pex directly, which depends on a shebang and an executable bit. Nothing later +; in the codelab depends on it, so a failure here is recorded without blocking what follows. +[python_intro::adding-modules/b4.2] +matches = plz-out/bin/src/main.pex +blocking = false + +; Adds ModuleDir to the plugin section plz init plugin python wrote. +[python_intro::third-party-dependencies/b2] +matches = .plzconfig +mode = merge + +; Conditional: only "if you encounter an error eg. no such option: --system". Not an instruction +; to follow unconditionally. +[python_intro::third-party-dependencies/b3] +matches = [plugin "python"] +kind = ignore + +; ---- puku --------------------------------------------------------------------------------- + +; A [buildconfig] entry to add to the .plzconfig plz init wrote. +[puku::initialising-your-project-and-running-puku-with-please/b2] +matches = .plzconfig +mode = merge + +; "Uncomment and edit the following lines" - an edit with no verbatim reading. Carried out, it +; pins Please 17.22.0, so please.exe would try to replace itself with an upstream release that +; has no Windows build, and the check would stop testing the binary it was given. +[puku::initialising-your-project-and-running-puku-with-please/b3] +matches = [please] +kind = ignore + +; The Puku alias every later step uses as `plz puku`. Introduced as "optional but convenient", +; so no heading names the file; it is a fragment for .plzconfig. +[puku::initialising-your-project-and-running-puku-with-please/b4] +matches = [Alias "puku"] +kind = file:.plzconfig +mode = merge + +; Adds ModFile to the plugin section plz init plugin go wrote. +[puku::initialising-your-project-and-running-puku-with-please/b7] +matches = .plzconfig +mode = merge + +; The default [build] path, shown for information. +[puku::initialising-your-project-and-running-puku-with-please/b8] +matches = /usr/local/bin:/usr/bin:/bin +kind = illustration + +; An example [build] path for Go installed by Homebrew on macOS. Conditional, and Unix-only; the +; codelab's only Windows note is to use where.exe to find Go, with no path form to put here. +[puku::initialising-your-project-and-running-puku-with-please/b10] +matches = [Build] +kind = ignore + +; The same example for Go under /usr/local/go. +[puku::initialising-your-project-and-running-puku-with-please/b11] +matches = [Build] +kind = ignore + +; Bash syntax: an inline environment prefix is a command name to PowerShell. Later steps need +; the standard library this installs, but blocking on it would hide every other finding in the +; codelab behind a line whose failure is already understood. +[puku::initialising-your-project-and-running-puku-with-please/b12] +matches = GODEBUG="installgoroot=all" go install std +blocking = false + +; Bash syntax, as above. The sync after it still runs. +[puku::adding-and-updating-modules/b7.1] +matches = GOTOOLCHAIN=local go get github.com/google/uuid@v1.6.0 +blocking = false + +; Bash syntax, as above. +[puku::adding-and-updating-modules/b8.1] +matches = GOTOOLCHAIN=local go get -u github.com/google/uuid +blocking = false + +; Troubleshooting advice with as a placeholder. Not something to run as written. +[puku::adding-and-updating-modules/b9] +matches = go get +skip = placeholder + +; The same advice for a different error message. +[puku::adding-and-updating-modules/b10] +matches = go get +skip = placeholder + +; github.com/example/module is a stand-in, not a real module, and `go get -u` on it two blocks +; later fails on every platform. The four blocks demonstrate go.mod directives on it and are +; skipped together; the real scenario with google/uuid that follows them runs. +[puku::stop-a-module-from-updating/b1] +matches = go mod edit -exclude github.com/example/module@v2.0.0 +skip = placeholder + +; As above. +[puku::stop-a-module-from-updating/b2] +matches = go mod edit -dropexclude github.com/example/module@v2.0.0 +skip = placeholder + +; As above. +[puku::stop-a-module-from-updating/b3] +matches = go mod edit -replace github.com/example/module=github.com/example/module@v1.5.0 +skip = placeholder + +; As above. +[puku::stop-a-module-from-updating/b4] +matches = go mod edit -dropreplace github.com/example/module +skip = placeholder + +; //third_party/go:module_name is a placeholder, as the prose around it says. +[puku::removing-modules/b1] +matches = plz query revdeps //third_party/go:module_name --level=-1 | grep -v //third_party/go +skip = placeholder + +; github.com/example/module again: dropping a requirement that was never added. +[puku::removing-modules/b2] +matches = go mod edit -droprequire github.com/example/module +skip = placeholder + +; Examples of how a BUILD file could look, for a package (mylib) the codelab never creates. +[puku::using-new-modules/b1] +matches = go_library( +kind = illustration + +; As above: an alternative form of go_repo, not the one the codelab built. +[puku::using-new-modules/b2] +matches = go_repo( +kind = illustration + +; As above. +[puku::using-new-modules/b3] +matches = go_library( +kind = illustration + +; Watches the tree until interrupted. It never exits by itself, on any platform. +[puku::using-new-modules/b4] +matches = plz puku watch //src/... +skip = interactive + +; ---- k8s ---------------------------------------------------------------------------------- + +; "Add a go toolchain to `third_party/go/BUILD`" - no trailing colon, so the prose rule does not +; read it, and the fence says go when the content is a BUILD file. +[k8s::creating-a-service/b2] +matches = go_toolchain( +kind = file:third_party/go/BUILD + +; "And configure the plugin:" - a fragment for the plugin section plz init plugin go wrote. +[k8s::creating-a-service/b3] +matches = [Plugin "go"] +kind = file:.plzconfig +mode = merge + +; Starts an HTTP server in the foreground, then relies on bash job control and pkill to reach +; and stop it. A non-interactive runner cannot do that on any platform: the first command never +; returns. +[k8s::creating-a-service/b6] +matches = plz run //hello_service:hello_service && curl localhost:8000 +skip = interactive + +; Loads the image into a Docker daemon. +[k8s::building-a-docker-image/b6] +matches = plz run //common/docker:base +needs = docker + +; "Create a `hello_service/k8s/Dockerfile` for our hello service:" wraps onto a second line, so the +; path is not on the line the prose rule reads. A real file the image build below depends on. +[k8s::using-our-base-image/b1] +matches = FROM //common/docker:base +kind = file:hello_service/k8s/Dockerfile + +; Starts a minikube cluster, whose default driver is Docker. +[k8s::local-testing-with-minikube/b2] +matches = plz run //third_party/binary:minikube -- start +needs = docker + +; bash command substitution feeding eval. No PowerShell reading. +[k8s::local-testing-with-minikube/b3] +matches = eval $(plz run //third_party/binary:minikube -- docker-env) +skip = unix-shell + +; Loads the image and applies the manifests to the cluster. +[k8s::local-testing-with-minikube/b4] +needs = docker, kubectl + +; Holds a port-forward open in the foreground and curls through it. +[k8s::local-testing-with-minikube/b5] +matches = kubectl port-forward service/hello-svc 8000:8000 && curl localhost:8000 +skip = interactive + +; Builds and pushes to the cluster. +[k8s::please-deploy/b1] +matches = plz run sequential --include docker-build --include k8s-push //hello_service/... +needs = docker, kubectl + +; The deploy alias, for .plzconfig. +[k8s::please-deploy/b2] +matches = .plzconfig +mode = merge + +; The same deploy, through the alias. +[k8s::please-deploy/b3] +matches = plz deploy //hello_service/... +needs = docker, kubectl + +; An example rule in a discussion section, for a package the codelab never creates. +[k8s::docker-build-and-build-systems/b1] +matches = docker_image( +kind = illustration + +; ---- using_plugins ------------------------------------------------------------------------ + +; Shows what plz init plugin go has already written. +[using_plugins::how-to-install-a-plugin/b3] +matches = [parse] +kind = illustration + +; Shows the plugins/BUILD that was generated, pinned at v1.29.0. Writing it would replace +; whichever revision plz init plugin just chose with an older one. +[using_plugins::how-to-install-a-plugin/b4] +matches = plugin_repo( +kind = illustration diff --git a/test/windows/known_failures.txt b/test/windows/known_failures.txt new file mode 100644 index 0000000000..3b68bd31da --- /dev/null +++ b/test/windows/known_failures.txt @@ -0,0 +1,5 @@ +# Tests that are known to fail on Windows. One "name" or "name::TestCase" per line, +# with a comment above each saying why and linking whatever tracks it. +# +# A test listed here that starts passing fails the job too, so that this list only ever +# shrinks. Nothing is in it yet; the first native run is what fills it in. diff --git a/test/windows/offline_repo/.plzconfig b/test/windows/offline_repo/.plzconfig new file mode 100644 index 0000000000..5f291f6328 --- /dev/null +++ b/test/windows/offline_repo/.plzconfig @@ -0,0 +1,20 @@ +; A repo that asks for a plugin the ordinary way, so that the only thing under test is where +; the plugin comes from. Named BUILD_FILE so the outer repo doesn't parse this as one of its +; own packages. +[parse] +; BUILD_FILE so the outer repo doesn't parse this as one of its own packages, and BUILD as +; well because a subrepo inherits this list: without it the plugin's own build files, which +; are named BUILD, cannot be found once it has been unpacked. Nothing here is called BUILD. +BuildFileName = BUILD_FILE +BuildFileName = BUILD +preloadsubincludes = ///shell//build_defs:shell + +[Plugin "shell"] +Target = //plugins:shell + +; No dir cache. Please's cache is content-addressed and lands outside the test's tmp dir, so +; leaving it on lets a run replay artifacts an earlier, differently-built binary produced - +; which has already caused one false pass during this port. Here it would also let a plugin +; that was once downloaded stand in for the bundled one, which is the whole point. +[cache] +dir = diff --git a/test/windows/offline_repo/BUILD_FILE b/test/windows/offline_repo/BUILD_FILE new file mode 100644 index 0000000000..130126bd5f --- /dev/null +++ b/test/windows/offline_repo/BUILD_FILE @@ -0,0 +1,13 @@ +# An sh_binary rather than a genrule: building one needs the plugin's build defs, the bundled +# arcat to write its payload, and busybox to run the action, so a single target covers the +# whole chain. +sh_binary( + name = "greet", + main = "greet.sh", + deps = [":lib"], +) + +sh_library( + name = "lib", + src = "lib.sh", +) diff --git a/test/windows/offline_repo/greet.sh b/test/windows/offline_repo/greet.sh new file mode 100644 index 0000000000..ff7af02cf1 --- /dev/null +++ b/test/windows/offline_repo/greet.sh @@ -0,0 +1,3 @@ +#!/bin/sh +. "$(dirname "$0")/lib.sh" +echo "$GREETING" diff --git a/test/windows/offline_repo/lib.sh b/test/windows/offline_repo/lib.sh new file mode 100644 index 0000000000..eaf1dcc35e --- /dev/null +++ b/test/windows/offline_repo/lib.sh @@ -0,0 +1 @@ +GREETING="offline ok" diff --git a/test/windows/offline_repo/plugins/BUILD_FILE b/test/windows/offline_repo/plugins/BUILD_FILE new file mode 100644 index 0000000000..6be10a9666 --- /dev/null +++ b/test/windows/offline_repo/plugins/BUILD_FILE @@ -0,0 +1,8 @@ +# Stock, with the revision the plugin is released as rather than the one that is bundled. The +# bundled archive answers for whatever revision is asked for, which is what makes an ordinary +# repo work offline and is also the footgun documented in plugin_revisions.txt. +plugin_repo( + name = "shell", + plugin = "shell-rules", + revision = "v0.2.1", +) diff --git a/test/windows/python/BUILD b/test/windows/python/BUILD new file mode 100644 index 0000000000..36e1ed22e1 --- /dev/null +++ b/test/windows/python/BUILD @@ -0,0 +1,30 @@ +# The subject of //test/windows:pex_test. It is built for windows_amd64 and run under Wine; +# nothing runs it here. +python_test( + name = "pex_test", + srcs = ["pex_test.py"], + data = [":data"], + labels = ["manual"], + visibility = ["//test/windows:all"], + deps = ["//third_party/python:six"], +) + +# The test reads this at the path it has here, so the Wine rule has to place it itself: a +# python_test's own data doesn't come along when another rule depends on the test. +filegroup( + name = "data", + srcs = ["data.txt"], + test_only = True, + visibility = ["//test/windows:all"], +) + +# Windows has no exec(), so the .pex preamble runs the interpreter as a child process and has to +# pass its exit status back itself. Getting that wrong would make every failing python_binary +# look like it succeeded. +python_binary( + name = "exit_code", + labels = ["manual"], + main = "exit_code.py", + test_only = True, + visibility = ["//test/windows:all"], +) diff --git a/test/windows/python/data.txt b/test/windows/python/data.txt new file mode 100644 index 0000000000..cf7af95867 --- /dev/null +++ b/test/windows/python/data.txt @@ -0,0 +1 @@ +hello from a data file diff --git a/test/windows/python/exit_code.py b/test/windows/python/exit_code.py new file mode 100644 index 0000000000..4ace8c740d --- /dev/null +++ b/test/windows/python/exit_code.py @@ -0,0 +1,6 @@ +"""Exits with the code it is given, so that something can check it arrives.""" + +import sys + +if __name__ == '__main__': + sys.exit(int(sys.argv[1])) diff --git a/test/windows/python/pex_test.py b/test/windows/python/pex_test.py new file mode 100644 index 0000000000..fe98cd84f7 --- /dev/null +++ b/test/windows/python/pex_test.py @@ -0,0 +1,42 @@ +"""What a .pex has to get right on Windows, none of which is visible on Linux. + +The interesting part is not the assertions - it is that this runs at all. Getting here means +please_pex prepended a preamble Windows will execute, that preamble found a Python interpreter +and handed it the .pex, and Python imported this module out of the zip. +""" + +import os +import sys +import unittest + + +class PexTest(unittest.TestCase): + def test_running_on_windows(self): + """The whole point: this is a Windows interpreter, not the host one.""" + self.assertEqual('nt', os.name) + self.assertEqual('win32', sys.platform) + + def test_imported_from_the_zip(self): + """sys.path[0] is the .pex itself, and this module came out of it.""" + self.assertTrue(sys.argv[0].endswith('.pex.exe'), sys.argv[0]) + self.assertIn('.pex.exe', __file__) + + def test_reads_a_data_file(self): + """Data files land beside the .pex rather than inside it.""" + with open('test/windows/python/data.txt') as f: + self.assertEqual('hello from a data file', f.read().strip()) + + def test_third_party_import(self): + """Third-party code is imported from a directory inside the zip, by a meta path hook. + + Setting that hook up scans the zip for distribution metadata, whose member names are + always /-separated whoever wrote them. Building the pattern for that out of os.sep made + it a backslash here, which the regex compiler read as an escape - so every .pex died on + startup, before any of its own code ran. + """ + import six + self.assertTrue(six.PY3) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/windows/run_codelabs.ps1 b/test/windows/run_codelabs.ps1 new file mode 100644 index 0000000000..ef1b8e8c25 --- /dev/null +++ b/test/windows/run_codelabs.ps1 @@ -0,0 +1,459 @@ +<# +.SYNOPSIS + Replays the published codelabs on Windows, and records what a person following them would hit. + +.DESCRIPTION + //test/windows:codelab_plan reduces docs/codelabs/*.md to an ordered list of steps: files to + write, commands to run, directories to change into. This replays that list with the Windows + release, the way a person reading https://please.build/codelabs.html on Windows would. + + It answers a different question from run_native_tests.ps1 and run_native_probes.ps1. Those ask + whether Please works on Windows. This asks whether the documentation does, and most of what it + finds is not a bug in Please: bash syntax PowerShell does not accept, Unix tools that are not + there, and plugins whose tools have no Windows release. That is the finding, and it is recorded + rather than worked around. Nothing here edits a codelab to make it pass. + + Each command is handed to pwsh exactly as the codelab writes it, as an encoded command so that + no quoting of this script's stands between the text and the parser. The shell is the subject + under test, not plumbing: a reader types these lines into PowerShell, so that is where they run. + + Every step ends as one of: + + PASS it exited zero, and contained the sidecar's assert text if there was one + FAIL it did not + KNOWN it failed, and codelab_known_failures.txt says so, with a reason + SKIPPED the sidecar says it cannot run here, or it needs a tool this machine lacks + BLOCKED an earlier step in the same codelab failed, so this one was never reached + + BLOCKED is never counted as a failure. Without it one missing plugin tool in go_intro would + manufacture a dozen more failures, and the one entry that matters would drown. + + What a codelab shows a command printing is compared and reported, but never fails a step. The + codelabs' output is full of timings and a randomly chosen greeting, and asserting on it would + produce flakes that discredit the whole check. + +.EXAMPLE + # On Linux, before pushing: parses the plan, resolves the known failures, prints what would run. + pwsh ./test/windows/run_codelabs.ps1 -DryRun -Plan plz-out/gen/test/windows/codelab_plan.json ` + -KnownFailures test/windows/codelab_known_failures.txt +#> +param( + # codelab_plan.json, as built by //test/windows:codelab_plan. + [Parameter(Mandatory)][string]$Plan, + # A directory holding the release zip. Not needed for -DryRun. + [string]$Release = '', + [string]$Logs = '', + # One "codelab_id" or "codelab_id::step-key" per line, with a comment above each saying why. + [string]$KnownFailures = '', + # Run only these codelabs. For reproducing one locally. + [string[]]$Only = @(), + # Per command, unless the sidecar gives one. A plugin download and a Go toolchain can both + # land in a single step. + [int]$TimeoutSeconds = 600, + # Execute nothing: check the bookkeeping and print the plan. + [switch]$DryRun +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# The codelabs chain commands with &&, which Windows PowerShell 5.1 rejects outright. A person on +# Windows today has pwsh 7, and so does windows-latest; running under 5.1 would report findings +# about a shell nobody should be using. +if ($PSVersionTable.PSVersion.Major -lt 7) { + throw "run_codelabs.ps1 needs PowerShell 7 (pwsh); this is $($PSVersionTable.PSVersion)" +} + +# Taken once, up front: each codelab points TEMP somewhere of its own, and GetTempPath follows it. +$temp = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { [IO.Path]::GetTempPath() } +if (-not $Logs) { $Logs = Join-Path $temp 'logs' } +New-Item -ItemType Directory -Force -Path $Logs | Out-Null +$problems = [Collections.Generic.List[string]]::new() + +function Write-Summary([string] $Text) { + if ($env:GITHUB_STEP_SUMMARY) { Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value $Text } + else { Write-Host $Text } +} + +# The same format, and the same rules, as run_native_tests.ps1's. +function Read-KnownFailures([string] $Path) { + $known = @{} + if (-not $Path -or -not (Test-Path $Path)) { return $known } + foreach ($line in Get-Content $Path) { + $trimmed = $line.Trim() + if (-not $trimmed -or $trimmed.StartsWith('#')) { continue } + $known[$trimmed] = $true + } + return $known +} + +function Get-LogName([string] $Key) { return ($Key -replace '[^A-Za-z0-9_-]+', '_') + '.log' } + +# --- what a step needs, decided on this machine ---------------------------------------------- + +# Presence alone is not the question. windows-latest has a docker, but it runs Windows containers, +# and every image in the codelabs is Linux; and it has a kubectl, with no cluster behind it. Either +# would pass a Get-Command check and then fail for a reason that says nothing about the codelab. +$needCache = @{} +function Test-Need([string] $Need) { + if ($needCache.ContainsKey($Need)) { return $needCache[$Need] } + $result = switch ($Need) { + 'docker' { + if (-not (Get-Command docker -EA SilentlyContinue)) { 'docker is not installed' } + else { + $os = (& docker info --format '{{.OSType}}' 2>$null | Out-String).Trim() + if ($os -ne 'linux') { "docker runs $(if ($os) { $os } else { 'no' }) containers, and the codelab's images are Linux" } + else { '' } + } + } + 'kubectl' { + if (-not (Get-Command kubectl -EA SilentlyContinue)) { 'kubectl is not installed' } + else { + & kubectl cluster-info --request-timeout=5s *> $null + if ($LASTEXITCODE -ne 0) { 'kubectl has no cluster to talk to' } else { '' } + } + } + default { $null } + } + $needCache[$Need] = $result + return $result +} + +# --- running one command -------------------------------------------------------------------- + +function Invoke-Command-Step($Step, [string] $WorkDir, [string] $LogPath) { + $timeout = if ($Step.PSObject.Properties['timeout'] -and $Step.timeout) { $Step.timeout } else { $TimeoutSeconds } + $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($Step.command)) + $proc = Start-Process -FilePath $script:Pwsh ` + -ArgumentList @('-NoProfile', '-NonInteractive', '-OutputFormat', 'Text', '-EncodedCommand', $encoded) ` + -WorkingDirectory $WorkDir -NoNewWindow -PassThru ` + -RedirectStandardOutput "$LogPath.out" -RedirectStandardError "$LogPath.err" ` + -RedirectStandardInput $script:EmptyInput + $timedOut = -not $proc.WaitForExit($timeout * 1000) + if ($timedOut) { + # The whole tree: a plz that has started a build action leaves children behind, and a + # child holding plz-out open stops the next codelab's directory being removed. + try { $proc.Kill($true) } catch { } + $proc.WaitForExit() + } + # ExitCode is not always populated until the object is refreshed, and a null would read as a + # pass. See run_native_probes.ps1. + $proc.Refresh() + $output = @(Get-Content -LiteralPath "$LogPath.out", "$LogPath.err" -EA SilentlyContinue) + Set-Content -LiteralPath $LogPath -Value (@("> $($Step.command)", " in $WorkDir", '') + $output) + Remove-Item -LiteralPath "$LogPath.out", "$LogPath.err" -EA SilentlyContinue + return [pscustomobject]@{ + ExitCode = if ($timedOut) { $null } elseif ($null -ne $proc.ExitCode) { $proc.ExitCode } else { 0 } + TimedOut = $timedOut + Timeout = $timeout + Output = $output + } +} + +# Merges a .plzconfig fragment the way a reader following the codelab edits the file: a key the +# section already has is replaced, a new key goes into its section, and a new section is appended. +# +# Appending the fragment verbatim was tried first, and it manufactured a failure on the first +# native run: plz init plugin go already writes GoTool, the codelab's fragment sets it again, and a +# plugin section refuses a repeated key where core config quietly takes the last one. A key repeated +# on purpose to extend a list would be replaced here rather than added to; no codelab fragment has +# one. +function Merge-PlzConfig([string] $Existing, [string] $Fragment) { + $lines = [Collections.Generic.List[string]]::new() + if ($Existing) { $lines.AddRange([string[]]($Existing.TrimEnd("`r", "`n") -split "`r?`n")) } + + # Section names are case-insensitive in this format; a subsection's quoted name is not. + function Get-SectionKey([string] $Header) { + if ($Header -notmatch '^\s*\[\s*([^\s"\]]+)\s*(?:"([^"]*)")?\s*\]') { return $null } + return "$($Matches[1].ToLowerInvariant())|$($Matches[2])" + } + # Where a section starts, and the index after its last non-blank line. + function Find-Section([string] $Key) { + for ($i = 0; $i -lt $lines.Count; $i++) { + # -cne: PowerShell compares case-insensitively by default, and the subsection half + # of the key must not be. + if ((Get-SectionKey $lines[$i]) -cne $Key) { continue } + $end = $i + 1 + for ($j = $i + 1; $j -lt $lines.Count -and -not $lines[$j].TrimStart().StartsWith('['); $j++) { + if ($lines[$j].Trim()) { $end = $j + 1 } + } + return @($i, $end) + } + return $null + } + + $section = $null + foreach ($raw in ($Fragment -split "`r?`n")) { + $line = $raw.Trim() + if (-not $line -or $line.StartsWith(';') -or $line.StartsWith('#')) { continue } + $key = Get-SectionKey $line + if ($key) { + $section = $key + if (-not (Find-Section $key)) { + if ($lines.Count -gt 0 -and $lines[$lines.Count - 1].Trim()) { $lines.Add('') } + $lines.Add($line) + } + continue + } + if (-not $section -or $line -notmatch '^([^=;#]+?)\s*=') { continue } + $name = $Matches[1].Trim() + $start, $end = Find-Section $section + $replaced = $false + for ($i = $start + 1; $i -lt $end; $i++) { + if ($lines[$i] -match '^\s*([^=;#]+?)\s*=' -and $Matches[1].Trim() -ieq $name) { + $lines[$i] = $line + $replaced = $true + break + } + } + if (-not $replaced) { $lines.Insert($end, $line) } + } + return ($lines -join "`n") + "`n" +} + +# How much of what the codelab shows this command printing actually appeared. Advisory only. +function Compare-Expected($Step, $Output) { + if (-not $Step.PSObject.Properties['expected_output'] -or -not $Step.expected_output) { return '' } + $got = ($Output | ForEach-Object { $_.Trim() }) -join "`n" + $missing = @($Step.expected_output | Where-Object { $_.Trim() -and -not $got.Contains($_.Trim()) }) + if ($missing.Count -eq 0) { return '' } + return "$($missing.Count) of $(@($Step.expected_output).Count) lines the codelab shows did not appear (advisory)" +} + +# --- setup ---------------------------------------------------------------------------------- + +$planDoc = Get-Content -Raw -LiteralPath $Plan | ConvertFrom-Json +$known = Read-KnownFailures $KnownFailures + +# An entry naming nothing is also caught on Linux, by //test/windows/codelab_script/script:script_test. +# Checked again here so that a plan and a failures list from different commits cannot pass quietly. +$planNames = @{} +foreach ($c in $planDoc.codelabs) { + $planNames[$c.id] = $true + foreach ($s in $c.steps) { $planNames[$s.key] = $true } +} +foreach ($k in $known.Keys) { + if (-not $planNames.ContainsKey($k)) { + $problems.Add("$k is in $KnownFailures but names nothing in the plan") + } +} + +if (-not $DryRun) { + if (-not $Release) { throw '-Release is required unless -DryRun is given' } + $zip = Get-ChildItem -Path $Release -Filter 'please_*.zip' | Select-Object -First 1 + if (-not $zip) { throw "No please_*.zip in $Release" } + $install = Join-Path $temp 'codelab-install' + if (Test-Path $install) { Remove-Item -Recurse -Force $install } + Expand-Archive -Path $zip.FullName -DestinationPath $install + $pleaseDir = Join-Path $install 'please' + if (-not (Test-Path (Join-Path $pleaseDir 'plz.cmd'))) { throw "No plz.cmd in $($zip.Name)" } + + # On the PATH, not invoked by path. The codelabs say `plz`, package/Install.md tells a Windows + # user to put this directory on their PATH, and doing the same here is also the only thing + # anywhere that runs plz.cmd natively. If it mangles arguments, that is a finding. + $env:PATH = "$pleaseDir$([IO.Path]::PathSeparator)$env:PATH" + $script:Pwsh = (Get-Process -Id $PID).Path + $script:EmptyInput = Join-Path $temp 'codelab-empty-stdin' + Set-Content -LiteralPath $script:EmptyInput -Value $null -NoNewline + + # Through the PATH, as every codelab step will be. If plz.cmd cannot even report a version, + # say so in one line rather than as a stack trace, and let the codelabs show how far it gets. + Write-Host '::group::plz --version' + try { + & plz --version 2>&1 | Write-Host + if ($LASTEXITCODE -ne 0) { $problems.Add("plz --version exited $LASTEXITCODE through the PATH") } + } catch { + $problems.Add("plz --version could not run through the PATH: $_") + } + Write-Host '::endgroup::' +} + +$utf8 = [Text.UTF8Encoding]::new($false) +$rows = [Collections.Generic.List[object]]::new() +$details = [Collections.Generic.List[string]]::new() + +# --- the codelabs --------------------------------------------------------------------------- + +foreach ($codelab in $planDoc.codelabs) { + if ($Only.Count -gt 0 -and $codelab.id -notin $Only) { continue } + $counts = [ordered]@{ PASS = 0; FAIL = 0; KNOWN = 0; SKIPPED = 0; BLOCKED = 0 } + $total = if ($codelab.blocks.PSObject.Properties['total']) { $codelab.blocks.total } else { 0 } + + if ($codelab.PSObject.Properties['not_runnable'] -and $codelab.not_runnable) { + $rows.Add([pscustomobject]@{ Id = $codelab.id; Blocks = $total; Steps = 0; Counts = $counts; Note = "not runnable: $($codelab.not_runnable)" }) + continue + } + + Write-Host "::group::$($codelab.id) - $($codelab.title)" + $root = Join-Path $temp "codelabs\$($codelab.id)" + $home_ = Join-Path $temp "codelabs\$($codelab.id)-home" + + if (-not $DryRun) { + foreach ($d in $root, $home_) { + if (Test-Path $d) { Remove-Item -Recurse -Force $d } + } + New-Item -ItemType Directory -Force -Path $root, "$home_\AppData\Local", "$home_\Temp" | Out-Null + # A home of its own, beside the working tree rather than in it, so ~/.please and the + # caches land somewhere a `tree -a` does not see. LOCALAPPDATA is not optional: Please's + # content-addressed cache has produced false passes twice in this port, and here it would + # let one codelab replay what an earlier one built. Nothing above $root holds a .plzconfig, + # which is what keeps plz init from stopping to ask whether to continue. + $env:HOME = $home_ + $env:USERPROFILE = $home_ + $env:LOCALAPPDATA = "$home_\AppData\Local" + $env:TEMP = "$home_\Temp" + $env:TMP = "$home_\Temp" + } + + $cwd = $root + $blockedBy = '' + $ran = 0 + + foreach ($step in $codelab.steps) { + $key = $step.key + $log = Join-Path $Logs (Get-LogName $key) + $outcome = '' + $note = '' + + if ($step.kind -eq 'skip') { + $outcome = 'SKIPPED' + $note = "$($step.reason): $($step.detail)" + } elseif ($blockedBy) { + $outcome = 'BLOCKED' + $note = "after $blockedBy" + } elseif ($DryRun) { + $what = switch ($step.kind) { + 'run' { $step.command } + 'file' { "$($step.mode) $($step.path)" } + 'chdir' { "cd $($step.dir)" } + } + Write-Host (" {0,-60} {1,-5} {2}" -f $key, $step.kind, $what) + if ($step.kind -eq 'run') { $ran++ } + continue + } else { + switch ($step.kind) { + 'chdir' { + $target = Join-Path $cwd $step.dir + if (Test-Path -LiteralPath $target -PathType Container) { + $cwd = (Resolve-Path -LiteralPath $target).Path + $outcome = 'PASS' + } else { + # A reader whose earlier step did not create it is stuck here too. + $outcome = 'FAIL' + $note = "no directory $($step.dir) in $cwd" + } + } + 'file' { + $path = Join-Path $cwd $step.path + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $path) | Out-Null + # Not Set-Content: its encoding differs between PowerShell versions, and a + # .plzconfig carrying a byte-order mark does not parse. + if ($step.mode -eq 'merge') { + $existing = if (Test-Path -LiteralPath $path) { [IO.File]::ReadAllText($path) } else { '' } + [IO.File]::WriteAllText($path, (Merge-PlzConfig $existing $step.content), $utf8) + } else { + [IO.File]::WriteAllText($path, "$($step.content)`n", $utf8) + } + $outcome = 'PASS' + } + 'run' { + $missing = @() + if ($step.PSObject.Properties['needs'] -and $step.needs) { + foreach ($need in $step.needs) { + $why = Test-Need $need + if ($null -eq $why) { + $problems.Add("${key}: unknown need '$need'; run_codelabs.ps1 has no way to check for it") + $missing += "unknown need $need" + } elseif ($why) { + $missing += $why + } + } + } + if ($missing.Count -gt 0) { + $outcome = 'SKIPPED' + $note = "tool-missing: $($missing -join '; ')" + break + } + Write-Host "> $($step.command)" + $ran++ + $r = Invoke-Command-Step $step $cwd $log + $r.Output | Select-Object -Last 40 | Write-Host + $assert = if ($step.PSObject.Properties['assert']) { $step.assert } else { '' } + if ($r.TimedOut) { + $outcome = 'FAIL' + $note = "timed out after $($r.Timeout)s" + } elseif ($r.ExitCode -ne 0) { + $outcome = 'FAIL' + $note = "exited $($r.ExitCode)" + } elseif ($assert -and -not (($r.Output -join "`n").Contains($assert))) { + $outcome = 'FAIL' + $note = "exited 0 but did not print '$assert'" + } else { + $outcome = 'PASS' + $note = Compare-Expected $step $r.Output + } + } + } + } + + if ($outcome -eq 'FAIL' -and ($known.ContainsKey($key) -or $known.ContainsKey($codelab.id))) { + $outcome = 'KNOWN' + } + if ($outcome -eq 'PASS' -and $known.ContainsKey($key)) { + $problems.Add("$key is in $KnownFailures but passed; remove it") + } + $nonBlocking = $step.PSObject.Properties['non_blocking'] -and $step.non_blocking + if ($outcome -in 'FAIL', 'KNOWN' -and -not $nonBlocking) { + $blockedBy = $key + } + if ($outcome -eq 'FAIL') { + $problems.Add("$key $note") + } + + $counts[$outcome]++ + if ($outcome -ne 'PASS' -or $note) { + $details.Add("$outcome $key$(if ($note) { " - $note" })") + } + Write-Host "$outcome $key$(if ($note) { " - $note" })" + } + + # The analogue of run_native_tests.ps1's "the bundle produced no runnable tests". A codelab that + # ran nothing and was not declared unrunnable has had its commands lost somewhere between the + # Markdown and here, and reporting it as a clean pass would be the worst possible answer. + if ($ran -eq 0 -and $counts.BLOCKED -eq 0 -and $counts.SKIPPED -eq 0) { + $problems.Add("$($codelab.id) ran no commands, and codelab_steps.conf does not say it has none to run") + } + if ($known.ContainsKey($codelab.id) -and $counts.KNOWN -eq 0 -and -not $DryRun) { + $problems.Add("$($codelab.id) is in $KnownFailures but nothing in it failed; remove it") + } + Write-Host '::endgroup::' + $rows.Add([pscustomobject]@{ Id = $codelab.id; Blocks = $total; Steps = @($codelab.steps).Count; Counts = $counts; Note = '' }) +} + +# --- report --------------------------------------------------------------------------------- + +Write-Summary "## Codelabs on Windows`n" +if ($DryRun) { Write-Summary "Dry run: nothing was executed.`n" } +Write-Summary '| Codelab | Blocks | Steps | Passed | Failed | Known | Skipped | Blocked | |' +Write-Summary '|---|---:|---:|---:|---:|---:|---:|---:|---|' +foreach ($r in $rows) { + $c = $r.Counts + Write-Summary "| $($r.Id) | $($r.Blocks) | $($r.Steps) | $($c.PASS) | $($c.FAIL) | $($c.KNOWN) | $($c.SKIPPED) | $($c.BLOCKED) | $($r.Note) |" +} +if ($details.Count -gt 0) { + # Printed as the step key first so a line can go straight into codelab_known_failures.txt. + Write-Summary "`n
    Every step that did not simply pass`n" + Write-Summary '```' + foreach ($d in $details) { Write-Summary $d } + Write-Summary '```' + Write-Summary '
    ' +} + +if ($problems.Count -gt 0) { + Write-Summary "`n### Problems`n" + foreach ($p in $problems) { Write-Summary "- $p" } + Write-Host "`n$($problems.Count) problem(s):" + foreach ($p in $problems) { Write-Host " $p" } + exit 1 +} +Write-Host "`nNo unexpected results." diff --git a/test/windows/run_native_probes.ps1 b/test/windows/run_native_probes.ps1 new file mode 100644 index 0000000000..64f77e7674 --- /dev/null +++ b/test/windows/run_native_probes.ps1 @@ -0,0 +1,138 @@ +<# +.SYNOPSIS + Runs the Windows release against a test repo, and probes the things Wine cannot show. + +.DESCRIPTION + The unit tests in the bundle cover Please's own code. This covers the release as an + artifact, and the failure classes docs/design/windows/05-testing-strategy.md lists as + invisible under Wine: files held open on teardown, and path length. + + Case-insensitivity and the symlink copy fallback are deliberately not here. They belong in + Go tests in src/fs, where they ride the bundle and are written in the language the fix will + be written in. +#> +param( + # A directory holding the release zip. + [Parameter(Mandatory)][string]$Release, + [string]$Logs = "$env:RUNNER_TEMP\logs", + # How many times to build and clean in a row. One build does not find a sharing violation. + [int]$Rebuilds = 5 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path +New-Item -ItemType Directory -Force -Path $Logs | Out-Null +$problems = @() + +function Write-Summary([string] $Text) { + if ($env:GITHUB_STEP_SUMMARY) { Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value $Text } + else { Write-Host $Text } +} + +function Invoke-Plz([string] $WorkDir, [string[]] $PlzArgs, [string] $LogName) { + $log = Join-Path $Logs $LogName + Push-Location $WorkDir + try { + # Not the repository checkout: .plzconfig_windows_amd64 names MinGW tools that are not + # on this machine, and a native plz reads it. The fixtures carry their own config. + $proc = Start-Process -FilePath $script:PleaseExe -ArgumentList $PlzArgs ` + -NoNewWindow -PassThru -RedirectStandardOutput "$log.out" -RedirectStandardError "$log.err" + $proc.WaitForExit() + $proc.Refresh() + Get-Content -LiteralPath "$log.out", "$log.err" -EA SilentlyContinue | Set-Content -LiteralPath $log + Get-Content -LiteralPath $log | Write-Host + # Start-Process does not always populate ExitCode until the object is refreshed, and + # a null here would read as a failure. + if ($null -ne $proc.ExitCode) { return $proc.ExitCode } + return 0 + } finally { Pop-Location } +} + +# --- the release itself ------------------------------------------------------------------- + +$zip = Get-ChildItem -Path $Release -Filter 'please_*.zip' | Select-Object -First 1 +if (-not $zip) { throw "No please_*.zip in $Release" } +$install = Join-Path $env:RUNNER_TEMP 'install' +if (Test-Path $install) { Remove-Item -Recurse -Force $install } +Expand-Archive -Path $zip.FullName -DestinationPath $install +$script:PleaseExe = Join-Path $install 'please\please.exe' +if (-not (Test-Path $script:PleaseExe)) { throw "No please.exe in $($zip.Name)" } + +Write-Host "::group::plz --version" +$version = & $script:PleaseExe --version 2>&1 | Out-String +Write-Host $version +Write-Host '::endgroup::' +Write-Summary "## Windows release`n`n``$($version.Trim())`` from ``$($zip.Name)```n" + +# --- a real build, compared byte for byte --------------------------------------------------- + +$work = Join-Path $env:RUNNER_TEMP 'smoke' +if (Test-Path $work) { Remove-Item -Recurse -Force $work } +Copy-Item -Recurse (Join-Path $repoRoot 'test\windows\smoke_repo') $work + +Write-Host "::group::build //:pipeline" +$code = Invoke-Plz $work @('build', '//:pipeline') 'smoke_build.log' +Write-Host '::endgroup::' +if ($code -ne 0) { + $problems += "building //:pipeline exited $code" +} else { + # Compared line by line rather than as bytes: the build action's output is whatever busybox + # wrote, and the expectation came out of git, so only the content is meant to match. + $got = Get-Content (Join-Path $work 'plz-out\gen\sorted.txt') + $want = Get-Content (Join-Path $work 'expected_sorted.txt') + if (Compare-Object $got $want) { + $problems += "//:pipeline produced $($got -join ',') rather than $($want -join ',')" + } +} + +# --- files held open on teardown ------------------------------------------------------------ + +# Windows refuses to delete or rename a file another process has open, and Wine is more +# permissive. This is the single most likely source of real-Windows-only failures, and it hits +# where Please works hardest: plz-out/tmp teardown and RemoveAll. One build never finds it; +# repetition under a live virus scanner sometimes does. +Write-Host "::group::$Rebuilds builds with a clean between each" +for ($i = 1; $i -le $Rebuilds; $i++) { + $code = Invoke-Plz $work @('clean') "clean_$i.log" + if ($code -ne 0) { $problems += "plz clean exited $code on run $i" } + $code = Invoke-Plz $work @('build', '//:pipeline') "rebuild_$i.log" + if ($code -ne 0) { $problems += "rebuild $i exited $code" } +} +Write-Host '::endgroup::' + +# --- long paths ----------------------------------------------------------------------------- + +# MAX_PATH is 260 unless long-path support is on and the binary opted in by manifest. Go +# prefixes absolute paths with \\?\ by itself, so the interesting failure is not in Please but +# in what it hands busybox as a command line, which gets no such treatment - which is exactly +# the pipe-and-redirect action this fixture builds. +$padding = 'w' * 60 +$deep = Join-Path $env:RUNNER_TEMP "long\$padding\$padding\$padding" +if (Test-Path (Join-Path $env:RUNNER_TEMP 'long')) { + Remove-Item -Recurse -Force (Join-Path $env:RUNNER_TEMP 'long') +} +New-Item -ItemType Directory -Force -Path $deep | Out-Null +$deepRepo = Join-Path $deep 'repo' +Copy-Item -Recurse (Join-Path $repoRoot 'test\windows\smoke_repo') $deepRepo +Write-Host "::group::build at a $($deepRepo.Length)-character path" +$code = Invoke-Plz $deepRepo @('build', '//:pipeline') 'long_path.log' +Write-Host '::endgroup::' +if ($code -ne 0) { + # Recorded rather than fatal on the first pass: whether this is expected to work depends on + # LongPathsEnabled, which the workflow prints. + $problems += "building at a $($deepRepo.Length)-character path exited $code" +} + +# --- report --------------------------------------------------------------------------------- + +if ($problems.Count -gt 0) { + Write-Summary "`n### Probe failures`n" + foreach ($p in $problems) { Write-Summary "- $p" } + Write-Host "`n$($problems.Count) probe failure(s):" + foreach ($p in $problems) { Write-Host " $p" } + exit 1 +} +Write-Summary "`nThe release built a repo, survived $Rebuilds clean-and-rebuild cycles, and built at a long path." +Write-Host "`nAll probes passed." diff --git a/test/windows/run_native_tests.ps1 b/test/windows/run_native_tests.ps1 new file mode 100644 index 0000000000..07e838da96 --- /dev/null +++ b/test/windows/run_native_tests.ps1 @@ -0,0 +1,183 @@ +<# +.SYNOPSIS + Runs the cross-built Windows test binaries natively, out of the bundle. + +.DESCRIPTION + The counterpart of wine_go_test for a real Windows machine. //test/windows:native_test_bundle + packages the same test binaries and the same data that run under Wine on Linux; this runs + them here, where the answers actually count. See docs/design/windows/05-testing-strategy.md + for what Wine cannot show and why that matters. + + Everything this sets up mirrors what wine_go_test sets up, minus Wine: the test's own + directory as the working directory, $DATA pointing at its data, and the bundled busybox on + the PATH for the tests that run build actions. +#> +param( + # The extracted bundle: manifest.txt, shell/, tests//. + [Parameter(Mandatory)][string]$Bundle, + [string]$Logs = "$env:RUNNER_TEMP\logs", + # One "name" or "name::TestCase" per line, with a comment above each saying why. A known + # failure that starts passing is also a failure, which is what stops this becoming a + # dumping ground. + [string]$KnownFailures = '', + # Run only these entries. For reproducing one failure locally. + [string[]]$Only = @(), + [int]$TimeoutSeconds = 600 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Read-KnownFailures([string] $Path) { + $known = @{} + if (-not $Path -or -not (Test-Path $Path)) { return $known } + foreach ($line in Get-Content $Path) { + $trimmed = $line.Trim() + if (-not $trimmed -or $trimmed.StartsWith('#')) { continue } + $known[$trimmed] = $true + } + return $known +} + +# Go's -test.v marks each case with a line like "--- FAIL: TestFoo (0.05s)". Subtests come +# through the same way, indented, which is why the pattern allows leading whitespace. +function Get-Cases([string] $Path) { + $cases = @() + foreach ($line in Get-Content -LiteralPath $Path -ErrorAction SilentlyContinue) { + if ($line -match '^\s*--- (PASS|FAIL|SKIP): (\S+)') { + $cases += [pscustomobject]@{ Result = $Matches[1]; Name = $Matches[2] } + } + } + return $cases +} + +function Write-Summary([string] $Text) { + if ($env:GITHUB_STEP_SUMMARY) { Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value $Text } + else { Write-Host $Text } +} + +$bundleDir = (Resolve-Path $Bundle).Path +$manifest = Join-Path $bundleDir 'manifest.txt' +if (-not (Test-Path $manifest)) { throw "No manifest.txt in $bundleDir; is that the bundle?" } +New-Item -ItemType Directory -Force -Path $Logs | Out-Null + +$known = Read-KnownFailures $KnownFailures +$names = Get-Content $manifest | Where-Object { $_.Trim() } +if ($Only.Count -gt 0) { $names = $names | Where-Object { $Only -contains $_ } } + +$rows = @() +$problems = @() + +foreach ($name in $names) { + $dir = Join-Path $bundleDir "tests\$name" + if (-not (Test-Path $dir)) { + # The manifest is written at parse time and the directories at build time, so this + # means a test was dropped between the two rather than that it failed. + $problems += "$name is in manifest.txt but has no directory in the bundle" + continue + } + + Write-Host "::group::$name" + Push-Location $dir + try { + # Please rewrites every backslash in every environment value on Windows + # (BuildEnv.normalisePathSeparators), so hand these over already normalised. Getting it + # wrong produces failures that look like port bugs and are not. + $here = $dir -replace '\\', '/' + foreach ($v in 'TEST_DIR', 'TMP_DIR', 'TMPDIR', 'HOME', 'USERPROFILE', 'TEMP', 'TMP') { + Set-Item -Path "env:$v" -Value $here + } + # Not something Please itself redirects, but without it a test that runs a build shares + # the machine's directory cache, and a cached artifact from another run is exactly how + # this port has produced a false pass before. + $env:LOCALAPPDATA = $here + + $dataFile = Join-Path $dir 'DATA.txt' + $env:DATA = if (Test-Path $dataFile) { (Get-Content -Raw $dataFile).Trim() } else { '' } + + if (Test-Path (Join-Path $dir 'NEEDS_SHELL')) { + # Its own directory on the PATH rather than the working directory, which is how an + # install has it and what Go's exec will agree to run. + $env:PATH = (Join-Path $bundleDir 'shell') + [IO.Path]::PathSeparator + $env:PATH + } + + $out = Join-Path $Logs "$name.out" + $err = Join-Path $Logs "$name.err" + $proc = Start-Process -FilePath (Join-Path $dir 'test.exe') ` + -ArgumentList '-test.v' -NoNewWindow -PassThru ` + -RedirectStandardOutput $out -RedirectStandardError $err + if (-not $proc.WaitForExit($TimeoutSeconds * 1000)) { + # A hung test would otherwise hold the job open for hours. Kill the tree: these + # binaries start children of their own. + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $problems += "$name timed out after ${TimeoutSeconds}s" + $rows += [pscustomobject]@{ Test = $name; Pass = 0; Fail = 0; Skip = 0; Status = 'TIMEOUT' } + continue + } + # Start-Process does not always populate ExitCode until the object is refreshed, and + # a null here would read as non-zero and fail a test that passed. + $proc.Refresh() + $code = if ($null -ne $proc.ExitCode) { $proc.ExitCode } else { 0 } + + # Panics go to stderr and belong with the output they interrupted. + $log = Join-Path $Logs "$name.log" + Get-Content -LiteralPath $out, $err -ErrorAction SilentlyContinue | Set-Content -LiteralPath $log + Get-Content -LiteralPath $log | Write-Host + + $cases = Get-Cases $log + $failed = @($cases | Where-Object { $_.Result -eq 'FAIL' }) + $passed = @($cases | Where-Object { $_.Result -eq 'PASS' }) + $skipped = @($cases | Where-Object { $_.Result -eq 'SKIP' }) + + foreach ($case in $failed) { + $key = "$name::$($case.Name)" + if ($known.ContainsKey($key) -or $known.ContainsKey($name)) { continue } + $problems += $key + Write-Host "::error title=$name::$($case.Name) failed" + } + foreach ($case in $passed) { + $key = "$name::$($case.Name)" + if ($known.ContainsKey($key)) { + $problems += "$key is in $KnownFailures but passed; remove it" + } + } + # A binary that dies without reporting a single case - a panic in TestMain, a missing + # DLL - would otherwise look like a clean run with nothing in it. + if ($code -ne 0 -and $failed.Count -eq 0 -and -not $known.ContainsKey($name)) { + $problems += "$name exited $code with no failing case; see $name.log" + Write-Host "::error title=$name::exited $code without reporting a failure" + } + + $status = if ($failed.Count -gt 0 -or $code -ne 0) { 'FAIL' } else { 'ok' } + $rows += [pscustomobject]@{ + Test = $name; Pass = $passed.Count; Fail = $failed.Count + Skip = $skipped.Count; Status = $status + } + } finally { + Pop-Location + Write-Host '::endgroup::' + } +} + +Write-Summary "## Windows unit tests`n" +Write-Summary '| Test | Pass | Fail | Skip | |' +Write-Summary '|---|---:|---:|---:|---|' +foreach ($row in $rows) { + Write-Summary "| $($row.Test) | $($row.Pass) | $($row.Fail) | $($row.Skip) | $($row.Status) |" +} +if ($rows.Count -gt 0) { + $totals = $rows | Measure-Object -Property Pass, Fail, Skip -Sum + Write-Summary "`n$($rows.Count) binaries, $($totals[0].Sum) passed, $($totals[1].Sum) failed, $($totals[2].Sum) skipped." +} else { + Write-Summary "`nNo test binaries ran at all." + $problems += 'the bundle produced no runnable tests' +} + +if ($problems.Count -gt 0) { + Write-Summary "`n### Unexpected`n" + foreach ($p in $problems) { Write-Summary "- $p" } + Write-Host "`n$($problems.Count) unexpected result(s):" + foreach ($p in $problems) { Write-Host " $p" } + exit 1 +} +Write-Host "`nAll $($rows.Count) test binaries behaved as expected." diff --git a/test/windows/shell/BUILD b/test/windows/shell/BUILD new file mode 100644 index 0000000000..1970960f91 --- /dev/null +++ b/test/windows/shell/BUILD @@ -0,0 +1,18 @@ +# The subject of //test/windows:sh_binary_test. Built for windows_amd64 and run under Wine; +# nothing builds these here, hence the manual labels. +sh_library( + name = "lib", + src = "lib.sh", + labels = ["manual"], +) + +# On Windows this comes out as greet.cmd: a batch preamble that unpacks the zip appended to it +# and hands the script to the bundled busybox. It sources the library out of the unpacked +# payload, takes an argument and exits non-zero, so the wrapper has to get all three right. +sh_binary( + name = "greet", + labels = ["manual"], + main = "greet.sh", + visibility = ["//test/windows:all"], + deps = [":lib"], +) diff --git a/test/windows/shell/greet.sh b/test/windows/shell/greet.sh new file mode 100644 index 0000000000..84df506927 --- /dev/null +++ b/test/windows/shell/greet.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# The subject of //test/windows:sh_binary_test. Sources a library that only exists because the +# payload was unpacked, and finds it relative to $0, which has to be the file that was run +# rather than this one - the same arrangement as on Unix, where they are one file. +set -eu + +lib="$(dirname "$0")/test/windows/shell/lib.sh" +. "$lib" + +echo "$GREETING $1" + +# Scribble on the unpacked library, so that a second run in the same directory has to replace +# it. Anything the payload leaves behind is a build output and so read-only, and a read-only +# file on Windows cannot be written or replaced at all - which would show up here as either +# this line failing or the next run reading the wrong greeting. +echo 'GREETING="stale"' > "$lib" + +exit "$2" diff --git a/test/windows/shell/lib.sh b/test/windows/shell/lib.sh new file mode 100644 index 0000000000..318e2f67a8 --- /dev/null +++ b/test/windows/shell/lib.sh @@ -0,0 +1 @@ +GREETING="hello from" diff --git a/test/windows/smoke_repo/.plzconfig b/test/windows/smoke_repo/.plzconfig new file mode 100644 index 0000000000..abbbc77ec8 --- /dev/null +++ b/test/windows/smoke_repo/.plzconfig @@ -0,0 +1,9 @@ +; Named BUILD_FILE so the outer repo doesn't parse this as one of its own packages. +[parse] +BuildFileName = BUILD_FILE + +; No dir cache. Please's cache is content-addressed and lands outside the test's tmp dir, so +; leaving it on lets a run replay artifacts an earlier, differently-built binary produced - +; which has already caused one false pass during this port. +[cache] +dir = diff --git a/test/windows/smoke_repo/BUILD_FILE b/test/windows/smoke_repo/BUILD_FILE new file mode 100644 index 0000000000..d65edf8bc7 --- /dev/null +++ b/test/windows/smoke_repo/BUILD_FILE @@ -0,0 +1,11 @@ +# The shell smoke test from docs/design/windows/02-shell-and-build-actions.md: a pipe, a +# redirect and two applets in one build action. +genrule( + name = "pipeline", + srcs = [ + "a.txt", + "b.txt", + ], + outs = ["sorted.txt"], + cmd = "cat $SRCS | sort > $OUT", +) diff --git a/test/windows/smoke_repo/a.txt b/test/windows/smoke_repo/a.txt new file mode 100644 index 0000000000..0ae7ef4fca --- /dev/null +++ b/test/windows/smoke_repo/a.txt @@ -0,0 +1,2 @@ +delta +bravo diff --git a/test/windows/smoke_repo/b.txt b/test/windows/smoke_repo/b.txt new file mode 100644 index 0000000000..8b5861158d --- /dev/null +++ b/test/windows/smoke_repo/b.txt @@ -0,0 +1,2 @@ +charlie +alpha diff --git a/test/windows/smoke_repo/expected_sorted.txt b/test/windows/smoke_repo/expected_sorted.txt new file mode 100644 index 0000000000..bfdfbcdb37 --- /dev/null +++ b/test/windows/smoke_repo/expected_sorted.txt @@ -0,0 +1,4 @@ +alpha +bravo +charlie +delta diff --git a/third_party/binary/BUILD b/third_party/binary/BUILD index 0c6b12eb4e..cf57e01bdc 100644 --- a/third_party/binary/BUILD +++ b/third_party/binary/BUILD @@ -20,3 +20,43 @@ remote_file( binary = True, url = f"https://github.com/please-build/puku/releases/download/v{PUKU_VERSION}/puku-{PUKU_VERSION}-{CONFIG.OS}_{CONFIG.ARCH}", ) + +# The shell and coreutils that build actions run in on Windows. Windows ships nothing that can +# execute a build action, so Please bundles this rather than requiring an install; see +# docs/design/windows/02-shell-and-build-actions.md. +# +# This is busybox-w64, a third-party fork of busybox (https://frippery.org/busybox/), not an +# upstream busybox release. It is GPL-2.0, so the release notes it separately. +BUSYBOX_VERSION = "FRP-6075-g169694ebd" + +remote_file( + name = "busybox", + out = "busybox.exe", + binary = True, + hashes = ["07bb1e5b095b00d68a695481f9240879f33c5724b40aa2308f999d54ed78f075"], + licences = ["GPL-2.0"], + url = f"https://frippery.org/files/busybox/busybox-w64-{BUSYBOX_VERSION}.exe", + visibility = [ + "//package:all", + "//test/windows:all", + ], +) + +# A Python for the Wine tests to run .pex files with. This is the embeddable package from +# python.org: python.exe, its DLLs and a zipped standard library, with nothing to install. +# +# It is here rather than as a toolchain because that is all it is for. Nothing Please builds +# depends on it, and no part of the Windows release ships it - a user brings their own Python, +# the same as on any other platform. +PYTHON_WINDOWS_VERSION = "3.11.9" + +remote_file( + name = "python-windows", + out = "python-windows", + extract = True, + hashes = ["009d6bf7e3b2ddca3d784fa09f90fe54336d5b60f0e0f305c37f400bf83cfd3b"], + licences = ["PSF-2.0"], + test_only = True, + url = f"https://www.python.org/ftp/python/{PYTHON_WINDOWS_VERSION}/python-{PYTHON_WINDOWS_VERSION}-embed-amd64.zip", + visibility = ["//test/windows:all"], +) diff --git a/third_party/go/BUILD b/third_party/go/BUILD index 7fbb6b9cad..405ed2b435 100644 --- a/third_party/go/BUILD +++ b/third_party/go/BUILD @@ -374,6 +374,26 @@ go_repo( version = "v0.0.0-20210914205149-d1177395e3b8", ) +# arcat, and the two of its requirements nothing else here already pins. Please normally +# downloads a prebuilt arcat release keyed by platform, and there is none for Windows; building +# it from source is how the offline Windows release gets one. See +# docs/design/windows/08-offline-release.md. +go_repo( + module = "github.com/xi2/xz", + version = "v0.0.0-20171230120015-48954b6210f8", +) + +go_repo( + module = "github.com/please-build/ar", + version = "v0.0.0-20251128102243-20fe5956df94", +) + +go_repo( + module = "github.com/please-build/arcat", + patch = ["arcat_windows_rename.patch"], + version = "v1.3.1", +) + go_repo( module = "github.com/klauspost/cpuid/v2", version = "v2.4.0", diff --git a/third_party/go/arcat_windows_rename.patch b/third_party/go/arcat_windows_rename.patch new file mode 100644 index 0000000000..0614313bc3 --- /dev/null +++ b/third_party/go/arcat_windows_rename.patch @@ -0,0 +1,51 @@ +Two Windows fixes for arcat, both of which stop it producing a usable zip at all. + +1. The output file is created with ioutil.TempFile, which returns it open, and the handle is + never closed - zip.NewFile opens the same path again and closes only its own. Unix does not + care that a file being renamed is still open. Windows refuses: + + panic: Failed to rename output file: rename .\arcat-1624372107 ...: Sharing violation. + + which is every zip arcat is asked to write, so nothing needing one builds. + +2. Zip member names are always /-separated, and filepath.WalkDir hands back the OS separator. + On Windows every name added by a directory walk therefore carried backslashes, so --rename_dir + and --strip_prefix silently matched nothing and the members came out with names no reader + splits into directories. The rest of writer.go already assumes slashes - samePaths uses + path.IsAbs, not filepath.IsAbs. + +Carried here rather than upstream because there is nowhere to push it to yet. See +docs/design/windows/08-offline-release.md. + +--- a/main.go ++++ b/main.go +@@ -173,6 +173,9 @@ + tempFile, err := ioutil.TempFile(".", "arcat-") + must(err) + filename := tempFile.Name() ++ // Windows will not rename a file that is still open, and zip.NewFile opens this path ++ // again for itself, so this handle has no further use. ++ must(tempFile.Close()) + + f := zip.NewFile(filename, opts.Zip.Strict) + f.RenameDirs = opts.Zip.RenameDirs +--- a/zip/writer.go ++++ b/zip/writer.go +@@ -215,6 +215,10 @@ + if err != nil { + return err + } ++ // A zip member name is always /-separated, whatever the host separator is. Everything ++ // downstream of here - the rename and strip-prefix options, and the names written into ++ // the archive - depends on that. ++ path = filepath.ToSlash(path) + mode := entry.Type() + if path != f.input && ((mode & fs.ModeSymlink) == fs.ModeSymlink) { + if resolved, err := filepath.EvalSymlinks(path); err != nil { +@@ -303,7 +307,7 @@ + // AddFiles walks the given directory and adds any zip files (determined by suffix) that it finds within. + func (f *File) AddFiles(in string) error { +- f.input = in ++ f.input = filepath.ToSlash(in) + return filepath.WalkDir(in, f.walk) + } diff --git a/tools/images/build.sh b/tools/images/build.sh index f3b7c8316c..2affc7a5e2 100755 --- a/tools/images/build.sh +++ b/tools/images/build.sh @@ -5,7 +5,7 @@ set -euvo pipefail tag=$(date +%Y%m%d) reporoot=$(plz query reporoot) -images=("alpine" "freebsd_builder" "ubuntu" "ubuntu_alt") +images=("alpine" "freebsd_builder" "ubuntu" "ubuntu_alt" "windows_builder") for image in ${images[@]}; do cd "$reporoot/tools/images/$image" diff --git a/tools/images/windows_builder/Dockerfile b/tools/images/windows_builder/Dockerfile new file mode 100644 index 0000000000..b6cb7713fc --- /dev/null +++ b/tools/images/windows_builder/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:noble +LABEL org.opencontainers.image.authors="please thoughtmachine net" +LABEL org.opencontainers.image.source=https://github.com/thought-machine/please + +# A few miscellaneous dependencies. MinGW is here for the C/C++ rules, which cross-compile to +# Windows with it; the Go cross-build itself needs no C toolchain. Wine runs the results, which +# is the only way anything about the port's runtime behaviour gets checked before there is a +# Windows machine. It needs the i386 architecture enabled even for a 64-bit-only install. +RUN dpkg --add-architecture i386 && apt-get update && \ + apt-get install -y curl git gcc xz-utils g++-mingw-w64-x86-64 wine wine64 && apt-get clean + +# Go +RUN curl -fsSL https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz | tar -xzC /usr/local +RUN ln -s /usr/local/go/bin/go /usr/local/bin/go && ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt +RUN GOOS=windows go install std + +WORKDIR /tmp diff --git a/tools/misc/gen_release.py b/tools/misc/gen_release.py index 40335a06e5..b8a8c214da 100755 --- a/tools/misc/gen_release.py +++ b/tools/misc/gen_release.py @@ -52,6 +52,7 @@ def __init__(self, github_token:str, dry_run:bool=False): self.known_content_types = { '.gz': 'application/gzip', '.xz': 'application/x-xz', + '.zip': 'application/zip', '.asc': 'text/plain', '.sha256': 'text/plain', } @@ -111,6 +112,8 @@ def _arch(self, artifact:str) -> str: return f'darwin_{cpu}' elif 'freebsd' in artifact: return f'freebsd_{cpu}' + elif 'windows' in artifact: + return f'windows_{cpu}' return f'linux_{cpu}' def sign_pgp(self, artifact:str) -> str: diff --git a/tools/misc/get_plz.ps1 b/tools/misc/get_plz.ps1 new file mode 100644 index 0000000000..c9a7adfad4 --- /dev/null +++ b/tools/misc/get_plz.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS + Downloads a precompiled copy of Please and installs it. + +.DESCRIPTION + The Windows counterpart of get_plz.sh, served from the same bucket and run the same way: + + irm https://get.please.build/get_plz.ps1 | iex + + Kept deliberately parallel to that script rather than clever, so the two can be read side by + side. The differences are all forced: the release is a .zip rather than a tarball, because + Windows has no guaranteed tar; the short name is a plz.cmd shim rather than a symlink, + because symlinks need Developer Mode; and the install is linked up a level by hard-linking + or copying, for the same reason. +#> + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +# This fork publishes GitHub Releases rather than to the bucket upstream uses, because CircleCI +# only runs the publishing job on thought-machine/please. The asset names carry the platform, +# which is the layout gen_release.py already produces for the GitHub half of a release. +# +# Point PLZ_DOWNLOAD_BASE at 'https://get.please.build' to use the upstream bucket instead; the +# path shape differs, so the script picks the right one from the base. +$UrlBase = if ($env:PLZ_DOWNLOAD_BASE) { + $env:PLZ_DOWNLOAD_BASE +} else { + 'https://github.com/PeterNeiss/please/releases/download' +} + +if ($env:PROCESSOR_ARCHITECTURE -eq 'AMD64' -or $env:PROCESSOR_ARCHITEW6432 -eq 'AMD64') { + $Arch = 'amd64' +} else { + Write-Error "Please does not support the $env:PROCESSOR_ARCHITECTURE architecture on Windows." + exit 1 +} + +# A GitHub release has no latest_version file; the redirect on /releases/latest names the tag. +if ($UrlBase -like '*github.com*') { + $Repo = ($UrlBase -replace '/releases/download$', '') + $Latest = (Invoke-WebRequest -UseBasicParsing "$Repo/releases/latest" -MaximumRedirection 0 -ErrorAction SilentlyContinue).Headers.Location + if (-not $Latest) { $Latest = (Invoke-WebRequest -UseBasicParsing "$Repo/releases/latest").BaseResponse.RequestMessage.RequestUri.AbsoluteUri } + $Version = ($Latest -split '/')[-1] -replace '^v', '' +} else { + $Version = (Invoke-WebRequest -UseBasicParsing "$UrlBase/latest_version").Content.Trim() +} +$Location = Join-Path $env:USERPROFILE '.please' +$Dir = Join-Path $Location $Version +$Zip = Join-Path ([System.IO.Path]::GetTempPath()) "please_$Version.zip" + +Write-Host "Downloading Please $Version..." -ForegroundColor Green +if (Test-Path $Dir) { Remove-Item -Recurse -Force $Dir } +New-Item -ItemType Directory -Force -Path $Dir | Out-Null +# The two layouts differ: a release keeps everything under one tag with the platform in the +# filename, the bucket keeps a directory per platform and version. +$Url = if ($UrlBase -like '*github.com*') { + "$UrlBase/v$Version/please_${Version}_windows_${Arch}.zip" +} else { + "$UrlBase/windows_${Arch}/$Version/please_$Version.zip" +} +Invoke-WebRequest -UseBasicParsing $Url -OutFile $Zip + +# The zip holds everything under a please/ directory, which is the layer the tarball strips with +# --strip-components=1. Expand-Archive has no equivalent, so unpack and move up. +$Staging = Join-Path ([System.IO.Path]::GetTempPath()) "please_$Version" +if (Test-Path $Staging) { Remove-Item -Recurse -Force $Staging } +Expand-Archive -Path $Zip -DestinationPath $Staging +Move-Item (Join-Path $Staging 'please\*') $Dir +Remove-Item -Recurse -Force $Staging, $Zip + +# Link it all back up a directory. Symlinks need Developer Mode on Windows, so hard-link where +# we can and copy where we can't; this is the same choice the self-updater and pleasew.ps1 make. +foreach ($file in Get-ChildItem -File $Dir) { + $link = Join-Path $Location $file.Name + if (Test-Path $link) { Remove-Item -Force $link } + try { + New-Item -ItemType HardLink -Path $link -Target $file.FullName -ErrorAction Stop | Out-Null + } catch { + Copy-Item -Force $file.FullName $link + } +} + +Write-Host "Please installed to $Location" -ForegroundColor Green +Write-Host "Add it to your PATH to use plz from anywhere:" +Write-Host " [Environment]::SetEnvironmentVariable('Path', `"`$env:Path;$Location`", 'User')" diff --git a/tools/please_shim/BUILD b/tools/please_shim/BUILD index e255e67dd4..d7a151d01c 100644 --- a/tools/please_shim/BUILD +++ b/tools/please_shim/BUILD @@ -17,6 +17,7 @@ go_binary( "//src/cli/logging", "//src/core", "//src/fs", + "//src/process", "//src/update", "//src/version", ], diff --git a/tools/please_shim/main.go b/tools/please_shim/main.go index 6a3fb1fc2e..7fd67628b8 100644 --- a/tools/please_shim/main.go +++ b/tools/please_shim/main.go @@ -7,7 +7,6 @@ import ( "os/exec" "path/filepath" "strings" - "syscall" "github.com/thought-machine/go-flags" @@ -15,6 +14,7 @@ import ( "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" + "github.com/thought-machine/please/src/process" "github.com/thought-machine/please/src/update" "github.com/thought-machine/please/src/version" ) @@ -214,7 +214,7 @@ func main() { } resolvePleaseLocation(config) - state.pleaseExecutable = filepath.Join(config.Please.Location, "please") + state.pleaseExecutable = filepath.Join(config.Please.Location, "please"+fs.ExeSuffix) // Install Please if not found. if !fs.FileExists(state.pleaseExecutable) { @@ -226,7 +226,7 @@ func main() { command := cli.ActiveFullCommand(parser.Command) maybeUpdatePlease(state, command == "update") - if err := syscall.Exec(state.pleaseExecutable, os.Args, os.Environ()); err != nil { + if err := process.ExecReplace(state.pleaseExecutable, os.Args, os.Environ()); err != nil { log.Fatalf("Failed to execute Please: %s", err) } }