ci(windows): check the installed app like a new user, and the uninstall

.github/workflows/windows-bundles.yml

@@ -78,34 +78,17 @@ jobs:
             $global:LASTEXITCODE = 0
           }
 
-      # Install the NSIS setup silently (per user) and start the installed app:
-      # the window needs WebView2, which the runner image has. Informational: a
-      # runner has no real desktop session, so a failure here is a warning, not
-      # a red build.
+      # Install the NSIS setup silently (per user), start the app as a new user
+      # would, close it and uninstall it: scripts\windows-smoke.ps1 (server,
+      # platform, first run, window, still running after 20 s, closing stops the
+      # server, uninstalling keeps the data). Informational: a runner has no real
+      # desktop session, so a failed check is an error annotation on the run
+      # page, not a red build.
       - name: Smoke test, install and start the app
         continue-on-error: true
         run: |
           $setup = Get-ChildItem "desktop/src-tauri/target/release/bundle/nsis/*.exe" | Select-Object -First 1
-          Start-Process -FilePath $setup.FullName -ArgumentList "/S" -Wait
-          $app = Get-ChildItem "$env:LOCALAPPDATA\ConductLoop" -Recurse -Filter *.exe | Where-Object { $_.Name -notmatch "unins" }
-          $app | ForEach-Object { "{0}  {1} bytes" -f $_.FullName, $_.Length }
-          $main = $app | Where-Object { $_.Name -ne "conductloop-server.exe" } | Select-Object -First 1
-          if (-not $main -or -not ($app | Where-Object { $_.Name -eq "conductloop-server.exe" })) {
-            throw "the installed app has no main executable or no conductloop-server.exe"
-          }
-          $proc = Start-Process -FilePath $main.FullName -PassThru
-          try {
-            $ok = $false
-            for ($i = 0; $i -lt 60 -and -not $ok; $i++) {
-              Start-Sleep -Seconds 2
-              try { $ok = (Invoke-WebRequest -UseBasicParsing -Uri "http://127.0.0.1:7777/api/health" -TimeoutSec 3).StatusCode -eq 200 } catch { }
-            }
-            if (-not $ok) { throw "the installed app did not start its server on :7777" }
-            "the installed app started its server" | Tee-Object -Append -FilePath $env:GITHUB_STEP_SUMMARY
-          } finally {
-            taskkill /T /F /PID $proc.Id | Out-Null
-            $global:LASTEXITCODE = 0
-          }
+          scripts\windows-smoke.ps1 -Setup $setup.FullName -Out smoke
 
       - name: Upload the NSIS installer
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
@@ -122,3 +105,12 @@ jobs:
           path: desktop/src-tauri/target/release/bundle/msi/*.msi
           if-no-files-found: error
           retention-days: 3
+
+      - name: Upload the smoke-test screenshot and log
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: conductloop-windows-smoke
+          path: smoke/
+          if-no-files-found: warn
+          retention-days: 3

CHANGELOG.md

@@ -4,6 +4,7 @@ User-facing changes to ConductLoop (the desktop app and its local server), newes
 
 ## Unreleased
 
+- CI only: the "Windows bundles" workflow now runs `scripts/windows-smoke.ps1` on the NSIS installer: it installs it silently, starts the app as a new user, and checks the server, that `/api/platform` says Windows with the in-app browser and code execution off, the first run and welcome page, the window (with a screenshot), that the app and its server are still running 20 s later, that closing the window stops the server, and that **uninstalling keeps the user's data** (it lives in `%LOCALAPPDATA%\ConductLoop\data`, inside the install folder). Still informational (a runner has no real desktop session), but each failed check is now an error annotation on the run page, and the screenshot and log are kept for 3 days.
 - CI only: the Linux bundles workflow now **starts the AppImage and the .deb** it builds, on Ubuntu 22.04 (the build image) and on ubuntu-latest, on a virtual display with an empty HOME and no keyring. `scripts/linux-smoke.sh` checks that the local server answers, that `/api/platform` says Linux with the in-app browser off, that it is a first run with the welcome page, that a ConductLoop window is on screen, and that closing the app stops the server; screenshots and logs are kept for 3 days. Until now the Linux packages were only built, never run. The script also works by hand on a Linux machine.
 - CI only: a "Windows (compile and tests)" workflow checks on every relevant pull request that the desktop wrapper still compiles for Windows and that the Windows-safe part of the Python tests passes there. It builds no installer, signs nothing, publishes nothing and is not a required check. Windows is still not supported.
 - Windows: a CI workflow, "Windows bundles", builds the NSIS installer and the .msi on `windows-latest` and keeps them for 3 days as downloadable artifacts. It only runs when started by hand (it is not a check of pull requests), signs nothing and publishes nothing. CI only: the app does not change.

scripts/windows-smoke.ps1

@@ -0,0 +1,187 @@
+<#
+Smoke-test the Windows installer of the desktop app: install it, start it the
+way a new user would, close it, uninstall it.
+
+  scripts\windows-smoke.ps1 -Setup path\to\ConductLoop_<v>_x64-setup.exe [-Out dir] [-Wait 120]
+
+Checks, in order (each prints a check mark or a cross; the exit status is 1 if
+any failed):
+  1. the NSIS setup installs silently, per user, with conductloop-desktop.exe
+     and conductloop-server.exe;
+  2. the local server answers /api/health on 127.0.0.1:7777 within -Wait s;
+  3. /api/platform says windows, with the in-app browser and code execution off;
+  4. it is a first run: /api/onboarding/state is fresh and /welcome is served;
+  5. the app has a "ConductLoop" window (and a screenshot is saved);
+  6. 20 s later the app and its server are still running (parent_watch must
+     not end anything on Windows);
+  7. closing the window stops the server: the port is free and no
+     conductloop-server.exe is left within 15 s;
+  8. uninstalling (silently) keeps the user's data: the app keeps its data in
+     %LOCALAPPDATA%\ConductLoop\data, inside the install folder.
+Writes windows-smoke.png and windows-smoke.log in -Out (default: the current
+folder). Talks only to 127.0.0.1:7777 with GETs. Publishes nothing.
+Windows x86_64 only; meant for a disposable machine or CI runner (it installs
+and uninstalls ConductLoop for the current user).
+#>
+[CmdletBinding()]
+param(
+  [Parameter(Mandatory = $true)][string]$Setup,
+  [string]$Out = (Get-Location).Path,
+  [int]$Wait = 120
+)
+$ErrorActionPreference = "Stop"
+$Base = "http://127.0.0.1:7777"
+$InstallDir = Join-Path $env:LOCALAPPDATA "ConductLoop"
+$DataDir = Join-Path $InstallDir "data"
+New-Item -ItemType Directory -Force -Path $Out | Out-Null
+$Log = Join-Path $Out "windows-smoke.log"
+$Shot = Join-Path $Out "windows-smoke.png"
+$script:failures = 0
+
+function Report([string]$line) {
+  Write-Host $line
+  Add-Content -Path $Log -Value $line
+  if ($env:GITHUB_STEP_SUMMARY) { Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value "- $line" }
+}
+function Ok([string]$what) { Report "✓ $what" }
+function Bad([string]$what) {
+  Report "✗ $what"
+  # An annotation shows on the run page even when the step may not fail the job.
+  if ($env:GITHUB_ACTIONS) { Write-Host "::error title=Windows smoke test::$what" }
+  $script:failures++
+}
+function Get-Json([string]$path) {
+  try { return Invoke-RestMethod -Uri "$Base$path" -TimeoutSec 5 } catch { return $null }
+}
+function Test-Up {
+  try { return (Invoke-WebRequest -UseBasicParsing -Uri "$Base/api/health" -TimeoutSec 2).StatusCode -eq 200 } catch { return $false }
+}
+function Get-Servers { @(Get-Process -Name "conductloop-server" -ErrorAction SilentlyContinue) }
+
+if (-not (Test-Path -LiteralPath $Setup)) { throw "no installer at '$Setup'" }
+if (Test-Up) { throw "something already answers on ${Base}: stop it first" }
+if ($env:GITHUB_STEP_SUMMARY) {
+  Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value "### Windows smoke test ($((Get-CimInstance Win32_OperatingSystem).Caption))"
+}
+
+# 1. Install.
+Start-Process -FilePath (Resolve-Path -LiteralPath $Setup).Path -ArgumentList "/S" -Wait
+$desktop = Join-Path $InstallDir "conductloop-desktop.exe"
+$server = Join-Path $InstallDir "conductloop-server.exe"
+if ((Test-Path $desktop) -and (Test-Path $server)) {
+  Ok "installed per user in $InstallDir"
+} else {
+  Bad "the installer did not put conductloop-desktop.exe and conductloop-server.exe in $InstallDir"
+  Get-ChildItem $InstallDir -Recurse -ErrorAction SilentlyContinue | Select-Object -First 20 | ForEach-Object { Write-Host "    $($_.FullName)" }
+  exit 1
+}
+
+# 2. The server comes up.
+$started = Get-Date
+$app = Start-Process -FilePath $desktop -PassThru
+$up = $false
+for ($i = 0; $i -lt $Wait -and -not $up; $i++) {
+  Start-Sleep -Seconds 1
+  $up = Test-Up
+  if ($app.HasExited) { break }
+}
+if ($up) {
+  $health = Get-Json "/api/health"
+  Ok ("server answered in {0:N0} s as {1}" -f ((Get-Date) - $started).TotalSeconds, $health.version)
+} else {
+  Bad ("the server did not answer within {0} s (app {1})" -f $Wait, $(if ($app.HasExited) { "exited with $($app.ExitCode)" } else { "running" }))
+  exit 1
+}
+
+# 3. It knows it is on Windows.
+$p = Get-Json "/api/platform"
+foreach ($k in "browser", "code_execution", "keychain") {
+  $v = $p.$k
+  $line = ("    {0}: available={1} {2} {3}" -f $k, $v.available, $v.reason, $v.message).TrimEnd()
+  Report $line
+}
+if ($p.platform -eq "windows" -and $p.browser.available -eq $false -and $p.code_execution.available -eq $false) {
+  Ok "/api/platform: windows, in-app browser and code execution off"
+} else {
+  Bad "/api/platform does not say windows with the browser and code execution off (platform=$($p.platform))"
+}
+
+# 4. A first run.
+$state = Get-Json "/api/onboarding/state"
+$welcome = try { (Invoke-WebRequest -UseBasicParsing -Uri "$Base/welcome" -TimeoutSec 5).Content } catch { "" }
+if ($state.fresh -eq $true -and $welcome -match "ConductLoop") {
+  Ok "first run: onboarding is fresh and /welcome is served"
+} else {
+  Bad "first run: onboarding not fresh or /welcome missing"
+}
+
+# 5. A window, and what it shows.
+Start-Sleep -Seconds 8
+$proc = Get-Process -Id $app.Id -ErrorAction SilentlyContinue
+if ($proc -and $proc.MainWindowTitle -eq "ConductLoop") {
+  Ok "a ConductLoop window is open"
+} else {
+  Bad "no ConductLoop window (title: '$($proc.MainWindowTitle)')"
+}
+try {
+  Add-Type -AssemblyName System.Windows.Forms, System.Drawing
+  $bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+  $bitmap = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height
+  $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+  $graphics.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)
+  $bitmap.Save($Shot, [System.Drawing.Imaging.ImageFormat]::Png)
+  Write-Host "    screenshot: $Shot"
+} catch {
+  Write-Host "    (no screenshot: $($_.Exception.Message))"
+}
+
+# 6. Still running a little later.
+Start-Sleep -Seconds 20
+if (-not $app.HasExited -and (Test-Up) -and (Get-Servers).Count -ge 1) {
+  Ok "20 s later the app and its server are still running"
+} else {
+  Bad "the app or its server stopped on its own (app exited: $($app.HasExited))"
+}
+
+# 7. Closing the window stops the server.
+$closed = $false
+if (-not $app.HasExited) { $app.Refresh(); $closed = $app.CloseMainWindow() }
+if (-not $closed) { Stop-Process -Id $app.Id -Force -ErrorAction SilentlyContinue }
+$freed = $null
+for ($i = 1; $i -le 15 -and $null -eq $freed; $i++) {
+  Start-Sleep -Seconds 1
+  if (-not (Test-Up) -and (Get-Servers).Count -eq 0) { $freed = $i }
+}
+$how = $(if ($closed) { "closing the window" } else { "ending the app" })
+if ($null -ne $freed) {
+  Ok "$how stopped its server ($freed s)"
+} else {
+  Bad "the server kept running 15 s after $how"
+  Get-Servers | ForEach-Object { Write-Host "    left: pid $($_.Id), started $($_.StartTime)" }
+}
+Get-Process -Name "conductloop-desktop", "conductloop-server" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
+
+# 8. Uninstalling keeps the user's data.
+$db = Join-Path $DataDir "ops.db"
+if (-not (Test-Path $db)) {
+  Bad "no database at $db after a run: the data folder is not where expected"
+} else {
+  $uninstaller = Get-ChildItem $InstallDir -Filter "uninstall*.exe" -ErrorAction SilentlyContinue | Select-Object -First 1
+  if (-not $uninstaller) {
+    Bad "no uninstaller in $InstallDir"
+  } else {
+    Start-Process -FilePath $uninstaller.FullName -ArgumentList "/S" -Wait
+    # The NSIS uninstaller copies itself elsewhere and returns at once: wait for it.
+    for ($i = 0; $i -lt 30 -and (Test-Path $desktop); $i++) { Start-Sleep -Seconds 1 }
+    if (Test-Path $desktop) {
+      Bad "the uninstaller did not remove the app"
+    } elseif (Test-Path $db) {
+      Ok "uninstalling removed the app and kept the user's data ($db)"
+    } else {
+      Bad "uninstalling DELETED the user's data ($db is gone)"
+    }
+  }
+}
+
+if ($script:failures -gt 0) { exit 1 }
+exit 0

tests/test_windows_bundles_workflow.py

@@ -48,14 +48,19 @@ def test_it_builds_with_the_script_on_windows():
         assert action in uses
 
 
+def _uploads():
+    return [s for s in JOB["steps"] if str(s.get("uses", "")).startswith("actions/upload-artifact@")]
+
+
 def test_both_installers_are_uploaded_for_three_days():
-    uploads = [s for s in JOB["steps"] if str(s.get("uses", "")).startswith("actions/upload-artifact@")]
-    assert len(uploads) == 2
-    for step in uploads:
-        assert step["with"]["retention-days"] == 3
+    installers = [s for s in _uploads() if s["with"]["name"] in ("conductloop-windows-nsis", "conductloop-windows-msi")]
+    assert len(installers) == 2
+    for step in installers:
         assert step["with"]["if-no-files-found"] == "error"
-    paths = " ".join(s["with"]["path"] for s in uploads)
+    paths = " ".join(s["with"]["path"] for s in installers)
     assert "bundle/nsis/*.exe" in paths and "bundle/msi/*.msi" in paths
+    for step in _uploads():
+        assert step["with"]["retention-days"] == 3
 
 
 def test_it_publishes_nothing_and_takes_no_secret():
@@ -78,6 +83,33 @@ def test_the_installed_app_smoke_test_cannot_turn_the_build_red():
     assert len(server) == 1 and not server[0].get("continue-on-error")
 
 
+def test_the_installed_app_is_checked_by_the_smoke_script_and_its_output_kept():
+    smoke = next(s for s in JOB["steps"] if "install and start" in s.get("name", ""))
+    assert "scripts\\windows-smoke.ps1 -Setup" in smoke["run"]
+    shots = [s for s in _uploads() if s["with"]["name"] == "conductloop-windows-smoke"]
+    assert len(shots) == 1 and shots[0].get("if") == "${{ !cancelled() }}"
+    assert shots[0]["with"]["path"] == "smoke/"
+
+
+SMOKE = ROOT / "scripts" / "windows-smoke.ps1"
+
+
+def test_the_smoke_script_only_reads_the_local_server():
+    code = "\n".join(line for line in SMOKE.read_text(encoding="utf-8").splitlines()
+                     if not line.lstrip().startswith("#"))
+    assert set(re.findall(r"https?://([^/\s\"'`]+)", code)) == {"127.0.0.1:7777"}
+    assert "-Method" not in code and "-Body" not in code  # GETs only
+    for word in ("gh ", "git ", "Invoke-Expression", "iex "):
+        assert word not in code, word
+
+
+def test_the_smoke_script_checks_what_matters_on_windows():
+    code = SMOKE.read_text(encoding="utf-8")
+    for needle in ('"windows"', "code_execution.available -eq $false", "CloseMainWindow",
+                   "Start-Sleep -Seconds 20", "uninstall", "ops.db", '"/S"'):
+        assert needle in code, needle
+
+
 def test_the_script_it_calls_exists():
     assert (ROOT / "scripts" / "build-windows.ps1").is_file()