Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,10 @@ Status: in progress. The design is in `specs/next-ci-reliability` of
since when, so the first sighting is remembered in a state file) or a run queued for `--queue-min`
(default 15). A finding is reported once and again only after it has cleared; `--ignore NAME` mutes
a runner that is offline on purpose; an unreadable answer is skipped, never reported. Exit code 1
when it reported something.
when it reported something. `report/install-runner-watch.ps1` registers it as the scheduled task
`CI runner watch` (every five minutes, hidden, the owner's `gh` login, toasts through
`workstation-maintenance\notify.ps1`); run it from a normal PowerShell window of the owner's
session, with `-Ignore <runner>` for a runner that is offline on purpose, and `-Remove` to retire it.
`--until 2026-10-14T11:00` ends the period in the past, so a baseline can be reproduced after later
changes; `--limit` raises the number of runs read per repository (default 300).

Expand Down
78 changes: 78 additions & 0 deletions report/install-runner-watch.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Installs the runner watchdog on the desktop: every five minutes, one cycle of runner_watch.py, and a
# Windows toast when a runner of the organisation is offline or a run waits too long for one. Run it
# from a normal PowerShell window of the owner's session: scheduled tasks created from a sandboxed
# or elevated shell can be private to that shell.
#
# powershell -NoProfile -ExecutionPolicy Bypass -File report\install-runner-watch.ps1
# powershell -NoProfile -ExecutionPolicy Bypass -File report\install-runner-watch.ps1 -Remove
#
# The scripts are copied to $Destination, so the task does not depend on which branch a checkout is
# on; run this again after runner_watch.py changes. The task uses the owner's `gh` login; nothing
# else is stored. State lives in $Destination\runner-watch.json.
param(
[string]$Organisation = 'naindev-contrast3d',
[string]$Destination = 'D:\Development\Tools\ci-train\watch',
[string]$Notify = 'D:\Development\Tools\workstation-maintenance\notify.ps1',
# Runners that are offline on purpose and must never raise a toast.
[string[]]$Ignore = @(),
[int]$OfflineMinutes = 10,
[int]$QueueMinutes = 15,
[switch]$Remove
)

$ErrorActionPreference = 'Stop'
$taskName = 'CI runner watch'

if ($Remove) {
Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue
Write-Host "Task '$taskName' removed. Files in $Destination are left in place."
exit 0
}

if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { throw 'gh is not installed' }
gh auth status *> $null
if ($LASTEXITCODE -ne 0) { throw 'gh is not signed in: run gh auth login' }
if (-not (Test-Path $Notify)) { throw "notify.ps1 not found at $Notify (pass -Notify)" }

# The real pythonw.exe, not the `pyw` launcher (see train\install.ps1 for why).
$python = (py -3 -c "import sys;print(sys.executable)")
$pythonw = Join-Path (Split-Path $python) 'pythonw.exe'
if (-not (Test-Path $pythonw)) { throw "pythonw.exe not found at $pythonw" }

New-Item -ItemType Directory -Force -Path $Destination | Out-Null
foreach ($file in 'runner_watch.py', 'ci_report.py') {
Copy-Item -Force (Join-Path $PSScriptRoot $file) (Join-Path $Destination $file)
}
$script = Join-Path $Destination 'runner_watch.py'
$state = Join-Path $Destination 'runner-watch.json'

$arguments = "`"$script`" --org $Organisation --notify `"$Notify`" --state `"$state`"" +
" --offline-min $OfflineMinutes --queue-min $QueueMinutes"
foreach ($name in $Ignore) { $arguments += " --ignore `"$name`"" }

# pythonw: no console window flashing every five minutes. No -WorkingDirectory (see the train).
$action = New-ScheduledTaskAction -Execute $pythonw -Argument $arguments
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
-RepetitionInterval (New-TimeSpan -Minutes 5)
$settings = New-ScheduledTaskSettingsSet `
-MultipleInstances IgnoreNew `
-ExecutionTimeLimit (New-TimeSpan -Minutes 3) `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries `
-StartWhenAvailable
$principal = New-ScheduledTaskPrincipal -UserId "$env:USERDOMAIN\$env:USERNAME" `
-LogonType Interactive -RunLevel Limited

Register-ScheduledTask `
-TaskName $taskName `
-Action $action `
-Trigger $trigger `
-Settings $settings `
-Principal $principal `
-Description "Toast when a runner of $Organisation is offline for $OfflineMinutes min or a run is queued for $QueueMinutes min." `
-Force | Out-Null

Write-Host "Task '$taskName' installed: every 5 minutes."
Write-Host " State: $state"
Write-Host " Try now: py -3 `"$script`" --org $Organisation --state `"$state`""
Write-Host " Remove: report\install-runner-watch.ps1 -Remove"
Loading