From 1c38c29abbb3eac7d50ad30681a4a8d510b8b900 Mon Sep 17 00:00:00 2001 From: Oleksandr Kuzminskyi Date: Wed, 22 Jul 2026 13:06:25 -0700 Subject: [PATCH] Stop unattended-upgrades from cancelling GitHub Actions jobs; patch warm-pool runners Two related fixes for the self-hosted actions-runner fleet, both scoped to the runner role (development, sandbox, and global copies kept in sync, as #277 did). The design notes live in .claude/plans/TASK-runner-unattended-upgrades-cancels-jobs.md. 1. unattended-upgrades no longer cancels running jobs. Ubuntu's daily unattended-upgrades runs a needrestart pass that restarts every service linked against an updated library -- including actions-runner.service. Restarting the runner mid-job cancels the running GitHub job (the job dies; the instance stays healthy), and a job killed mid `terraform apply` leaks test infrastructure. The kill point is random because apt-daily-upgrade.timer has a large RandomizedDelaySec. profile::github_runner::service now: - drops /etc/needrestart/conf.d/90-actions-runner.conf with a TARGETED override ($nrconf{override_rc} => 0 for actions-runner.service only), so patching continues and every other service still auto-restarts -- only the runner is exempt; and - installs gha-restart-when-idle.timer, which restarts the runner to load the patched libraries only when it is idle (no Runner.Worker) AND a restart is actually pending (a "(deleted)" library mapping in the process tree) -- ground truth that does not depend on how needrestart lists an exempt service. This deliberately lives in the runner role, not the shared profile::unattended_upgrades, so it cannot collide with the Elasticsearch needrestart drop-in (profile::elastic::service): the two roles never share a host, each owns a distinctly-named conf.d file, and needrestart is declared via ensure_packages. Elastic uses a GLOBAL list-only mode; the runner uses a targeted per-service override -- different mechanisms, no conflict. 2. Warm-pool runners are patched before they hibernate. The runner ASG uses a Hibernated warm pool. A hibernated instance never runs the daily upgrade timer, and a hibernation resume is not a boot (systemd boot units do not re-run), so pooled instances would serve jobs on stale libraries. profile::github_runner now applies pending security upgrades once during provisioning -- before the instance hibernates -- via a per-boot exec (tmpfs marker, written only on success so a failure retries). Fresh launches driven by max_instance_lifetime re-run it, bounding staleness. Running it at provisioning (not at resume) keeps the fast hibernation resume untouched. Not yet exercised on a live runner: the /proc/maps, pgrep and hibernation mechanics are Linux-only. puppet-lint, shellcheck and perl-syntax all pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...runner-unattended-upgrades-cancels-jobs.md | 101 ++++++++++++++++++ debian/changelog | 6 ++ .../gha-restart-when-idle.service | 6 ++ .../github_runner/gha-restart-when-idle.sh | 42 ++++++++ .../github_runner/gha-restart-when-idle.timer | 10 ++ .../needrestart-90-actions-runner.conf | 22 ++++ .../profile/manifests/github_runner.pp | 22 ++++ .../manifests/github_runner/service.pp | 66 ++++++++++++ .../gha-restart-when-idle.service | 6 ++ .../github_runner/gha-restart-when-idle.sh | 42 ++++++++ .../github_runner/gha-restart-when-idle.timer | 10 ++ .../needrestart-90-actions-runner.conf | 22 ++++ .../profile/manifests/github_runner.pp | 22 ++++ .../manifests/github_runner/service.pp | 66 ++++++++++++ .../gha-restart-when-idle.service | 6 ++ .../github_runner/gha-restart-when-idle.sh | 42 ++++++++ .../github_runner/gha-restart-when-idle.timer | 10 ++ .../needrestart-90-actions-runner.conf | 22 ++++ modules/profile/manifests/github_runner.pp | 22 ++++ .../manifests/github_runner/service.pp | 66 ++++++++++++ 20 files changed, 611 insertions(+) create mode 100644 .claude/plans/TASK-runner-unattended-upgrades-cancels-jobs.md create mode 100644 environments/development/modules/profile/files/github_runner/gha-restart-when-idle.service create mode 100644 environments/development/modules/profile/files/github_runner/gha-restart-when-idle.sh create mode 100644 environments/development/modules/profile/files/github_runner/gha-restart-when-idle.timer create mode 100644 environments/development/modules/profile/files/github_runner/needrestart-90-actions-runner.conf create mode 100644 environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.service create mode 100644 environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.sh create mode 100644 environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.timer create mode 100644 environments/sandbox/modules/profile/files/github_runner/needrestart-90-actions-runner.conf create mode 100644 modules/profile/files/github_runner/gha-restart-when-idle.service create mode 100644 modules/profile/files/github_runner/gha-restart-when-idle.sh create mode 100644 modules/profile/files/github_runner/gha-restart-when-idle.timer create mode 100644 modules/profile/files/github_runner/needrestart-90-actions-runner.conf diff --git a/.claude/plans/TASK-runner-unattended-upgrades-cancels-jobs.md b/.claude/plans/TASK-runner-unattended-upgrades-cancels-jobs.md new file mode 100644 index 0000000..4a30234 --- /dev/null +++ b/.claude/plans/TASK-runner-unattended-upgrades-cancels-jobs.md @@ -0,0 +1,101 @@ +# Task: unattended-upgrades restarts actions-runner mid-job, cancelling CI jobs + +## Symptom + +GitHub Actions jobs on the self-hosted runner fleet (`actions-runner-…` ASG, +us-west-1) get cancelled mid-run with `##[error] The operation was canceled`, +followed by "Terminate orphan process" cleanup of `make`/`pytest`/`terraform`. +The **runner instance stays alive and healthy** — it is the *job* that dies, not +the box. Because the job dies mid-`terraform apply`, pytest never runs its +teardown, so **AWS test resources leak** and have to be cleaned up by hand. + +It happens at **random points** in a run (seen ~20 min in during a Puppet wait, +and ~90 s in during `terraform apply`), which made it look like flaky infra. + +Concrete case: run `terraform-aws-openvpn` PR #81, job on runner `ip-10-0-1-59` +(`i-0ee5dd25d62d63956`) — instance was `InService` + `ProtectedFromScaleIn: true` +the whole time, yet the job was cancelled at 18:32:24Z. + +## Root cause (confirmed from the runner's own logs) + +Ubuntu's **`apt-daily-upgrade.service` (unattended-upgrades)** fires on a +randomized daily timer, and the post-upgrade **`needrestart`** pass restarts every +service linked against an updated shared library — **including +`actions-runner.service`**. Restarting the runner agent mid-job cancels the job. + +systemd journal on the runner (`i-0ee5dd25d62d63956`, all UTC): + +``` +18:31:48 systemd[1]: Starting apt-daily-upgrade.service - Daily apt upgrade and clean activities... +18:32:10 systemd[1]: Stopping actions-runner.service - GitHub self-hosted runner... <-- killed here +18:32:33 systemd[1]: actions-runner.service: Deactivated / Stopped / Started +``` +(Same pass also bounced fwupd, packagekit, snapd, postfix, rpcbind — the usual +needrestart list after a libssl/glibc-class update.) + +Runner listener log at the same moment: +``` +18:32:11Z ERR BrokerServer System.ObjectDisposedException: ... 'System.Net.Sockets.NetworkStream' +18:32:11Z INFO JobDispatcher Shutting down JobDispatcher... cancel any running job. +18:32:29Z INFO JobDispatcher finish job request ... with result: Canceled +18:32:30Z INFO Listener Runner execution been cancelled. +``` + +The randomized timer (`apt-daily-upgrade.timer` has a large `RandomizedDelaySec`) +explains the random kill-point across runs. This is **not** scale-in / instance +termination — an earlier idle instance (`i-0e300d…`) was scaled in separately and +was a red herring; the job ran on a healthy protected runner. + +## Fix (goal: keep patching, never restart the runner mid-job) + +We WANT unattended-upgrades to keep applying security patches. We do NOT want it +to restart `actions-runner.service`. Two parts: + +### 1. Exclude actions-runner from needrestart auto-restart +Drop a needrestart override (Puppet-managed file on the runner): + +```perl +# /etc/needrestart/conf.d/actions-runner.conf +# Keep unattended-upgrades patching the box, but never restart the CI runner +# mid-job — restarting actions-runner.service cancels the running GitHub job. +$nrconf{override_rc} = { + qr(^actions-runner\.service$) => 0, # 0 = skip restart, 1 = force +}; +``` +(Confirm `override_rc` semantics on the installed needrestart version — noble ships +a recent one; `=> 0` = do-not-restart is the documented knob.) + +### 2. Restart the runner only when idle (so patches still land) +Excluding it means the runner keeps running the pre-upgrade libraries until it is +restarted some other way. Restart `actions-runner.service` **only when no job is +running**, e.g.: +- from the job-completed hook (`gha_postrun.sh` / `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`), or +- a small systemd timer / cron that restarts the service only if the runner is idle + (no active Worker / job). + +Leave `Unattended-Upgrade::Automatic-Reboot` as-is unless kernel-update reboots are +also cancelling jobs — same idea applies (reboot only when idle). + +## Acceptance + +- unattended-upgrades still runs and applies package updates on the runner. +- A `needrestart` pass triggered by an upgrade does **not** stop/restart + `actions-runner.service` while a job is running (verify: run an upgrade during a + job; job survives; journal shows no `Stopping actions-runner.service`). +- The runner still gets restarted (to pick up patched libs) when idle. +- No more mid-run `The operation was canceled` from this cause; no leaked test infra. + +## Scope / notes + +- Where this lives: the Puppet profile/module that builds the `actions-runner` + AMI / configures the runner instances (this repo, or the runner module referenced + from `aws-control`). Find the class managing `actions-runner.service` and add the + needrestart drop-in + idle-restart there. +- Unrelated to terraform-aws-openvpn PR #81 — that change is correct; it was just + the victim that surfaced this. Validate #81 locally with `make test-clean` (self- + destroys its stack) rather than waiting on the fleet. +- Separately worth a look (lower priority, not the cause here): the fleet flaps + (`IdleRunnersTooHigh`/`Low` oscillating desired 1↔2 every ~15 min), and the + scale-in protection model is job-start-triggered (`gha_prerun.sh` calls + `ih-aws autoscaling scale-in enable-protection`, giving up if already + `Terminating:Wait`). Not what cancelled these jobs, but a source of churn. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index e939979..3c9c7cc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +puppet-code (0.1.0-1build318) noble; urgency=medium + + * commit event. see changes history in git log + + -- root Wed, 22 Jul 2026 20:06:27 +0000 + puppet-code (0.1.0-1build317) noble; urgency=medium * commit event. see changes history in git log diff --git a/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.service b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.service new file mode 100644 index 0000000..70978fb --- /dev/null +++ b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart the GitHub Actions runner to load patched libraries, only when idle + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/gha-restart-when-idle.sh diff --git a/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.sh b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.sh new file mode 100644 index 0000000..991bd82 --- /dev/null +++ b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# gha-restart-when-idle.sh -- managed by Puppet (profile::github_runner::service). +# +# Companion to /etc/needrestart/conf.d/90-actions-runner.conf, which stops +# unattended-upgrades from restarting actions-runner.service mid-job. That keeps +# running jobs alive, but leaves the runner process still mapping the +# pre-upgrade shared libraries. Run periodically by gha-restart-when-idle.timer, +# this restarts the runner so those patched libraries actually get loaded -- +# but ONLY when the runner is idle and ONLY when a restart is actually pending. +set -euo pipefail + +SERVICE=actions-runner.service + +# A job in flight always has a Runner.Worker process; idle never does. +running_a_job() { pgrep -f 'Runner\.Worker' >/dev/null 2>&1; } + +# 1. Never touch a runner that is executing a job. +running_a_job && exit 0 + +# 2. Is a restart actually pending? True when the runner (or a child) still maps +# a shared library that an upgrade has since replaced on disk -- a "(deleted)" +# mapping in /proc/PID/maps. This is ground truth, independent of how +# needrestart chooses to list a service we have told it not to restart. +main_pid=$(systemctl show -p MainPID --value "$SERVICE" 2>/dev/null || echo 0) +[ "${main_pid:-0}" -gt 0 ] || exit 0 # not running -> nothing to restart + +pending=0 +for pid in "$main_pid" \ + $(pgrep -P "$main_pid" 2>/dev/null || true) \ + $(pgrep -f 'Runner\.Listener' 2>/dev/null || true); do + if grep -qsE '\.so[^ ]* \(deleted\)$' "/proc/${pid}/maps" 2>/dev/null; then + pending=1 + break + fi +done +[ "$pending" -eq 1 ] || exit 0 + +# 3. Re-check idle at the last moment to narrow the small race with a job that +# may have started since step 1, then restart to load the patched libraries. +running_a_job && exit 0 +logger -t gha-restart-when-idle "patched libraries pending and runner idle -> restarting ${SERVICE}" +systemctl restart "$SERVICE" diff --git a/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.timer b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.timer new file mode 100644 index 0000000..ee2a987 --- /dev/null +++ b/environments/development/modules/profile/files/github_runner/gha-restart-when-idle.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodically restart the idle GitHub Actions runner to load patched libraries + +[Timer] +OnBootSec=15min +OnUnitActiveSec=15min +Unit=gha-restart-when-idle.service + +[Install] +WantedBy=timers.target diff --git a/environments/development/modules/profile/files/github_runner/needrestart-90-actions-runner.conf b/environments/development/modules/profile/files/github_runner/needrestart-90-actions-runner.conf new file mode 100644 index 0000000..c536de6 --- /dev/null +++ b/environments/development/modules/profile/files/github_runner/needrestart-90-actions-runner.conf @@ -0,0 +1,22 @@ +# /etc/needrestart/conf.d/90-actions-runner.conf +# Managed by Puppet (profile::github_runner::service). Do not edit. +# +# unattended-upgrades (profile::unattended_upgrades) patches the box; its +# post-upgrade needrestart pass would restart actions-runner.service to pick up +# updated shared libraries. Restarting the runner mid-job cancels the running +# GitHub Actions job -- the job dies (the instance stays healthy), and any +# half-provisioned test infrastructure leaks. Skip the runner here so a job is +# never cancelled by a patch. gha-restart-when-idle.timer restarts the runner +# out of band, only when it is idle, so the patched libraries still get loaded. +# +# Unlike profile::elastic::service, which sets a GLOBAL list-only mode +# ($nrconf{restart} = 'l') on Elasticsearch nodes, this is a TARGETED override: +# every other service on the runner still auto-restarts normally -- only the +# runner itself is exempt. The two never share a host and own distinct conf.d +# files, so they cannot conflict. +# +# override_rc maps a service-name regex to a restart flag: 0 = never restart, +# 1 = always restart. +$nrconf{override_rc} = { + qr(^actions-runner\.service$) => 0, +}; diff --git a/environments/development/modules/profile/manifests/github_runner.pp b/environments/development/modules/profile/manifests/github_runner.pp index d2b8603..7f0e444 100644 --- a/environments/development/modules/profile/manifests/github_runner.pp +++ b/environments/development/modules/profile/manifests/github_runner.pp @@ -51,4 +51,26 @@ group => $group, } + # Warm-pool runners are HIBERNATED right after provisioning, so the daily + # unattended-upgrades timer never runs while they sit in the pool -- and a + # hibernation resume is not a boot, so systemd boot units do not re-run on the + # warm->hot transition either. Apply pending security upgrades ONCE here, + # during provisioning, before the instance hibernates, so every runner enters + # the warm pool already patched. Fresh launches (driven by the ASG's + # max_instance_lifetime) re-run this on each new instance, which bounds how + # stale a pooled instance can be. In-service runners keep getting the daily + # timer (and profile::github_runner::service keeps that from cancelling jobs). + # + # The marker lives on tmpfs (/run, cleared on a real boot) so this runs once + # per boot rather than on every Puppet apply, and is written only on success, + # so a failed upgrade simply retries on the next apply. This applies Ubuntu + # security updates, which do not depend on the InfraHouse repos. + exec { 'gha-boot-security-upgrade': + command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done', + path => '/usr/bin:/bin:/usr/sbin:/sbin', + unless => 'test -f /run/gha-boot-upgrade.done', + timeout => 1200, + require => Class['profile::unattended_upgrades'], + } + } diff --git a/environments/development/modules/profile/manifests/github_runner/service.pp b/environments/development/modules/profile/manifests/github_runner/service.pp index 6ff6018..91015c2 100644 --- a/environments/development/modules/profile/manifests/github_runner/service.pp +++ b/environments/development/modules/profile/manifests/github_runner/service.pp @@ -153,4 +153,70 @@ user => 'root', minute => '*/5', } + + # Security patching without cancelling jobs. + # + # profile::unattended_upgrades patches every host fleet-wide. Its post-upgrade + # needrestart pass restarts services linked against updated libraries -- and + # left to itself it restarts actions-runner.service, which cancels whatever + # job is running (the job dies mid-run; the instance stays healthy). We keep + # patching but exempt the runner from that automatic restart, then restart it + # ourselves only when it is idle so the patched libraries still get loaded. + # + # This deliberately lives in this role-specific class, NOT in the shared + # profile::unattended_upgrades, so it cannot collide with the Elasticsearch + # needrestart drop-in (profile::elastic::service): the two roles never share a + # host, and each owns a distinctly-named /etc/needrestart/conf.d file. + # needrestart is declared via ensure_packages for the same collision-safety. + stdlib::ensure_packages(['needrestart']) + + $restart_when_idle_script = '/usr/local/bin/gha-restart-when-idle.sh' + $restart_when_idle_service = '/etc/systemd/system/gha-restart-when-idle.service' + $restart_when_idle_timer = '/etc/systemd/system/gha-restart-when-idle.timer' + + file { '/etc/needrestart/conf.d/90-actions-runner.conf': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/needrestart-90-actions-runner.conf', + require => Package['needrestart'], + } + + file { $restart_when_idle_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.sh', + } + + file { $restart_when_idle_service: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.service', + notify => Exec['daemon-reload'], + } + + file { $restart_when_idle_timer: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.timer', + notify => Exec['daemon-reload'], + } + + service { 'gha-restart-when-idle.timer': + ensure => running, + enable => true, + require => [ + File[$restart_when_idle_script], + File[$restart_when_idle_service], + File[$restart_when_idle_timer], + Exec['daemon-reload'], + ], + } } diff --git a/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.service b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.service new file mode 100644 index 0000000..70978fb --- /dev/null +++ b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart the GitHub Actions runner to load patched libraries, only when idle + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/gha-restart-when-idle.sh diff --git a/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.sh b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.sh new file mode 100644 index 0000000..991bd82 --- /dev/null +++ b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# gha-restart-when-idle.sh -- managed by Puppet (profile::github_runner::service). +# +# Companion to /etc/needrestart/conf.d/90-actions-runner.conf, which stops +# unattended-upgrades from restarting actions-runner.service mid-job. That keeps +# running jobs alive, but leaves the runner process still mapping the +# pre-upgrade shared libraries. Run periodically by gha-restart-when-idle.timer, +# this restarts the runner so those patched libraries actually get loaded -- +# but ONLY when the runner is idle and ONLY when a restart is actually pending. +set -euo pipefail + +SERVICE=actions-runner.service + +# A job in flight always has a Runner.Worker process; idle never does. +running_a_job() { pgrep -f 'Runner\.Worker' >/dev/null 2>&1; } + +# 1. Never touch a runner that is executing a job. +running_a_job && exit 0 + +# 2. Is a restart actually pending? True when the runner (or a child) still maps +# a shared library that an upgrade has since replaced on disk -- a "(deleted)" +# mapping in /proc/PID/maps. This is ground truth, independent of how +# needrestart chooses to list a service we have told it not to restart. +main_pid=$(systemctl show -p MainPID --value "$SERVICE" 2>/dev/null || echo 0) +[ "${main_pid:-0}" -gt 0 ] || exit 0 # not running -> nothing to restart + +pending=0 +for pid in "$main_pid" \ + $(pgrep -P "$main_pid" 2>/dev/null || true) \ + $(pgrep -f 'Runner\.Listener' 2>/dev/null || true); do + if grep -qsE '\.so[^ ]* \(deleted\)$' "/proc/${pid}/maps" 2>/dev/null; then + pending=1 + break + fi +done +[ "$pending" -eq 1 ] || exit 0 + +# 3. Re-check idle at the last moment to narrow the small race with a job that +# may have started since step 1, then restart to load the patched libraries. +running_a_job && exit 0 +logger -t gha-restart-when-idle "patched libraries pending and runner idle -> restarting ${SERVICE}" +systemctl restart "$SERVICE" diff --git a/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.timer b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.timer new file mode 100644 index 0000000..ee2a987 --- /dev/null +++ b/environments/sandbox/modules/profile/files/github_runner/gha-restart-when-idle.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodically restart the idle GitHub Actions runner to load patched libraries + +[Timer] +OnBootSec=15min +OnUnitActiveSec=15min +Unit=gha-restart-when-idle.service + +[Install] +WantedBy=timers.target diff --git a/environments/sandbox/modules/profile/files/github_runner/needrestart-90-actions-runner.conf b/environments/sandbox/modules/profile/files/github_runner/needrestart-90-actions-runner.conf new file mode 100644 index 0000000..c536de6 --- /dev/null +++ b/environments/sandbox/modules/profile/files/github_runner/needrestart-90-actions-runner.conf @@ -0,0 +1,22 @@ +# /etc/needrestart/conf.d/90-actions-runner.conf +# Managed by Puppet (profile::github_runner::service). Do not edit. +# +# unattended-upgrades (profile::unattended_upgrades) patches the box; its +# post-upgrade needrestart pass would restart actions-runner.service to pick up +# updated shared libraries. Restarting the runner mid-job cancels the running +# GitHub Actions job -- the job dies (the instance stays healthy), and any +# half-provisioned test infrastructure leaks. Skip the runner here so a job is +# never cancelled by a patch. gha-restart-when-idle.timer restarts the runner +# out of band, only when it is idle, so the patched libraries still get loaded. +# +# Unlike profile::elastic::service, which sets a GLOBAL list-only mode +# ($nrconf{restart} = 'l') on Elasticsearch nodes, this is a TARGETED override: +# every other service on the runner still auto-restarts normally -- only the +# runner itself is exempt. The two never share a host and own distinct conf.d +# files, so they cannot conflict. +# +# override_rc maps a service-name regex to a restart flag: 0 = never restart, +# 1 = always restart. +$nrconf{override_rc} = { + qr(^actions-runner\.service$) => 0, +}; diff --git a/environments/sandbox/modules/profile/manifests/github_runner.pp b/environments/sandbox/modules/profile/manifests/github_runner.pp index d2b8603..7f0e444 100644 --- a/environments/sandbox/modules/profile/manifests/github_runner.pp +++ b/environments/sandbox/modules/profile/manifests/github_runner.pp @@ -51,4 +51,26 @@ group => $group, } + # Warm-pool runners are HIBERNATED right after provisioning, so the daily + # unattended-upgrades timer never runs while they sit in the pool -- and a + # hibernation resume is not a boot, so systemd boot units do not re-run on the + # warm->hot transition either. Apply pending security upgrades ONCE here, + # during provisioning, before the instance hibernates, so every runner enters + # the warm pool already patched. Fresh launches (driven by the ASG's + # max_instance_lifetime) re-run this on each new instance, which bounds how + # stale a pooled instance can be. In-service runners keep getting the daily + # timer (and profile::github_runner::service keeps that from cancelling jobs). + # + # The marker lives on tmpfs (/run, cleared on a real boot) so this runs once + # per boot rather than on every Puppet apply, and is written only on success, + # so a failed upgrade simply retries on the next apply. This applies Ubuntu + # security updates, which do not depend on the InfraHouse repos. + exec { 'gha-boot-security-upgrade': + command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done', + path => '/usr/bin:/bin:/usr/sbin:/sbin', + unless => 'test -f /run/gha-boot-upgrade.done', + timeout => 1200, + require => Class['profile::unattended_upgrades'], + } + } diff --git a/environments/sandbox/modules/profile/manifests/github_runner/service.pp b/environments/sandbox/modules/profile/manifests/github_runner/service.pp index 6ff6018..91015c2 100644 --- a/environments/sandbox/modules/profile/manifests/github_runner/service.pp +++ b/environments/sandbox/modules/profile/manifests/github_runner/service.pp @@ -153,4 +153,70 @@ user => 'root', minute => '*/5', } + + # Security patching without cancelling jobs. + # + # profile::unattended_upgrades patches every host fleet-wide. Its post-upgrade + # needrestart pass restarts services linked against updated libraries -- and + # left to itself it restarts actions-runner.service, which cancels whatever + # job is running (the job dies mid-run; the instance stays healthy). We keep + # patching but exempt the runner from that automatic restart, then restart it + # ourselves only when it is idle so the patched libraries still get loaded. + # + # This deliberately lives in this role-specific class, NOT in the shared + # profile::unattended_upgrades, so it cannot collide with the Elasticsearch + # needrestart drop-in (profile::elastic::service): the two roles never share a + # host, and each owns a distinctly-named /etc/needrestart/conf.d file. + # needrestart is declared via ensure_packages for the same collision-safety. + stdlib::ensure_packages(['needrestart']) + + $restart_when_idle_script = '/usr/local/bin/gha-restart-when-idle.sh' + $restart_when_idle_service = '/etc/systemd/system/gha-restart-when-idle.service' + $restart_when_idle_timer = '/etc/systemd/system/gha-restart-when-idle.timer' + + file { '/etc/needrestart/conf.d/90-actions-runner.conf': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/needrestart-90-actions-runner.conf', + require => Package['needrestart'], + } + + file { $restart_when_idle_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.sh', + } + + file { $restart_when_idle_service: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.service', + notify => Exec['daemon-reload'], + } + + file { $restart_when_idle_timer: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.timer', + notify => Exec['daemon-reload'], + } + + service { 'gha-restart-when-idle.timer': + ensure => running, + enable => true, + require => [ + File[$restart_when_idle_script], + File[$restart_when_idle_service], + File[$restart_when_idle_timer], + Exec['daemon-reload'], + ], + } } diff --git a/modules/profile/files/github_runner/gha-restart-when-idle.service b/modules/profile/files/github_runner/gha-restart-when-idle.service new file mode 100644 index 0000000..70978fb --- /dev/null +++ b/modules/profile/files/github_runner/gha-restart-when-idle.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restart the GitHub Actions runner to load patched libraries, only when idle + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/gha-restart-when-idle.sh diff --git a/modules/profile/files/github_runner/gha-restart-when-idle.sh b/modules/profile/files/github_runner/gha-restart-when-idle.sh new file mode 100644 index 0000000..991bd82 --- /dev/null +++ b/modules/profile/files/github_runner/gha-restart-when-idle.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# gha-restart-when-idle.sh -- managed by Puppet (profile::github_runner::service). +# +# Companion to /etc/needrestart/conf.d/90-actions-runner.conf, which stops +# unattended-upgrades from restarting actions-runner.service mid-job. That keeps +# running jobs alive, but leaves the runner process still mapping the +# pre-upgrade shared libraries. Run periodically by gha-restart-when-idle.timer, +# this restarts the runner so those patched libraries actually get loaded -- +# but ONLY when the runner is idle and ONLY when a restart is actually pending. +set -euo pipefail + +SERVICE=actions-runner.service + +# A job in flight always has a Runner.Worker process; idle never does. +running_a_job() { pgrep -f 'Runner\.Worker' >/dev/null 2>&1; } + +# 1. Never touch a runner that is executing a job. +running_a_job && exit 0 + +# 2. Is a restart actually pending? True when the runner (or a child) still maps +# a shared library that an upgrade has since replaced on disk -- a "(deleted)" +# mapping in /proc/PID/maps. This is ground truth, independent of how +# needrestart chooses to list a service we have told it not to restart. +main_pid=$(systemctl show -p MainPID --value "$SERVICE" 2>/dev/null || echo 0) +[ "${main_pid:-0}" -gt 0 ] || exit 0 # not running -> nothing to restart + +pending=0 +for pid in "$main_pid" \ + $(pgrep -P "$main_pid" 2>/dev/null || true) \ + $(pgrep -f 'Runner\.Listener' 2>/dev/null || true); do + if grep -qsE '\.so[^ ]* \(deleted\)$' "/proc/${pid}/maps" 2>/dev/null; then + pending=1 + break + fi +done +[ "$pending" -eq 1 ] || exit 0 + +# 3. Re-check idle at the last moment to narrow the small race with a job that +# may have started since step 1, then restart to load the patched libraries. +running_a_job && exit 0 +logger -t gha-restart-when-idle "patched libraries pending and runner idle -> restarting ${SERVICE}" +systemctl restart "$SERVICE" diff --git a/modules/profile/files/github_runner/gha-restart-when-idle.timer b/modules/profile/files/github_runner/gha-restart-when-idle.timer new file mode 100644 index 0000000..ee2a987 --- /dev/null +++ b/modules/profile/files/github_runner/gha-restart-when-idle.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodically restart the idle GitHub Actions runner to load patched libraries + +[Timer] +OnBootSec=15min +OnUnitActiveSec=15min +Unit=gha-restart-when-idle.service + +[Install] +WantedBy=timers.target diff --git a/modules/profile/files/github_runner/needrestart-90-actions-runner.conf b/modules/profile/files/github_runner/needrestart-90-actions-runner.conf new file mode 100644 index 0000000..c536de6 --- /dev/null +++ b/modules/profile/files/github_runner/needrestart-90-actions-runner.conf @@ -0,0 +1,22 @@ +# /etc/needrestart/conf.d/90-actions-runner.conf +# Managed by Puppet (profile::github_runner::service). Do not edit. +# +# unattended-upgrades (profile::unattended_upgrades) patches the box; its +# post-upgrade needrestart pass would restart actions-runner.service to pick up +# updated shared libraries. Restarting the runner mid-job cancels the running +# GitHub Actions job -- the job dies (the instance stays healthy), and any +# half-provisioned test infrastructure leaks. Skip the runner here so a job is +# never cancelled by a patch. gha-restart-when-idle.timer restarts the runner +# out of band, only when it is idle, so the patched libraries still get loaded. +# +# Unlike profile::elastic::service, which sets a GLOBAL list-only mode +# ($nrconf{restart} = 'l') on Elasticsearch nodes, this is a TARGETED override: +# every other service on the runner still auto-restarts normally -- only the +# runner itself is exempt. The two never share a host and own distinct conf.d +# files, so they cannot conflict. +# +# override_rc maps a service-name regex to a restart flag: 0 = never restart, +# 1 = always restart. +$nrconf{override_rc} = { + qr(^actions-runner\.service$) => 0, +}; diff --git a/modules/profile/manifests/github_runner.pp b/modules/profile/manifests/github_runner.pp index d2b8603..7f0e444 100644 --- a/modules/profile/manifests/github_runner.pp +++ b/modules/profile/manifests/github_runner.pp @@ -51,4 +51,26 @@ group => $group, } + # Warm-pool runners are HIBERNATED right after provisioning, so the daily + # unattended-upgrades timer never runs while they sit in the pool -- and a + # hibernation resume is not a boot, so systemd boot units do not re-run on the + # warm->hot transition either. Apply pending security upgrades ONCE here, + # during provisioning, before the instance hibernates, so every runner enters + # the warm pool already patched. Fresh launches (driven by the ASG's + # max_instance_lifetime) re-run this on each new instance, which bounds how + # stale a pooled instance can be. In-service runners keep getting the daily + # timer (and profile::github_runner::service keeps that from cancelling jobs). + # + # The marker lives on tmpfs (/run, cleared on a real boot) so this runs once + # per boot rather than on every Puppet apply, and is written only on success, + # so a failed upgrade simply retries on the next apply. This applies Ubuntu + # security updates, which do not depend on the InfraHouse repos. + exec { 'gha-boot-security-upgrade': + command => 'apt-get update -qq && unattended-upgrade && touch /run/gha-boot-upgrade.done', + path => '/usr/bin:/bin:/usr/sbin:/sbin', + unless => 'test -f /run/gha-boot-upgrade.done', + timeout => 1200, + require => Class['profile::unattended_upgrades'], + } + } diff --git a/modules/profile/manifests/github_runner/service.pp b/modules/profile/manifests/github_runner/service.pp index 6ff6018..91015c2 100644 --- a/modules/profile/manifests/github_runner/service.pp +++ b/modules/profile/manifests/github_runner/service.pp @@ -153,4 +153,70 @@ user => 'root', minute => '*/5', } + + # Security patching without cancelling jobs. + # + # profile::unattended_upgrades patches every host fleet-wide. Its post-upgrade + # needrestart pass restarts services linked against updated libraries -- and + # left to itself it restarts actions-runner.service, which cancels whatever + # job is running (the job dies mid-run; the instance stays healthy). We keep + # patching but exempt the runner from that automatic restart, then restart it + # ourselves only when it is idle so the patched libraries still get loaded. + # + # This deliberately lives in this role-specific class, NOT in the shared + # profile::unattended_upgrades, so it cannot collide with the Elasticsearch + # needrestart drop-in (profile::elastic::service): the two roles never share a + # host, and each owns a distinctly-named /etc/needrestart/conf.d file. + # needrestart is declared via ensure_packages for the same collision-safety. + stdlib::ensure_packages(['needrestart']) + + $restart_when_idle_script = '/usr/local/bin/gha-restart-when-idle.sh' + $restart_when_idle_service = '/etc/systemd/system/gha-restart-when-idle.service' + $restart_when_idle_timer = '/etc/systemd/system/gha-restart-when-idle.timer' + + file { '/etc/needrestart/conf.d/90-actions-runner.conf': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/needrestart-90-actions-runner.conf', + require => Package['needrestart'], + } + + file { $restart_when_idle_script: + ensure => file, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.sh', + } + + file { $restart_when_idle_service: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.service', + notify => Exec['daemon-reload'], + } + + file { $restart_when_idle_timer: + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/github_runner/gha-restart-when-idle.timer', + notify => Exec['daemon-reload'], + } + + service { 'gha-restart-when-idle.timer': + ensure => running, + enable => true, + require => [ + File[$restart_when_idle_script], + File[$restart_when_idle_service], + File[$restart_when_idle_timer], + Exec['daemon-reload'], + ], + } }