From 82f4804556cc23eafef66111ce07ae08ea8dc87d Mon Sep 17 00:00:00 2001 From: nathan Date: Sat, 30 May 2026 11:56:04 -0400 Subject: [PATCH 1/2] Persist runner state across restarts The runner stored compose files and .env data in /tmp while the systemd unit ran with PrivateTmp=true, which gives the service a fresh tmpfs on every (re)start. As a result, compose projects and other deploy artifacts were destroyed whenever the runner restarted. - Move compose projects from /tmp to /var/lib/lockethq/compose/, matching the existing persistent convention used by the proxy module. - Set PrivateTmp=false and add StateDirectory=lockethq in the systemd unit so /var/lib/lockethq is created/owned by systemd and never cleared on restart. - "Update runner" now re-pushes the systemd unit and runs daemon-reload before restart, so unit changes reach already-installed servers. - Add a best-effort startup migration that moves any leftover /tmp/lockethq-compose-* projects into the persistent location, with a recursive copy fallback for cross-filesystem (EXDEV) moves. Co-Authored-By: Claude Opus 4.8 --- app/src-tauri/src/ssh_install.rs | 15 ++++- runner/lockethq-runner.service | 8 ++- runner/src/docker.rs | 97 +++++++++++++++++++++++++++++++- runner/src/main.rs | 4 ++ 4 files changed, 120 insertions(+), 4 deletions(-) diff --git a/app/src-tauri/src/ssh_install.rs b/app/src-tauri/src/ssh_install.rs index a89600a..5f5f093 100644 --- a/app/src-tauri/src/ssh_install.rs +++ b/app/src-tauri/src/ssh_install.rs @@ -242,9 +242,22 @@ pub async fn update_runner( other => return Err(AppError::Ssh(format!("unsupported arch: {other}"))), }; upload(&session, "/usr/local/bin/lockethq-runner", binary, "0755", sudo).await?; + + // Refresh the systemd unit too — updates may ship unit changes (e.g. + // PrivateTmp/StateDirectory tweaks) that the binary alone can't apply. + // daemon-reload picks up the new unit before we restart. + let unit = include_str!("../../../runner/lockethq-runner.service"); + upload( + &session, + "/etc/systemd/system/lockethq-runner.service", + unit.as_bytes(), + "0644", + sudo, + ).await?; + let (code, _, err) = exec( &session, - &format!("{sudo} systemctl restart lockethq-runner && echo updated"), + &format!("{sudo} systemctl daemon-reload && {sudo} systemctl restart lockethq-runner && echo updated"), ).await?; if code != 0 { return Err(AppError::Ssh(format!("restart failed: {err}"))); diff --git a/runner/lockethq-runner.service b/runner/lockethq-runner.service index c3dddfe..eeddb6b 100644 --- a/runner/lockethq-runner.service +++ b/runner/lockethq-runner.service @@ -9,11 +9,17 @@ ExecStart=/usr/local/bin/lockethq-runner --config /etc/lockethq/runner.toml Restart=always RestartSec=2 User=root +# Persist runner state (compose files, .env, traefik config) across restarts. +# systemd creates/owns /var/lib/lockethq and never clears it on restart. +StateDirectory=lockethq # Hardening — runner needs docker.sock access, so keep root but drop other caps. NoNewPrivileges=true ProtectSystem=full ProtectHome=true -PrivateTmp=true +# PrivateTmp must stay off: a private /tmp is a fresh tmpfs on every (re)start, +# so anything the runner writes under /tmp is destroyed when the service +# restarts. Persistent state lives under /var/lib/lockethq (see StateDirectory). +PrivateTmp=false [Install] WantedBy=multi-user.target diff --git a/runner/src/docker.rs b/runner/src/docker.rs index f09d414..158258a 100644 --- a/runner/src/docker.rs +++ b/runner/src/docker.rs @@ -18,6 +18,99 @@ use lockethq_shared::{ use std::collections::HashMap; use tokio::sync::mpsc; +/// Persistent root for runner-owned state. Lives under `/var/lib` (not `/tmp`) +/// so compose files, `.env` files, and other deploy artifacts survive a runner +/// restart — a private `/tmp` is wiped on every restart. Mirrors the location +/// used by the proxy module (`/var/lib/lockethq/traefik`). +pub const STATE_ROOT: &str = "/var/lib/lockethq"; + +/// Directory holding the compose project for `project` (compose YAML + `.env`). +/// Persisted under [`STATE_ROOT`] so it is never destroyed on restart. +fn compose_project_dir(project: &str) -> std::path::PathBuf { + std::path::Path::new(STATE_ROOT) + .join("compose") + .join(project) +} + +/// One-time migration for boxes upgraded from a build that stored compose +/// projects in `/tmp` (`lockethq-compose-`). Older runners ran with +/// `PrivateTmp=true`, so those dirs were already wiped on restart — but if the +/// runner is updated *without* a restart in between, the files may still be +/// present. Move any survivors into the persistent location so a later +/// `compose down`/recreate can still find the compose file. Best-effort: logs +/// and continues on any error so a bad entry never blocks startup. +pub async fn migrate_legacy_compose_dirs() { + let tmp = std::env::temp_dir(); + let mut entries = match tokio::fs::read_dir(&tmp).await { + Ok(e) => e, + Err(_) => return, // no /tmp to scan — nothing to do + }; + let dest_base = std::path::Path::new(STATE_ROOT).join("compose"); + while let Ok(Some(entry)) = entries.next_entry().await { + let name = entry.file_name(); + let name = name.to_string_lossy(); + let Some(project) = name.strip_prefix("lockethq-compose-") else { + continue; + }; + if !entry.file_type().await.map(|t| t.is_dir()).unwrap_or(false) { + continue; + } + let dest = dest_base.join(project); + if tokio::fs::try_exists(&dest).await.unwrap_or(false) { + // Already migrated (or a newer deploy recreated it) — leave the + // canonical copy in place and drop the stale /tmp one. + let _ = tokio::fs::remove_dir_all(entry.path()).await; + continue; + } + if let Err(e) = tokio::fs::create_dir_all(&dest_base).await { + tracing::warn!("compose migration: mkdir {dest_base:?} failed: {e}"); + return; + } + // `rename` fails with EXDEV across filesystems (/tmp tmpfs → + // /var/lib disk), so fall back to a recursive copy + delete. + let moved = match tokio::fs::rename(entry.path(), &dest).await { + Ok(()) => true, + Err(_) => match copy_dir_recursive(&entry.path(), &dest).await { + Ok(()) => { + let _ = tokio::fs::remove_dir_all(entry.path()).await; + true + } + Err(e) => { + tracing::warn!( + "compose migration: copying '{project}' to {dest:?} failed: {e}" + ); + false + } + }, + }; + if moved { + tracing::info!("migrated legacy compose project '{project}' to {dest:?}"); + } + } +} + +/// Recursively copy `src` into `dst` (creating `dst`). Used by the legacy +/// compose migration when a cross-filesystem `rename` isn't possible. +fn copy_dir_recursive<'a>( + src: &'a std::path::Path, + dst: &'a std::path::Path, +) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + tokio::fs::create_dir_all(dst).await?; + let mut rd = tokio::fs::read_dir(src).await?; + while let Some(entry) = rd.next_entry().await? { + let from = entry.path(); + let to = dst.join(entry.file_name()); + if entry.file_type().await?.is_dir() { + copy_dir_recursive(&from, &to).await?; + } else { + tokio::fs::copy(&from, &to).await?; + } + } + Ok(()) + }) +} + pub async fn connect() -> Result { Docker::connect_with_local_defaults() .context("connecting to docker daemon (is /var/run/docker.sock accessible?)") @@ -610,7 +703,7 @@ pub async fn compose_up_stream( }).to_string()) .await; } - let dir = std::env::temp_dir().join(format!("lockethq-compose-{safe}")); + let dir = compose_project_dir(&safe); if let Err(e) = tokio::fs::create_dir_all(&dir).await { let _ = tx.send(serde_json::json!({"event":"error","message":format!("mkdir: {e}")}).to_string()).await; return Ok(()); @@ -756,7 +849,7 @@ pub async fn compose_up( "compose project '{requested}' already in use; deploying as '{safe}' to avoid clobbering existing containers" ); } - let dir = std::env::temp_dir().join(format!("lockethq-compose-{safe}")); + let dir = compose_project_dir(&safe); tokio::fs::create_dir_all(&dir).await?; let yaml_path = dir.join("docker-compose.yml"); { diff --git a/runner/src/main.rs b/runner/src/main.rs index 8283009..886e8bc 100644 --- a/runner/src/main.rs +++ b/runner/src/main.rs @@ -47,6 +47,10 @@ async fn main() -> Result<()> { let docker = docker::connect().await?; tracing::info!("connected to docker daemon"); + // Move any compose projects left in /tmp by older builds into the + // persistent state dir so they survive restarts. + docker::migrate_legacy_compose_dirs().await; + let state = Arc::new(routes::AppState { docker, token: cfg.token.clone(), From 84ec076d17a019cb14f7d161da15ca2d771662de Mon Sep 17 00:00:00 2001 From: nathan Date: Sat, 30 May 2026 12:01:37 -0400 Subject: [PATCH 2/2] ci: build runner binary before cargo check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo check --workspace` compiles the Tauri app crate, whose build script resolves the `binaries/*` resource glob from tauri.conf.json. Those runner binaries are gitignored release artifacts, so in CI the glob matched nothing and the build script failed with "glob pattern binaries/* ... didn't match any files" — CI had been red on main for this reason regardless of the change under test. Build the runner and stage it at app/src-tauri/binaries/ before the check so the bundled resource genuinely exists. This also verifies the runner compiles to a binary as a side effect. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb212f4..5b15071 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,16 @@ jobs: libayatana-appindicator3-dev \ librsvg2-dev \ patchelf + - name: Build runner binary + # The app's tauri.conf.json bundles `binaries/*` as resources, but those + # runner binaries are gitignored build artifacts only produced at release + # time. Without them the tauri build script fails with "glob pattern + # binaries/* ... didn't match any files". Build the real runner and stage + # it where the app expects it; this also verifies the runner compiles. + run: | + cargo build -p lockethq-runner + mkdir -p app/src-tauri/binaries + cp target/debug/lockethq-runner app/src-tauri/binaries/lockethq-runner-x86_64 - name: cargo check run: cargo check --workspace --all-targets