From 131c694629b473055eb7fdbd48c38c03d224c3af Mon Sep 17 00:00:00 2001 From: x Date: Mon, 9 Nov 2026 16:25:12 -0500 Subject: [PATCH] fix(desktop-gpui): recover from a lost GPU device on Linux without crashing in the NVIDIA driver --- .github/workflows/ci.yml | 1 + .../workflows/linux-desktop-reliability.yml | 15 ++ .../patches/zed-device-lost.patch | 193 ++++++++++++++++++ scripts/linux-desktop-harness/README.md | 5 + .../scenario-device-loss.sh | 62 ++++++ scripts/prepare-gpui-dependency.sh | 1 + scripts/prepare-gpui-dependency.test.mjs | 2 + 7 files changed, 279 insertions(+) create mode 100644 apps/desktop-gpui/patches/zed-device-lost.patch create mode 100644 scripts/linux-desktop-harness/scenario-device-loss.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa48a739940..fc8568943d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -492,6 +492,7 @@ jobs: if [[ "$RUNNER_OS" == "Linux" ]]; then WGPU_BACKEND=vulkan cargo +1.95.0 test -p gpui_wgpu --lib wgpu_atlas -- --nocapture WGPU_BACKEND=gl cargo +1.95.0 test -p gpui_wgpu --lib wgpu_atlas -- --nocapture + cargo +1.95.0 test -p gpui_linux --lib -- unmapped_frame retained_visibility elif [[ "$RUNNER_OS" == "macOS" ]]; then cargo +1.95.0 test -p gpui_wgpu --lib wgpu_atlas -- --nocapture # Run under lldb: a SIGSEGV in these tests (1 run in about 36 so diff --git a/.github/workflows/linux-desktop-reliability.yml b/.github/workflows/linux-desktop-reliability.yml index 28bd9180670..8742d083aa2 100644 --- a/.github/workflows/linux-desktop-reliability.yml +++ b/.github/workflows/linux-desktop-reliability.yml @@ -412,6 +412,21 @@ jobs: grep -qx "$line" "$RUNNER_TEMP/out/hyprland-screenshot/result.txt" done + # A GPU device lost while the main window is hidden is found by the + # frame that maps the window again. The window must still come back + # (GitHub #3946). + - name: Hyprland GPU device lost while the window is hidden + env: + SCENARIO: device-loss + run: | + sudo -E env DEV_BIN="$RUNNER_TEMP/build/cap-gpui" DEV_LIBS="$RUNNER_TEMP/build/libs" \ + bash scripts/linux-desktop-harness/run-desktop.sh hyprland "$RUNNER_TEMP/build/cap-gpui" \ + "$RUNNER_TEMP/out/hyprland-device-loss" + cat "$RUNNER_TEMP/out/hyprland-device-loss/result.txt" + for line in first_shown=yes hidden=yes shown_again=yes device_lost=1 restore_failed=0 cap_alive=yes panics=0; do + grep -qx "$line" "$RUNNER_TEMP/out/hyprland-device-loss/result.txt" + done + # The hosted runner can build v4l2loopback; Blacksmith's kernel cannot. - name: Install Linux desktop dependencies uses: ./.github/actions/install-desktop-deps diff --git a/apps/desktop-gpui/patches/zed-device-lost.patch b/apps/desktop-gpui/patches/zed-device-lost.patch new file mode 100644 index 00000000000..2ac7f094b5c --- /dev/null +++ b/apps/desktop-gpui/patches/zed-device-lost.patch @@ -0,0 +1,193 @@ +diff --git a/crates/gpui_linux/src/linux/wayland/window.rs b/crates/gpui_linux/src/linux/wayland/window.rs +index e09e75a..8dd40de 100644 +--- a/crates/gpui_linux/src/linux/wayland/window.rs ++++ b/crates/gpui_linux/src/linux/wayland/window.rs +@@ -97,6 +97,19 @@ struct InProgressConfigure { + /// new role. Compositors that send one do so within a frame or two. + const REMAP_CONFIGURE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + ++/// How long an unmapped window waits before drawing again after a frame that ++/// presented nothing. ++const UNMAPPED_FRAME_RETRY: std::time::Duration = std::time::Duration::from_millis(50); ++ ++/// Whether a frame that presented nothing has to be drawn again without a ++/// frame callback. Compositors send those only for a mapped surface, so when ++/// the frame that would have attached the first buffer finds the GPU device ++/// lost, or is spent recovering from that, nothing else asks for the next one ++/// and the window stays invisible for good (GitHub #3946). ++fn unmapped_frame_needs_retry(presented: bool, buffer_attached: bool, render_owed: bool) -> bool { ++ !presented && !buffer_attached && render_owed ++} ++ + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum VisibilityPhase { + Visible, +@@ -257,6 +270,7 @@ pub struct WaylandWindowState { + hovered: bool, + pub(crate) force_render_after_recovery: bool, + renderer_presented: bool, ++ frame_retry_scheduled: bool, + /// A buffer is committed on the surface: it is mapped. + buffer_attached: bool, + /// The renderer's surface was retired by an unmap and must be recreated +@@ -827,6 +841,7 @@ impl WaylandWindowState { + hovered: false, + force_render_after_recovery: false, + renderer_presented: false, ++ frame_retry_scheduled: false, + buffer_attached: false, + surface_retired: false, + in_progress_window_controls: None, +@@ -2310,7 +2325,11 @@ impl PlatformWindow for WaylandWindow { + WaylandWindowStatePtr::visibility_sync(&state, transition, true); + } + +- if state.renderer.needs_redraw() { ++ // A draw that found the device lost presented nothing: the next frame ++ // recovers and has to render even if nothing else changed. ++ if state.renderer.needs_redraw() ++ || (!state.renderer_presented && state.renderer.device_lost()) ++ { + state.force_render_after_recovery = true; + } + } +@@ -2331,7 +2350,33 @@ impl PlatformWindow for WaylandWindow { + state.surface.commit(); + } + ++ let retry = !state.frame_retry_scheduled ++ && unmapped_frame_needs_retry( ++ state.renderer_presented, ++ state.buffer_attached, ++ state.force_render_after_recovery, ++ ); + state.renderer_presented = false; ++ if retry { ++ state.frame_retry_scheduled = true; ++ let background = state ++ .client ++ .get_client() ++ .borrow() ++ .common ++ .background_executor ++ .clone(); ++ let foreground = state.globals.executor.clone(); ++ drop(state); ++ let window = self.0.clone(); ++ foreground ++ .spawn(async move { ++ background.timer(UNMAPPED_FRAME_RETRY).await; ++ window.state.borrow_mut().frame_retry_scheduled = false; ++ window.frame(); ++ }) ++ .detach(); ++ } + } + + fn sprite_atlas(&self) -> Arc { +@@ -2771,6 +2816,27 @@ mod tests { + } + } + ++#[cfg(test)] ++mod unmapped_frame_retry_tests { ++ use super::unmapped_frame_needs_retry; ++ ++ #[test] ++ fn a_first_frame_lost_to_gpu_recovery_is_drawn_again() { ++ assert!(unmapped_frame_needs_retry(false, false, true)); ++ } ++ ++ #[test] ++ fn a_mapped_surface_waits_for_its_frame_callback() { ++ assert!(!unmapped_frame_needs_retry(false, true, true)); ++ } ++ ++ #[test] ++ fn a_presented_or_idle_frame_is_not_retried() { ++ assert!(!unmapped_frame_needs_retry(true, false, true)); ++ assert!(!unmapped_frame_needs_retry(false, false, false)); ++ } ++} ++ + #[cfg(test)] + mod retained_visibility_tests { + use super::{RetainedVisibility, VisibilityPhase}; +diff --git a/crates/gpui_wgpu/src/wgpu_renderer.rs b/crates/gpui_wgpu/src/wgpu_renderer.rs +index 72ce028..da38590 100644 +--- a/crates/gpui_wgpu/src/wgpu_renderer.rs ++++ b/crates/gpui_wgpu/src/wgpu_renderer.rs +@@ -1220,6 +1220,13 @@ impl WgpuRenderer { + return false; + } + ++ if device_loss_injected() { ++ log::error!("wgpu device lost: injected through GPUI_INJECT_DEVICE_LOSS"); ++ self.device_lost ++ .store(true, std::sync::atomic::Ordering::SeqCst); ++ return false; ++ } ++ + let last_error = self.last_error.lock().unwrap().take(); + if let Some(error) = last_error { + self.failed_frame_count += 1; +@@ -2275,18 +2282,37 @@ impl WgpuRenderer { + + // Drop old resources to release Arc/Arc and GPU resources + self.resources = None; +- *gpu_context.borrow_mut() = None; ++ ++ // The lost context stays in place until its replacement exists, ++ // and the replacement is made on the same instance. NVIDIA's ++ // Vulkan driver cannot have a second instance created and the ++ // first destroyed while Wayland surfaces are in use: the next ++ // surface query or swapchain creation on the survivor calls a ++ // null pointer inside the driver (GitHub #3946). One instance for ++ // the life of the app never runs that teardown, and a failed ++ // attempt retries with it. ++ let (instance, was_software) = match gpu_context.borrow().as_ref() { ++ Some(context) => ( ++ context.instance.clone(), ++ context.adapter.get_info().device_type == wgpu::DeviceType::Cpu, ++ ), ++ None => (WgpuContext::instance(Box::new(window.clone())), false), ++ }; + + // Wait briefly for the GPU driver to stabilize, then try to + // recreate the context without software renderers. If this fails + // the caller should request another frame and retry — the real GPU + // may need more time to come back (e.g. after suspend/resume). ++ // A session that was on a software renderer has no GPU to wait ++ // for, and would otherwise never recover. + std::thread::sleep(std::time::Duration::from_millis(350)); + +- let instance = WgpuContext::instance(Box::new(window.clone())); + let surface = create_surface(&instance, window_handle.as_raw())?; +- let new_context = +- WgpuContext::new_rejecting_software(instance, &surface, self.compositor_gpu)?; ++ let new_context = if was_software { ++ WgpuContext::new(instance, &surface, self.compositor_gpu)? ++ } else { ++ WgpuContext::new_rejecting_software(instance, &surface, self.compositor_gpu)? ++ }; + *gpu_context.borrow_mut() = Some(new_context); + surface + } else { +@@ -2324,6 +2350,17 @@ impl WgpuRenderer { + } + } + ++/// Test hook: `GPUI_INJECT_DEVICE_LOSS` names a file, and creating that file ++/// makes the next frame end the way a frame does when the driver reports the ++/// device lost. Real losses need a driver fault, which tests cannot ask for. ++fn device_loss_injected() -> bool { ++ static TRIGGER: std::sync::LazyLock> = ++ std::sync::LazyLock::new(|| std::env::var_os("GPUI_INJECT_DEVICE_LOSS").map(Into::into)); ++ TRIGGER ++ .as_ref() ++ .is_some_and(|trigger| std::fs::remove_file(trigger).is_ok()) ++} ++ + #[cfg(not(target_family = "wasm"))] + fn create_surface( + instance: &wgpu::Instance, diff --git a/scripts/linux-desktop-harness/README.md b/scripts/linux-desktop-harness/README.md index af183b426c6..66858a8b30b 100644 --- a/scripts/linux-desktop-harness/README.md +++ b/scripts/linux-desktop-harness/README.md @@ -63,6 +63,11 @@ portal's dialog aborts in the container). It checks a PNG of the display's size desktop's wallpaper as its background (`CAP_GPUI_AUTO_DESKTOP_BACKGROUND=1`). GNOME keeps its default JPEG XL wallpaper; `WALLPAPER=swaybg` starts swaybg with a PNG first, as Omarchy does on Hyprland. +`SCENARIO=device-loss` (Hyprland) closes Cap's main window, loses the GPU +device (`GPUI_INJECT_DEVICE_LOSS` names a file, and creating it makes the +renderer's next frame end as it does when the driver reports the device lost) +and launches Cap again: the frame that finds the loss is the one that would +have mapped the window, and the window must still come back (GitHub #3946). `SCENARIO=register` launches Cap from a shell `ROUNDS` times, each time once with its `so.cap.Cap.desktop` entry removed and once with it in place, and counts the launches that registered the app id with the portals. diff --git a/scripts/linux-desktop-harness/scenario-device-loss.sh b/scripts/linux-desktop-harness/scenario-device-loss.sh new file mode 100644 index 00000000000..a8a8cb328ed --- /dev/null +++ b/scripts/linux-desktop-harness/scenario-device-loss.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# The GPU device is lost while Cap's main window is hidden, so the loss is +# found by the frame that would have mapped the window again (GitHub #3946): +# close the main window, lose the device, launch Cap again, which asks the +# running instance for its window, and check the window comes back. +# GPUI_INJECT_DEVICE_LOSS names a file; creating it makes the renderer's next +# frame end as it does when the driver reports the device lost. +# Prints key=value results to $HARNESS_OUT/result.txt. +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +export XDG_RUNTIME_DIR=/run/user/$(id -u) +export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus +out=${HARNESS_OUT:-/out} +eval "$(systemctl --user show-environment | grep -E "^(WAYLAND_DISPLAY|XDG_SESSION_TYPE|XDG_CURRENT_DESKTOP|XDG_SESSION_DESKTOP|LIBGL_ALWAYS_SOFTWARE|HYPRLAND_INSTANCE_SIGNATURE)=" | sed 's/^/export /')" +: >"$out/result.txt" +result() { printf '%s=%s\n' "$1" "$2" >>"$out/result.txt"; } +main_windows() { + hyprctl -j clients | python3 -c 'import json, sys +print(sum(1 for client in json.load(sys.stdin) if client["title"] == "Cap" and client["mapped"]))' +} +wait_main_windows() { + for _ in $(seq $(($2 * 4))); do + [[ $(main_windows) == "$1" ]] && return 0 + sleep 0.25 + done + return 1 +} + +# A profile past onboarding, so the window that opens is the main one. +mkdir -p "$out/profile" +cat >"$out/profile/store" <<'JSON' +{ + "general_settings": { + "hasCompletedStartup": true, + "hasCompletedOnboarding": true, + "enableNotifications": false, + "enableTelemetry": false + } +} +JSON +trigger=$out/device-loss +pid=$(bash "$here/launch-cap.sh" cap GPUI_INJECT_DEVICE_LOSS="$trigger") +if ! wait_main_windows 1 60; then + result first_shown no + hyprctl -j clients >>"$out/actions.log" 2>&1 + exit 0 +fi +result first_shown yes +sleep 2 +hyprctl dispatch 'hl.dsp.window.close({ window = "title:^Cap$" })' >>"$out/actions.log" 2>&1 +result hidden "$(wait_main_windows 0 10 && echo yes || echo no)" +sleep 1 +touch "$trigger" +bash "$here/launch-cap.sh" again >/dev/null +result shown_again "$(wait_main_windows 1 20 && echo yes || echo no)" +sleep 1 +result device_lost "$(grep -ac 'injected through GPUI_INJECT_DEVICE_LOSS' "$out/cap.log")" +result recovered "$(grep -ac 'GPU recovery complete' "$out/cap.log")" +result restore_failed "$(grep -ac 'Could not restore retained window visibility' "$out/cap.log")" +result cap_alive "$(kill -0 "$pid" 2>/dev/null && echo yes || echo no)" +result panics "$(grep -ac 'panicked at' "$out/cap.log")" +cat "$out/result.txt" diff --git a/scripts/prepare-gpui-dependency.sh b/scripts/prepare-gpui-dependency.sh index b2b9199b86b..a2ec5858a5b 100755 --- a/scripts/prepare-gpui-dependency.sh +++ b/scripts/prepare-gpui-dependency.sh @@ -14,6 +14,7 @@ patch_paths=( "apps/desktop-gpui/patches/zed-closed-window-events.patch" "apps/desktop-gpui/patches/zed-glass.patch" "apps/desktop-gpui/patches/zed-perf.patch" + "apps/desktop-gpui/patches/zed-device-lost.patch" ) patch_files=("${patch_paths[@]/#/$repo_root/}") base_revision="5d1f83d9f27a19bec1fb241dc33b42238af9cf8d" diff --git a/scripts/prepare-gpui-dependency.test.mjs b/scripts/prepare-gpui-dependency.test.mjs index 151a6db640d..1b3e774a958 100644 --- a/scripts/prepare-gpui-dependency.test.mjs +++ b/scripts/prepare-gpui-dependency.test.mjs @@ -156,6 +156,8 @@ test("dependency verification handles overlapping patches without changing the c patch("zed-glass.patch"); writeFileSync(path.join(upstream, "perf.rs"), "faster frames\n"); patch("zed-perf.patch"); + writeFileSync(path.join(upstream, "recovery.rs"), "one instance\n"); + patch("zed-device-lost.patch"); const script = path.join(cap, "scripts/prepare-gpui-dependency.sh"); writeFileSync( script,