fix(macos): stop HUD dialogs painting a grey rectangle over the desktop

On macOS an owned message box is a sheet, and AppKit dims the whole
owning window behind it. The HUD is a ~907x696 transparent window around
a ~60px bar, so every dialog it owned greyed a large invisible area of
the desktop.

On macOS the transparent overlays (HUD, source selector, countdown) no
longer own their message boxes: an unowned alert is app-modal and sits
above their floating level. Windows and Linux keep the owner, where an
unowned dialog opens behind the always-on-top HUD.

The two permission dialogs this first covered in ipc/handlers.ts are
gone since #735 replaced them with the permissions window; what remains
are main.ts's dialogs (About, updates, diagnostics), all of which fall
back to the HUD as their owner.

Author: EtienneLescot

electron/main.ts

@@ -5,7 +5,6 @@ import {
 	app,
 	BrowserWindow,
 	clipboard,
-	dialog,
 	ipcMain,
 	Menu,
 	nativeImage,
@@ -59,6 +58,7 @@ import {
 	registerIpcHandlers,
 } from "./ipc/handlers";
 import { installMainProcessErrorGuards } from "./main-process-errors";
+import { showMessageBoxOver } from "./messageBox";
 import {
 	registerPermissionsIpc,
 	showPermissionsWindow,
@@ -465,14 +465,15 @@ function channelAllowsUpdateCheck(): boolean {
 /** Message boxes must be owned by a window. The HUD is `alwaysOnTop` and `skipTaskbar`
  *  (electron/windows.ts), so an unowned dialog opens *behind* it on Windows and most Linux
  *  WMs, with no taskbar entry to recover it — the user sees a button flash and nothing else.
- *  Mirrors what ipc/handlers.ts already does for its own dialogs. */
+ *  Mirrors what ipc/handlers.ts already does for its own dialogs. On macOS the HUD is
+ *  skipped as an owner (see messageBox.ts). */
 function showMessageBox(options: Electron.MessageBoxOptions) {
 	const visible = (win: BrowserWindow | null) =>
 		win && !win.isDestroyed() && win.isVisible() ? win : null;
 	// A modal owned by a hidden window may never be drawn, so an unowned dialog is the safer
 	// fallback when the HUD has been closed to the tray.
 	const parent = visible(BrowserWindow.getFocusedWindow()) ?? visible(mainWindow);
-	return parent ? dialog.showMessageBox(parent, options) : dialog.showMessageBox(options);
+	return showMessageBoxOver(parent, options);
 }
 
 function aboutFacts(): AboutFacts {

electron/messageBox.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("electron", () => ({ dialog: { showMessageBox: vi.fn() } }));
+
+import type { BrowserWindow } from "electron";
+import { markSheetless, messageBoxOwner } from "./messageBox";
+
+function fakeWindow(destroyed = false) {
+	return { isDestroyed: () => destroyed } as unknown as BrowserWindow;
+}
+
+describe("messageBoxOwner", () => {
+	it("shows a transparent overlay's dialog unowned on macOS, where a sheet greys the whole window", () => {
+		const hud = fakeWindow();
+		markSheetless(hud);
+
+		expect(messageBoxOwner(hud, "darwin")).toBeNull();
+	});
+
+	it("keeps the overlay as owner elsewhere, where an unowned dialog opens behind it", () => {
+		const hud = fakeWindow();
+		markSheetless(hud);
+
+		expect(messageBoxOwner(hud, "win32")).toBe(hud);
+		expect(messageBoxOwner(hud, "linux")).toBe(hud);
+	});
+
+	it("keeps an opaque window as owner on macOS", () => {
+		const editor = fakeWindow();
+
+		expect(messageBoxOwner(editor, "darwin")).toBe(editor);
+	});
+
+	it("never attaches to a missing or destroyed window", () => {
+		expect(messageBoxOwner(null, "win32")).toBeNull();
+		expect(messageBoxOwner(fakeWindow(true), "win32")).toBeNull();
+	});
+});

electron/messageBox.ts

@@ -0,0 +1,42 @@
+import { type BrowserWindow, dialog } from "electron";
+
+/**
+ * Transparent windows that must not own a dialog on macOS.
+ *
+ * On macOS an owned message box is a sheet, and AppKit dims the whole owning window behind
+ * it. The HUD and the other overlays are mostly invisible padding around what they draw
+ * (the HUD is ~900x700 for a bar ~60 px tall), so the dim paints a grey rectangle over the
+ * desktop, far larger than anything the user sees of the app.
+ *
+ * Elsewhere they keep owning their dialogs: on Windows and most Linux WMs an unowned
+ * dialog opens behind these `alwaysOnTop` windows, with no taskbar entry to recover it.
+ * macOS has no such problem -- an unowned alert is app-modal and sits at the modal-panel
+ * level, above the HUD's floating one.
+ */
+const sheetlessWindows = new WeakSet<BrowserWindow>();
+
+export function markSheetless(win: BrowserWindow): void {
+	sheetlessWindows.add(win);
+}
+
+/** The window a message box should be attached to, or null to show it unowned. */
+export function messageBoxOwner(
+	parent: BrowserWindow | null | undefined,
+	platform: NodeJS.Platform = process.platform,
+): BrowserWindow | null {
+	if (!parent || parent.isDestroyed()) {
+		return null;
+	}
+	if (platform === "darwin" && sheetlessWindows.has(parent)) {
+		return null;
+	}
+	return parent;
+}
+
+export function showMessageBoxOver(
+	parent: BrowserWindow | null | undefined,
+	options: Electron.MessageBoxOptions,
+): Promise<Electron.MessageBoxReturnValue> {
+	const owner = messageBoxOwner(parent);
+	return owner ? dialog.showMessageBox(owner, options) : dialog.showMessageBox(options);
+}

electron/windows.ts

@@ -19,6 +19,7 @@ import {
 	sameRect,
 } from "./hudWindowBounds";
 import { followAcrossSpaces } from "./macSpaces";
+import { markSheetless } from "./messageBox";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -447,6 +448,7 @@ export function createHudOverlayWindow(): BrowserWindow {
 		});
 	}
 
+	markSheetless(win);
 	return win;
 }
 
@@ -600,6 +602,7 @@ export function createSourceSelectorWindow(): BrowserWindow {
 		});
 	}
 
+	markSheetless(win);
 	return win;
 }
 
@@ -651,6 +654,7 @@ export function createCountdownOverlayWindow(): BrowserWindow {
 		});
 	}
 
+	markSheetless(win);
 	return win;
 }