diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 1a926718f..16823723a 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -593,6 +593,11 @@ interface Window { message?: string; error?: string; }>; + getKeystrokeTelemetry: (videoPath?: string) => Promise<{ + success: boolean; + samples: KeystrokeTelemetryPoint[]; + message?: string; + }>; setCursorTelemetry: ( videoPath: string | undefined, samples: CursorTelemetryPoint[], @@ -969,6 +974,12 @@ interface CursorTelemetryPoint { | "not-allowed"; } +interface KeystrokeTelemetryPoint { + timeMs: number; + key: string; + modifiers: Array<"meta" | "ctrl" | "alt" | "shift">; +} + interface SystemCursorAsset { dataUrl: string; hotspotX: number; diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index 2c5cf8f17..a5060bde7 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -29,3 +29,5 @@ export const COMPANION_AUDIO_LAYOUTS = [ export const CURSOR_TELEMETRY_VERSION = 2; export const CURSOR_SAMPLE_INTERVAL_MS = 33; export const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz +export const KEYSTROKE_TELEMETRY_VERSION = 1; +export const MAX_KEYSTROKE_SAMPLES = 10_000; diff --git a/electron/ipc/cursor/interaction.test.ts b/electron/ipc/cursor/interaction.test.ts index 4ea662414..ca2fa752e 100644 --- a/electron/ipc/cursor/interaction.test.ts +++ b/electron/ipc/cursor/interaction.test.ts @@ -11,10 +11,19 @@ vi.mock("electron", () => ({ }, })); +import { + activeKeystrokeSamples, + setActiveKeystrokeSamples, + setCursorCaptureStartTimeMs, + setIsKeystrokeCaptureActive, +} from "../state"; import { repairBundledUiohookBinaryForCurrentArch, + resolveUiohookKeyToken, shouldStartGlobalInteractionHook, } from "./interaction"; +import { recordKeystroke, resetKeystrokeRepeatState } from "./keystrokeTelemetry"; +import { resetCursorCaptureClock } from "./telemetry"; describe("shouldStartGlobalInteractionHook", () => { it("does not start the synchronous uiohook event tap on macOS", () => { @@ -27,6 +36,81 @@ describe("shouldStartGlobalInteractionHook", () => { }); }); +describe("resolveUiohookKeyToken", () => { + const keyTable = { + Enter: 28, + Escape: 1, + ArrowLeft: 57419, + ArrowRight: 57421, + Left: 100, + PageUp: 3657, + A: 30, + C: 46, + Comma: 51, + Ctrl: 29, + CtrlRight: 3613, + Alt: 56, + AltRight: 3640, + ShiftRight: 54, + Meta: 3675, + MetaRight: 3676, + }; + + it("aliases uiohook names onto the overlay token vocabulary", () => { + expect(resolveUiohookKeyToken(28, keyTable)).toBe("enter"); + expect(resolveUiohookKeyToken(1, keyTable)).toBe("esc"); + expect(resolveUiohookKeyToken(57419, keyTable)).toBe("arrowleft"); + expect(resolveUiohookKeyToken(100, keyTable)).toBe("arrowleft"); + expect(resolveUiohookKeyToken(3657, keyTable)).toBe("pageup"); + expect(resolveUiohookKeyToken(30, keyTable)).toBe("a"); + expect(resolveUiohookKeyToken(46, keyTable)).toBe("c"); + expect(resolveUiohookKeyToken(51, keyTable)).toBe("comma"); + expect(resolveUiohookKeyToken(99999, keyTable)).toBeNull(); + }); + + it("folds right-hand modifiers onto the same tokens as the left-hand keys", () => { + expect(resolveUiohookKeyToken(29, keyTable)).toBe("ctrl"); + expect(resolveUiohookKeyToken(3613, keyTable)).toBe("ctrl"); + expect(resolveUiohookKeyToken(56, keyTable)).toBe("alt"); + expect(resolveUiohookKeyToken(3640, keyTable)).toBe("alt"); + expect(resolveUiohookKeyToken(54, keyTable)).toBe("shift"); + expect(resolveUiohookKeyToken(3675, keyTable)).toBe("meta"); + expect(resolveUiohookKeyToken(3676, keyTable)).toBe("meta"); + expect(resolveUiohookKeyToken(57421, keyTable)).toBe("arrowright"); + }); +}); + +describe("keystroke repeat collapse", () => { + afterEach(() => { + vi.restoreAllMocks(); + setIsKeystrokeCaptureActive(false); + setActiveKeystrokeSamples([]); + resetKeystrokeRepeatState(); + resetCursorCaptureClock(); + }); + + it("bumps last-seen time and emits once for a held key", () => { + setIsKeystrokeCaptureActive(true); + setCursorCaptureStartTimeMs(1_000); + setActiveKeystrokeSamples([]); + resetKeystrokeRepeatState(); + resetCursorCaptureClock(); + + const now = vi.spyOn(Date, "now"); + now.mockReturnValue(1_100); + recordKeystroke("c", ["meta"]); + now.mockReturnValue(1_120); + recordKeystroke("c", ["meta"]); + + expect(activeKeystrokeSamples).toHaveLength(1); + expect(activeKeystrokeSamples[0]).toMatchObject({ key: "c", modifiers: ["meta"] }); + + now.mockReturnValue(1_200); + recordKeystroke("c", ["meta"]); + expect(activeKeystrokeSamples).toHaveLength(2); + }); +}); + describe("repairBundledUiohookBinaryForCurrentArch", () => { const tempRoots: string[] = []; diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index 47c42437f..50d09aecf 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -5,6 +5,7 @@ import { hasLoggedInteractionHookFailure, interactionCaptureCleanup, isCursorCaptureActive, + isKeystrokeCaptureActive, lastLeftClick, setHasLoggedInteractionHookFailure, setInteractionCaptureCleanup, @@ -13,10 +14,13 @@ import { } from "../state"; import type { CursorInteractionType, + HookKeyboardEvent, HookMouseEvent, + KeystrokeModifier, UiohookLike, UiohookModuleNamespace, } from "../types"; +import { recordKeystroke } from "./keystrokeTelemetry"; import { getCursorCaptureElapsedMs, getHookCursorScreenPoint, @@ -187,6 +191,75 @@ export function shouldStartGlobalInteractionHook(platform: NodeJS.Platform = pro return platform !== "darwin"; } +const KEY_TOKEN_ALIASES: Record = { + return: "enter", + enter: "enter", + escape: "esc", + arrow_left: "arrowleft", + left: "arrowleft", + arrow_right: "arrowright", + right: "arrowright", + arrow_up: "arrowup", + up: "arrowup", + arrow_down: "arrowdown", + down: "arrowdown", + page_up: "pageup", + page_down: "pagedown", + ctrlright: "ctrl", + altright: "alt", + altgr: "alt", + shiftright: "shift", + metaright: "meta", +}; + +export function resolveUiohookKeyToken( + keycode: number, + keyTable: Record, +): string | null { + if (!Number.isFinite(keycode)) { + return null; + } + + let name: string | null = null; + for (const [key, code] of Object.entries(keyTable)) { + if (code === keycode) { + name = key; + break; + } + } + if (!name) { + return null; + } + + const token = name.toLowerCase(); + return KEY_TOKEN_ALIASES[token] ?? token; +} + +function loadUiohookKeyTable(moduleExports: UiohookModuleNamespace): Record { + const table = moduleExports.UiohookKey; + if (!table || typeof table !== "object") { + return {}; + } + return table; +} + +function modifiersFromHookEvent(event: HookKeyboardEvent): KeystrokeModifier[] { + const modifiers: KeystrokeModifier[] = []; + if (event.metaKey) { + modifiers.push("meta"); + } + if (event.ctrlKey) { + modifiers.push("ctrl"); + } + if (event.altKey) { + modifiers.push("alt"); + } + if (event.shiftKey) { + modifiers.push("shift"); + } + return modifiers; +} + export function recordCursorMouseDown(button: 1 | 2 | 3) { if (!isCursorCaptureActive || isCursorCapturePaused()) { return; @@ -264,7 +337,7 @@ export async function startInteractionCapture() { } if (!hook || typeof hook.on !== "function" || typeof hook.start !== "function") { - console.log("[CursorTelemetry] hook unusable — aborting interaction capture"); + console.log("[CursorTelemetry] hook unusable aborting interaction capture"); return; } @@ -289,11 +362,29 @@ export async function startInteractionCapture() { setLinuxCursorScreenPoint({ x: point.x, y: point.y, updatedAt: Date.now() }); }; + const keyTable = isKeystrokeCaptureActive + ? loadUiohookKeyTable(nodeRequire("uiohook-napi") as UiohookModuleNamespace) + : {}; + const captureKeys = isKeystrokeCaptureActive; + const onKeyDown = (event: HookKeyboardEvent) => { + if (typeof event.keycode !== "number") { + return; + } + const token = resolveUiohookKeyToken(event.keycode, keyTable); + if (!token) { + return; + } + recordKeystroke(token, modifiersFromHookEvent(event)); + }; + hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); } + if (captureKeys) { + hook.on("keydown", onKeyDown); + } setInteractionCaptureCleanup(() => { try { @@ -303,12 +394,18 @@ export async function startInteractionCapture() { if (process.platform === "linux") { hook.off("mousemove", onMouseMove); } + if (captureKeys) { + hook.off("keydown", onKeyDown); + } } else if (typeof hook.removeListener === "function") { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); } + if (captureKeys) { + hook.removeListener("keydown", onKeyDown); + } } } catch { // ignore listener cleanup errors diff --git a/electron/ipc/cursor/keystrokeTelemetry.test.ts b/electron/ipc/cursor/keystrokeTelemetry.test.ts new file mode 100644 index 000000000..4609f471d --- /dev/null +++ b/electron/ipc/cursor/keystrokeTelemetry.test.ts @@ -0,0 +1,250 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { KEYSTROKE_TELEMETRY_VERSION, MAX_KEYSTROKE_SAMPLES } from "../constants"; + +const { writeFile, rm, readAppSetting } = vi.hoisted(() => ({ + writeFile: vi.fn(), + rm: vi.fn(), + readAppSetting: vi.fn(), +})); + +vi.mock("node:fs/promises", () => ({ + default: { + writeFile, + rm, + }, +})); + +vi.mock("electron", () => ({ + app: { + getPath: vi.fn(() => "/tmp"), + }, +})); + +vi.mock("../../appSettingsStore", () => ({ + readAppSetting, +})); + +vi.mock("../utils", () => ({ + getKeystrokePathForVideo: vi.fn(() => "/tmp/recording.keystrokes.json"), + getTelemetryPathForVideo: vi.fn(() => "/tmp/recording.cursor.json"), + normalizeVideoSourcePath: (videoPath?: string | null) => { + if (typeof videoPath !== "string") { + return null; + } + const trimmed = videoPath.trim(); + return trimmed ? trimmed : null; + }, + getScreen: vi.fn(() => ({ + getCursorScreenPoint: () => ({ x: 0, y: 0 }), + getPrimaryDisplay: () => ({ scaleFactor: 1 }), + getDisplayNearestPoint: () => ({ bounds: { x: 0, y: 0, width: 1, height: 1 } }), + getAllDisplays: () => [], + })), +})); + +import { + pendingKeystrokeSamples, + setActiveKeystrokeSamples, + setPendingKeystrokeSamples, +} from "../state"; +import { + isExplicitKeystrokeTelemetryPathDenied, + isKeystrokeCaptureEnabledFromPrefs, + normalizeKeystrokeTelemetrySamples, + persistPendingKeystrokeTelemetry, + resetKeystrokeRepeatState, + snapshotKeystrokeTelemetryForPersistence, + writeKeystrokeTelemetry, +} from "./keystrokeTelemetry"; + +const recordingRegisterSource = readFileSync( + fileURLToPath(new URL("../register/recording.ts", import.meta.url)), + "utf8", +); +const macRecordingSource = readFileSync( + fileURLToPath(new URL("../recording/mac.ts", import.meta.url)), + "utf8", +); + +describe("keystroke telemetry", () => { + beforeEach(() => { + writeFile.mockReset(); + rm.mockReset(); + readAppSetting.mockReset(); + readAppSetting.mockReturnValue(null); + setActiveKeystrokeSamples([]); + setPendingKeystrokeSamples([]); + resetKeystrokeRepeatState(); + }); + + it("drops junk samples and clamps negative time", () => { + expect( + normalizeKeystrokeTelemetrySamples([ + null, + "c", + { timeMs: -10, key: "c", modifiers: ["meta"] }, + { timeMs: 20, key: "", modifiers: ["meta"] }, + { timeMs: 30, key: " ", modifiers: ["meta"] }, + ]), + ).toEqual([{ timeMs: 0, key: "c", modifiers: ["meta"] }]); + }); + + it("filters modifiers to the allow-list in stable order", () => { + expect( + normalizeKeystrokeTelemetrySamples([ + { + timeMs: 10, + key: "C", + modifiers: ["shift", "super", "ctrl", "ctrl", "meta", "alt"], + }, + ]), + ).toEqual([{ timeMs: 10, key: "c", modifiers: ["meta", "ctrl", "alt", "shift"] }]); + }); + + it("sorts samples by timeMs", () => { + expect( + normalizeKeystrokeTelemetrySamples([ + { timeMs: 30, key: "v", modifiers: ["meta"] }, + { timeMs: 10, key: "c", modifiers: ["meta"] }, + ]).map((sample) => sample.timeMs), + ).toEqual([10, 30]); + }); + + it("caps samples and drops bare modifier keys", () => { + const samples = [ + { timeMs: 1, key: "shift", modifiers: [] }, + { timeMs: 2, key: "meta", modifiers: [] }, + { timeMs: 3, key: "cmd", modifiers: [] }, + ...Array.from({ length: MAX_KEYSTROKE_SAMPLES + 5 }, (_, index) => ({ + timeMs: index + 10, + key: "a", + modifiers: [], + })), + ]; + const normalized = normalizeKeystrokeTelemetrySamples(samples); + expect(normalized).toHaveLength(MAX_KEYSTROKE_SAMPLES); + expect(normalized[0]?.timeMs).toBe(10); + expect(normalized.some((sample) => sample.key === "shift")).toBe(false); + }); + + it("drops right-hand uiohook modifier names as bare keys", () => { + expect( + normalizeKeystrokeTelemetrySamples([ + { timeMs: 1, key: "ctrlright", modifiers: ["ctrl"] }, + { timeMs: 2, key: "altright", modifiers: ["alt"] }, + { timeMs: 3, key: "metaright", modifiers: ["meta"] }, + { timeMs: 4, key: "c", modifiers: ["ctrl"] }, + ]), + ).toEqual([{ timeMs: 4, key: "c", modifiers: ["ctrl"] }]); + }); + + it("removes the sidecar when writing an empty payload", async () => { + await writeKeystrokeTelemetry("/tmp/recording.mp4", []); + expect(rm).toHaveBeenCalledWith("/tmp/recording.keystrokes.json", { force: true }); + expect(writeFile).not.toHaveBeenCalled(); + }); + + it("writes a versioned sidecar for nonempty samples", async () => { + const samples = [{ timeMs: 10, key: "c", modifiers: ["meta"] }]; + await writeKeystrokeTelemetry("/tmp/recording.mp4", samples); + expect(writeFile).toHaveBeenCalledWith( + "/tmp/recording.keystrokes.json", + JSON.stringify({ version: KEYSTROKE_TELEMETRY_VERSION, samples }, null, 2), + "utf-8", + ); + expect(rm).not.toHaveBeenCalled(); + }); + + it("does not delete a sidecar when persistPending is empty", async () => { + setPendingKeystrokeSamples([{ timeMs: 10, key: "c", modifiers: ["meta"] }]); + await persistPendingKeystrokeTelemetry("/tmp/recording.mp4"); + expect(writeFile).toHaveBeenCalledTimes(1); + + writeFile.mockClear(); + rm.mockClear(); + await persistPendingKeystrokeTelemetry("/tmp/recording.mp4"); + expect(rm).not.toHaveBeenCalled(); + expect(writeFile).not.toHaveBeenCalled(); + }); + + it("writes then clears pending when persistPending is nonempty", async () => { + setPendingKeystrokeSamples([{ timeMs: 10, key: "c", modifiers: ["meta"] }]); + await persistPendingKeystrokeTelemetry("/tmp/recording.mp4"); + expect(writeFile).toHaveBeenCalledTimes(1); + expect(pendingKeystrokeSamples).toEqual([]); + }); + + it("merges active samples into pending on snapshot", () => { + snapshotKeystrokeTelemetryForPersistence(); + expect(pendingKeystrokeSamples).toEqual([]); + + setActiveKeystrokeSamples([{ timeMs: 10, key: "c", modifiers: ["meta"] }]); + snapshotKeystrokeTelemetryForPersistence(); + expect(pendingKeystrokeSamples).toEqual([{ timeMs: 10, key: "c", modifiers: ["meta"] }]); + + setActiveKeystrokeSamples([ + { timeMs: 10, key: "c", modifiers: ["meta"] }, + { timeMs: 20, key: "v", modifiers: ["meta"] }, + ]); + snapshotKeystrokeTelemetryForPersistence(); + expect(pendingKeystrokeSamples).toEqual([ + { timeMs: 10, key: "c", modifiers: ["meta"] }, + { timeMs: 20, key: "v", modifiers: ["meta"] }, + ]); + }); + + it("persists keystrokes from finalizeStoredVideo and recording stop", () => { + expect(macRecordingSource).toContain("persistPendingKeystrokeTelemetry"); + expect(recordingRegisterSource).toContain("snapshotKeystrokeTelemetryForPersistence"); + expect(recordingRegisterSource).toContain("persistPendingKeystrokeTelemetry"); + expect(recordingRegisterSource).not.toContain("set-keystroke-telemetry"); + }); + + it("does not deny omitted keystroke telemetry paths", () => { + expect(isExplicitKeystrokeTelemetryPathDenied(undefined, () => false)).toBe(false); + }); + + it("denies explicit paths that fail the allowlist", () => { + expect(isExplicitKeystrokeTelemetryPathDenied("/tmp/secret.mp4", () => false)).toBe(true); + }); + + it("allows explicit paths that pass the allowlist", () => { + expect(isExplicitKeystrokeTelemetryPathDenied("/tmp/recording.mp4", () => true)).toBe( + false, + ); + }); + + it("treats blank explicit keystroke telemetry paths as omitted", () => { + expect(isExplicitKeystrokeTelemetryPathDenied("", () => false)).toBe(false); + expect(isExplicitKeystrokeTelemetryPathDenied(" ", () => false)).toBe(false); + }); + + it("authorizes get-keystroke-telemetry senders without wrapping cursor telemetry", () => { + expect(recordingRegisterSource).toContain("get-keystroke-telemetry"); + expect(recordingRegisterSource).not.toContain("set-keystroke-telemetry"); + expect(recordingRegisterSource).toContain( + 'ipcMain.handle("get-keystroke-telemetry", async (event, videoPath?: string)', + ); + expect(recordingRegisterSource).toContain("BrowserWindow.fromWebContents"); + expect(recordingRegisterSource).toContain( + 'ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string)', + ); + }); + + it("treats missing prefs as capture off", () => { + readAppSetting.mockReturnValue(null); + expect(isKeystrokeCaptureEnabledFromPrefs()).toBe(false); + }); + + it("treats a non-boolean enabled flag as capture off", () => { + readAppSetting.mockReturnValue({ keystrokeOverlay: { enabled: "yes" } }); + expect(isKeystrokeCaptureEnabledFromPrefs()).toBe(false); + }); + + it("enables capture only when overlay.enabled is true", () => { + readAppSetting.mockReturnValue({ keystrokeOverlay: { enabled: true } }); + expect(isKeystrokeCaptureEnabledFromPrefs()).toBe(true); + }); +}); diff --git a/electron/ipc/cursor/keystrokeTelemetry.ts b/electron/ipc/cursor/keystrokeTelemetry.ts new file mode 100644 index 000000000..42e2b118a --- /dev/null +++ b/electron/ipc/cursor/keystrokeTelemetry.ts @@ -0,0 +1,229 @@ +import fs from "node:fs/promises"; +import { readAppSetting } from "../../appSettingsStore"; +import { KEYSTROKE_TELEMETRY_VERSION, MAX_KEYSTROKE_SAMPLES } from "../constants"; +import { + activeKeystrokeSamples, + isKeystrokeCaptureActive, + pendingKeystrokeSamples, + setPendingKeystrokeSamples, +} from "../state"; +import type { KeystrokeModifier, KeystrokeTelemetryPoint } from "../types"; +import { getKeystrokePathForVideo, normalizeVideoSourcePath } from "../utils"; +import { getCursorCaptureElapsedMs, isCursorCapturePaused } from "./telemetry"; + +const EDITOR_PREFERENCES_SETTING_KEY = "recordly.editor.preferences"; +const KEY_REPEAT_COLLAPSE_MS = 45; +const MODIFIER_ORDER: KeystrokeModifier[] = ["meta", "ctrl", "alt", "shift"]; +const BARE_MODIFIER_KEYS = new Set([ + "shift", + "shiftright", + "rshift", + "ctrl", + "ctrlright", + "rctrl", + "alt", + "altright", + "ralt", + "altgr", + "option", + "meta", + "metaright", + "rmeta", + "cmd", + "command", + "win", + "windows", + "super", +]); + +let lastSeenKeyToken: string | null = null; +let lastSeenTimeMs = Number.NEGATIVE_INFINITY; + +export function isBareModifierKey(token: string): boolean { + return BARE_MODIFIER_KEYS.has(token); +} + +export function resetKeystrokeRepeatState() { + lastSeenKeyToken = null; + lastSeenTimeMs = Number.NEGATIVE_INFINITY; +} + +export function shouldCollapseKeyRepeat( + token: string, + timeMs: number, + lastToken: string | null, + lastTimeMs: number, + windowMs = KEY_REPEAT_COLLAPSE_MS, +): boolean { + return lastToken === token && timeMs - lastTimeMs < windowMs; +} + +export function noteKeystrokeRepeat(token: string, timeMs: number): boolean { + const collapse = shouldCollapseKeyRepeat(token, timeMs, lastSeenKeyToken, lastSeenTimeMs); + lastSeenTimeMs = timeMs; + if (!collapse) { + lastSeenKeyToken = token; + } + return collapse; +} + +export function isKeystrokeCaptureEnabledFromPrefs(): boolean { + const prefs = readAppSetting(EDITOR_PREFERENCES_SETTING_KEY); + if (!prefs || typeof prefs !== "object" || Array.isArray(prefs)) { + return false; + } + const overlay = (prefs as { keystrokeOverlay?: unknown }).keystrokeOverlay; + if (!overlay || typeof overlay !== "object" || Array.isArray(overlay)) { + return false; + } + return (overlay as { enabled?: unknown }).enabled === true; +} + +export function isExplicitKeystrokeTelemetryPathDenied( + videoPath: string | undefined, + isAllowedPath: (candidatePath: string) => boolean, +): boolean { + const explicitVideoPath = normalizeVideoSourcePath(videoPath); + return Boolean(explicitVideoPath && !isAllowedPath(explicitVideoPath)); +} + +function normalizeModifiers(raw: unknown): KeystrokeModifier[] { + if (!Array.isArray(raw)) { + return []; + } + const seen = new Set(); + for (const value of raw) { + if (value === "meta" || value === "ctrl" || value === "alt" || value === "shift") { + seen.add(value); + } + } + return MODIFIER_ORDER.filter((modifier) => seen.has(modifier)); +} + +export function normalizeKeystrokeTelemetrySamples(rawSamples: unknown): KeystrokeTelemetryPoint[] { + const samples = Array.isArray(rawSamples) + ? rawSamples + : Array.isArray((rawSamples as { samples?: unknown[] } | null | undefined)?.samples) + ? ((rawSamples as { samples: unknown[] }).samples ?? []) + : []; + + return samples + .filter((sample: unknown): sample is Record => + Boolean(sample && typeof sample === "object" && !Array.isArray(sample)), + ) + .flatMap((sample) => { + if (typeof sample.timeMs !== "number" || !Number.isFinite(sample.timeMs)) { + return []; + } + if (typeof sample.key !== "string") { + return []; + } + const key = sample.key.trim().toLowerCase(); + if (!key || isBareModifierKey(key)) { + return []; + } + return [ + { + timeMs: Math.max(0, sample.timeMs), + key, + modifiers: normalizeModifiers(sample.modifiers), + }, + ]; + }) + .sort((left, right) => left.timeMs - right.timeMs) + .slice(0, MAX_KEYSTROKE_SAMPLES); +} + +export async function writeKeystrokeTelemetry(videoPath: string, samples: unknown) { + const telemetryPath = getKeystrokePathForVideo(videoPath); + const normalizedSamples = normalizeKeystrokeTelemetrySamples(samples); + + if (normalizedSamples.length === 0) { + await fs.rm(telemetryPath, { force: true }); + return normalizedSamples; + } + + await fs.writeFile( + telemetryPath, + JSON.stringify( + { version: KEYSTROKE_TELEMETRY_VERSION, samples: normalizedSamples }, + null, + 2, + ), + "utf-8", + ); + + return normalizedSamples; +} + +export function pushKeystrokeSample(point: KeystrokeTelemetryPoint) { + if (!isKeystrokeCaptureActive || isCursorCapturePaused()) { + return; + } + + activeKeystrokeSamples.push({ + timeMs: Math.max(0, point.timeMs), + key: point.key, + modifiers: point.modifiers, + }); + + if (activeKeystrokeSamples.length > MAX_KEYSTROKE_SAMPLES) { + activeKeystrokeSamples.shift(); + } +} + +function omitOwnModifier(token: string, modifiers: KeystrokeModifier[]): KeystrokeModifier[] { + if (token === "meta" || token === "ctrl" || token === "alt" || token === "shift") { + return modifiers.filter((modifier) => modifier !== token); + } + return modifiers; +} + +export function recordKeystroke(token: string, modifiers: KeystrokeModifier[]) { + if (!isKeystrokeCaptureActive || isCursorCapturePaused()) { + return; + } + + const key = token.trim().toLowerCase(); + if (!key || isBareModifierKey(key)) { + return; + } + + const timeMs = getCursorCaptureElapsedMs(); + if (noteKeystrokeRepeat(key, timeMs)) { + return; + } + + pushKeystrokeSample({ + timeMs, + key, + modifiers: omitOwnModifier(key, normalizeModifiers(modifiers)), + }); +} + +export function snapshotKeystrokeTelemetryForPersistence() { + if (activeKeystrokeSamples.length === 0) { + return; + } + + if (pendingKeystrokeSamples.length === 0) { + setPendingKeystrokeSamples([...activeKeystrokeSamples]); + return; + } + + const lastPendingTimeMs = + pendingKeystrokeSamples[pendingKeystrokeSamples.length - 1]?.timeMs ?? -1; + setPendingKeystrokeSamples([ + ...pendingKeystrokeSamples, + ...activeKeystrokeSamples.filter((sample) => sample.timeMs > lastPendingTimeMs), + ]); +} + +export async function persistPendingKeystrokeTelemetry(videoPath: string) { + if (pendingKeystrokeSamples.length === 0) { + return; + } + + await writeKeystrokeTelemetry(videoPath, pendingKeystrokeSamples); + setPendingKeystrokeSamples([]); +} diff --git a/electron/ipc/cursor/monitor.test.ts b/electron/ipc/cursor/monitor.test.ts new file mode 100644 index 000000000..7b3940dc2 --- /dev/null +++ b/electron/ipc/cursor/monitor.test.ts @@ -0,0 +1,67 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ + app: { + getPath: vi.fn(() => "/tmp"), + }, + BrowserWindow: { + getAllWindows: () => [], + }, +})); + +import { parseKeystrokeMonitorLine } from "./monitor"; + +const nativeMonitorSource = readFileSync( + fileURLToPath(new URL("../../native/NativeCursorMonitor.swift", import.meta.url)), + "utf8", +); + +describe("parseKeystrokeMonitorLine", () => { + it("parses KEY:down:c:meta", () => { + expect(parseKeystrokeMonitorLine("KEY:down:c:meta")).toEqual({ + key: "c", + modifiers: ["meta"], + }); + }); + + it("parses empty modifiers", () => { + expect(parseKeystrokeMonitorLine("KEY:down:enter:")).toEqual({ + key: "enter", + modifiers: [], + }); + }); + + it("ignores malformed lines", () => { + expect(parseKeystrokeMonitorLine("KEY:down:C:meta")).toBeNull(); + expect(parseKeystrokeMonitorLine("KEY:up:c:meta")).toBeNull(); + expect(parseKeystrokeMonitorLine("STATE:arrow")).toBeNull(); + expect(parseKeystrokeMonitorLine("INTERACTION:mousedown:1")).toBeNull(); + }); + + it("parses named punctuation tokens", () => { + expect(parseKeystrokeMonitorLine("KEY:down:comma:")).toEqual({ + key: "comma", + modifiers: [], + }); + }); +}); + +describe("native cursor monitor keystroke tap", () => { + it("retries the mouse-only tap when the combined key mask cannot be created", () => { + expect(nativeMonitorSource).toContain("installListenOnlyTap(combinedMask)"); + expect(nativeMonitorSource).toContain("installListenOnlyTap(mouseOnlyMask)"); + expect(nativeMonitorSource).toContain( + "Keyboard event tap unavailable; keystroke telemetry disabled", + ); + expect(nativeMonitorSource).toContain( + "Mouse interaction event tap unavailable; click telemetry disabled", + ); + }); + + it("emits named punctuation tokens from the Swift key map", () => { + expect(nativeMonitorSource).toContain('return "comma"'); + expect(nativeMonitorSource).toContain('return "period"'); + }); +}); diff --git a/electron/ipc/cursor/monitor.ts b/electron/ipc/cursor/monitor.ts index 8a507d56d..a849a4616 100644 --- a/electron/ipc/cursor/monitor.ts +++ b/electron/ipc/cursor/monitor.ts @@ -11,8 +11,33 @@ import { setNativeCursorMonitorOutputBuffer, setNativeCursorMonitorProcess, } from "../state"; -import type { CursorVisualType } from "../types"; +import type { CursorVisualType, KeystrokeModifier } from "../types"; import { recordCursorMouseDown, recordCursorMouseUp } from "./interaction"; +import { recordKeystroke } from "./keystrokeTelemetry"; + +const KEYSTROKE_MONITOR_LINE = /^KEY:down:([a-z0-9]+):([a-z,]*)$/; +const MODIFIER_ORDER: KeystrokeModifier[] = ["meta", "ctrl", "alt", "shift"]; + +export function parseKeystrokeMonitorLine( + line: string, +): { key: string; modifiers: KeystrokeModifier[] } | null { + const match = line.match(KEYSTROKE_MONITOR_LINE); + if (!match) { + return null; + } + + const seen = new Set(); + for (const value of match[2].split(",")) { + if (value === "meta" || value === "ctrl" || value === "alt" || value === "shift") { + seen.add(value); + } + } + + return { + key: match[1], + modifiers: MODIFIER_ORDER.filter((modifier) => seen.has(modifier)), + }; +} export function emitCursorStateChanged(cursorType: CursorVisualType) { BrowserWindow.getAllWindows().forEach((window) => { @@ -39,6 +64,12 @@ export function handleCursorMonitorStdout(chunk: Buffer) { continue; } + const keystroke = parseKeystrokeMonitorLine(line); + if (keystroke) { + recordKeystroke(keystroke.key, keystroke.modifiers); + continue; + } + const match = line.match(/^STATE:(.+)$/); if (!match) continue; const next = match[1].trim() as CursorVisualType; @@ -84,7 +115,7 @@ export function stopNativeCursorMonitor() { setNativeCursorMonitorOutputBuffer(""); } -export async function startNativeCursorMonitor() { +export async function startNativeCursorMonitor(options?: { captureKeys?: boolean }) { stopNativeCursorMonitor(); if (process.platform !== "darwin" && process.platform !== "win32") { @@ -113,7 +144,9 @@ export async function startNativeCursorMonitor() { let proc: ReturnType | null; try { - proc = spawn(helperPath, [], { + const spawnArgs = + process.platform === "darwin" && options?.captureKeys ? ["--capture-keys"] : []; + proc = spawn(helperPath, spawnArgs, { stdio: ["pipe", "pipe", "pipe"], }); } catch (spawnError) { diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 57a9e6881..08044802c 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -1,6 +1,10 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; import fs from "node:fs/promises"; import { BrowserWindow } from "electron"; +import { + persistPendingKeystrokeTelemetry, + snapshotKeystrokeTelemetryForPersistence, +} from "../cursor/keystrokeTelemetry"; import { persistPendingCursorTelemetry, snapshotCursorTelemetryForPersistence, @@ -263,6 +267,7 @@ export async function finalizeStoredVideo(videoPath: string) { } snapshotCursorTelemetryForPersistence(); + snapshotKeystrokeTelemetryForPersistence(); setCurrentVideoPath(videoPath); setCurrentProjectPath(null); try { @@ -270,6 +275,11 @@ export async function finalizeStoredVideo(videoPath: string) { } catch (error) { console.warn("[mac-stop] Failed to persist cursor telemetry:", error); } + try { + await persistPendingKeystrokeTelemetry(videoPath); + } catch (error) { + console.warn("Failed to persist keystroke telemetry", error); + } if (isAutoRecordingPath(videoPath)) { await pruneAutoRecordings([videoPath]); } diff --git a/electron/ipc/recording/prune.ts b/electron/ipc/recording/prune.ts index 29787e8e2..d03b26996 100644 --- a/electron/ipc/recording/prune.ts +++ b/electron/ipc/recording/prune.ts @@ -10,6 +10,7 @@ import { } from "../constants"; import { currentVideoPath } from "../state"; import { + getKeystrokePathForVideo, getRecordingsDir, getTelemetryPathForVideo, isAutoRecordingPath, @@ -170,6 +171,7 @@ export async function pruneAutoRecordings(exemptPaths: string[] = []) { try { await fs.rm(entry.filePath, { force: true }); await fs.rm(getTelemetryPathForVideo(entry.filePath), { force: true }); + await fs.rm(getKeystrokePathForVideo(entry.filePath), { force: true }); // Clean up companion audio files left from recording (macOS .m4a, Windows .wav) const base = entry.filePath.replace(/\.(mp4|mov|webm)$/i, ""); const companionSuffixes = Array.from( diff --git a/electron/ipc/register/project.ts b/electron/ipc/register/project.ts index ff309a19a..bc30bb71c 100644 --- a/electron/ipc/register/project.ts +++ b/electron/ipc/register/project.ts @@ -747,6 +747,7 @@ export function registerProjectHandlers() { // Also delete the cursor telemetry sidecar if it exists const telemetryPath = getTelemetryPathForVideo(resolvedPath); await fs.unlink(telemetryPath).catch(() => undefined); + await fs.unlink(`${resolvedPath}.keystrokes.json`).catch(() => undefined); const currentResolved = currentVideoPath ? await fs.realpath(currentVideoPath).catch(() => currentVideoPath) : null; diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 2c50e3976..98ce8783d 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -21,6 +21,14 @@ import { import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; +import { + isExplicitKeystrokeTelemetryPathDenied, + isKeystrokeCaptureEnabledFromPrefs, + normalizeKeystrokeTelemetrySamples, + persistPendingKeystrokeTelemetry, + resetKeystrokeRepeatState, + snapshotKeystrokeTelemetryForPersistence, +} from "../cursor/keystrokeTelemetry"; import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor"; import { normalizeCursorTelemetrySamples, @@ -44,7 +52,7 @@ import { getSystemCursorHelperSourcePath, getWindowsCaptureExePath, } from "../paths/binaries"; -import { rememberApprovedLocalReadPath } from "../project/manager"; +import { isAllowedLocalReadPath, rememberApprovedLocalReadPath } from "../project/manager"; import { getBrowserMicSidecarFilters, shouldKeepRecordingAudioSidecars, @@ -104,6 +112,7 @@ import { nativeScreenRecordingActive, selectedSource, setActiveCursorSamples, + setActiveKeystrokeSamples, setCachedSystemCursorAssets, setCachedSystemCursorAssetsSourceMtimeMs, setCursorCaptureStartTimeMs, @@ -112,6 +121,7 @@ import { setFfmpegCaptureTargetPath, setFfmpegScreenRecordingActive, setIsCursorCaptureActive, + setIsKeystrokeCaptureActive, setLastLeftClick, setLinuxCursorScreenPoint, setNativeCaptureMicrophonePath, @@ -123,6 +133,7 @@ import { setNativeCaptureTargetPath, setNativeScreenRecordingActive, setPendingCursorSamples, + setPendingKeystrokeSamples, setWindowsCaptureOutputBuffer, setWindowsCapturePaused, setWindowsCaptureProcess, @@ -145,6 +156,7 @@ import { } from "../state"; import type { CursorTelemetryPoint, NativeMacRecordingOptions, SelectedSource } from "../types"; import { + getKeystrokePathForVideo, getMacPrivacySettingsUrl, getRecordingsDir, getScreen, @@ -1039,6 +1051,12 @@ export function registerRecordingHandlers( error, ); } + snapshotKeystrokeTelemetryForPersistence(); + try { + await persistPendingKeystrokeTelemetry(finalVideoPath); + } catch (error) { + console.warn("Failed to persist keystroke telemetry", error); + } return { success: true, path: finalVideoPath }; } catch (error) { @@ -1868,7 +1886,12 @@ export function registerRecordingHandlers( stopCursorCapture(); stopInteractionCapture(); startWindowBoundsCapture(); - void startNativeCursorMonitor(); + setActiveKeystrokeSamples([]); + setPendingKeystrokeSamples([]); + const captureKeys = isKeystrokeCaptureEnabledFromPrefs(); + setIsKeystrokeCaptureActive(captureKeys); + resetKeystrokeRepeatState(); + void startNativeCursorMonitor({ captureKeys }); setIsCursorCaptureActive(true); setActiveCursorSamples([]); setPendingCursorSamples([]); @@ -1890,6 +1913,9 @@ export function registerRecordingHandlers( resetCursorCaptureClock(); snapshotCursorTelemetryForPersistence(); setActiveCursorSamples([]); + setIsKeystrokeCaptureActive(false); + snapshotKeystrokeTelemetryForPersistence(); + setActiveKeystrokeSamples([]); } const source = selectedSource || { name: "Screen" }; @@ -1946,6 +1972,49 @@ export function registerRecordingHandlers( } }); + ipcMain.handle("get-keystroke-telemetry", async (event, videoPath?: string) => { + const senderWindow = BrowserWindow.fromWebContents(event.sender); + if (!senderWindow || senderWindow.isDestroyed()) { + return { + success: false, + samples: [], + message: "Failed to load keystroke telemetry", + }; + } + + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); + if (!targetVideoPath) { + return { success: true, samples: [] }; + } + + if (isExplicitKeystrokeTelemetryPathDenied(videoPath, isAllowedLocalReadPath)) { + return { + success: false, + samples: [], + message: "Failed to load keystroke telemetry", + }; + } + + const telemetryPath = getKeystrokePathForVideo(targetVideoPath); + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = parseJsonWithByteOrderMark(content); + const samples = normalizeKeystrokeTelemetrySamples(parsed); + return { success: true, samples }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { success: true, samples: [] }; + } + console.warn("Failed to load keystroke telemetry", error); + return { + success: false, + samples: [], + message: "Failed to load keystroke telemetry", + }; + } + }); + ipcMain.handle( "set-cursor-telemetry", async (_, videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..76e42db2f 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -3,6 +3,7 @@ import type { CursorInteractionType, CursorTelemetryPoint, CursorVisualType, + KeystrokeTelemetryPoint, NativeCaptureDiagnostics, RecordingSessionData, SelectedSource, @@ -81,6 +82,9 @@ export let cursorCapturePauseStartedAtMs: number | null = null; export let activeCursorSamples: CursorTelemetryPoint[] = []; export let pendingCursorSamples: CursorTelemetryPoint[] = []; export let isCursorCaptureActive = false; +export let activeKeystrokeSamples: KeystrokeTelemetryPoint[] = []; +export let pendingKeystrokeSamples: KeystrokeTelemetryPoint[] = []; +export let isKeystrokeCaptureActive = false; export let interactionCaptureCleanup: (() => void) | null = null; export let hasLoggedInteractionHookFailure = false; export let lastLeftClick: { timeMs: number; cx: number; cy: number } | null = null; @@ -254,6 +258,15 @@ export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { export function setIsCursorCaptureActive(v: boolean) { isCursorCaptureActive = v; } +export function setActiveKeystrokeSamples(v: KeystrokeTelemetryPoint[]) { + activeKeystrokeSamples = v; +} +export function setPendingKeystrokeSamples(v: KeystrokeTelemetryPoint[]) { + pendingKeystrokeSamples = v; +} +export function setIsKeystrokeCaptureActive(v: boolean) { + isKeystrokeCaptureActive = v; +} export function setInteractionCaptureCleanup(v: (() => void) | null) { interactionCaptureCleanup = v; } diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..c7c06c1be 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -106,6 +106,14 @@ export interface CursorTelemetryPoint { cursorType?: CursorVisualType; } +export type KeystrokeModifier = "meta" | "ctrl" | "alt" | "shift"; + +export interface KeystrokeTelemetryPoint { + timeMs: number; + key: string; + modifiers: KeystrokeModifier[]; +} + export type NativeMacWindowSource = { id: string; name: string; @@ -120,7 +128,7 @@ export type NativeMacWindowSource = { height?: number; }; -export type HookEventName = "mousedown" | "mouseup" | "mousemove"; +export type HookEventName = "mousedown" | "mouseup" | "mousemove" | "keydown"; export type HookMouseEvent = { button?: number; @@ -139,7 +147,15 @@ export type HookMouseEvent = { }; }; -export type HookEventListener = (event: HookMouseEvent) => void; +export type HookKeyboardEvent = { + keycode?: number; + shiftKey?: boolean; + ctrlKey?: boolean; + altKey?: boolean; + metaKey?: boolean; +}; + +export type HookEventListener = (event: HookMouseEvent & HookKeyboardEvent) => void; export type UiohookLike = { on: (eventName: HookEventName, listener: HookEventListener) => void; @@ -153,6 +169,7 @@ export type UiohookModuleNamespace = { uIOhook?: UiohookLike; uiohook?: UiohookLike; Uiohook?: UiohookLike; + UiohookKey?: Record; default?: UiohookLike | UiohookModuleNamespace; }; diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 23960f209..6acb767fc 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -67,6 +67,10 @@ export function getTelemetryPathForVideo(videoPath: string) { return `${videoPath}.cursor.json`; } +export function getKeystrokePathForVideo(videoPath: string) { + return `${videoPath}.keystrokes.json`; +} + export function isAutoRecordingPath(filePath: string) { return path.basename(filePath).startsWith(AUTO_RECORDING_PREFIX); } diff --git a/electron/native/NativeCursorMonitor.swift b/electron/native/NativeCursorMonitor.swift index 18c644150..354d9b8a1 100644 --- a/electron/native/NativeCursorMonitor.swift +++ b/electron/native/NativeCursorMonitor.swift @@ -406,6 +406,141 @@ if CommandLine.arguments.contains("--export-images") { exit(0) } +func focusedIsSecureTextField() -> Bool { + guard let element = focusedElement() else { + return false + } + let subrole = attributeString(element, kAXSubroleAttribute) + return subrole == (kAXSecureTextFieldSubrole as String) || subrole == "AXSecureTextField" +} + +func isModifierOnlyKeyCode(_ keyCode: Int64) -> Bool { + switch keyCode { + case 0x36, 0x37, 0x38, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E: + return true + default: + return false + } +} + +func keystrokeToken(forKeyCode keyCode: Int64) -> String? { + switch keyCode { + case 0x00: return "a" + case 0x0B: return "b" + case 0x08: return "c" + case 0x02: return "d" + case 0x0E: return "e" + case 0x03: return "f" + case 0x05: return "g" + case 0x04: return "h" + case 0x22: return "i" + case 0x26: return "j" + case 0x28: return "k" + case 0x25: return "l" + case 0x2E: return "m" + case 0x2D: return "n" + case 0x1F: return "o" + case 0x23: return "p" + case 0x0C: return "q" + case 0x0F: return "r" + case 0x01: return "s" + case 0x11: return "t" + case 0x20: return "u" + case 0x09: return "v" + case 0x0D: return "w" + case 0x07: return "x" + case 0x10: return "y" + case 0x06: return "z" + case 0x1D: return "0" + case 0x12: return "1" + case 0x13: return "2" + case 0x14: return "3" + case 0x15: return "4" + case 0x17: return "5" + case 0x16: return "6" + case 0x1A: return "7" + case 0x1C: return "8" + case 0x19: return "9" + case 0x24, 0x4C: return "enter" + case 0x35: return "esc" + case 0x30: return "tab" + case 0x31: return "space" + case 0x33: return "backspace" + case 0x75: return "delete" + case 0x7B: return "arrowleft" + case 0x7C: return "arrowright" + case 0x7E: return "arrowup" + case 0x7D: return "arrowdown" + case 0x73: return "home" + case 0x77: return "end" + case 0x74: return "pageup" + case 0x79: return "pagedown" + case 0x7A: return "f1" + case 0x78: return "f2" + case 0x63: return "f3" + case 0x76: return "f4" + case 0x60: return "f5" + case 0x61: return "f6" + case 0x62: return "f7" + case 0x64: return "f8" + case 0x65: return "f9" + case 0x6D: return "f10" + case 0x67: return "f11" + case 0x6F: return "f12" + case 0x69: return "f13" + case 0x6B: return "f14" + case 0x71: return "f15" + case 0x6A: return "f16" + case 0x40: return "f17" + case 0x4F: return "f18" + case 0x50: return "f19" + case 0x2B: return "comma" + case 0x2F: return "period" + case 0x1B: return "minus" + case 0x18: return "equal" + case 0x2C: return "slash" + case 0x29: return "semicolon" + case 0x27: return "quote" + case 0x21: return "bracketleft" + case 0x1E: return "bracketright" + case 0x2A: return "backslash" + case 0x32: return "backquote" + default: + return nil + } +} + +func emitKeystrokeIfNeeded(_ event: CGEvent) { + if event.getIntegerValueField(.keyboardEventAutorepeat) != 0 { + return + } + if focusedIsSecureTextField() { + return + } + let keyCode = event.getIntegerValueField(.keyboardEventKeycode) + if isModifierOnlyKeyCode(keyCode) { + return + } + guard let token = keystrokeToken(forKeyCode: keyCode) else { + return + } + var mods: [String] = [] + if event.flags.contains(.maskCommand) { + mods.append("meta") + } + if event.flags.contains(.maskControl) { + mods.append("ctrl") + } + if event.flags.contains(.maskAlternate) { + mods.append("alt") + } + if event.flags.contains(.maskShift) { + mods.append("shift") + } + print("KEY:down:\(token):\(mods.joined(separator: ","))") + fflush(stdout) +} + func mouseInteractionCallback( proxy: CGEventTapProxy, type: CGEventType, @@ -439,6 +574,9 @@ func mouseInteractionCallback( } action = "mouseup" button = 3 + case .keyDown: + emitKeystrokeIfNeeded(event) + return Unmanaged.passUnretained(event) default: return Unmanaged.passUnretained(event) } @@ -448,6 +586,30 @@ func mouseInteractionCallback( return Unmanaged.passUnretained(event) } +func eventMask(for types: [CGEventType]) -> CGEventMask { + types.reduce(CGEventMask(0)) { mask, type in + mask | (CGEventMask(1) << type.rawValue) + } +} + +func installListenOnlyTap(_ mask: CGEventMask) -> Bool { + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: mask, + callback: mouseInteractionCallback, + userInfo: nil + ), + let eventTapSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else { + return false + } + CFRunLoopAddSource(CFRunLoopGetMain(), eventTapSource, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + return true +} + +let captureKeys = CommandLine.arguments.contains("--capture-keys") let mouseEventTypes: [CGEventType] = [ .leftMouseDown, .leftMouseUp, @@ -456,24 +618,18 @@ let mouseEventTypes: [CGEventType] = [ .otherMouseDown, .otherMouseUp, ] -let mouseEventMask = mouseEventTypes.reduce(CGEventMask(0)) { mask, type in - mask | (CGEventMask(1) << type.rawValue) -} -let mouseEventTap = CGEvent.tapCreate( - tap: .cgSessionEventTap, - place: .headInsertEventTap, - options: .listenOnly, - eventsOfInterest: mouseEventMask, - callback: mouseInteractionCallback, - userInfo: nil -) -if let mouseEventTap, - let eventTapSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, mouseEventTap, 0) { - CFRunLoopAddSource(CFRunLoopGetMain(), eventTapSource, .commonModes) - CGEvent.tapEnable(tap: mouseEventTap, enable: true) -} else { - fputs("Mouse interaction event tap unavailable; click telemetry disabled\n", stderr) - fflush(stderr) +let mouseOnlyMask = eventMask(for: mouseEventTypes) +let combinedMask = captureKeys + ? mouseOnlyMask | (CGEventMask(1) << CGEventType.keyDown.rawValue) + : mouseOnlyMask +if !installListenOnlyTap(combinedMask) { + if captureKeys, installListenOnlyTap(mouseOnlyMask) { + fputs("Keyboard event tap unavailable; keystroke telemetry disabled\n", stderr) + fflush(stderr) + } else { + fputs("Mouse interaction event tap unavailable; click telemetry disabled\n", stderr) + fflush(stderr) + } } var lastState = "" diff --git a/electron/preload.ts b/electron/preload.ts index 990ee7a8f..364e789f8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -611,6 +611,9 @@ contextBridge.exposeInMainWorld("electronAPI", { getCursorTelemetry: (videoPath?: string) => { return ipcRenderer.invoke("get-cursor-telemetry", videoPath); }, + getKeystrokeTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke("get-keystroke-telemetry", videoPath); + }, setCursorTelemetry: (videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { return ipcRenderer.invoke("set-cursor-telemetry", videoPath, samples); }, diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 8b289e3f5..66a880fd8 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -90,6 +90,11 @@ import { DEFAULT_ZOOM_OUT_DURATION_MS, } from "./types"; import { fromCursorSwaySliderValue, toCursorSwaySliderValue } from "./videoPlayback/cursorSway"; +import { + DEFAULT_KEYSTROKE_OVERLAY, + type KeystrokeOverlayPosition, + type KeystrokeOverlaySettings, +} from "./videoPlayback/keystrokeOverlay/keystrokeTypes"; import { isZeroPadding } from "./videoPlayback/layoutUtils"; import { supportsPreviewPlaybackRate } from "./videoPlayback/playbackRate"; import { @@ -612,6 +617,9 @@ interface SettingsPanelProps { onWebcamChange?: (webcam: WebcamOverlaySettings) => void; onUploadWebcam?: () => void; onClearWebcam?: () => void; + keystrokeOverlay?: KeystrokeOverlaySettings; + onKeystrokeOverlayChange?: (overlay: KeystrokeOverlaySettings) => void; + keystrokeSampleCount?: number; padding?: Padding; onPaddingChange?: (padding: Padding) => void; cropRegion?: CropRegion; @@ -677,6 +685,25 @@ const WEBCAM_POSITION_PRESETS: Array<{ { preset: "bottom-right", label: "↘" }, ]; +const KEYSTROKE_OVERLAY_POSITIONS: Array<{ + position: KeystrokeOverlayPosition; + labelKey: string; + fallback: string; +}> = [ + { position: "bottom-left", labelKey: "keystrokes.positionBottomLeft", fallback: "Bottom left" }, + { + position: "bottom-center", + labelKey: "keystrokes.positionBottomCenter", + fallback: "Bottom center", + }, + { + position: "bottom-right", + labelKey: "keystrokes.positionBottomRight", + fallback: "Bottom right", + }, + { position: "top-center", labelKey: "keystrokes.positionTopCenter", fallback: "Top center" }, +]; + type CursorStyleOption = { value: CursorStyle; label: string }; type WallpaperTile = { @@ -1060,6 +1087,9 @@ export function SettingsPanel({ onWebcamChange, onUploadWebcam, onClearWebcam, + keystrokeOverlay = DEFAULT_KEYSTROKE_OVERLAY, + onKeystrokeOverlayChange, + keystrokeSampleCount = 0, padding = DEFAULT_PADDING, onPaddingChange, cropRegion, @@ -1639,6 +1669,14 @@ export function SettingsPanel({ onWebcamChange({ ...defaultWebcam }); }; + const resetKeystrokesSection = () => { + onKeystrokeOverlayChange?.({ ...initialEditorPreferences.keystrokeOverlay }); + }; + + const updateKeystrokeOverlay = (patch: Partial) => { + onKeystrokeOverlayChange?.({ ...keystrokeOverlay, ...patch }); + }; + const resetCropSection = () => { onCropChange?.(DEFAULT_CROP_REGION); }; @@ -3486,6 +3524,127 @@ export function SettingsPanel({ ); + case "keystrokes": + return ( +
+
+
+ + {tSettings("sections.keystrokes", "Keystrokes")} + + +
+
+ {tSettings("captions.enabled", "Show")} + + updateKeystrokeOverlay({ enabled }) + } + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+
+ {keystrokeOverlay.enabled ? ( +
+
+
+ {tSettings("keystrokes.mode", "Mode")} +
+ { + if (value === "shortcuts" || value === "all") { + updateKeystrokeOverlay({ mode: value }); + } + }} + className="grid grid-cols-2 gap-1.5" + aria-label={tSettings("keystrokes.mode", "Mode")} + > + + {tSettings("keystrokes.modeShortcuts", "Shortcuts")} + + + {tSettings("keystrokes.modeAll", "All typing")} + + +
+
+
+ {tSettings("keystrokes.position", "Position")} +
+
+ {KEYSTROKE_OVERLAY_POSITIONS.map((option) => { + const isActive = + keystrokeOverlay.position === option.position; + return ( + + ); + })} +
+
+ updateKeystrokeOverlay({ size })} + formatValue={(v) => `${v.toFixed(2)}×`} + parseInput={(text) => parseFloat(text.replace(/×$/, ""))} + /> + {keystrokeSampleCount === 0 ? ( +
+

+ {tSettings( + "keystrokes.empty", + "This recording has no keystrokes. Leave Show on when you record to capture them.", + )} +

+
+ ) : null} +
+ ) : null} +
+ ); case "webcam": return (
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 6d22666a1..020cc8e53 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -290,6 +290,7 @@ export default function VideoEditor() { }); const { cursor: { effectiveCursorTelemetry }, + keystrokeSamples, projection, audio, captionCommands, @@ -320,6 +321,7 @@ export default function VideoEditor() { effectiveSpeedRegions, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeSamples, effectiveShowCursor, cursorTelemetrySourcePath, hasCaptionsForSidecar, @@ -364,6 +366,7 @@ export default function VideoEditor() { setNativeCaptureUnavailableModalOpen, handleUploadWebcam, handleClearWebcam, + keystrokeSampleCount: keystrokeSamples.length, }); return ( void; onAnnotationSizeChange?: (id: string, size: { width: number; height: number }) => void; cursorTelemetry?: CursorTelemetryPoint[]; + keystrokeSamples?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; showCursor?: boolean; cursorStyle?: CursorStyle; cursorSize?: number; @@ -345,6 +353,8 @@ const VideoPlayback = forwardRef( onAnnotationPositionChange, onAnnotationSizeChange, cursorTelemetry = [], + keystrokeSamples = [], + keystrokeOverlay = DEFAULT_KEYSTROKE_OVERLAY, showCursor = false, cursorStyle = DEFAULT_CURSOR_STYLE, cursorSize = DEFAULT_CURSOR_SIZE, @@ -530,9 +540,7 @@ const VideoPlayback = forwardRef( }, []); const initializePixiRenderer = useCallback( - async ( - container: HTMLDivElement, - ): Promise => { + async (container: HTMLDivElement): Promise => { const backendOrder: PixiPreviewBackend[] = ["webgl", "webgpu"]; const attempts: PixiRendererAttempt[] = []; @@ -2668,6 +2676,13 @@ const VideoPlayback = forwardRef( ) : null} +
; type TimelineState = ReturnType; @@ -13,6 +14,7 @@ type BuildExportRenderOptionsInput = { effectiveSpeedRegions: SpeedRegion[]; effectiveZoomRegions: ZoomRegion[]; effectiveCursorTelemetry: CursorTelemetryPoint[]; + keystrokeTelemetry: KeystrokeTelemetryPoint[]; effectiveShowCursor: boolean; previewWidth: number; previewHeight: number; @@ -26,6 +28,7 @@ export function buildExportRenderOptions({ effectiveSpeedRegions, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeTelemetry, effectiveShowCursor, previewWidth, previewHeight, @@ -63,6 +66,8 @@ export function buildExportRenderOptions({ autoCaptionSettings: timeline.autoCaptionSettings, zoomRegions: effectiveZoomRegions, cursorTelemetry: effectiveCursorTelemetry, + keystrokeTelemetry, + keystrokeOverlay: appearance.keystrokeOverlay, showCursor: effectiveShowCursor, cursorStyle: appearance.cursorStyle, cursorSize: appearance.cursorSize, diff --git a/src/components/video-editor/export/exportRunnerSupport.ts b/src/components/video-editor/export/exportRunnerSupport.ts index 11c8cf8e6..e250a429e 100644 --- a/src/components/video-editor/export/exportRunnerSupport.ts +++ b/src/components/video-editor/export/exportRunnerSupport.ts @@ -9,6 +9,7 @@ import type { useTimelineState } from "../state/useTimelineState"; import type { CursorTelemetryPoint, SpeedRegion, ZoomRegion } from "../types"; import type { VideoPlaybackRef } from "../VideoPlayback"; import { summarizeErrorMessage } from "../videoEditorUtils"; +import type { KeystrokeTelemetryPoint } from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; import type { PendingExportSave } from "./exportPersistence"; import type { useExportSession } from "./useExportSession"; import type { useExportSettings } from "./useExportSettings"; @@ -26,6 +27,7 @@ export type ExportRunnerInput = { effectiveSpeedRegions: SpeedRegion[]; effectiveZoomRegions: ZoomRegion[]; effectiveCursorTelemetry: CursorTelemetryPoint[]; + keystrokeTelemetry: KeystrokeTelemetryPoint[]; effectiveShowCursor: boolean; ensureSupportedMp4SourceDimensions: ( frameRate: ReturnType["mp4FrameRate"], diff --git a/src/components/video-editor/export/useEditorExportController.ts b/src/components/video-editor/export/useEditorExportController.ts index ec6aa3f77..aace91305 100644 --- a/src/components/video-editor/export/useEditorExportController.ts +++ b/src/components/video-editor/export/useEditorExportController.ts @@ -6,6 +6,7 @@ import type { useAppearanceState } from "../state/useAppearanceState"; import type { useTimelineState } from "../state/useTimelineState"; import type { CursorTelemetryPoint, SpeedRegion, ZoomRegion } from "../types"; import type { VideoPlaybackRef } from "../VideoPlayback"; +import type { KeystrokeTelemetryPoint } from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; import { useExportDialogActions } from "./useExportDialogActions"; import type { useExportDimensions } from "./useExportDimensions"; import { useExportMessages } from "./useExportMessages"; @@ -35,6 +36,7 @@ type Input = { effectiveSpeedRegions: SpeedRegion[]; effectiveZoomRegions: ZoomRegion[]; effectiveCursorTelemetry: CursorTelemetryPoint[]; + keystrokeSamples: KeystrokeTelemetryPoint[]; effectiveShowCursor: boolean; cursorTelemetrySourcePath: string | null; hasCaptionsForSidecar: boolean; @@ -58,6 +60,7 @@ export function useEditorExportController(input: Input) { effectiveSpeedRegions: input.effectiveSpeedRegions, effectiveZoomRegions: input.effectiveZoomRegions, effectiveCursorTelemetry: input.effectiveCursorTelemetry, + keystrokeTelemetry: input.keystrokeSamples, effectiveShowCursor: input.effectiveShowCursor, ensureSupportedMp4SourceDimensions: input.dimensions.ensureSupportedMp4SourceDimensions, captionSidecarPayload: input.captionSidecarPayload, diff --git a/src/components/video-editor/export/useExportRunner.ts b/src/components/video-editor/export/useExportRunner.ts index 37edbdba4..d68031ddc 100644 --- a/src/components/video-editor/export/useExportRunner.ts +++ b/src/components/video-editor/export/useExportRunner.ts @@ -39,6 +39,7 @@ export function useExportRunner(input: ExportRunnerInput) { effectiveSpeedRegions, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeTelemetry, effectiveShowCursor, ensureSupportedMp4SourceDimensions, captionSidecarPayload, @@ -141,6 +142,7 @@ export function useExportRunner(input: ExportRunnerInput) { effectiveSpeedRegions, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeTelemetry, effectiveShowCursor, previewWidth, previewHeight, @@ -287,6 +289,7 @@ export function useExportRunner(input: ExportRunnerInput) { effectiveSpeedRegions, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeTelemetry, effectiveShowCursor, previewWidth, previewHeight, diff --git a/src/components/video-editor/hooks/useKeystrokeTelemetry.ts b/src/components/video-editor/hooks/useKeystrokeTelemetry.ts new file mode 100644 index 000000000..d1aa83df2 --- /dev/null +++ b/src/components/video-editor/hooks/useKeystrokeTelemetry.ts @@ -0,0 +1,65 @@ +import { useEffect, useRef, useState } from "react"; +import type { KeystrokeTelemetryPoint } from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; + +export function useKeystrokeTelemetry(videoSourcePath: string | null) { + const [keystrokeSamples, setKeystrokeSamples] = useState([]); + const pendingRetryTimeoutRef = useRef(null); + + useEffect(() => { + let mounted = true; + let retryAttempts = 0; + const scheduleRetry = () => { + if (!mounted || retryAttempts >= 12) { + return; + } + retryAttempts += 1; + pendingRetryTimeoutRef.current = window.setTimeout(() => { + pendingRetryTimeoutRef.current = null; + if (mounted) { + void load(); + } + }, 350); + }; + async function load() { + if (!videoSourcePath) { + if (mounted) { + setKeystrokeSamples([]); + } + return; + } + try { + const result = await window.electronAPI.getKeystrokeTelemetry(videoSourcePath); + if (!mounted) { + return; + } + const samples = result.success ? result.samples : []; + setKeystrokeSamples(samples); + if (samples.length === 0) { + scheduleRetry(); + } + } catch { + console.warn("Unable to load keystroke telemetry"); + if (!mounted) { + return; + } + setKeystrokeSamples([]); + scheduleRetry(); + } + } + + if (pendingRetryTimeoutRef.current !== null) { + window.clearTimeout(pendingRetryTimeoutRef.current); + pendingRetryTimeoutRef.current = null; + } + void load(); + return () => { + mounted = false; + if (pendingRetryTimeoutRef.current !== null) { + window.clearTimeout(pendingRetryTimeoutRef.current); + pendingRetryTimeoutRef.current = null; + } + }; + }, [videoSourcePath]); + + return { keystrokeSamples }; +} diff --git a/src/components/video-editor/hooks/useTimelineEditingController.ts b/src/components/video-editor/hooks/useTimelineEditingController.ts index 083340f9a..c7fbc7b5d 100644 --- a/src/components/video-editor/hooks/useTimelineEditingController.ts +++ b/src/components/video-editor/hooks/useTimelineEditingController.ts @@ -18,6 +18,7 @@ import { useCursorTelemetry } from "./useCursorTelemetry"; import { useEditorGlobalInteractions } from "./useEditorGlobalInteractions"; import { useEditorPlaybackControls } from "./useEditorPlaybackControls"; import { useFreshRecordingAutoZoom } from "./useFreshRecordingAutoZoom"; +import { useKeystrokeTelemetry } from "./useKeystrokeTelemetry"; import { useTimelineProjection } from "./useTimelineProjection"; import { useZoomRegionCommands } from "./useZoomRegionCommands"; @@ -74,6 +75,7 @@ export function useTimelineEditingController(input: Input) { pendingFreshRecordingAutoZoomPathRef: input.pendingFreshRecordingAutoZoomPathRef, autoSuggestedVideoPathRef: input.autoSuggestedVideoPathRef, }); + const { keystrokeSamples } = useKeystrokeTelemetry(input.videoSourcePath); const projection = useTimelineProjection({ timeline, duration: input.duration, @@ -218,6 +220,7 @@ export function useTimelineEditingController(input: Input) { return { cursor, + keystrokeSamples, projection, audio, playback, diff --git a/src/components/video-editor/layout/EditorPreviewPanel.tsx b/src/components/video-editor/layout/EditorPreviewPanel.tsx index 1c7ca4869..710de830c 100644 --- a/src/components/video-editor/layout/EditorPreviewPanel.tsx +++ b/src/components/video-editor/layout/EditorPreviewPanel.tsx @@ -34,6 +34,7 @@ import type { useAppearanceState } from "../state/useAppearanceState"; import type { useTimelineState } from "../state/useTimelineState"; import type { TimelineEditorHandle } from "../timeline/TimelineEditor"; import type { VideoPlaybackRef } from "../VideoPlayback"; +import type { KeystrokeTelemetryPoint } from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; import { EditorVideoPreview } from "./EditorVideoPreview"; type Props = { @@ -58,6 +59,7 @@ type Props = { zoomCommands: ReturnType; annotationCommands: ReturnType; effectiveCursorTelemetry: ReturnType["cursorTelemetry"]; + keystrokeSamples: KeystrokeTelemetryPoint[]; effectiveShowCursor: boolean; isCropped: boolean; handleOpenCropEditor: () => void; @@ -100,6 +102,7 @@ export function EditorPreviewPanel(props: Props) { zoomCommands, annotationCommands, effectiveCursorTelemetry, + keystrokeSamples, effectiveShowCursor, isCropped, handleOpenCropEditor, @@ -192,6 +195,7 @@ export function EditorPreviewPanel(props: Props) { audio={audio} effectiveZoomRegions={projection.effectiveZoomRegions} effectiveCursorTelemetry={effectiveCursorTelemetry} + keystrokeSamples={keystrokeSamples} effectiveShowCursor={effectiveShowCursor} setDuration={setDuration} setIsPreviewReady={setIsPreviewReady} diff --git a/src/components/video-editor/layout/EditorShell.tsx b/src/components/video-editor/layout/EditorShell.tsx index e85928280..313850674 100644 --- a/src/components/video-editor/layout/EditorShell.tsx +++ b/src/components/video-editor/layout/EditorShell.tsx @@ -78,6 +78,7 @@ export function EditorShell(props: Props) { } = projectController; const { cursor, + keystrokeSamples, projection, audio, playback, @@ -209,6 +210,7 @@ export function EditorShell(props: Props) { zoomCommands={zoomCommands} annotationCommands={annotationCommands} effectiveCursorTelemetry={cursor.effectiveCursorTelemetry} + keystrokeSamples={keystrokeSamples} effectiveShowCursor={effectiveShowCursor} isCropped={ui.isCropped} handleOpenCropEditor={ui.handleOpenCropEditor} diff --git a/src/components/video-editor/layout/EditorSidebar.tsx b/src/components/video-editor/layout/EditorSidebar.tsx index d73720b68..453ecf205 100644 --- a/src/components/video-editor/layout/EditorSidebar.tsx +++ b/src/components/video-editor/layout/EditorSidebar.tsx @@ -3,6 +3,7 @@ import { ClosedCaptioning, Cursor, Gear, + Keyboard, PuzzlePiece, Sparkle, UserCircle, @@ -28,6 +29,11 @@ export function EditorSidebar({ t, activeSection, setActiveSection, settingsPane () => [ { id: "scene" as const, label: t("settings.sections.scene", "Scene"), icon: Sparkle }, { id: "cursor" as const, label: t("settings.sections.cursor", "Cursor"), icon: Cursor }, + { + id: "keystrokes" as const, + label: t("settings.sections.keystrokes", "Keystrokes"), + icon: Keyboard, + }, { id: "webcam" as const, label: t("settings.sections.webcam", "Webcam"), icon: Camera }, { id: "captions" as const, diff --git a/src/components/video-editor/layout/EditorVideoPreview.tsx b/src/components/video-editor/layout/EditorVideoPreview.tsx index ae0fe3e96..17c213f8a 100644 --- a/src/components/video-editor/layout/EditorVideoPreview.tsx +++ b/src/components/video-editor/layout/EditorVideoPreview.tsx @@ -5,6 +5,7 @@ import type { useAppearanceState } from "../state/useAppearanceState"; import type { useTimelineState } from "../state/useTimelineState"; import type { CursorTelemetryPoint, ZoomRegion } from "../types"; import VideoPlayback, { type VideoPlaybackRef } from "../VideoPlayback"; +import type { KeystrokeTelemetryPoint } from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; type PlaybackProps = ComponentProps; type Handlers = Pick< @@ -31,6 +32,7 @@ type Props = { audio: ReturnType; effectiveZoomRegions: ZoomRegion[]; effectiveCursorTelemetry: CursorTelemetryPoint[]; + keystrokeSamples: KeystrokeTelemetryPoint[]; effectiveShowCursor: boolean; setDuration: Dispatch>; setIsPreviewReady: Dispatch>; @@ -54,6 +56,7 @@ export function EditorVideoPreview({ audio, effectiveZoomRegions, effectiveCursorTelemetry, + keystrokeSamples, effectiveShowCursor, setDuration, setIsPreviewReady, @@ -103,6 +106,8 @@ export function EditorVideoPreview({ autoCaptionSettings={timeline.autoCaptionSettings} selectedAnnotationId={timeline.selectedAnnotationId} cursorTelemetry={effectiveCursorTelemetry} + keystrokeSamples={keystrokeSamples} + keystrokeOverlay={appearance.keystrokeOverlay} showCursor={effectiveShowCursor} cursorStyle={appearance.cursorStyle} cursorSize={appearance.cursorSize} diff --git a/src/components/video-editor/layout/useEditorSettingsPanelProps.ts b/src/components/video-editor/layout/useEditorSettingsPanelProps.ts index 5ea46b498..8d8561173 100644 --- a/src/components/video-editor/layout/useEditorSettingsPanelProps.ts +++ b/src/components/video-editor/layout/useEditorSettingsPanelProps.ts @@ -38,6 +38,7 @@ type Input = { setNativeCaptureUnavailableModalOpen: Dispatch>; handleUploadWebcam: () => void; handleClearWebcam: () => void; + keystrokeSampleCount: number; }; export function useEditorSettingsPanelProps(input: Input): ComponentProps { @@ -67,6 +68,7 @@ export function useEditorSettingsPanelProps(input: Input): ComponentProps region.id === timeline.selectedZoomId, @@ -186,6 +188,9 @@ export function useEditorSettingsPanelProps(input: Input): ComponentProps settings)(appearance.webcam), + keystrokeOverlay: appearance.keystrokeOverlay, aspectRatio, exportEncodingMode: exportSettings.exportEncodingMode, exportBackendPreference: exportSettings.exportBackendPreference, @@ -144,6 +145,7 @@ export function useVideoEditorPresets({ ...snapshot.webcam, sourcePath: current.sourcePath, })); + appearance.setKeystrokeOverlay({ ...snapshot.keystrokeOverlay }); setAspectRatio(snapshot.aspectRatio); exportSettings.setExportEncodingMode(snapshot.exportEncodingMode); exportSettings.setExportBackendPreference(snapshot.exportBackendPreference); diff --git a/src/components/video-editor/project/useProjectLifecycle.ts b/src/components/video-editor/project/useProjectLifecycle.ts index 1f0a0f320..c331efb9e 100644 --- a/src/components/video-editor/project/useProjectLifecycle.ts +++ b/src/components/video-editor/project/useProjectLifecycle.ts @@ -166,6 +166,7 @@ export function useProjectLifecycle(input: Input) { appearance.setPadding(editor.padding); appearance.setCropRegion(editor.cropRegion); appearance.setWebcam(editor.webcam); + appearance.setKeystrokeOverlay(editor.keystrokeOverlay); timeline.setZoomRegions(editor.zoomRegions); timeline.setTrimRegions(editor.trimRegions); timeline.setClipRegions(editor.clipRegions); diff --git a/src/components/video-editor/project/useProjectSnapshotModel.ts b/src/components/video-editor/project/useProjectSnapshotModel.ts index e138cad17..66efca9a5 100644 --- a/src/components/video-editor/project/useProjectSnapshotModel.ts +++ b/src/components/video-editor/project/useProjectSnapshotModel.ts @@ -114,6 +114,7 @@ export function useProjectSnapshotModel({ padding: appearance.padding, cropRegion: appearance.cropRegion, webcam: appearance.webcam, + keystrokeOverlay: appearance.keystrokeOverlay, zoomRegions: timeline.zoomRegions, trimRegions: timeline.trimRegions, clipRegions: timeline.clipRegions, @@ -176,6 +177,7 @@ export function useProjectSnapshotModel({ appearance.padding, appearance.cropRegion, appearance.webcam, + appearance.keystrokeOverlay, timeline.zoomRegions, timeline.trimRegions, timeline.clipRegions, diff --git a/src/components/video-editor/projectPersistence.test.ts b/src/components/video-editor/projectPersistence.test.ts index 7b585cd3b..93ac4bb74 100644 --- a/src/components/video-editor/projectPersistence.test.ts +++ b/src/components/video-editor/projectPersistence.test.ts @@ -126,4 +126,33 @@ describe("normalizeProjectEditor", () => { expect(editor.webcam.height).toBe(80); expect(editor.webcam.roundness).toBeCloseTo(4.34, 1); }); + + it("defaults and clamps keystroke overlay settings", () => { + expect(normalizeProjectEditor({}).keystrokeOverlay).toEqual({ + enabled: false, + mode: "shortcuts", + position: "bottom-left", + size: 1, + }); + + const editor = normalizeProjectEditor({ + keystrokeOverlay: { + enabled: "yes", + mode: "words", + position: "top-left", + size: 9, + } as never, + }); + expect(editor.keystrokeOverlay).toEqual({ + enabled: false, + mode: "shortcuts", + position: "bottom-left", + size: 2, + }); + expect( + normalizeProjectEditor({ + keystrokeOverlay: { size: Number.NaN } as never, + }).keystrokeOverlay.size, + ).toBe(1); + }); }); diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 75154893c..1a165e441 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -72,6 +72,10 @@ import { type ZoomRegion, type ZoomTransitionEasing, } from "./types"; +import { + DEFAULT_KEYSTROKE_OVERLAY, + type KeystrokeOverlaySettings, +} from "./videoPlayback/keystrokeOverlay/keystrokeTypes"; import { convertLegacyWebcamRadiusToRoundness, normalizeWebcamCropRegion } from "./webcamOverlay"; export const PROJECT_VERSION = 2; @@ -141,6 +145,7 @@ export interface ProjectEditorState { autoCaptions: CaptionCue[]; autoCaptionSettings: AutoCaptionSettings; webcam: WebcamOverlaySettings; + keystrokeOverlay: KeystrokeOverlaySettings; aspectRatio: AspectRatio; sourceAudioTrackSettingsByClip?: Record; defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; @@ -791,6 +796,10 @@ export function normalizeProjectEditor(editor: Partial): Pro const webcam: Partial = editor.webcam && typeof editor.webcam === "object" ? editor.webcam : {}; + const keystrokeOverlay: Partial = + editor.keystrokeOverlay && typeof editor.keystrokeOverlay === "object" + ? editor.keystrokeOverlay + : {}; const webcamSourcePath = typeof webcam.sourcePath === "string" ? webcam.sourcePath : null; const legacyZoomScaleEffect = isFiniteNumber( (webcam as Partial<{ zoomScaleEffect: number }>).zoomScaleEffect, @@ -1042,6 +1051,26 @@ export function normalizeProjectEditor(editor: Partial): Pro ? clamp(webcam.margin, 0, 96) : DEFAULT_WEBCAM_MARGIN, }, + keystrokeOverlay: { + enabled: + typeof keystrokeOverlay.enabled === "boolean" + ? keystrokeOverlay.enabled + : DEFAULT_KEYSTROKE_OVERLAY.enabled, + mode: + keystrokeOverlay.mode === "shortcuts" || keystrokeOverlay.mode === "all" + ? keystrokeOverlay.mode + : DEFAULT_KEYSTROKE_OVERLAY.mode, + position: + keystrokeOverlay.position === "bottom-left" || + keystrokeOverlay.position === "bottom-center" || + keystrokeOverlay.position === "bottom-right" || + keystrokeOverlay.position === "top-center" + ? keystrokeOverlay.position + : DEFAULT_KEYSTROKE_OVERLAY.position, + size: isFiniteNumber(keystrokeOverlay.size) + ? clamp(keystrokeOverlay.size, 0.5, 2) + : DEFAULT_KEYSTROKE_OVERLAY.size, + }, sourceAudioTrackSettingsByClip: editor.sourceAudioTrackSettingsByClip && typeof editor.sourceAudioTrackSettingsByClip === "object" diff --git a/src/components/video-editor/state/useAppearanceState.ts b/src/components/video-editor/state/useAppearanceState.ts index 6dbfd4b16..d8661b32b 100644 --- a/src/components/video-editor/state/useAppearanceState.ts +++ b/src/components/video-editor/state/useAppearanceState.ts @@ -22,6 +22,10 @@ import { DEFAULT_ZOOM_OUT_DURATION_MS, DEFAULT_ZOOM_OUT_EASING, } from "../types"; +import { + DEFAULT_KEYSTROKE_OVERLAY, + type KeystrokeOverlaySettings, +} from "../videoPlayback/keystrokeOverlay/keystrokeTypes"; export function useAppearanceState(preferences: EditorPreferences) { const [wallpaper, setWallpaper] = useState(preferences.wallpaper); @@ -113,6 +117,9 @@ export function useAppearanceState(preferences: EditorPreferences) { const [webcam, setWebcam] = useState( preferences.webcam ?? DEFAULT_WEBCAM_OVERLAY, ); + const [keystrokeOverlay, setKeystrokeOverlay] = useState( + preferences.keystrokeOverlay ?? DEFAULT_KEYSTROKE_OVERLAY, + ); const [resolvedWebcamVideoUrl, setResolvedWebcamVideoUrl] = useState(null); return { @@ -198,6 +205,8 @@ export function useAppearanceState(preferences: EditorPreferences) { setCropRegion, webcam, setWebcam, + keystrokeOverlay, + setKeystrokeOverlay, resolvedWebcamVideoUrl, setResolvedWebcamVideoUrl, }; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index d5dc8714f..1aa03dabb 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -109,6 +109,7 @@ export function normalizeCursorClickEffectColor( export type EditorEffectSection = | "scene" | "cursor" + | "keystrokes" | "captions" | "caption" | "webcam" diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/KeystrokeOverlay.tsx b/src/components/video-editor/videoPlayback/keystrokeOverlay/KeystrokeOverlay.tsx new file mode 100644 index 000000000..4634b91d3 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/KeystrokeOverlay.tsx @@ -0,0 +1,112 @@ +import { getSquircleSvgPath } from "@/lib/geometry/squircle"; +import { + CAPTION_FONT_WEIGHT, + getCaptionPadding, + getCaptionScaledFontSize, + getCaptionScaledRadius, +} from "../../captionStyle"; +import { getDefaultCaptionFontFamily } from "../../types"; +import type { KeystrokeOverlaySettings, KeystrokeTelemetryPoint } from "./keystrokeTypes"; +import { visibleKeycaps } from "./visibleKeycaps"; + +type KeystrokeOverlayProps = { + samples: readonly KeystrokeTelemetryPoint[]; + settings: KeystrokeOverlaySettings; + timeMs: number; + overlayWidth: number; + isGap: boolean; +}; + +function applyKeycapPillClip(element: HTMLDivElement | null, radius: number) { + if (!element) { + return; + } + const path = getSquircleSvgPath({ + x: 0, + y: 0, + width: element.offsetWidth, + height: element.offsetHeight, + radius, + }); + if (!path) { + return; + } + element.style.clipPath = `path('${path}')`; + element.style.setProperty("-webkit-clip-path", `path('${path}')`); +} + +export function KeystrokeOverlay({ + samples, + settings, + timeMs, + overlayWidth, + isGap, +}: KeystrokeOverlayProps) { + const fontSize = Math.min( + 64, + Math.max(10, getCaptionScaledFontSize(30 * settings.size, overlayWidth, 62)), + ); + const padding = getCaptionPadding(fontSize); + const radius = getCaptionScaledRadius(17.5, fontSize); + const groups = isGap || !settings.enabled ? [] : visibleKeycaps(samples, timeMs, settings.mode); + if (groups.length === 0) { + return null; + } + + const margin = fontSize * 1.1; + const isTop = settings.position === "top-center"; + const isRight = settings.position === "bottom-right"; + const isLeft = settings.position === "bottom-left"; + + return ( +
+ {groups.map((group) => ( +
+ {group.labels.map((label, index) => ( +
applyKeycapPillClip(element, radius)} + style={{ + background: "rgba(0,0,0,0.9)", + boxSizing: "border-box", + color: "#fff", + fontFamily: getDefaultCaptionFontFamily(), + fontSize, + fontWeight: CAPTION_FONT_WEIGHT, + lineHeight: 1, + padding: `${padding.y}px ${padding.x}px`, + whiteSpace: "nowrap", + }} + > + {label} +
+ ))} +
+ ))} +
+ ); +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.test.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.test.ts new file mode 100644 index 000000000..c2d124746 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { keystrokeLabels } from "./keystrokeLabels"; + +describe("keystrokeLabels", () => { + it("renders meta+c as command glyph then C", () => { + expect(keystrokeLabels("c", ["meta"])).toEqual(["⌘", "C"]); + }); + + it("renders ctrl+c as Ctrl then C", () => { + expect(keystrokeLabels("c", ["ctrl"])).toEqual(["Ctrl", "C"]); + }); + + it("renders enter as Enter", () => { + expect(keystrokeLabels("enter", [])).toEqual(["Enter"]); + }); + + it("renders named punctuation as a glyph", () => { + expect(keystrokeLabels("period", [])).toEqual(["."]); + }); +}); diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.ts new file mode 100644 index 000000000..3e81380a7 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeLabels.ts @@ -0,0 +1,82 @@ +import type { KeystrokeModifier } from "./keystrokeTypes"; + +const MODIFIER_ORDER: KeystrokeModifier[] = ["meta", "ctrl", "alt", "shift"]; + +const SPECIAL_KEY_LABELS: Record = { + enter: "Enter", + esc: "Esc", + tab: "Tab", + backspace: "Backspace", + delete: "Delete", + space: "Space", + arrowleft: "←", + arrowright: "→", + arrowup: "↑", + arrowdown: "↓", + home: "Home", + end: "End", + pageup: "PgUp", + pagedown: "PgDn", +}; + +const PUNCTUATION_LABELS: Record = { + comma: ",", + period: ".", + minus: "-", + equal: "=", + slash: "/", + semicolon: ";", + quote: "'", + bracketleft: "[", + bracketright: "]", + backslash: "\\", + backquote: "`", +}; + +function isFunctionKey(key: string) { + return /^f([1-9]|1[0-9])$/.test(key); +} + +export function isCharacterKey(key: string) { + return key === "space" || key.length === 1 || key in PUNCTUATION_LABELS; +} + +function modifierLabel(modifier: KeystrokeModifier, modifiers: readonly KeystrokeModifier[]) { + if (modifier === "meta") { + return "⌘"; + } + if (modifier === "ctrl") { + return "Ctrl"; + } + if (modifier === "alt") { + return "Alt"; + } + return modifiers.includes("meta") ? "⇧" : "Shift"; +} + +function keyLabel(key: string) { + const special = SPECIAL_KEY_LABELS[key]; + if (special) { + return special; + } + const punctuation = PUNCTUATION_LABELS[key]; + if (punctuation) { + return punctuation; + } + if (isFunctionKey(key) || key.length === 1) { + return key.toUpperCase(); + } + return key.charAt(0).toUpperCase() + key.slice(1); +} + +export function keystrokeLabels(key: string, modifiers: readonly KeystrokeModifier[]): string[] { + const present = new Set(modifiers); + const labels: string[] = []; + for (const modifier of MODIFIER_ORDER) { + if (present.has(modifier)) { + labels.push(modifierLabel(modifier, modifiers)); + } + } + labels.push(keyLabel(key)); + return labels; +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts new file mode 100644 index 000000000..6c6e63ed4 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts @@ -0,0 +1,33 @@ +export type KeystrokeModifier = "meta" | "ctrl" | "alt" | "shift"; +export type KeystrokeOverlayMode = "shortcuts" | "all"; +export type KeystrokeOverlayPosition = + | "bottom-left" + | "bottom-center" + | "bottom-right" + | "top-center"; + +export interface KeystrokeOverlaySettings { + enabled: boolean; + mode: KeystrokeOverlayMode; + position: KeystrokeOverlayPosition; + size: number; // 0.5–2 +} + +export const DEFAULT_KEYSTROKE_OVERLAY: KeystrokeOverlaySettings = { + enabled: false, + mode: "shortcuts", + position: "bottom-left", + size: 1, +}; + +export interface KeystrokeTelemetryPoint { + timeMs: number; + key: string; // lowercase token: "c" | "enter" | "arrowleft" | "f12" + modifiers: KeystrokeModifier[]; +} + +export interface KeycapGroup { + id: string; // `${timeMs}:${key}:${modsSorted}` + labels: string[]; + opacity: number; +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.test.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.test.ts new file mode 100644 index 000000000..4aa3929f0 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import type { KeystrokeTelemetryPoint } from "./keystrokeTypes"; +import { KEYCAP_FADE_MS, KEYCAP_HOLD_MS, visibleKeycaps } from "./visibleKeycaps"; + +function point( + timeMs: number, + key: string, + modifiers: KeystrokeTelemetryPoint["modifiers"] = [], +): KeystrokeTelemetryPoint { + return { timeMs, key, modifiers }; +} + +describe("visibleKeycaps", () => { + it("hides a bare letter in shortcuts mode and shows meta+c", () => { + const samples = [point(100, "c"), point(120, "c", ["meta"])]; + expect(visibleKeycaps(samples, 120, "shortcuts").map((group) => group.labels)).toEqual([ + ["⌘", "C"], + ]); + }); + + it("holds at full opacity then fades linearly", () => { + const samples = [point(0, "enter")]; + expect(visibleKeycaps(samples, 0, "shortcuts")[0]?.opacity).toBe(1); + expect(visibleKeycaps(samples, KEYCAP_HOLD_MS, "shortcuts")[0]?.opacity).toBe(1); + expect( + visibleKeycaps(samples, KEYCAP_HOLD_MS + KEYCAP_FADE_MS / 2, "shortcuts")[0]?.opacity, + ).toBeCloseTo(0.5); + expect(visibleKeycaps(samples, KEYCAP_HOLD_MS + KEYCAP_FADE_MS, "shortcuts")).toEqual([]); + }); + + it("keeps the four newest groups", () => { + const samples = [ + point(10, "enter"), + point(20, "tab"), + point(30, "esc"), + point(40, "backspace"), + point(50, "delete"), + ]; + expect(visibleKeycaps(samples, 50, "shortcuts").map((group) => group.id)).toEqual([ + "20:tab:", + "30:esc:", + "40:backspace:", + "50:delete:", + ]); + }); + + it("keeps stable ids when a later extra event arrives", () => { + const first = [point(10, "enter"), point(20, "tab"), point(30, "esc")]; + const before = visibleKeycaps(first, 30, "shortcuts").map((group) => group.id); + const after = visibleKeycaps([...first, point(40, "delete")], 40, "shortcuts").map( + (group) => group.id, + ); + expect(before).toEqual(["10:enter:", "20:tab:", "30:esc:"]); + expect(after).toEqual(["10:enter:", "20:tab:", "30:esc:", "40:delete:"]); + }); + + it("does not merge typing into a word", () => { + const samples = [ + point(10, "h"), + point(80, "e"), + point(150, "l"), + point(220, "l"), + point(290, "o"), + ]; + expect(visibleKeycaps(samples, 290, "all").map((group) => group.labels)).toEqual([ + ["E"], + ["L"], + ["L"], + ["O"], + ]); + }); + + it("shows named punctuation in all typing and hides it in shortcuts", () => { + const samples = [point(100, "period")]; + expect(visibleKeycaps(samples, 100, "all").map((group) => group.labels)).toEqual([["."]]); + expect(visibleKeycaps(samples, 100, "shortcuts")).toEqual([]); + }); +}); diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.ts new file mode 100644 index 000000000..59b8d3cca --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps.ts @@ -0,0 +1,98 @@ +import { isCharacterKey, keystrokeLabels } from "./keystrokeLabels"; +import type { + KeycapGroup, + KeystrokeModifier, + KeystrokeOverlayMode, + KeystrokeTelemetryPoint, +} from "./keystrokeTypes"; + +export const KEYCAP_HOLD_MS = 500; +export const KEYCAP_FADE_MS = 150; +export const KEYCAP_DEDUPE_MS = 45; +export const MAX_VISIBLE_GROUPS = 4; + +const SHORTCUT_KEYS = new Set([ + "enter", + "esc", + "tab", + "backspace", + "delete", + "arrowleft", + "arrowright", + "arrowup", + "arrowdown", + "home", + "end", + "pageup", + "pagedown", +]); + +function isFunctionKey(key: string) { + return /^f([1-9]|1[0-9])$/.test(key); +} + +function hasNonShiftModifier(modifiers: readonly KeystrokeModifier[]) { + return modifiers.some( + (modifier) => modifier === "ctrl" || modifier === "meta" || modifier === "alt", + ); +} + +function isShortcutEvent(event: KeystrokeTelemetryPoint) { + if (hasNonShiftModifier(event.modifiers)) { + return true; + } + return SHORTCUT_KEYS.has(event.key) || isFunctionKey(event.key); +} + +function isVisibleInMode(event: KeystrokeTelemetryPoint, mode: KeystrokeOverlayMode) { + if (isShortcutEvent(event)) { + return true; + } + return mode === "all" && isCharacterKey(event.key); +} + +function eventSignature(key: string, modifiers: readonly KeystrokeModifier[]) { + return `${key}:${modifiers.join("+")}`; +} + +function groupOpacity(ageMs: number) { + if (ageMs <= KEYCAP_HOLD_MS) { + return 1; + } + return Math.max(0, 1 - (ageMs - KEYCAP_HOLD_MS) / KEYCAP_FADE_MS); +} + +export function visibleKeycaps( + samples: readonly KeystrokeTelemetryPoint[], + timeMs: number, + mode: KeystrokeOverlayMode, +): KeycapGroup[] { + const windowEnd = timeMs; + const windowStart = timeMs - (KEYCAP_HOLD_MS + KEYCAP_FADE_MS); + const inWindow: KeystrokeTelemetryPoint[] = []; + for (const event of samples) { + if (!isVisibleInMode(event, mode)) { + continue; + } + if (event.timeMs > windowEnd || event.timeMs <= windowStart) { + continue; + } + const signature = eventSignature(event.key, event.modifiers); + const duplicate = inWindow.some( + (kept) => + eventSignature(kept.key, kept.modifiers) === signature && + event.timeMs - kept.timeMs < KEYCAP_DEDUPE_MS, + ); + if (duplicate) { + continue; + } + inWindow.push(event); + } + + const groups = inWindow.map((event) => ({ + id: `${event.timeMs}:${event.key}:${event.modifiers.join("+")}`, + labels: keystrokeLabels(event.key, event.modifiers), + opacity: groupOpacity(timeMs - event.timeMs), + })); + return groups.slice(-MAX_VISIBLE_GROUPS); +} diff --git a/src/i18n/locales/de/settings.json b/src/i18n/locales/de/settings.json index d0837fae2..89b03a280 100644 --- a/src/i18n/locales/de/settings.json +++ b/src/i18n/locales/de/settings.json @@ -158,10 +158,23 @@ "caption": "Untertitel", "zoom": "Zoom", "cursor": "Cursor", + "keystrokes": "Tastenanschläge", "webcam": "Webcam", "frame": "Bild", "crop": "Zuschneiden" }, + "keystrokes": { + "mode": "Modus", + "modeShortcuts": "Tastenkürzel", + "modeAll": "Alle Eingaben", + "position": "Position", + "positionBottomLeft": "Unten links", + "positionBottomCenter": "Unten mittig", + "positionBottomRight": "Unten rechts", + "positionTopCenter": "Oben mittig", + "size": "Größe", + "empty": "Diese Aufnahme enthält keine Tastenanschläge. Lassen Sie Anzeigen beim Aufnehmen eingeschaltet, um sie zu erfassen." + }, "captions": { "selectOnTimeline": "Wählen Sie einen Untertitel auf der Zeitleiste aus, um ihn zu bearbeiten.", "enabled": "Anzeigen", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 596109cb3..2d7e3e049 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -164,10 +164,23 @@ "caption": "Caption", "zoom": "Zoom", "cursor": "Cursor", + "keystrokes": "Keystrokes", "webcam": "Webcam", "frame": "Frame", "crop": "Crop" }, + "keystrokes": { + "mode": "Mode", + "modeShortcuts": "Shortcuts", + "modeAll": "All typing", + "position": "Position", + "positionBottomLeft": "Bottom left", + "positionBottomCenter": "Bottom center", + "positionBottomRight": "Bottom right", + "positionTopCenter": "Top center", + "size": "Size", + "empty": "This recording has no keystrokes. Leave Show on when you record to capture them." + }, "captions": { "selectOnTimeline": "Select a caption on the timeline to edit it.", "enabled": "Show", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 3510d6bbb..676966bf4 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -158,10 +158,23 @@ "caption": "Subtítulo", "zoom": "Zoom", "cursor": "Cursor", + "keystrokes": "Pulsaciones", "webcam": "Cámara", "frame": "Marco", "crop": "Recorte" }, + "keystrokes": { + "mode": "Modo", + "modeShortcuts": "Atajos", + "modeAll": "Toda la escritura", + "position": "Posición", + "positionBottomLeft": "Inferior izquierda", + "positionBottomCenter": "Inferior centro", + "positionBottomRight": "Inferior derecha", + "positionTopCenter": "Superior centro", + "size": "Tamaño", + "empty": "Esta grabación no tiene pulsaciones. Deja Mostrar activado al grabar para capturarlas." + }, "captions": { "selectOnTimeline": "Selecciona un subtítulo en la línea de tiempo para editarlo.", "enabled": "Mostrar", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index d7e4e6753..fba427370 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -158,10 +158,23 @@ "caption": "Sous-titre", "zoom": "Zoom", "cursor": "Curseur", + "keystrokes": "Frappes", "webcam": "Webcam", "frame": "Cadre", "crop": "Recadrage" }, + "keystrokes": { + "mode": "Mode", + "modeShortcuts": "Raccourcis", + "modeAll": "Toute saisie", + "position": "Position", + "positionBottomLeft": "Bas gauche", + "positionBottomCenter": "Bas centre", + "positionBottomRight": "Bas droit", + "positionTopCenter": "Haut centre", + "size": "Taille", + "empty": "Cet enregistrement n'a pas de frappes. Laissez Afficher activé pendant l'enregistrement pour les capturer." + }, "captions": { "selectOnTimeline": "Sélectionnez un sous-titre sur la timeline pour le modifier.", "enabled": "Afficher", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 8027ac02c..800be1c79 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -158,10 +158,23 @@ "caption": "Sottotitolo", "zoom": "Zoom", "cursor": "Cursore", + "keystrokes": "Tasti", "webcam": "Webcam", "frame": "Cornice", "crop": "Ritaglio" }, + "keystrokes": { + "mode": "Modalità", + "modeShortcuts": "Scorciatoie", + "modeAll": "Tutta la digitazione", + "position": "Posizione", + "positionBottomLeft": "In basso a sinistra", + "positionBottomCenter": "In basso al centro", + "positionBottomRight": "In basso a destra", + "positionTopCenter": "In alto al centro", + "size": "Dimensione", + "empty": "Questa registrazione non ha tasti. Lascia Mostra attivo durante la registrazione per catturarli." + }, "captions": { "selectOnTimeline": "Seleziona un sottotitolo sulla timeline per modificarlo.", "enabled": "Mostra", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index b2063381b..b302bc0cd 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -158,10 +158,23 @@ "caption": "자막", "zoom": "확대", "cursor": "커서", + "keystrokes": "키 입력", "webcam": "웹캠", "frame": "프레임", "crop": "자르기" }, + "keystrokes": { + "mode": "모드", + "modeShortcuts": "단축키", + "modeAll": "모든 입력", + "position": "위치", + "positionBottomLeft": "왼쪽 아래", + "positionBottomCenter": "가운데 아래", + "positionBottomRight": "오른쪽 아래", + "positionTopCenter": "가운데 위", + "size": "크기", + "empty": "이 녹화에는 키 입력이 없습니다. 캡처하려면 녹화할 때 표시를 켜 두세요." + }, "captions": { "selectOnTimeline": "타임라인에서 자막을 선택하여 편집하세요.", "enabled": "표시", diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 4fda91805..2195f65e2 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -158,10 +158,23 @@ "caption": "Ondertitel", "zoom": "Zoom", "cursor": "Cursor", + "keystrokes": "Toetsaanslagen", "webcam": "Webcam", "frame": "Kader", "crop": "Bijsnijden" }, + "keystrokes": { + "mode": "Modus", + "modeShortcuts": "Sneltoetsen", + "modeAll": "Alle invoer", + "position": "Positie", + "positionBottomLeft": "Linksonder", + "positionBottomCenter": "Middenonder", + "positionBottomRight": "Rechtsonder", + "positionTopCenter": "Middenboven", + "size": "Grootte", + "empty": "Deze opname heeft geen toetsaanslagen. Laat Tonen aan tijdens het opnemen om ze vast te leggen." + }, "captions": { "selectOnTimeline": "Selecteer een ondertitel op de tijdlijn om deze te bewerken.", "enabled": "Tonen", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 1cb1b72ee..890e8c61f 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -158,10 +158,23 @@ "caption": "Legenda", "zoom": "Zoom", "cursor": "Cursor", + "keystrokes": "Teclas", "webcam": "Webcam", "frame": "Quadro", "crop": "Corte" }, + "keystrokes": { + "mode": "Modo", + "modeShortcuts": "Atalhos", + "modeAll": "Toda a digitação", + "position": "Posição", + "positionBottomLeft": "Inferior esquerdo", + "positionBottomCenter": "Inferior centro", + "positionBottomRight": "Inferior direito", + "positionTopCenter": "Superior centro", + "size": "Tamanho", + "empty": "Esta gravação não tem teclas. Deixe Mostrar ativado ao gravar para capturá-las." + }, "captions": { "selectOnTimeline": "Selecione uma legenda na linha do tempo para editá-la.", "enabled": "Mostrar", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 7ec21ff1c..99c278e1a 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -158,10 +158,23 @@ "caption": "Субтитр", "zoom": "Зум", "cursor": "Курсор", + "keystrokes": "Клавиши", "webcam": "Веб-камера", "frame": "Рамка", "crop": "Обрезать видео" }, + "keystrokes": { + "mode": "Режим", + "modeShortcuts": "Сочетания", + "modeAll": "Весь ввод", + "position": "Положение", + "positionBottomLeft": "Слева внизу", + "positionBottomCenter": "По центру внизу", + "positionBottomRight": "Справа внизу", + "positionTopCenter": "По центру вверху", + "size": "Размер", + "empty": "В этой записи нет нажатий клавиш. Оставьте «Показать» включённым при записи, чтобы сохранить их." + }, "captions": { "selectOnTimeline": "Выберите субтитры на таймлайне, чтобы отредактировать их.", "enabled": "Показать", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 053d10951..ff9278308 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -158,10 +158,23 @@ "caption": "字幕编辑", "zoom": "缩放", "cursor": "光标", + "keystrokes": "按键", "webcam": "摄像头", "frame": "边框", "crop": "裁剪" }, + "keystrokes": { + "mode": "模式", + "modeShortcuts": "快捷键", + "modeAll": "全部输入", + "position": "位置", + "positionBottomLeft": "左下", + "positionBottomCenter": "底部居中", + "positionBottomRight": "右下", + "positionTopCenter": "顶部居中", + "size": "大小", + "empty": "此录制没有按键。录制时请保持显示开启以捕获按键。" + }, "captions": { "selectOnTimeline": "在时间轴上选择字幕进行编辑。", "enabled": "显示", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 7de885204..4dc1ace54 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -158,10 +158,23 @@ "caption": "字幕編輯", "zoom": "縮放", "cursor": "游標", + "keystrokes": "按鍵", "webcam": "網路攝影機", "frame": "外框", "crop": "裁切" }, + "keystrokes": { + "mode": "模式", + "modeShortcuts": "快捷鍵", + "modeAll": "全部輸入", + "position": "位置", + "positionBottomLeft": "左下", + "positionBottomCenter": "底部置中", + "positionBottomRight": "右下", + "positionTopCenter": "頂部置中", + "size": "大小", + "empty": "此錄製沒有按鍵。錄製時請保持顯示開啟以擷取按鍵。" + }, "captions": { "selectOnTimeline": "在時間軸上選擇字幕進行編輯。", "enabled": "顯示", diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 640bd39aa..ccad1e9e7 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -31,6 +31,10 @@ import { PixiCursorOverlay, preloadCursorAssets, } from "@/components/video-editor/videoPlayback/cursorRenderer"; +import type { + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; import { computePaddedLayout, scalePreviewBorderRadius, @@ -74,9 +78,9 @@ import { isVideoWallpaperSource } from "@/lib/wallpapers"; import { renderAnnotations } from "./annotationRenderer"; import { renderCaptions } from "./captionRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; +import { renderKeystrokes } from "./keystrokeRenderer"; import { resolveMediaElementSource } from "./localMediaSource"; - interface FrameRenderConfig { timelineEffects?: boolean; width: number; @@ -108,6 +112,8 @@ interface FrameRenderConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; @@ -1528,6 +1534,17 @@ export class FrameRenderer { timestamp / 1000, ); } + + if (this.config.keystrokeOverlay && this.compositeCtx) { + renderKeystrokes( + this.compositeCtx, + this.config.keystrokeTelemetry ?? [], + this.config.keystrokeOverlay, + this.config.width, + this.config.height, + cursorTimeMs, + ); + } } private updateLayout(): void { diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 2df730000..649edf326 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -3,8 +3,8 @@ import type { AnnotationRegion, AutoCaptionSettings, CaptionCue, - CursorClickEffectStyle, CropRegion, + CursorClickEffectStyle, CursorStyle, CursorTelemetryPoint, Padding, @@ -15,6 +15,10 @@ import type { ZoomRegion, ZoomTransitionEasing, } from "@/components/video-editor/types"; +import type { + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; import { FrameRenderer } from "./frameRenderer"; import { StreamingVideoDecoder } from "./streamingDecoder"; import type { @@ -64,6 +68,8 @@ interface GifExporterConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -165,6 +171,8 @@ export function buildGifFrameRendererConfig( annotationRegions: config.annotationRegions, autoCaptions: config.autoCaptions, autoCaptionSettings: config.autoCaptionSettings, + keystrokeTelemetry: config.keystrokeTelemetry, + keystrokeOverlay: config.keystrokeOverlay, speedRegions: config.speedRegions, previewWidth: config.previewWidth, previewHeight: config.previewHeight, diff --git a/src/lib/exporter/keystrokeRenderer.test.ts b/src/lib/exporter/keystrokeRenderer.test.ts new file mode 100644 index 000000000..a3cd65861 --- /dev/null +++ b/src/lib/exporter/keystrokeRenderer.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_KEYSTROKE_OVERLAY } from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; +import { renderKeystrokes } from "./keystrokeRenderer"; + +function createContext() { + return { + save: vi.fn(), + restore: vi.fn(), + fill: vi.fn(), + fillText: vi.fn(), + beginPath: vi.fn(), + moveTo: vi.fn(), + lineTo: vi.fn(), + closePath: vi.fn(), + measureText: vi.fn((text: string) => ({ width: String(text).length * 10 })), + font: "", + fillStyle: "", + textAlign: "left" as const, + textBaseline: "alphabetic" as const, + globalAlpha: 1, + }; +} + +describe("renderKeystrokes", () => { + it("does not draw when the overlay is disabled", () => { + const ctx = createContext(); + renderKeystrokes( + ctx as unknown as CanvasRenderingContext2D, + [{ timeMs: 0, key: "c", modifiers: ["meta"] }], + { ...DEFAULT_KEYSTROKE_OVERLAY, enabled: false }, + 960, + 540, + 0, + ); + expect(ctx.fillText).not.toHaveBeenCalled(); + }); + + it("does not draw when there are no samples", () => { + const ctx = createContext(); + renderKeystrokes( + ctx as unknown as CanvasRenderingContext2D, + [], + { ...DEFAULT_KEYSTROKE_OVERLAY, enabled: true }, + 960, + 540, + 0, + ); + expect(ctx.fillText).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/exporter/keystrokeRenderer.ts b/src/lib/exporter/keystrokeRenderer.ts new file mode 100644 index 000000000..85321bd37 --- /dev/null +++ b/src/lib/exporter/keystrokeRenderer.ts @@ -0,0 +1,143 @@ +import { + CAPTION_FONT_WEIGHT, + getCaptionPadding, + getCaptionScaledFontSize, + getCaptionScaledRadius, +} from "@/components/video-editor/captionStyle"; +import { getDefaultCaptionFontFamily } from "@/components/video-editor/types"; +import type { + KeycapGroup, + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; +import { visibleKeycaps } from "@/components/video-editor/videoPlayback/keystrokeOverlay/visibleKeycaps"; +import { drawSquircleOnCanvas } from "@/lib/geometry/squircle"; + +type MeasuredPill = { + label: string; + width: number; +}; + +type MeasuredGroup = { + group: KeycapGroup; + pills: MeasuredPill[]; + width: number; +}; + +function measureGroups( + ctx: CanvasRenderingContext2D, + groups: readonly KeycapGroup[], + paddingX: number, + pillGap: number, +): MeasuredGroup[] { + return groups.map((group) => { + const pills = group.labels.map((label) => ({ + label, + width: ctx.measureText(label).width + paddingX * 2, + })); + const width = pills.reduce( + (total, pill, index) => total + pill.width + (index > 0 ? pillGap : 0), + 0, + ); + return { group, pills, width }; + }); +} + +function drawGroupRow( + ctx: CanvasRenderingContext2D, + row: MeasuredGroup, + originX: number, + originY: number, + pillHeight: number, + paddingX: number, + radius: number, + pillGap: number, +) { + ctx.save(); + ctx.globalAlpha *= row.group.opacity; + let x = originX; + for (const pill of row.pills) { + ctx.fillStyle = "rgba(0,0,0,0.9)"; + drawSquircleOnCanvas(ctx, { + x, + y: originY, + width: pill.width, + height: pillHeight, + radius, + }); + ctx.fill(); + ctx.fillStyle = "#fff"; + ctx.fillText(pill.label, x + paddingX, originY + pillHeight / 2); + x += pill.width + pillGap; + } + ctx.restore(); +} + +export function renderKeystrokes( + ctx: CanvasRenderingContext2D, + samples: KeystrokeTelemetryPoint[], + settings: KeystrokeOverlaySettings, + width: number, + height: number, + timeMs: number, +): void { + if (!settings.enabled || samples.length === 0) { + return; + } + + const groups = visibleKeycaps(samples, timeMs, settings.mode); + if (groups.length === 0) { + return; + } + + const fontSize = Math.min( + 64, + Math.max(10, getCaptionScaledFontSize(30 * settings.size, width, 62)), + ); + const padding = getCaptionPadding(fontSize); + const radius = getCaptionScaledRadius(17.5, fontSize); + const margin = fontSize * 1.1; + const groupGap = fontSize * 0.35; + const pillGap = fontSize * 0.22; + const pillHeight = fontSize + padding.y * 2; + + ctx.save(); + ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; + ctx.textAlign = "left"; + ctx.textBaseline = "middle"; + + const newestFirst = measureGroups(ctx, groups, padding.x, pillGap).slice().reverse(); + const isTop = settings.position === "top-center"; + const isRight = settings.position === "bottom-right"; + const isLeft = settings.position === "bottom-left"; + + if (isTop) { + let y = margin; + for (const row of newestFirst) { + drawGroupRow( + ctx, + row, + (width - row.width) / 2, + y, + pillHeight, + padding.x, + radius, + pillGap, + ); + y += pillHeight + groupGap; + } + } else { + let y = height - margin; + for (const row of newestFirst) { + const x = isLeft + ? margin + : isRight + ? width - margin - row.width + : (width - row.width) / 2; + drawGroupRow(ctx, row, x, y - pillHeight, pillHeight, padding.x, radius, pillGap); + y -= pillHeight + groupGap; + } + } + + ctx.restore(); +} diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index 79c2caa13..737028cb8 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -312,7 +312,7 @@ describe("ModernFrameRenderer blur export path", () => { renderer.annotationScaleFactor = 1; renderer.annotationAssets = { imageCache: new Map() }; - await renderer.composeBlurAnnotationFrame(500); + await renderer.composeBlurAnnotationFrame(500, 500); expect(renderAnnotations).toHaveBeenCalledTimes(1); expect(renderer.getCanvas()).not.toBe(sourceCanvas); diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 629f25210..0f2628a34 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -40,6 +40,10 @@ import { PixiCursorOverlay, preloadCursorAssets, } from "@/components/video-editor/videoPlayback/cursorRenderer"; +import type { + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; import { computePaddedLayout, scalePreviewBorderRadius, @@ -87,6 +91,7 @@ import { renderAnnotationToCanvas, } from "./annotationRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; +import { renderKeystrokes } from "./keystrokeRenderer"; import { resolveMediaElementSource } from "./localMediaSource"; import { getShadowFilterPadding, @@ -127,6 +132,8 @@ interface FrameRenderConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; @@ -1477,7 +1484,28 @@ export class FrameRenderer { context.restore(); } - private async composeBlurAnnotationFrame(timeMs: number): Promise { + private paintKeystrokeOverlay(context: CanvasRenderingContext2D, timeMs: number) { + if (!this.config.keystrokeOverlay) { + return; + } + renderKeystrokes( + context, + this.config.keystrokeTelemetry ?? [], + this.config.keystrokeOverlay, + this.config.width, + this.config.height, + timeMs, + ); + } + + private hasKeystrokeOverlay() { + return Boolean( + this.config.keystrokeOverlay?.enabled && + (this.config.keystrokeTelemetry?.length ?? 0) > 0, + ); + } + + private async composeBlurAnnotationFrame(timeMs: number, cursorTimeMs: number): Promise { if (!this.app) { this.outputCanvasOverride = null; return; @@ -1510,6 +1538,7 @@ export class FrameRenderer { ); this.drawCaptionOverlay(context); + this.paintKeystrokeOverlay(context, cursorTimeMs); this.outputCanvasOverride = canvas; } @@ -2954,10 +2983,10 @@ export class FrameRenderer { this.updateAnnotationLayer(timeMs); this.updateCaptionLayer(timestamp / 1000); this.updateWebcamOverlay(); - await this.renderOutput(timeMs); + await this.renderOutput(timeMs, cursorTimeMs); } - private async renderOutput(timeMs: number): Promise { + private async renderOutput(timeMs: number, cursorTimeMs: number): Promise { if (this.hasActiveBlurAnnotations(timeMs)) { const annotationContainerVisible = this.annotationContainer?.visible ?? true; const captionContainerVisible = this.captionContainer?.visible ?? true; @@ -2978,12 +3007,27 @@ export class FrameRenderer { this.captionContainer.visible = captionContainerVisible; } - await this.composeBlurAnnotationFrame(timeMs); + await this.composeBlurAnnotationFrame(timeMs, cursorTimeMs); return; } - this.outputCanvasOverride = null; this.app!.render(); + if (!this.hasKeystrokeOverlay()) { + this.outputCanvasOverride = null; + return; + } + + const compositeState = this.ensureExportCompositeCanvas(); + if (!compositeState) { + this.outputCanvasOverride = null; + return; + } + + const { canvas, context } = compositeState; + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(this.app!.canvas as HTMLCanvasElement, 0, 0); + this.paintKeystrokeOverlay(context, cursorTimeMs); + this.outputCanvasOverride = canvas; } private updateLayout(): void { diff --git a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts index b06d0f80a..884bd02a0 100644 --- a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts +++ b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts @@ -428,6 +428,38 @@ describe("ModernVideoExporter native static-layout eligibility", () => { ]); }); + it("skips native static layout when the keystroke overlay is enabled with samples", () => { + const exporter = createExporter({ + keystrokeOverlay: { + enabled: true, + mode: "shortcuts", + position: "bottom-left", + size: 1, + }, + keystrokeTelemetry: [{ timeMs: 0, key: "c", modifiers: ["meta"] }], + }); + + expect( + exporter.getNativeStaticLayoutSkipReasons({ audioMode: "none" }, videoInfo, 60), + ).toContain("unsupported-keystroke-overlay"); + }); + + it("does not skip native static layout for an enabled overlay with no samples", () => { + const exporter = createExporter({ + keystrokeOverlay: { + enabled: true, + mode: "shortcuts", + position: "bottom-left", + size: 1, + }, + keystrokeTelemetry: [], + }); + + expect( + exporter.getNativeStaticLayoutSkipReasons({ audioMode: "none" }, videoInfo, 60), + ).not.toContain("unsupported-keystroke-overlay"); + }); + it("reports invalid crop geometry instead of passing native export bad coordinates", () => { const exporter = createExporter({ cropRegion: { x: 0, y: 0, width: 0, height: 1 }, diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 81137b267..f987100d4 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -21,6 +21,10 @@ import { DEFAULT_WEBCAM_ROUNDNESS } from "@/components/video-editor/types"; import { createCursorFollowCameraState } from "@/components/video-editor/videoPlayback/cursorFollowCamera"; import { buildNativeCursorAtlas } from "@/components/video-editor/videoPlayback/cursorRenderer"; import { getCursorViewportScale } from "@/components/video-editor/videoPlayback/cursorScale"; +import type { + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; import { computePaddedLayout, scalePreviewBorderRadius, @@ -122,6 +126,8 @@ interface VideoExporterConfig extends ExportConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -631,6 +637,8 @@ export class ModernVideoExporter { annotationRegions: this.config.annotationRegions, autoCaptions: this.config.autoCaptions, autoCaptionSettings: this.config.autoCaptionSettings, + keystrokeTelemetry: this.config.keystrokeTelemetry, + keystrokeOverlay: this.config.keystrokeOverlay, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, @@ -1746,6 +1754,12 @@ export class ModernVideoExporter { if ((this.config.autoCaptions ?? []).length > 0) { reasons.push("unsupported-caption-overlay"); } + if ( + this.config.keystrokeOverlay?.enabled && + (this.config.keystrokeTelemetry?.length ?? 0) > 0 + ) { + reasons.push("unsupported-keystroke-overlay"); + } if (this.config.webcam?.enabled) { // Native GPU compositors use a different corner and shadow model. // Keep webcam exports on the shared renderer used by preview. diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index d58642c72..ce6b0591b 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -1,4 +1,3 @@ -import { requiresClipTimelineRendering } from "./clipTimeline"; import type { AnnotationRegion, AudioRegion, @@ -17,8 +16,13 @@ import type { ZoomRegion, ZoomTransitionEasing, } from "@/components/video-editor/types"; +import type { + KeystrokeOverlaySettings, + KeystrokeTelemetryPoint, +} from "@/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes"; import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; +import { requiresClipTimelineRendering } from "./clipTimeline"; import { buildEditedTrackSourceSegments, classifyEditedTrackStrategy } from "./editedTrackStrategy"; import { advanceFinalizationProgress, @@ -73,6 +77,8 @@ interface VideoExporterConfig extends ExportConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeTelemetryPoint[]; + keystrokeOverlay?: KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -244,6 +250,8 @@ export class VideoExporter { annotationRegions: this.config.annotationRegions, autoCaptions: this.config.autoCaptions, autoCaptionSettings: this.config.autoCaptionSettings, + keystrokeTelemetry: this.config.keystrokeTelemetry, + keystrokeOverlay: this.config.keystrokeOverlay, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight,