Startup probe and image delivery: end on DA1, never type a late reply, resend undelivered uploads

CHANGELOG.md

@@ -29,6 +29,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Fixed
 
+- On a slow terminal connection or a busy machine, the terminal's reply to NMSh's startup image check no longer appears in the composer as `_Gi=31;OK\` and no longer turns images into character art. NMSh's startup check now ends as soon as the terminal has answered (instead of always waiting 80 ms), waits up to 0.75 s for a slow one, and still uses an answer that comes later.
+- A Kitty or Ghostty image drawn before NMSh had taken the screen (a welcome image that loaded during startup) is sent again instead of leaving blank space where the picture belongs.
 - A native welcome image (Kitty, Ghostty) stays a native image after the first key press, scroll or resize instead of turning into character art. A logo animation still stops at once and its frames are deleted; only the still remains.
 - With the Portable cursor renderer, NMSh's Kitty cursor setup no longer leaves Kitty's own `cursor_trail` on, which drew two trails.
 

src/app/TerminalApp.ts

@@ -29,7 +29,7 @@ import {dividerAnimated, MIN_CUSTOM_STOPS, TREATMENT_MOTION_LABELS, treatmentFor
 import {colorLevel} from '../presentation/capabilities.js';
 import type {TerminalFrame} from '../terminal/TerminalRenderer.js';
 import {detectTerminalHost} from '../host/terminalHost.js';
-import {probeHost} from '../host/probe.js';
+import {LATE_PROBE_WINDOW_MS, probeHost, takeLateProbeReplies} from '../host/probe.js';
 import {SessionPresetStore, PresetStartup, presetNeedsAcknowledgement, type SessionPreset} from '../session/SessionPresets.js';
 import {createPresetPanel, presetPanelKey, renderPresetPanel, type PresetPanel} from '../session/PresetPanel.js';
 import {MiseProjectService, detectMiseProject} from '../tools/MiseProject.js';
@@ -1518,6 +1518,8 @@ export class TerminalApp {
       this.host.capabilities = resolved.capabilities;
       this.renderer.setCapabilities(resolved.capabilities);
       earlyInput = resolved.input;
+      // A slow terminal can still answer: its late replies are taken out of the input (never typed) and used as evidence.
+      if (resolved.pending) this.lateProbeUntil = Date.now() + LATE_PROBE_WINDOW_MS;
     }
     this.journal = new SessionJournal(this.transcriptStore, this.promptConfiguration.sessionRetention,
       () => ({startCwd: this.presentationStartCwd, finalCwd: this.shellCwd, transcript: this.output.transcript(), live: this.liveLink()}),
@@ -1596,7 +1598,23 @@ export class TerminalApp {
     appendFileSync(join(tmpdir(), 'nmsh-key-debug.log'), `RAW hex=${hex} escaped=${JSON.stringify(data)}\n`);
   }
 
+  /** Until then, replies to the startup probe that arrive late are recognized (see `takeLateProbe`). */
+  private lateProbeUntil = 0;
+
+  private takeLateProbe(data: string): string {
+    if (Date.now() > this.lateProbeUntil) { this.lateProbeUntil = 0; return data; }
+    const late = takeLateProbeReplies(data, this.host.capabilities);
+    if (late.done) this.lateProbeUntil = 0;
+    if (late.capabilities) {
+      // Images read the live host capabilities; the renderer's own copy only chose input modes at entry and stays as is.
+      this.host.capabilities = late.capabilities;
+      if (!this.stopped) setImmediate(() => { if (!this.stopped) this.render(); });
+    }
+    return late.input;
+  }
+
   private readonly onInput = (data: string): void => {
+    if (this.lateProbeUntil) { data = this.takeLateProbe(data); if (!data) return; }
     // Before probes classify a read, or inside a native reader, input can already be sensitive.
     // Key diagnostics therefore observe only the idle composer, never startup or running-program input, nor typing held
     // at the prompt because it may be a reply to hidden input (#426).

src/graphics/service.ts

@@ -31,7 +31,8 @@ import {renderCells} from './unicode.js';
 export interface GraphicsHost {
   decision(): GraphicsDecision;
   /** Raw terminal bytes (an upload or a delete). Nothing may be written while a program owns the screen. */
-  send(data: string): void;
+  /** False when the bytes could not be written (the screen is not NMSh's yet): an upload is then not recorded. */
+  send(data: string): void | boolean;
   /** A private directory for converters (WebP). */
   scratchDir(): string;
   color(): ColorLevel;
@@ -248,10 +249,10 @@ export class GraphicsService {
       const png = encodePng(scaleBox(source, target.width, target.height), {level: 3});
       const id = this.allocateId();
       this.makeRoom(png.length);
-      this.host.send(`${kittyTransmit(png, id)}${kittyVirtualPlacement(id, box.columns, box.rows)}`);
+      const delivered = this.host.send(`${kittyTransmit(png, id)}${kittyVirtualPlacement(id, box.columns, box.rows)}`) !== false;
       upload = {id, bytes: png.length, owners: new Set(), at: 0};
-      this.uploads.set(key, upload);
-      this.uploadSize += png.length;
+      // An upload the terminal never received is not remembered: the next paint sends it again instead of empty cells.
+      if (delivered) { this.uploads.set(key, upload); this.uploadSize += png.length; }
     }
     upload.owners.add(request.owner);
     upload.at = ++this.clock;

src/host/probe.ts

@@ -1,6 +1,15 @@
 import type {TerminalCapabilities} from './capabilities.js';
 
 export const HOST_PROBE_TIMEOUT_MS = 80;
+/**
+ * The extended batch ends with DA1, which every VT-style terminal answers, and answers in order: its reply means every
+ * earlier reply has arrived, so the probe ends at once. This is only the wait for a terminal that is slow to answer (SSH,
+ * a busy machine); replies later than this are still recognized (see `takeLateProbeReplies`) and never become typing.
+ */
+export const HOST_PROBE_REPLY_TIMEOUT_MS = 750;
+/** How long after startup a reply to the startup probe is still recognized as one. */
+export const LATE_PROBE_WINDOW_MS = 10_000;
+const DA1_REPLY = /\u001b\[\?[\d;]{1,64}c/u;
 export const HOST_QUERY = '\u001b[?u\u001b[?2026$p';
 /**
  * Kitty graphics support query: a 1×1 RGB direct-data query (a=q) that the
@@ -16,8 +25,8 @@ export const GRAPHICS_QUERY = '\u001b_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\u001b\\';
  * alternate screen and leaves it, so nothing the person sees is touched, and it is sent only when asked for.
  */
 export const TEXT_SIZING_QUERY = '\u001b[?1049h\u001b[2J\u001b[H\u001b[6n\u001b]66;w=2; \u001b\\\u001b[6n\u001b]66;s=2; \u001b\\\u001b[6n\u001b[?1049l';
-/** DA1 (device attributes: `4` is Sixel), the cell size in pixels and the text area in pixels. */
-export const DISPLAY_QUERY = '\u001b[c\u001b[16t\u001b[14t';
+/** The cell size in pixels, the text area in pixels and DA1 (device attributes: `4` is Sixel), last as the end marker. */
+export const DISPLAY_QUERY = '\u001b[16t\u001b[14t\u001b[c';
 
 export interface ProbeOptions {
   graphics?: boolean;
@@ -84,9 +93,18 @@ export interface ProbeTransport {
   listen(receive: (data: string) => void): () => void;
 }
 
-/** One bounded parallel query batch per attachment; caller owns raw mode. */
+/** Whether the DA1 end marker is in `input` (outside a bracketed paste). */
+function sawEndMarker(input: string): boolean {
+  return DA1_REPLY.test(input.replace(/\u001b\[200~[\s\S]*?(?:\u001b\[201~|$)/gu, ''));
+}
+
+/**
+ * One bounded parallel query batch per attachment; caller owns raw mode. The extended batch ends as soon as the DA1 end
+ * marker arrives; `pending` says it did not arrive in time, so later replies may still come (see `takeLateProbeReplies`).
+ */
 export function probeHost(hints: Readonly<TerminalCapabilities>, transport: ProbeTransport,
-  timeoutMs = HOST_PROBE_TIMEOUT_MS, options: ProbeOptions = {}): Promise<ReturnType<typeof resolveProbeReplies>> {
+  timeoutMs?: number, options: ProbeOptions = {}): Promise<ReturnType<typeof resolveProbeReplies> & {pending: boolean}> {
+  const limit = options.extended ? HOST_PROBE_REPLY_TIMEOUT_MS : HOST_PROBE_TIMEOUT_MS;
   return new Promise(resolve => {
     let input = '';
     let finished = false;
@@ -96,14 +114,15 @@ export function probeHost(hints: Readonly<TerminalCapabilities>, transport: Prob
       finished = true;
       clearTimeout(timer);
       remove();
-      resolve(resolveProbeReplies(input, hints, options));
+      resolve({...resolveProbeReplies(input, hints, options), pending: Boolean(options.extended) && !sawEndMarker(input)});
     };
-    const timer = setTimeout(finish, Math.max(0, Math.min(HOST_PROBE_TIMEOUT_MS, timeoutMs)));
+    // A timer that fires late (a blocked event loop) runs before that turn's reads: let input already waiting arrive first.
+    const timer = setTimeout(() => setImmediate(finish), Math.max(0, Math.min(limit, timeoutMs ?? limit)));
     try {
       remove = transport.listen(data => {
         input += data;
-        // Stop collecting promptly for large paste/input bursts.
-        if (input.length >= 65536) finish();
+        // Stop collecting promptly for large paste/input bursts, and as soon as the end marker answers.
+        if (input.length >= 65536 || (options.extended && sawEndMarker(input))) finish();
       });
       // A transport can deliver buffered input synchronously from listen().
       if (finished) { remove(); return; }
@@ -112,3 +131,21 @@ export function probeHost(hints: Readonly<TerminalCapabilities>, transport: Prob
     } catch { finish(); }
   });
 }
+
+/** The Kitty graphics reply to the startup query (image id 31), whole. */
+const LATE_GRAPHICS_REPLY = /\u001b_Gi=31;([^\u001b]{0,128})\u001b\\/gu;
+
+/**
+ * Replies to the startup probe that arrive after it gave up: removed from `input` so they never become typing, with the
+ * evidence they carry. `done` means the DA1 end marker arrived (nothing more is outstanding).
+ */
+export function takeLateProbeReplies(input: string, capabilities: Readonly<TerminalCapabilities>): {input: string; capabilities?: TerminalCapabilities; done: boolean} {
+  let next: TerminalCapabilities | undefined;
+  const done = sawEndMarker(input);
+  const rest = input.replace(LATE_GRAPHICS_REPLY, (_match, message: string) => {
+    if (message === 'OK' && capabilities.graphicsProtocol !== 'kitty') next = {...(next ?? capabilities), graphicsProtocol: 'kitty'};
+    return '';
+  // DA1 and the pixel-size reports are never keys; removed, they cannot reach a program that has the screen by now.
+  }).replace(/\u001b\[\?[\d;]{1,64}c|\u001b\[[46];\d{1,5};\d{1,5}t/gu, '');
+  return {input: rest, ...(next ? {capabilities: next} : {}), done};
+}

src/terminal/TerminalRenderer.ts

@@ -179,8 +179,11 @@ export class TerminalRenderer {
   }
 
   /** Raw bytes for an owner of terminal-side objects (the graphics service uploading an image). Nothing while a program owns the screen. */
-  sendRaw(data: string): void {
-    if (this.active && !this.suspended && data) this.write(data);
+  /** Writes `data` now; false when nothing was written (not entered yet, or suspended), so callers can send it again later. */
+  sendRaw(data: string): boolean {
+    if (!this.active || this.suspended || !data) return false;
+    this.write(data);
+    return true;
   }
 
   /** The rows last written to the screen, for presentation-only captures (screen savers). Never sent anywhere. */

src/terminal/keys.ts

@@ -120,6 +120,9 @@ const SEQUENCES: Array<[string, Key['kind']]> = [
   ['\u001B[79;5u', 'toggleDetails'], // Kitty Ctrl+O (uppercase O)
 ];
 
+/** The start of a Kitty graphics reply split across reads (short: replies are), held until the rest arrives. */
+const HELD_GRAPHICS_REPLY = /^\u001B_G[^\u001B]{0,64}$/u;
+
 export function decodeKeys(input: string): Key[] {
   const keys: Key[] = [];
   let index = 0;
@@ -160,6 +163,12 @@ export function decodeKeys(input: string): Key[] {
       index += sequence[0].length;
       continue;
     }
+    // A Kitty graphics reply (APC G ... ST) is the terminal answering, never a key: Escape plus its text would be typed.
+    const apcMatch = /^\u001B_G[^\u001B]{0,4096}\u001B\\/u.exec(input.slice(index));
+    if (apcMatch) {
+      index += apcMatch[0].length;
+      continue;
+    }
     const csiMatch = /^\u001B\[[0-?]*[ -/]*[@-~]/.exec(input.slice(index));
     if (csiMatch) {
       // Unknown CSI sequence, consume and ignore
@@ -215,7 +224,8 @@ export class KeyDecoder {
    * delivered only with the next keystroke (ESC then → became Escape + Right).
    */
   get pendingEscape(): boolean {
-    return this.pasteBuffer === undefined && this.keyBuffer === '\u001B';
+    // A held start of a graphics reply is flushed the same way, so it can never swallow the keys typed after it.
+    return this.pasteBuffer === undefined && (this.keyBuffer === '\u001B' || HELD_GRAPHICS_REPLY.test(this.keyBuffer));
   }
 
   /** Decodes whatever is held as complete keys (a held lone ESC becomes Escape). */
@@ -250,7 +260,7 @@ export class KeyDecoder {
           const suffix = remaining.slice(escape);
           const known = [...SEQUENCES.map(([sequence]) => sequence), '\u001B[200~'];
           // Held only while it can still become a key; an endless parameter run is not held (bounded memory).
-          if (suffix.length <= 256 && ((known.some(sequence => sequence.startsWith(suffix)) && !known.includes(suffix)) || /^\u001B\[[0-?]*[ -/]*$/.test(suffix) || /^\u001BO$/.test(suffix))) {
+          if (suffix.length <= 256 && ((known.some(sequence => sequence.startsWith(suffix)) && !known.includes(suffix)) || /^\u001B\[[0-?]*[ -/]*$/.test(suffix) || /^\u001BO$/.test(suffix) || HELD_GRAPHICS_REPLY.test(suffix))) {
             this.keyBuffer = suffix;
             remaining = remaining.slice(0, escape);
           }

tests/graphicsService.test.ts

@@ -220,3 +220,18 @@ test('cropRaster clamps and copies exactly the region', () => {
   assert.ok(clamped.width >= 1 && clamped.height >= 1 && clamped.width <= 4);
   void BASELINE_CAPABILITIES;
 });
+
+test('an upload the terminal never received is sent again by the next paint', () => {
+  const sent: string[] = []; let open = false;
+  const decision: GraphicsDecision = {backend: 'kitty', evidence: 'query', inline: 'placeholder', unicode: 'half', cell: {width: 8, height: 16, source: 'estimated'}, reason: 'Kitty.'};
+  const service = new GraphicsService({decision: () => decision, send: data => { if (!open) return false; sent.push(data); return true; }, scratchDir: () => '.', color: () => 'truecolor', safeGlyphs: () => false});
+  const raster = {width: 16, height: 16, data: new Uint8ClampedArray(16 * 16 * 4).fill(200)};
+  const image = {key: 'undelivered', name: 'x', path: '/x.png', info: {format: 'png' as const, width: 16, height: 16, animated: false, alpha: true, bytes: 1}, raster, fileBytes: 1};
+  const box = {columns: 4, rows: 2, owner: 'o', surface: 'inline' as const};
+  service.place(image, box);
+  assert.equal(service.stats().uploads, 0, 'not remembered while the screen is not ours');
+  open = true;
+  service.place(image, box);
+  assert.equal(service.stats().uploads, 1);
+  assert.equal(sent.filter(d => d.includes('a=T') || d.includes('a=t')).length, 1, 'transmitted once the screen is ours');
+});

tests/hostProbeExtended.test.ts

@@ -1,7 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import {BASELINE_CAPABILITIES} from '../src/host/capabilities.js';
-import {DISPLAY_QUERY, HOST_QUERY, TEXT_SIZING_QUERY, probeHost, resolveProbeReplies} from '../src/host/probe.js';
+import {DISPLAY_QUERY, HOST_QUERY, TEXT_SIZING_QUERY, probeHost, resolveProbeReplies, takeLateProbeReplies} from '../src/host/probe.js';
 
 /** Replies as a terminal would send them, in the order the queries were written. */
 const cpr = (row: number, column: number) => `\u001b[${row};${column}R`;
@@ -72,3 +72,47 @@ test('a real exchange: replies arriving through the transport resolve the capabi
   assert.deepEqual(result.capabilities.cellPixels, {width: 10, height: 20});
   assert.equal(result.input, '');
 });
+
+test('the extended probe ends as soon as DA1, its last query, answers', async () => {
+  assert.ok(DISPLAY_QUERY.endsWith('\u001b[c'), 'DA1 is the end marker');
+  const started = Date.now();
+  let listener: (data: string) => void = () => {};
+  const transport = {
+    listen: (receive: (data: string) => void) => { listener = receive; return () => {}; },
+    write: () => { setImmediate(() => listener(`\u001b_Gi=31;OK\u001b\\\u001b[?1u\u001b[?2026;2$y\u001b[6;20;10t\u001b[?62;4c`)); },
+  };
+  const result = await probeHost({...BASELINE_CAPABILITIES}, transport, undefined, {extended: true, graphics: true});
+  assert.ok(Date.now() - started < 60, 'no fixed wait once the marker arrived');
+  assert.equal(result.pending, false);
+  assert.equal(result.input, '');
+  assert.equal(result.capabilities.graphicsProtocol, 'kitty');
+});
+
+test('replies later than the probe never become typing and still count as evidence', async () => {
+  const transport = {listen: () => () => {}, write: () => {}};
+  const result = await probeHost({...BASELINE_CAPABILITIES}, transport, 5, {extended: true, graphics: true});
+  assert.equal(result.pending, true, 'the end marker did not arrive');
+  assert.equal(result.capabilities.graphicsProtocol, 'none');
+  const late = takeLateProbeReplies(`ls\u001b_Gi=31;OK\u001b\\\u001b[?62;4c`, result.capabilities);
+  assert.equal(late.capabilities?.graphicsProtocol, 'kitty', 'a late OK still turns native images on');
+  assert.equal(late.done, true);
+  assert.doesNotMatch(late.input, /Gi=31|OK/, 'the reply is not typed');
+  assert.equal(late.input, 'ls', 'keys around it are kept; DA1 is removed too');
+  const error = takeLateProbeReplies(`\u001b_Gi=31;ENOTSUPPORTED:x\u001b\\`, {...BASELINE_CAPABILITIES, graphicsProtocol: 'kitty'});
+  assert.equal(error.input, ''); assert.equal(error.capabilities, undefined); assert.equal(error.done, false);
+});
+
+test('a late graphics reply after the screen is entered enables native images without restarting the renderer', async () => {
+  const {TerminalApp} = await import('../src/app/TerminalApp.js');
+  const app = new TerminalApp();
+  try {
+    Object.defineProperty(app, 'render', {value: () => {}});
+    app['host'].capabilities = {...app['host'].capabilities, graphicsProtocol: 'none'};
+    app['renderer']['active'] = true;
+    app['lateProbeUntil'] = Date.now() + 1000;
+    assert.doesNotThrow(() => app['onInput']('\u001b_Gi=31;OK\u001b\\\u001b[?62;4c'));
+    assert.equal(app['host'].capabilities.graphicsProtocol, 'kitty');
+    assert.equal(app['lateProbeUntil'], 0, 'DA1 closed the late window');
+    app['onInput']('ls'); assert.equal(app['editor'].text, 'ls', 'only the real keys were typed');
+  } finally { app['renderer']['active'] = false; app['stop'](0); app['session'].kill(); }
+});

tests/keys.test.ts

@@ -44,3 +44,18 @@ test('existing Kitty Ctrl mappings are unaffected by the new Escape entries', ()
   assert.deepEqual(decodeKeys('\u001B[99;5u'), [{kind: 'interrupt'}], 'Kitty Ctrl+C');
   assert.deepEqual(decodeKeys('\u001B[127u'), [{kind: 'backspace'}], 'Kitty Backspace (no modifier)');
 });
+
+test('a Kitty graphics reply is never decoded as Escape plus typed text, even split across reads', () => {
+  assert.deepEqual(decodeKeys('a\u001b_Gi=31;OK\u001b\\b'), [{kind: 'text', value: 'a'}, {kind: 'text', value: 'b'}]);
+  const decoder = new KeyDecoder();
+  assert.deepEqual(decoder.push('\u001b_Gi=31;O'), []);
+  assert.deepEqual(decoder.push('K\u001b\\x'), [{kind: 'text', value: 'x'}]);
+});
+
+test('a held start of a graphics reply is flushed like a lone Escape and never swallows later keys', () => {
+  const decoder = new KeyDecoder();
+  assert.deepEqual(decoder.push('\u001b_Gi=31'), []);
+  assert.equal(decoder.pendingEscape, true, 'the owner flushes it after the Escape pause');
+  assert.ok(decoder.flush().length > 0);
+  assert.deepEqual(decoder.push('ls'), [{kind: 'text', value: 'l'}, {kind: 'text', value: 's'}]);
+});