pysimlin: run the notebook journey on a kernel that answers

.github/workflows/ci.yaml

@@ -393,14 +393,19 @@ jobs:
         run: make -C src/pysimlin export-check
 
       # Screenshots, the trace, and the jupyter server log are what a failed
-      # journey needs to be understood.
+      # journey needs to be understood. They live in a dot-directory, and
+      # upload-artifact@v4 skips hidden files unless told otherwise (it then
+      # finds nothing at all). Nothing to upload is a warning, not an error:
+      # this step also runs when an earlier build step failed, before the
+      # journey wrote anything.
       - name: Upload journey output
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: pysimlin-e2e-output
           path: src/pysimlin/e2e/.output/
-          if-no-files-found: ignore
+          include-hidden-files: true
+          if-no-files-found: warn
 
   policy:
     name: Policy checks

src/pysimlin/CLAUDE.md

@@ -16,7 +16,7 @@ For Python-specific standards (thread safety, lock ordering), see [docs/dev/pyth
 - `simlin/model.py` -- `Model` wrapper class; proxies `path`/`revision`/`dirty`/`save()`/`reload()` to its project, holds `selection` (written by the widget), and the display surface: `diagram()` / `_svg_mimebundle()` (static SVG), `widget()` (a fresh `ModelWidget`), and `_repr_mimebundle_` (a fresh widget's bundle merged with the SVG so static renderers show the picture). Every display creates a new widget and the model caches none of them; the imports of `simlin.widget` are lazy (`_widget_core.import_widget_module`) so `import simlin` never pays for anywidget -- which is the OPTIONAL `notebook` extra (`pip install "pysimlin[notebook]"`; the dev and e2e extras include it): without it `widget()` raises `SimlinDependencyError` (also an `ImportError`) carrying `_widget_core.install_hint()` (`%pip` under Colab) and `_repr_mimebundle_` degrades to the SVG plus a plain-text repr that carries the same message plus a `RuntimeWarning` with it, never a traceback (the warning is attributed to the user's cell by `_widget_core.user_stacklevel()`, which skips every `simlin.*` and IPython/ipykernel frame -- a bare display reaches the repr through IPython's display formatter, and a warning attributed there would name `formatters.py`, one location for every display, and be deduped for the whole kernel session; the text/plain carries the hint too because Python's once-per-location filter can still swallow a re-run of the same cell). `_widget_core.install_hint()` detects Colab exactly as anywidget does, `"google.colab.output" in sys.modules`, importing nothing. `tests/test_widget_optional.py` pins that arm by blocking the import (`sys.modules[name] = None`); every other widget test runs with the extra installed. Why an extra and why still anywidget: its chain is a couple of dozen packages that MCP/script installs never displaying a model should not carry, and a homegrown widget stack cannot replace it -- VS Code's kernel needs `import ipywidgets` and marimo needs `anywidget.AnyWidget`, so the extra is what keeps every host
 - `simlin/widget.py` -- `ModelWidget(anywidget.AnyWidget)`, the imperative shell of the notebook editor. Traits `project_json`/`revision` (kernel-owned, always assigned together in one `hold_sync()`), `selection` (browser-owned, mirrored to `Model.selection`), `height`/`theme`/`read_only`. Requests and replies are custom messages, never traits: browser `{type:'wasm'}` -> bytes as the first binary buffer; browser `{type:'snapshot', id, base, json}` -> `Project._apply_snapshot` -> EXACTLY ONE reply, always, and every reply (`saved`/`rejected`, from whichever arm below, including a malformed snapshot's) echoes the request's `id` verbatim (any JSON value; omitted when the request carried none) -- the browser resolves a save only on the reply naming its request, since one model displayed in several views sees every reply in every view: applied (including a `SimlinWriteError`, where the change is real and a post-commit step failed -- the write, or notifying subscribers) -> traits `(exact json received, base+1)` in one `hold_sync()` that exits BEFORE `{type:'saved'}` is sent (+ a warn `notice` naming what failed; `model.save()` is suggested only when `write_failed`); not applied (stale base, parse failure, or a handler exception BEFORE anything applied) -> no trait writes (the project's real pair is re-pushed, a no-op when unchanged), `{type:'rejected'}` + a warn `notice`; a handler exception AFTER the snapshot applied is an ACCEPT like the write-failure case -- the exact bytes received are pushed at `base+1` (best effort) and `{type:'saved'}` is sent plus a warn `notice`, because a `rejected` there costs the view a remount (undo history and any local edits since are lost) for a change that is real; a failure AFTER the reply went out (`_SnapshotProgress.replied`) sends nothing more, and a `BaseException` (KeyboardInterrupt) gets the same one reply and is then re-raised; browser `{type:'oversize', bytes}` (it refused to send a snapshot whose wire size -- the text JSON-escaped, `_widget_core.snapshot_wire_size` -- exceeds the `max_snapshot_bytes` trait, default `_widget_core.MAX_SNAPSHOT_BYTES` = 8 MiB, chosen to stay under tornado's 10 MiB `websocket_max_message_size` with only the envelope/header left to add) -> a `RuntimeWarning` on the kernel's stderr (a comm handler runs outside any cell; JupyterLab shows it in the Log Console -- the toast is what the user sees) + the same warn `notice` the browser already showed, no reply; seeding or pushing a `project_json` above the cap warns once per widget (`_warn_if_oversize`, attributed to the user's cell by `_widget_core.user_stacklevel()` -- the display, `Model.widget()` call or `edit()` block -- with IPython's display-formatter frames skipped). Subscribes to `Project.on_change` (marshalled to `shell.kernel.io_loop.add_callback` when running under ipykernel, else direct) and pushes the project's current pair plus a notice for every change that is not one of its own accepted snapshots (`_own_revisions`, a SET of the revisions its accepts produced whose notification is still queued -- ipykernel 7 handles comm messages on a subshell thread while a cell runs, so several accepts can precede the first delivery); `close()` unsubscribes. Asset delivery is resolved ONCE at import from `SIMLIN_WIDGET_ASSET` (`bundled` | `inline` | http(s) URL) into `_ASSETS`; a missing asset never breaks `import simlin` -- creating a widget raises `SimlinAssetError`. The `inline` shim contract with the JS (`globalThis.__simlinWidgetInlineWasm = "<base64>";` on the line before the module) is documented on `_widget_core.inline_esm`
 - `simlin/_widget_core.py` -- the widget's pure decisions: `parse_asset_mode`/`inline_esm`/`missing_asset_message`, `parse_incoming` (message classification; malformed messages are `Unrecognised`, never exceptions), `plan_snapshot_reply` (accept / stale / raised arms), the snapshot size guard (`MAX_SNAPSHOT_BYTES` with its rationale, `snapshot_wire_size`, `oversize_warning`/`oversize_report_warning`/`oversize_notice`, `format_size` -- mirrored byte for byte by `snapshotWireSize`/`formatSize` in the JS, with one fixture list pinned in both suites), `is_own_change`, `notice_for_change` (one text per `ChangeSource`), `dispatch_for_shell`. `tests/test_widget_core.py` enumerates every arm; `tests/test_widget.py` drives the shell through a recording comm (no browser, no kernel)
-- `e2e/` -- the JupyterLab journey (Playwright; design AC4.2): `global-setup.ts` launches `jupyter lab` from this venv (`jupyter-server.ts`: OS-assigned port, random token, temporary config/data/runtime/root directories, a preflight that the interpreter imports `simlin` with staged widget assets and `jupyterlab`) and hands the URL/token/root to the spec through `SIMLIN_E2E_*` env vars; each notebook's kernel session is started on the server before the browser opens the notebook (`startNotebookSession`, tried `SESSION_START_ATTEMPTS` times), because Lab connects to a session that exists and leaves a notebook kernel-less for good when its own one start request fails; a cell is run only once Lab's own session context says its kernel is connected (`kernelState`, read from `window.jupyterapp`, which the server is launched to expose -- the toolbar's execution indicator reads `idle` before the session has connected to anything, and a run in that window races Lab's session start); each notebook opens in a JupyterLab workspace of its own and is confirmed by path before anything is clicked (`openNotebook`), because Lab restores a shared workspace's earlier notebooks before it opens the one the URL names, and a helper that means "the visible notebook" then runs a cell of the wrong one; `kernel-session.spec.ts` pins the session-start and kernel-state decisions without a browser; `notebook-editor.spec.ts` drives the real JupyterLab UI: `simlin.open` + display -> the Editor renders; a variable added and given an equation in the Editor -> the file on disk changes and a following `m.run()` cell reports it; `m.selection` names the selected variable; a Python `edit()` -> "Updated from Python" in the Editor; a second-process write -> "Updated on disk" and `m.revision` advanced, and after those remounts `m.selection` is `()` again. It is NOT part of `pytest`/pre-commit: `make e2e` runs it (CI: the `pysimlin-e2e` job); it needs the `e2e` extra (jupyterlab, in THIS venv so its share/jupyter carries the anywidget/ipywidgets labextensions), the CFFI extension, staged `simlin/_widget/` assets (`make assets`), and Playwright's chromium. `SIMLIN_E2E_PYTHON` picks another interpreter; `SIMLIN_E2E_KEEP_TMP` keeps the server tree; the server log (jupyter-lab.log) is written to the git-ignored output directory beside the spec (see `e2e/playwright.config.ts`)
+- `e2e/` -- the JupyterLab journey (Playwright; design AC4.2): `global-setup.ts` launches `jupyter lab` from this venv (`jupyter-server.ts`: OS-assigned port, random token, temporary config/data/runtime/root directories, a preflight that the interpreter imports `simlin` with staged widget assets and `jupyterlab`, and the runtime file read only once whole -- `readServerInfo`, since the server writes it in place) and hands the URL/token/root/server-log path to the spec through `SIMLIN_E2E_*` env vars; each notebook's kernel session is started on the server before the browser opens the notebook (`startNotebookSession`, tried `SESSION_START_ATTEMPTS` times), because Lab connects to a session that exists and leaves a notebook kernel-less for good when its own one start request fails; a cell is run only once Lab's own session context says its kernel is connected (`kernelState`, read from `window.jupyterapp`, which the server is launched to expose -- the toolbar's execution indicator reads `idle` before the session has connected to anything, and a run in that window races Lab's session start); a cell run is an action with a checked effect: `runCell` waits for the prompt to show a new execution count (`parsePrompt`/`cellRan`) and otherwise fails at the run with `unrunCellReport` (`[*]` = sent and unanswered by the kernel, no new count = Lab never ran it; the kernel's state and the server log's tail) rather than as an output that never appears; the `e2e` extra requires ipykernel >= 7.4, the first release that sends shell replies through the shell socket's ZMQStream (ipython/ipykernel#1529) -- before it a reply sent raw on that socket could strand a request arriving at the same moment, Lab's burst of requests when it connects a notebook made that likely on a notebook's first cell, and the kernel then sat idle with the cell at `[*]` for good; each notebook opens in a JupyterLab workspace of its own and is confirmed by path before anything is clicked (`openNotebook`), because Lab restores a shared workspace's earlier notebooks before it opens the one the URL names, and a helper that means "the visible notebook" then runs a cell of the wrong one; `kernel-session.spec.ts` pins the runtime-file, session-start, kernel-state, and cell-run decisions without a browser; `notebook-editor.spec.ts` drives the real JupyterLab UI: `simlin.open` + display -> the Editor renders; a variable added and given an equation in the Editor -> the file on disk changes and a following `m.run()` cell reports it; `m.selection` names the selected variable; a Python `edit()` -> "Updated from Python" in the Editor; a second-process write -> "Updated on disk" and `m.revision` advanced, and after those remounts `m.selection` is `()` again. It is NOT part of `pytest`/pre-commit: `make e2e` runs it (CI: the `pysimlin-e2e` job); it needs the `e2e` extra (jupyterlab, in THIS venv so its share/jupyter carries the anywidget/ipywidgets labextensions), the CFFI extension, staged `simlin/_widget/` assets (`make assets`), and Playwright's chromium. `SIMLIN_E2E_PYTHON` picks another interpreter; `SIMLIN_E2E_KEEP_TMP` keeps the server tree; the server log (jupyter-lab.log) is written to the git-ignored output directory beside the spec (see `e2e/playwright.config.ts`)
 - `scripts/check_notebook_export.py` -- executes an example notebook headless with nbconvert and checks both static-export arms of the display cell: with the stored widget state the HTML carries the widget view plus the embedded state (the `_esm` rides along), and with `metadata.widgets` stripped -- what GitHub or a notebook saved without state gives a static renderer -- nbconvert's `WidgetsDataTypeFilter` skips the widget-view mimetype and the HTML carries the SVG (the lab template embeds it as a base64 `<img>`). Its pure half is unit-tested in `tests/test_check_notebook_export.py`; the shell runs under `make export-check` and the CI `pysimlin-e2e` job, never as a pytest (the dev extra has no nbconvert and a skipped test proves nothing)
 - `docs/notebook-hosts.md` -- per-host checklist for the notebook editor (JupyterLab 4, Notebook 7, VS Code local/Remote-SSH, Colab, marimo) with each host's VERIFIED/UNVERIFIED status; only JupyterLab is verified (by `e2e/`). Update the status line in the same PR that runs a checklist. `examples/colab_quickstart.ipynb` is the Colab entry point (its `%pip` cell is skipped by `tests/test_examples.py`'s script run)
 - `simlin/_widget/` -- package data: `widget.js` and `libsimlin-browser.wasm`, build outputs of the `@simlin/notebook-widget` TypeScript package (git-ignored; see its README.md and `pyproject.toml` package-data)

src/pysimlin/e2e/global-setup.ts

@@ -19,4 +19,5 @@ export default async function globalSetup(): Promise<void> {
   process.env[ENV.url] = server.url;
   process.env[ENV.token] = server.token;
   process.env[ENV.rootDir] = server.rootDir;
+  process.env[ENV.logPath] = server.logPath;
 }

src/pysimlin/e2e/jupyter-server.ts

@@ -38,6 +38,7 @@ export const ENV = {
   url: 'SIMLIN_E2E_JUPYTER_URL',
   token: 'SIMLIN_E2E_JUPYTER_TOKEN',
   rootDir: 'SIMLIN_E2E_ROOT_DIR',
+  logPath: 'SIMLIN_E2E_SERVER_LOG',
 } as const;
 
 const here = __dirname;
@@ -93,7 +94,28 @@ export function userSettingsFiles(configDir: string): Array<{ file: string; cont
   ];
 }
 
-/** Read `url`/`token`/`root_dir` from a `jpserver-<pid>.json` runtime file. */
+/**
+ * Read `url`/`token`/`root_dir` from a `jpserver-<pid>.json` runtime file,
+ * or `undefined` while the file is still being written.
+ *
+ * The server writes the file in place (jupyter_server's `secure_write` opens
+ * the final path and `json.dump`s into it), so a reader that finds the file
+ * can read it truncated, or empty. Text that does not parse yet is that
+ * window; a file that parses but lacks a field is a real error.
+ */
+export function readServerInfo(json: string): ServerInfo | undefined {
+  try {
+    JSON.parse(json);
+  } catch (err) {
+    if (err instanceof SyntaxError) {
+      return undefined;
+    }
+    throw err;
+  }
+  return parseServerInfo(json);
+}
+
+/** Read `url`/`token`/`root_dir` from a complete `jpserver-<pid>.json` runtime file. */
 export function parseServerInfo(json: string): ServerInfo {
   const info = JSON.parse(json) as { url?: unknown; token?: unknown; root_dir?: unknown };
   if (typeof info.url !== 'string' || typeof info.token !== 'string' || typeof info.root_dir !== 'string') {
@@ -266,6 +288,77 @@ export function canRunCells(state: string): boolean {
   return state === 'idle' || state === 'busy';
 }
 
+/**
+ * A code cell's execution as its input prompt shows it: `null` for never run
+ * (`[ ]:`), `'running'` for sent to the kernel and not yet answered (`[*]:`),
+ * or the execution count the kernel replied with (`[3]:`).
+ */
+export type CellExecution = null | 'running' | number;
+
+/** Read a cell's input prompt text (`[ ]:`, `[*]:`, `[3]:`). */
+export function parsePrompt(text: string): CellExecution {
+  const m = /^\s*\[\s*([^\]]*?)\s*\]:?\s*$/.exec(text);
+  if (m === null) {
+    throw new Error(`not a cell input prompt: ${JSON.stringify(text)}`);
+  }
+  if (m[1] === '') {
+    return null;
+  }
+  if (m[1] === '*') {
+    return 'running';
+  }
+  if (/^\d+$/.test(m[1])) {
+    return Number(m[1]);
+  }
+  throw new Error(`not a cell input prompt: ${JSON.stringify(text)}`);
+}
+
+/**
+ * Whether a run of a cell whose prompt read `before` has been answered by the
+ * kernel, now that it reads `now`: the kernel's reply carries a new execution
+ * count, so a count that differs from the one the cell had is the kernel's
+ * word that it ran the cell. A cell that shows no count, or `[*]`, or the
+ * count it had before, has not been run by this press.
+ */
+export function cellRan(before: CellExecution, now: CellExecution): boolean {
+  return typeof now === 'number' && now !== before;
+}
+
+/**
+ * Why a cell run the journey asked for never came back, in terms that say
+ * which side to look at: a `[*]` prompt means JupyterLab sent the request and
+ * the kernel has not answered (look at the kernel and the server log), and a
+ * prompt with no new count means JupyterLab never ran the cell at all (look at
+ * the page: focus, a dialog, a cancelled run).
+ */
+export function unrunCellReport(facts: {
+  notebook: string;
+  index: number;
+  before: CellExecution;
+  now: CellExecution;
+  kernel: string;
+  waitedMs: number;
+  serverLogTail: string;
+}): string {
+  const shown = (e: CellExecution): string => (e === null ? '[ ]' : e === 'running' ? '[*]' : `[${e}]`);
+  const verdict =
+    facts.now === 'running'
+      ? 'JupyterLab sent it to the kernel, which has not answered'
+      : 'JupyterLab never ran it (no execution request went out, or the run was cancelled)';
+  return [
+    `cell ${facts.index} of ${facts.notebook} did not run within ${facts.waitedMs} ms: ${verdict}.`,
+    `prompt before the run: ${shown(facts.before)}; now: ${shown(facts.now)}; notebook kernel: ${facts.kernel}`,
+    `last lines of the jupyter server log:`,
+    facts.serverLogTail,
+  ].join('\n');
+}
+
+/** The last `lines` lines of `text`, for an error message. */
+export function tail(text: string, lines: number): string {
+  const all = text.trimEnd().split('\n');
+  return all.slice(Math.max(0, all.length - lines)).join('\n');
+}
+
 /** Launch JupyterLab and resolve once its REST API answers with our token. */
 export async function launchJupyterLab(): Promise<LaunchedServer> {
   const python = pythonExecutable();
@@ -322,7 +415,7 @@ export async function launchJupyterLab(): Promise<LaunchedServer> {
       if (files.length === 0) {
         return undefined;
       }
-      return parseServerInfo(fs.readFileSync(path.join(runtimeDir, files[0]), 'utf8'));
+      return readServerInfo(fs.readFileSync(path.join(runtimeDir, files[0]), 'utf8'));
     });
     const url = info.url;
     const bearer = info.token;