From e6ca762b71c00ca726efdcd2f86781e2fa3a12fd Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Fri, 23 Oct 2026 19:45:56 -0400 Subject: [PATCH 1/7] Add optional Jupyter kernel workers with private sandbox transport --- benchmarks/jupyter.py | 322 ++++ .../jupyter-macos-arm64-2026-10-23.json | 1405 +++++++++++++++++ docs/API.md | 19 +- docs/ARCHITECTURE.md | 6 + docs/JUPYTER.md | 290 ++++ docs/README.md | 1 + docs/RECORDING.md | 3 + docs/SANDBOX_CONFIGURATION.md | 4 +- docs/WORKER_PROTOCOL.md | 18 + docs/_quarto.yml | 2 + docs/benchmarks/jupyter.md | 100 ++ pyproject.toml | 1 + python/mcp_console/_client.py | 9 +- python/mcp_console/_sync.py | 7 +- scripts/validate_runtime_sources.py | 2 + src/cell.rs | 26 +- src/cli.rs | 12 + src/jupyter.rs | 101 ++ src/jupyter/linux.rs | 46 + src/jupyter/namespace_ipykernel.py | 27 + src/jupyter/private_ipc.sbpl | 10 + src/jupyter/worker.py | 536 +++++++ src/main.rs | 47 +- src/sandbox.rs | 6 +- src/server.rs | 43 +- src/server/arguments.rs | 6 + src/server/presentation.rs | 58 +- src/settings.rs | 42 +- src/transcript.rs | 18 +- src/transcript/event.rs | 2 + src/transcript/markdown.rs | 65 +- src/worker/coordinator.rs | 1 + src/worker_client/configuration.rs | 56 + src/worker_client/environment/inspection.rs | 6 +- tests/boundaries/_run.py | 1 + .../boundaries/cli/interface/test_jupyter.py | 139 ++ .../client_server/integrations/test_python.py | 49 + .../client_server/jupyter/_sandbox.py | 235 +++ .../client_server/jupyter/test_ark.py | 58 + .../jupyter/test_linux_sandbox.py | 82 + .../jupyter/test_macos_sandbox.py | 89 ++ .../client_server/jupyter/test_python.py | 395 +++++ .../jupyter/test_sandbox_inheritance.py | 115 ++ .../client_server/server/test_tools.py | 64 + tests/fixtures/jupyter_delayed_info.py | 29 + tests/fixtures/jupyter_sandbox_network.py | 45 + .../fixtures/jupyter_sandbox_network_linux.py | 45 + tests/fixtures/jupyter_startup.py | 12 + tests/fixtures/python_sync_server.py | 8 +- .../cli/interface/test_help/help.yaml | 11 + .../test_jupyter/jupyter_hides_code.yaml | 120 ++ ...r_rejects_unavailable_tool_parameters.yaml | 48 + .../jupyter_tool_configuration.yaml | 117 ++ .../test_jupyter/launch_options.yaml | 115 ++ ...clients_forward_jupyter_kernel_source.yaml | 28 + .../persistent_r_errors_and_plot_output.yaml | 47 + ...ate_network_preserves_default_sandbox.yaml | 152 ++ ...ate_network_preserves_default_sandbox.yaml | 146 ++ ...private_ipc_preserves_default_sandbox.yaml | 146 ++ ...private_ipc_preserves_default_sandbox.yaml | 160 ++ ...d_kernel_info_replies_survive_retries.yaml | 32 + ...restart_preserve_generation_ownership.yaml | 86 + ...cels_kernel_startup_and_reaps_process.yaml | 59 + ...rnel_with_explicit_socket_permissions.yaml | 16 + ...crash_recovery_and_process_retirement.yaml | 126 ++ ...ent_state_streams_errors_and_recovery.yaml | 153 ++ .../rich_output_and_prompted_input.yaml | 51 + ...l_startup_remains_pending_until_ready.yaml | 50 + ...up_warning_preserves_output_and_state.yaml | 28 + ...cted_filesystem_preserves_private_ipc.yaml | 155 ++ ...izes_and_lists_tools.jupyter-writable.yaml | 121 ++ ...alizes_and_lists_tools.jupyter.direct.yaml | 119 ++ .../initializes_and_lists_tools.jupyter.yaml | 120 ++ tests/support/jupyter.py | 169 ++ tests/support/requirements.py | 10 + 75 files changed, 6971 insertions(+), 77 deletions(-) create mode 100644 benchmarks/jupyter.py create mode 100644 benchmarks/results/jupyter-macos-arm64-2026-10-23.json create mode 100644 docs/JUPYTER.md create mode 100644 docs/benchmarks/jupyter.md create mode 100644 src/jupyter.rs create mode 100644 src/jupyter/linux.rs create mode 100644 src/jupyter/namespace_ipykernel.py create mode 100644 src/jupyter/private_ipc.sbpl create mode 100644 src/jupyter/worker.py create mode 100644 tests/boundaries/cli/interface/test_jupyter.py create mode 100644 tests/boundaries/client_server/jupyter/_sandbox.py create mode 100644 tests/boundaries/client_server/jupyter/test_ark.py create mode 100644 tests/boundaries/client_server/jupyter/test_linux_sandbox.py create mode 100644 tests/boundaries/client_server/jupyter/test_macos_sandbox.py create mode 100644 tests/boundaries/client_server/jupyter/test_python.py create mode 100644 tests/boundaries/client_server/jupyter/test_sandbox_inheritance.py create mode 100644 tests/fixtures/jupyter_delayed_info.py create mode 100644 tests/fixtures/jupyter_sandbox_network.py create mode 100644 tests/fixtures/jupyter_sandbox_network_linux.py create mode 100644 tests/fixtures/jupyter_startup.py create mode 100644 tests/snapshots/cli/interface/test_jupyter/jupyter_hides_code.yaml create mode 100644 tests/snapshots/cli/interface/test_jupyter/jupyter_rejects_unavailable_tool_parameters.yaml create mode 100644 tests/snapshots/cli/interface/test_jupyter/jupyter_tool_configuration.yaml create mode 100644 tests/snapshots/cli/interface/test_jupyter/launch_options.yaml create mode 100644 tests/snapshots/client_server/integrations/test_python/clients_forward_jupyter_kernel_source.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_ark/persistent_r_errors_and_plot_output.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_linux_sandbox/ark_private_network_preserves_default_sandbox.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_linux_sandbox/ipykernel_private_network_preserves_default_sandbox.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_macos_sandbox/ark_private_ipc_preserves_default_sandbox.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_macos_sandbox/ipykernel_private_ipc_preserves_default_sandbox.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/delayed_kernel_info_replies_survive_retries.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/interrupt_and_restart_preserve_generation_ownership.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/interrupt_cancels_kernel_startup_and_reaps_process.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/ipc_kernel_with_explicit_socket_permissions.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/kernel_crash_recovery_and_process_retirement.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/persistent_state_streams_errors_and_recovery.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/rich_output_and_prompted_input.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/slow_kernel_startup_remains_pending_until_ready.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_python/tcp_startup_warning_preserves_output_and_state.yaml create mode 100644 tests/snapshots/client_server/jupyter/test_sandbox_inheritance/unrestricted_filesystem_preserves_private_ipc.yaml create mode 100644 tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter-writable.yaml create mode 100644 tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml create mode 100644 tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml create mode 100644 tests/support/jupyter.py diff --git a/benchmarks/jupyter.py b/benchmarks/jupyter.py new file mode 100644 index 000000000..a39e51e03 --- /dev/null +++ b/benchmarks/jupyter.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Measure public MCP startup and warm assignment calls with installed runtimes. + +Run with Python 3.11+ after building Console and installing released dependencies: + + MCP_CONSOLE_TEST_JUPYTER_PYTHON=/path/to/python \ + MCP_CONSOLE_TEST_ARK=/path/to/ark \ + MCP_CONSOLE_TEST_RETICULATE_LIBRARY=/path/to/R/library \ + python3 benchmarks/jupyter.py --binary target/release/mcp-console > results.json + +Setup and version probes run outside timed regions. Each sample launches a new +Console server, negotiates MCP, completes x = 1, then completes thirty identical +warm cells. Variants use direct execution with normal Console recording. +Installed-runtime caches remain warm across launches; this is not a cold-cache +or package-installation benchmark. RSS is a simultaneous process-tree snapshot, +not peak memory or unique physical memory: shared pages can be counted repeatedly. +""" + +import argparse +import hashlib +import json +import os +import platform +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from support.client import McpClient +from support.execution import DIRECT +from support.r import r_test_environment +from support.resolvers import bare_runtime_environment + + +@dataclass(frozen=True) +class Variant: + name: str + field: str + kernel: str | None = None + transport: str | None = None + + +VARIANTS = ( + Variant("builtin-python", "python"), + Variant("ipykernel-ipc", "code", "ipykernel", "ipc"), + Variant("builtin-r", "r"), + Variant("ark-tcp", "code", "ark", "tcp"), +) + + +def installed_path(name: str, *, directory: bool = False) -> Path: + path = Path(os.environ[name]) + assert path.is_absolute(), (name, path) + assert path.is_dir() if directory else path.is_file(), (name, path) + return path + + +@contextmanager +def launch_settings( + variant: Variant, + python: Path, + ark: Path, + library: Path, + inherited: dict[str, str], +) -> Iterator[tuple[tuple[str, ...], dict[str, str], Path]]: + with tempfile.TemporaryDirectory(prefix="console-jupyter-benchmark-") as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + environment = bare_runtime_environment(inherited, library) + environment["RETICULATE_PYTHON"] = str(python) + arguments = ["--no-config"] + if variant.kernel is None: + arguments.extend( + [ + "-c", + f"languages=[{variant.field}]", + "-c", + "python=" + json.dumps(str(python)), + "-c", + "r.resolution=disabled", + "-c", + "r.packages=[]", + ] + ) + else: + name = "benchmark-" + variant.kernel + spec = root / "jupyter" / "kernels" / name + spec.mkdir(parents=True) + if variant.kernel == "ipykernel": + argv = [ + str(python), + "-Xfrozen_modules=off", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}", + ] + language = "python" + else: + argv = [ + str(ark), + "--connection_file", + "{connection_file}", + "--session-mode", + "notebook", + "--", + "--vanilla", + ] + language = "R" + environment["RUST_LOG"] = "error" + (spec / "kernel.json").write_text( + json.dumps({"argv": argv, "display_name": name, "language": language}), + encoding="utf-8", + ) + environment["JUPYTER_PATH"] = str(root / "jupyter") + environment.update( + JUPYTER_CONFIG_DIR=str(root / "jupyter-config"), + JUPYTER_RUNTIME_DIR=str(root / "jupyter-runtime"), + IPYTHONDIR=str(root / "ipython"), + ) + arguments.extend( + [ + "--jupyter-kernel", + name, + "--jupyter-python", + str(python), + "--jupyter-transport", + variant.transport, + ] + ) + yield DIRECT.serve(*arguments), environment, workspace + + +def process_rss(root_pid: int) -> dict: + ps = shutil.which("ps") + if ps is None: + return {"available": False, "reason": "ps is unavailable"} + observed = subprocess.run( + [ps, "-axo", "pid=,ppid=,rss=,comm="], + check=False, + capture_output=True, + text=True, + ) + if observed.returncode != 0: + return {"available": False, "reason": observed.stderr.strip()} + processes = {} + for line in observed.stdout.splitlines(): + pid, parent, rss, executable = line.split(None, 3) + processes[int(pid)] = { + "pid": int(pid), + "parent_pid": int(parent), + "rss_kib": int(rss), + "executable": executable, + } + assert root_pid in processes, "Console exited before its memory snapshot" + selected = {root_pid} + while children := { + pid + for pid, process in processes.items() + if process["parent_pid"] in selected and pid not in selected + }: + selected.update(children) + rows = [processes[pid] for pid in sorted(selected)] + return { + "available": True, + "total_rss_kib": sum(row["rss_kib"] for row in rows), + "process_count": len(rows), + "processes": rows, + } + + +def sample( + binary: Path, + variant: Variant, + python: Path, + ark: Path, + library: Path, + inherited: dict[str, str], + warm_cells: int, +) -> dict: + with launch_settings(variant, python, ark, library, inherited) as settings: + arguments, environment, workspace = settings + started = time.perf_counter() + with McpClient( + binary, arguments, environment, current_directory=workspace + ) as client: + client.initialize_and_list_tools() + client.expect("[done]", **{variant.field: "x = 1"}) + first_result_seconds = time.perf_counter() - started + warm = [] + for _ in range(warm_cells): + started = time.perf_counter() + client.expect("[done]", **{variant.field: "x = 1"}) + warm.append(time.perf_counter() - started) + rss = process_rss(client.process.pid) + client.finish() + return { + "first_result_seconds": first_result_seconds, + "warm_cell_seconds": warm, + "warm_median_seconds": statistics.median(warm), + "after_warm_rss": rss, + } + + +def version(command: list[str], environment: dict[str, str]) -> str: + result = subprocess.run( + command, env=environment, check=True, capture_output=True, text=True, timeout=30 + ) + return result.stdout.strip() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--binary", type=Path, default=ROOT / "target/release/mcp-console" + ) + parser.add_argument("--launches", type=int, default=5) + parser.add_argument("--warm-cells", type=int, default=30) + parser.add_argument( + "--variant", choices=[v.name for v in VARIANTS], action="append" + ) + options = parser.parse_args() + assert options.launches > 0 and options.warm_cells > 0 + variants = tuple( + v for v in VARIANTS if not options.variant or v.name in options.variant + ) + binary = options.binary.resolve() + assert binary.is_file(), binary + python = installed_path("MCP_CONSOLE_TEST_JUPYTER_PYTHON") + ark = installed_path("MCP_CONSOLE_TEST_ARK") + library = installed_path("MCP_CONSOLE_TEST_RETICULATE_LIBRARY", directory=True) + inherited, rscript = r_test_environment() + versions_environment = bare_runtime_environment(inherited, library) + dependencies = json.loads( + version( + [ + str(python), + "-c", + "import importlib.metadata as m, json, sys; print(json.dumps({'python': sys.version, **{n: m.version(n) for n in ('jupyter_client', 'ipykernel', 'pyzmq')}}))", + ], + versions_environment, + ) + ) + dependencies["ark"] = version([str(ark), "--version"], versions_environment) + dependencies["r_and_reticulate"] = version( + [ + str(rscript), + "--vanilla", + "-e", + 'cat(R.version.string, "\\nreticulate ", as.character(packageVersion("reticulate")), sep = "")', + ], + versions_environment, + ) + results = { + "method": { + "launches_per_variant": options.launches, + "warm_cells_per_launch": options.warm_cells, + "source": "x = 1", + "startup_interval": "Before MCP process launch through the first completed assignment, including MCP initialization and tool discovery.", + "warm_interval": "One complete public send call after the first assignment.", + "dependency_preparation": "None: existing Python executable and published R library selected; ir/uv absent from child PATH; R resolution disabled.", + "execution": "Direct, with normal Console recording in a new temporary workspace per launch.", + "cache_state": "Installed-runtime caches remain warm; each launch uses fresh Jupyter and IPython configuration directories.", + "memory": "ps RSS after warm cells for Console and all descendants; includes shared-page double counting, excludes benchmark client, is not peak RSS.", + "sample_order": "Round robin in listed variant order, one new launch per variant per round.", + }, + "host": {"platform": platform.platform(), "machine": platform.machine()}, + "binary": { + "path": str(binary), + "bytes": binary.stat().st_size, + "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + }, + "dependencies": dependencies, + "variants": { + variant.name: {"status": "ok", "samples": []} for variant in variants + }, + } + for _ in range(options.launches): + for variant in variants: + entry = results["variants"][variant.name] + if entry["status"] == "error": + continue + try: + entry["samples"].append( + sample( + binary, + variant, + python, + ark, + library, + inherited, + options.warm_cells, + ) + ) + except Exception as error: + entry["status"] = "error" + entry["error"] = f"{type(error).__name__}: {error}" + for entry in results["variants"].values(): + if entry["samples"]: + entry["first_result_median_seconds"] = statistics.median( + item["first_result_seconds"] for item in entry["samples"] + ) + entry["warm_cell_median_seconds"] = statistics.median( + duration + for item in entry["samples"] + for duration in item["warm_cell_seconds"] + ) + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/results/jupyter-macos-arm64-2026-10-23.json b/benchmarks/results/jupyter-macos-arm64-2026-10-23.json new file mode 100644 index 000000000..7ec93dbfe --- /dev/null +++ b/benchmarks/results/jupyter-macos-arm64-2026-10-23.json @@ -0,0 +1,1405 @@ +{ + "method": { + "launches_per_variant": 5, + "warm_cells_per_launch": 30, + "source": "x = 1", + "startup_interval": "Before MCP process launch through the first completed assignment, including MCP initialization and tool discovery.", + "warm_interval": "One complete public send call after the first assignment.", + "dependency_preparation": "None: existing Python executable and published R library selected; ir/uv absent from child PATH; R resolution disabled.", + "execution": "Direct, with normal Console recording in a new temporary workspace per launch.", + "cache_state": "Installed-runtime caches remain warm; each launch uses fresh Jupyter and IPython configuration directories.", + "memory": "ps RSS after warm cells for Console and all descendants; includes shared-page double counting, excludes benchmark client, is not peak RSS.", + "sample_order": "Round robin in listed variant order, one new launch per variant per round." + }, + "host": { + "platform": "macOS-26.7.1-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpu": "Apple M4 Max", + "logical_cpus": 16, + "memory_bytes": 137438953472 + }, + "binary": { + "path": "/target/release/mcp-console", + "bytes": 12375056, + "sha256": "8aa187a83b1b2764748c667867bdf4ae122462e42e4179c1fdf7075c8d452812" + }, + "dependencies": { + "python": "3.13.16 (main, Oct 3 2026, 00:54:36) [Clang 22.1.3 ]", + "jupyter_client": "8.10.0", + "ipykernel": "7.4.0", + "pyzmq": "27.2.0", + "ark": "Ark 0.1.252", + "r_and_reticulate": "R version 4.6.1 (2026-06-24)\nreticulate 1.47.0" + }, + "variants": { + "builtin-python": { + "status": "ok", + "samples": [ + { + "first_result_seconds": 1.1344942080322653, + "warm_cell_seconds": [ + 0.003383832983672619, + 0.003255167044699192, + 0.003087499993853271, + 0.0030999999726191163, + 0.003100540954619646, + 0.003028541919775307, + 0.0030795420752838254, + 0.0031009999802336097, + 0.003070542006753385, + 0.003036750014871359, + 0.003158499952405691, + 0.003114582970738411, + 0.0036150829400867224, + 0.0030720409704372287, + 0.0030765829142183065, + 0.003073291969485581, + 0.0031015410786494613, + 0.003023875062353909, + 0.0030890420312061906, + 0.0026267500361427665, + 0.0031624999828636646, + 0.0031728328904137015, + 0.0031077079474925995, + 0.003113957936875522, + 0.0031144999666139483, + 0.003627459052950144, + 0.002994750044308603, + 0.003072583000175655, + 0.0030757079366594553, + 0.003141124965623021 + ], + "warm_median_seconds": 0.003100270463619381, + "after_warm_rss": { + "available": true, + "total_rss_kib": 131680, + "process_count": 4, + "processes": [ + { + "pid": 39474, + "parent_pid": 39453, + "rss_kib": 10352, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39475, + "parent_pid": 39474, + "rss_kib": 5136, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39494, + "parent_pid": 39474, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39495, + "parent_pid": 39494, + "rss_kib": 111472, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1255172080127522, + "warm_cell_seconds": [ + 0.003393416991457343, + 0.0032669169595465064, + 0.0031347910407930613, + 0.0030811670003458858, + 0.0030771249439567327, + 0.0030380829703062773, + 0.0035125829745084047, + 0.0030574999982491136, + 0.0031286660814657807, + 0.0031095410231500864, + 0.0030544999754056334, + 0.002761665964499116, + 0.0030553750693798065, + 0.003076957887969911, + 0.00306608306709677, + 0.003029500017873943, + 0.0030564579647034407, + 0.0031394590623676777, + 0.0030756250489503145, + 0.003122666967101395, + 0.003113333019427955, + 0.0030437910463660955, + 0.0030618750024586916, + 0.0030792910838499665, + 0.0031416660640388727, + 0.003020165953785181, + 0.003422957961447537, + 0.002556583029218018, + 0.003044875105842948, + 0.0029949580784887075 + ], + "warm_median_seconds": 0.003076291468460113, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132064, + "process_count": 4, + "processes": [ + { + "pid": 39610, + "parent_pid": 39453, + "rss_kib": 10336, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39611, + "parent_pid": 39610, + "rss_kib": 5168, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39629, + "parent_pid": 39610, + "rss_kib": 4704, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39630, + "parent_pid": 39629, + "rss_kib": 111856, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1148887500166893, + "warm_cell_seconds": [ + 0.0035692910896614194, + 0.00333724997472018, + 0.0031439579324796796, + 0.0031327500473707914, + 0.003119833068922162, + 0.0018602090422064066, + 0.003089333069510758, + 0.00311633397359401, + 0.003115209052339196, + 0.0031144170789048076, + 0.0030327499844133854, + 0.0031096250750124454, + 0.003096416941843927, + 0.0031084580114111304, + 0.0031336250249296427, + 0.0031014999840408564, + 0.002554665901698172, + 0.0031168749555945396, + 0.0030830829637125134, + 0.0030894590308889747, + 0.003745042020455003, + 0.0031309169717133045, + 0.0031204998958855867, + 0.0030906670726835728, + 0.003083124989643693, + 0.0031156670302152634, + 0.0018390408949926496, + 0.0029921249952167273, + 0.003122083959169686, + 0.0030536659760400653 + ], + "warm_median_seconds": 0.0031120210769586265, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132064, + "process_count": 4, + "processes": [ + { + "pid": 39768, + "parent_pid": 39453, + "rss_kib": 10352, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39769, + "parent_pid": 39768, + "rss_kib": 5184, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39787, + "parent_pid": 39768, + "rss_kib": 4704, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39788, + "parent_pid": 39787, + "rss_kib": 111824, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1510181250050664, + "warm_cell_seconds": [ + 0.0033644590293988585, + 0.003216042066924274, + 0.003165000001899898, + 0.0031325409654527903, + 0.003215499920770526, + 0.0019393330439925194, + 0.003177083097398281, + 0.0031526669627055526, + 0.0031322500435635448, + 0.0037989590782672167, + 0.0031071659177541733, + 0.0031520839547738433, + 0.003179833060130477, + 0.003140999935567379, + 0.0031466250075027347, + 0.0032482920214533806, + 0.003166582901030779, + 0.003217624966055155, + 0.0031829579966142774, + 0.003127874922938645, + 0.0032885830150917172, + 0.0030964999459683895, + 0.0031465840293094516, + 0.003174666897393763, + 0.0030771660385653377, + 0.0030672920402139425, + 0.003375208005309105, + 0.001987875089980662, + 0.003104708041064441, + 0.0030817079823464155 + ], + "warm_median_seconds": 0.003152375458739698, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132096, + "process_count": 4, + "processes": [ + { + "pid": 39902, + "parent_pid": 39453, + "rss_kib": 10384, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39903, + "parent_pid": 39902, + "rss_kib": 5152, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39921, + "parent_pid": 39902, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39922, + "parent_pid": 39921, + "rss_kib": 111840, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.2585021670674905, + "warm_cell_seconds": [ + 0.00341916608158499, + 0.003321459051221609, + 0.003134791972115636, + 0.003095833002589643, + 0.0031303330324590206, + 0.003061374998651445, + 0.003052124986425042, + 0.0030371249886229634, + 0.0034897909499704838, + 0.00317295896820724, + 0.00326020794454962, + 0.003131708945147693, + 0.0018347500590607524, + 0.003045459045097232, + 0.0030697910115122795, + 0.0031202089739963412, + 0.0030227910028770566, + 0.0031500000040978193, + 0.0031041670590639114, + 0.00314679101575166, + 0.0031075410079210997, + 0.0031269999453797936, + 0.003106083022430539, + 0.0030869580805301666, + 0.0030373750487342477, + 0.0030009999172762036, + 0.0030068339547142386, + 0.0030918329721316695, + 0.003245958941988647, + 0.0030940830474719405 + ], + "warm_median_seconds": 0.0031051250407472253, + "after_warm_rss": { + "available": true, + "total_rss_kib": 131824, + "process_count": 4, + "processes": [ + { + "pid": 40070, + "parent_pid": 39453, + "rss_kib": 10320, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40071, + "parent_pid": 40070, + "rss_kib": 5136, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40089, + "parent_pid": 40070, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40090, + "parent_pid": 40089, + "rss_kib": 111648, + "executable": "/target/release/mcp-console" + } + ] + } + } + ], + "first_result_median_seconds": 1.1344942080322653, + "warm_cell_median_seconds": 0.0031076244777068496 + }, + "ipykernel-ipc": { + "status": "ok", + "samples": [ + { + "first_result_seconds": 0.4112006250070408, + "warm_cell_seconds": [ + 0.005798917030915618, + 0.005455916980281472, + 0.005770582938566804, + 0.006193791050463915, + 0.005688540986739099, + 0.005824416992254555, + 0.0055684170220047235, + 0.005320291966199875, + 0.005010792054235935, + 0.005324375000782311, + 0.005589124979451299, + 0.005333333974704146, + 0.005953832995146513, + 0.005849375040270388, + 0.006158042000606656, + 0.006202000076882541, + 0.006113290903158486, + 0.005992332939058542, + 0.005827166954986751, + 0.005252957926131785, + 0.005495833000168204, + 0.004331375006586313, + 0.005603625089861453, + 0.005917791975662112, + 0.005704124923795462, + 0.005824916996061802, + 0.006299957982264459, + 0.006008207914419472, + 0.005849792039953172, + 0.005968082929030061 + ], + "warm_median_seconds": 0.005811667011585087, + "after_warm_rss": { + "available": true, + "total_rss_kib": 115136, + "process_count": 4, + "processes": [ + { + "pid": 39506, + "parent_pid": 39453, + "rss_kib": 9808, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39507, + "parent_pid": 39506, + "rss_kib": 4864, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39508, + "parent_pid": 39507, + "rss_kib": 43712, + "executable": "" + }, + { + "pid": 39509, + "parent_pid": 39508, + "rss_kib": 56752, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.4525593330617994, + "warm_cell_seconds": [ + 0.0058431250508874655, + 0.005853542010299861, + 0.005546125001274049, + 0.005598500021733344, + 0.005626208032481372, + 0.005527790985070169, + 0.006049790885299444, + 0.005309583968482912, + 0.005575375049374998, + 0.005170500022359192, + 0.005452499957755208, + 0.005557958967983723, + 0.005544874933548272, + 0.00541487499140203, + 0.005638167029246688, + 0.005783125059679151, + 0.005971374921500683, + 0.005271125002764165, + 0.005622540949843824, + 0.0056857920717448, + 0.005747125018388033, + 0.0055031250230968, + 0.005648624966852367, + 0.005574292037636042, + 0.005512875039130449, + 0.005627417005598545, + 0.005883040954358876, + 0.005877209012396634, + 0.005793333984911442, + 0.00578699994366616 + ], + "warm_median_seconds": 0.005624374491162598, + "after_warm_rss": { + "available": true, + "total_rss_kib": 115280, + "process_count": 4, + "processes": [ + { + "pid": 39644, + "parent_pid": 39453, + "rss_kib": 9840, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39648, + "parent_pid": 39644, + "rss_kib": 4848, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39662, + "parent_pid": 39648, + "rss_kib": 43856, + "executable": "" + }, + { + "pid": 39665, + "parent_pid": 39662, + "rss_kib": 56736, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.37756395898759365, + "warm_cell_seconds": [ + 0.0057848329888656735, + 0.005673791980370879, + 0.005697875050827861, + 0.005660124938003719, + 0.005501999985426664, + 0.005408082972280681, + 0.0054963750299066305, + 0.0052998330211266875, + 0.005599708063527942, + 0.005612333072349429, + 0.005495375022292137, + 0.0053740410367026925, + 0.005313417059369385, + 0.005218207952566445, + 0.005489042028784752, + 0.0063853750471025705, + 0.005732416990213096, + 0.005608249921351671, + 0.0058224579552188516, + 0.005902000004425645, + 0.005541207967326045, + 0.005221667001023889, + 0.0057512500789016485, + 0.005889042047783732, + 0.005519790924154222, + 0.0056330839870497584, + 0.005584875005297363, + 0.005637500085867941, + 0.005426167044788599, + 0.005137874977663159 + ], + "warm_median_seconds": 0.005592291534412652, + "after_warm_rss": { + "available": true, + "total_rss_kib": 115040, + "process_count": 4, + "processes": [ + { + "pid": 39799, + "parent_pid": 39453, + "rss_kib": 9824, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39800, + "parent_pid": 39799, + "rss_kib": 4832, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39801, + "parent_pid": 39800, + "rss_kib": 43632, + "executable": "" + }, + { + "pid": 39802, + "parent_pid": 39801, + "rss_kib": 56752, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.384810124989599, + "warm_cell_seconds": [ + 0.005655167042277753, + 0.004154499969445169, + 0.005249292007647455, + 0.005507249967195094, + 0.004268749966286123, + 0.005651207990013063, + 0.005624666111543775, + 0.00527616695035249, + 0.005287292064167559, + 0.005260916077531874, + 0.005439499975182116, + 0.005316708004102111, + 0.00551912491209805, + 0.0051024999702349305, + 0.0054250420071184635, + 0.004058666992932558, + 0.005423500086180866, + 0.005195750040002167, + 0.005208916030824184, + 0.005331250024028122, + 0.005309707950800657, + 0.005373666994273663, + 0.005257166922092438, + 0.005390833015553653, + 0.005197250051423907, + 0.005251999944448471, + 0.005425167037174106, + 0.005593250039964914, + 0.005461334018036723, + 0.0053210839396342635 + ], + "warm_median_seconds": 0.005318895971868187, + "after_warm_rss": { + "available": true, + "total_rss_kib": 115744, + "process_count": 4, + "processes": [ + { + "pid": 39934, + "parent_pid": 39453, + "rss_kib": 9808, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39935, + "parent_pid": 39934, + "rss_kib": 4880, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39936, + "parent_pid": 39935, + "rss_kib": 44096, + "executable": "" + }, + { + "pid": 39937, + "parent_pid": 39936, + "rss_kib": 56960, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.37861758400686085, + "warm_cell_seconds": [ + 0.005607375060208142, + 0.005428000004030764, + 0.005310083040967584, + 0.005349499988369644, + 0.005450457916595042, + 0.005378208006732166, + 0.0053815830033272505, + 0.005305332946591079, + 0.0056183330016210675, + 0.005454250029288232, + 0.005278417025692761, + 0.005345833022147417, + 0.005286542000249028, + 0.005971333011984825, + 0.005577874951995909, + 0.004077999969013035, + 0.005295249982737005, + 0.005208084010519087, + 0.005188083043321967, + 0.005226125009357929, + 0.005233249976299703, + 0.005485582980327308, + 0.00521637499332428, + 0.005207665963098407, + 0.005187290953472257, + 0.005476083024404943, + 0.00543620903044939, + 0.003997999941930175, + 0.005346041987650096, + 0.005179500090889633 + ], + "warm_median_seconds": 0.0053279580315575, + "after_warm_rss": { + "available": true, + "total_rss_kib": 114944, + "process_count": 4, + "processes": [ + { + "pid": 40103, + "parent_pid": 39453, + "rss_kib": 9808, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40104, + "parent_pid": 40103, + "rss_kib": 4816, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40105, + "parent_pid": 40104, + "rss_kib": 43552, + "executable": "" + }, + { + "pid": 40106, + "parent_pid": 40105, + "rss_kib": 56768, + "executable": "" + } + ] + } + } + ], + "first_result_median_seconds": 0.384810124989599, + "warm_cell_median_seconds": 0.005502562504261732 + }, + "builtin-r": { + "status": "ok", + "samples": [ + { + "first_result_seconds": 1.1505356249399483, + "warm_cell_seconds": [ + 0.003351332969032228, + 0.00390008301474154, + 0.003235542098991573, + 0.003226041910238564, + 0.003253249917179346, + 0.003242083010263741, + 0.0027837080415338278, + 0.0028420418966561556, + 0.0039921250427141786, + 0.003219250007532537, + 0.0031452090479433537, + 0.003165624919347465, + 0.0031389580108225346, + 0.003055499983020127, + 0.003132832935079932, + 0.0031108749099075794, + 0.0026895420160144567, + 0.003153208992443979, + 0.0030744579853489995, + 0.003103082999587059, + 0.0030733749736100435, + 0.00313258299138397, + 0.0031068329699337482, + 0.0031169999856501818, + 0.0027477910043671727, + 0.003077792003750801, + 0.003167457995004952, + 0.0030804580310359597, + 0.0024820410180836916, + 0.003073125029914081 + ], + "warm_median_seconds": 0.0031247914885170758, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132448, + "process_count": 4, + "processes": [ + { + "pid": 39513, + "parent_pid": 39453, + "rss_kib": 10160, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39514, + "parent_pid": 39513, + "rss_kib": 5152, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39533, + "parent_pid": 39513, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39534, + "parent_pid": 39533, + "rss_kib": 112416, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1364903330104426, + "warm_cell_seconds": [ + 0.003305832971818745, + 0.0032245409674942493, + 0.003144332906231284, + 0.0036057090619578958, + 0.0056154580088332295, + 0.0037070419639348984, + 0.0031092079589143395, + 0.0031226660357788205, + 0.003057667054235935, + 0.0030462919967249036, + 0.003086916054598987, + 0.003100166912190616, + 0.00315912498626858, + 0.003013458917848766, + 0.0030646249651908875, + 0.0030437089735642076, + 0.003183332970365882, + 0.00302979105617851, + 0.003034000052139163, + 0.003041959018446505, + 0.003095082938671112, + 0.0031974170124158263, + 0.0033461250131949782, + 0.003230875008739531, + 0.0030793750192970037, + 0.003088042023591697, + 0.003047417034395039, + 0.0031014580745249987, + 0.003048375016078353, + 0.0030350839952006936 + ], + "warm_median_seconds": 0.003097624925430864, + "after_warm_rss": { + "available": true, + "total_rss_kib": 133024, + "process_count": 4, + "processes": [ + { + "pid": 39669, + "parent_pid": 39453, + "rss_kib": 10304, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39670, + "parent_pid": 39669, + "rss_kib": 5168, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39688, + "parent_pid": 39669, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39689, + "parent_pid": 39688, + "rss_kib": 112832, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1373379160650074, + "warm_cell_seconds": [ + 0.0034629589645192027, + 0.0032864169916138053, + 0.0032352919224649668, + 0.0028216250939294696, + 0.0031354579841718078, + 0.0030307499691843987, + 0.0030728339916095138, + 0.0030679579358547926, + 0.003082999959588051, + 0.0030567500507459044, + 0.00305499997921288, + 0.0030446669552475214, + 0.003042458090931177, + 0.0035673329839482903, + 0.0032754159765318036, + 0.003171792021021247, + 0.0033169580856338143, + 0.0032043750397861004, + 0.0031019169837236404, + 0.0030977920396253467, + 0.0031608750578016043, + 0.003147665993310511, + 0.003039209055714309, + 0.0030826249858364463, + 0.0032243330497294664, + 0.0030262910295277834, + 0.003055750043131411, + 0.002752874977886677, + 0.0030539999715983868, + 0.0030839579412713647 + ], + "warm_median_seconds": 0.0030834789504297078, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132896, + "process_count": 4, + "processes": [ + { + "pid": 39806, + "parent_pid": 39453, + "rss_kib": 10304, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39807, + "parent_pid": 39806, + "rss_kib": 5168, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39825, + "parent_pid": 39806, + "rss_kib": 4704, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39826, + "parent_pid": 39825, + "rss_kib": 112720, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.1516064580064267, + "warm_cell_seconds": [ + 0.0033205829095095396, + 0.0033381249522790313, + 0.003122915979474783, + 0.0032833339646458626, + 0.003184916917234659, + 0.0031016250140964985, + 0.003077874891459942, + 0.0031305419979617, + 0.0028272500494495034, + 0.0030798340449109674, + 0.0031027919612824917, + 0.003051459090784192, + 0.0030563330510631204, + 0.0030199169414117932, + 0.0030358340591192245, + 0.0030742500675842166, + 0.0030982919270172715, + 0.0031668340088799596, + 0.003120583016425371, + 0.00264879094902426, + 0.003117250045761466, + 0.003099541994743049, + 0.003049249993637204, + 0.0031055830186232924, + 0.003006207989528775, + 0.003025541896931827, + 0.003132999991066754, + 0.003065458033233881, + 0.0030500839930027723, + 0.0030366670107468963 + ], + "warm_median_seconds": 0.0030890629859641194, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132480, + "process_count": 4, + "processes": [ + { + "pid": 39941, + "parent_pid": 39453, + "rss_kib": 10320, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39942, + "parent_pid": 39941, + "rss_kib": 5136, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39962, + "parent_pid": 39941, + "rss_kib": 4704, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39963, + "parent_pid": 39962, + "rss_kib": 112320, + "executable": "/target/release/mcp-console" + } + ] + } + }, + { + "first_result_seconds": 1.19503545796033, + "warm_cell_seconds": [ + 0.003326167003251612, + 0.0031934159342199564, + 0.0032398339826613665, + 0.0030634159920737147, + 0.0031009590020403266, + 0.003033875022083521, + 0.0029955829959362745, + 0.0030789999291300774, + 0.0026607910403981805, + 0.0036102919839322567, + 0.0031075410079210997, + 0.0030454170191660523, + 0.0030827090376988053, + 0.0031403329921886325, + 0.0031136670149862766, + 0.0030812500044703484, + 0.0018530419329181314, + 0.0031032090773805976, + 0.003021541051566601, + 0.0030638331081718206, + 0.0018017920665442944, + 0.003042208030819893, + 0.0035797921009361744, + 0.0026322080520913005, + 0.0028629580046981573, + 0.0030792499892413616, + 0.00322862493339926, + 0.0030267080292105675, + 0.0030522909946739674, + 0.003460209001787007 + ], + "warm_median_seconds": 0.0030791249591857195, + "after_warm_rss": { + "available": true, + "total_rss_kib": 132912, + "process_count": 4, + "processes": [ + { + "pid": 40110, + "parent_pid": 39453, + "rss_kib": 10352, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40111, + "parent_pid": 40110, + "rss_kib": 5152, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40129, + "parent_pid": 40110, + "rss_kib": 4720, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40130, + "parent_pid": 40129, + "rss_kib": 112688, + "executable": "/target/release/mcp-console" + } + ] + } + } + ], + "first_result_median_seconds": 1.1505356249399483, + "warm_cell_median_seconds": 0.0030964374891482294 + }, + "ark-tcp": { + "status": "ok", + "samples": [ + { + "first_result_seconds": 0.9024117919616401, + "warm_cell_seconds": [ + 0.004792334046214819, + 0.004370582988485694, + 0.004266917007043958, + 0.004529042053036392, + 0.004490041057579219, + 0.0044727500062435865, + 0.004304457921534777, + 0.00436362496111542, + 0.004065832938067615, + 0.003048541024327278, + 0.004092207993380725, + 0.004146000021137297, + 0.004291916033253074, + 0.0038889170391485095, + 0.004111583926714957, + 0.004128333064727485, + 0.004395166994072497, + 0.002849832992069423, + 0.004082791041582823, + 0.004115500021725893, + 0.0043880000011995435, + 0.004172166925854981, + 0.004266332951374352, + 0.004144124919548631, + 0.004387791967019439, + 0.0041443329537287354, + 0.004070875002071261, + 0.004066042019985616, + 0.004402292077429593, + 0.004614791017957032 + ], + "warm_median_seconds": 0.0042192499386146665, + "after_warm_rss": { + "available": true, + "total_rss_kib": 167984, + "process_count": 4, + "processes": [ + { + "pid": 39545, + "parent_pid": 39453, + "rss_kib": 9792, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39546, + "parent_pid": 39545, + "rss_kib": 4800, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39547, + "parent_pid": 39546, + "rss_kib": 44464, + "executable": "" + }, + { + "pid": 39548, + "parent_pid": 39547, + "rss_kib": 108928, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.8935326670762151, + "warm_cell_seconds": [ + 0.004705584025941789, + 0.0042968339985236526, + 0.004460916970856488, + 0.004920499981380999, + 0.0047899160999804735, + 0.004242291091941297, + 0.005013416986912489, + 0.0047796249855309725, + 0.004167790990322828, + 0.004499958944506943, + 0.004896665923297405, + 0.004603874986059964, + 0.004734375048428774, + 0.004635249963030219, + 0.004597584018483758, + 0.004436583025380969, + 0.004559791064821184, + 0.0044285409385338426, + 0.0041805419605225325, + 0.004375832970254123, + 0.004253207938745618, + 0.004248459008522332, + 0.004219833062961698, + 0.004314459045417607, + 0.004297625040635467, + 0.004482791991904378, + 0.004672665963880718, + 0.004434000002220273, + 0.0043992920545861125, + 0.004413874936290085 + ], + "warm_median_seconds": 0.004448749998118728, + "after_warm_rss": { + "available": true, + "total_rss_kib": 168016, + "process_count": 4, + "processes": [ + { + "pid": 39700, + "parent_pid": 39453, + "rss_kib": 9792, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39701, + "parent_pid": 39700, + "rss_kib": 4832, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39702, + "parent_pid": 39701, + "rss_kib": 44480, + "executable": "" + }, + { + "pid": 39704, + "parent_pid": 39702, + "rss_kib": 108912, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.8768766249995679, + "warm_cell_seconds": [ + 0.0049030419904738665, + 0.004868416930548847, + 0.004402165999636054, + 0.004557250067591667, + 0.004290165961720049, + 0.005003041937015951, + 0.00473108293954283, + 0.004387625027447939, + 0.004371082992292941, + 0.004661125014536083, + 0.0045342500088736415, + 0.00448645802680403, + 0.004334000055678189, + 0.004252874990925193, + 0.004167458973824978, + 0.0042507919715717435, + 0.004255209001712501, + 0.004185125115327537, + 0.004594625090248883, + 0.004488999955356121, + 0.00432720803655684, + 0.004222000017762184, + 0.004257416934706271, + 0.004147375002503395, + 0.004102082923054695, + 0.004209749982692301, + 0.004231292055919766, + 0.004305958980694413, + 0.00427375000435859, + 0.0041252500377595425 + ], + "warm_median_seconds": 0.0043165835086256266, + "after_warm_rss": { + "available": true, + "total_rss_kib": 166208, + "process_count": 4, + "processes": [ + { + "pid": 39839, + "parent_pid": 39453, + "rss_kib": 9824, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39840, + "parent_pid": 39839, + "rss_kib": 4848, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39841, + "parent_pid": 39840, + "rss_kib": 44544, + "executable": "" + }, + { + "pid": 39842, + "parent_pid": 39841, + "rss_kib": 106992, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.9020907919621095, + "warm_cell_seconds": [ + 0.003556750016286969, + 0.0046062079491093755, + 0.004375541931949556, + 0.004369833972305059, + 0.004244333948008716, + 0.004410124965943396, + 0.004340042010881007, + 0.0042016670340672135, + 0.004181374912150204, + 0.0042497089598327875, + 0.004398999968543649, + 0.0042120839934796095, + 0.004246750031597912, + 0.004264916991814971, + 0.004452832974493504, + 0.004405667074024677, + 0.004646916058845818, + 0.0034041249891743064, + 0.004845792078413069, + 0.004560041939839721, + 0.004514458007179201, + 0.004548292024992406, + 0.004219167050905526, + 0.00407000002451241, + 0.00394099997356534, + 0.004057707963511348, + 0.0041929170256480575, + 0.004178833100013435, + 0.004314749967306852, + 0.0042247920064255595 + ], + "warm_median_seconds": 0.004257312975823879, + "after_warm_rss": { + "available": true, + "total_rss_kib": 167568, + "process_count": 4, + "processes": [ + { + "pid": 39976, + "parent_pid": 39453, + "rss_kib": 9808, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39977, + "parent_pid": 39976, + "rss_kib": 4800, + "executable": "/target/release/mcp-console" + }, + { + "pid": 39978, + "parent_pid": 39977, + "rss_kib": 44336, + "executable": "" + }, + { + "pid": 39979, + "parent_pid": 39978, + "rss_kib": 108624, + "executable": "" + } + ] + } + }, + { + "first_result_seconds": 0.8730657499982044, + "warm_cell_seconds": [ + 0.004804083029739559, + 0.004398791003040969, + 0.004363749991171062, + 0.004208249971270561, + 0.004202749929390848, + 0.0043712499318644404, + 0.00410925003234297, + 0.0039952920051291585, + 0.004068083944730461, + 0.0038820409681648016, + 0.004170542000792921, + 0.004052250063978136, + 0.00429108296521008, + 0.004242792027071118, + 0.004151166998781264, + 0.004154708003625274, + 0.004058084101416171, + 0.004148166975937784, + 0.004163541947491467, + 0.004010665928944945, + 0.004047750029712915, + 0.004615457961335778, + 0.004204375087283552, + 0.004083958105184138, + 0.004065208020620048, + 0.004099708050489426, + 0.004186167032457888, + 0.004071249975822866, + 0.0041617919923737645, + 0.0041600000113248825 + ], + "warm_median_seconds": 0.004157354007475078, + "after_warm_rss": { + "available": true, + "total_rss_kib": 165616, + "process_count": 4, + "processes": [ + { + "pid": 40141, + "parent_pid": 39453, + "rss_kib": 9824, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40142, + "parent_pid": 40141, + "rss_kib": 4816, + "executable": "/target/release/mcp-console" + }, + { + "pid": 40143, + "parent_pid": 40142, + "rss_kib": 44528, + "executable": "" + }, + { + "pid": 40144, + "parent_pid": 40143, + "rss_kib": 106448, + "executable": "" + } + ] + } + } + ], + "first_result_median_seconds": 0.8935326670762151, + "warm_cell_median_seconds": 0.004270333505701274 + } + }, + "source_revision": "79b55739d12592fb1dde0e2858325bad0056bac3" +} diff --git a/docs/API.md b/docs/API.md index 96d0d3d72..619a26b50 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,19 +10,24 @@ One MCP connection owns one persistent session. Send at most one complete source cell per call. Collect its result before submitting another cell. -| Argument | Meaning | -| -------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| `r`, `python`, `sql` | Optional source string. At most one may be non-null. The configured language selection determines which fields are exposed. | -| `timeout_ms` | Nonnegative integer; maximum observation wait, normally 60,000 milliseconds. It does not cancel execution. | -| `stdin` | Optional string of exact input bytes encoded as UTF-8. No newline is added. | -| `control` | `interrupt` or `restart`. | -| `requirements` | Dependency declaration or inspection action; see [Requirements](#requirements). | +| Argument | Meaning | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `r`, `python`, `sql` | Optional source string. At most one may be non-null. The configured language selection determines which fields are exposed. | +| `code` | Native source for the selected [Jupyter kernel](JUPYTER.md). Available only with `--jupyter-kernel`; cannot accompany another source field. | +| `timeout_ms` | Nonnegative integer; maximum observation wait, normally 60,000 milliseconds. It does not cancel execution. | +| `stdin` | Optional string of exact input bytes encoded as UTF-8. No newline is added. | +| `control` | `interrupt` or `restart`. | +| `requirements` | Dependency declaration or inspection action; see [Requirements](#requirements). | Unknown fields and incorrect types are errors. Omit unused arguments. A hidden language field is rejected even when its value is `null`. `timeout_ms` cannot be null. +Jupyter mode exposes `code` instead of the built-in language fields. +In Jupyter mode, `requirements` supports only `action: "get"`; install packages in the kernel environment before launch. +Kernel-specific input, display, interruption, and export limits are described in [Jupyter kernels](JUPYTER.md). + Examples below are **tool argument objects**, not complete JSON-RPC messages: ```json diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5c79034fa..20e04eb2c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,6 +34,11 @@ The frontend verifies the pinned companion before launch. On Unix it becomes the runner through `exec`; on Windows it waits for the runner. Console must not introduce another native supervisor that competes for the runner's resources. +The public [Jupyter mode](JUPYTER.md) uses `relay → Python adapter → local Jupyter kernel` inside the same execution boundary. +The compile-time embedded [adapter](../src/jupyter/worker.py) owns the kernel child, translates Jupyter messages into Console worker events, and reaps the kernel before its own normal exit. +The relay owns the adapter; native enforcement and descendant cleanup remain with the runner. +The installed kernelspec supplies the runtime and its packages, so this mode does not use Console's dependency preparation or built-in interpreter coordination. + ## Ownership | Owner | Responsibility | @@ -62,6 +67,7 @@ The adapter removes the file after synchronous materialization or the server's i One connection-owned background task discovers runtimes, prepares defaults, and prelaunches the built-in worker. MCP initialization, tool discovery, and pings do not wait. Workers supplied through internal testing hooks are launched on demand. +Jupyter adapters also launch on demand; tool discovery and dependency inspection do not start the kernel. With the `inherit_codex` sandbox profile, startup waits for the first `send` to capture client authority before creating any preparation or execution process. Capture happens in transport request order before asynchronous handler dispatch; a cancelled first request cannot let a later request select the connection's permissions. diff --git a/docs/JUPYTER.md b/docs/JUPYTER.md new file mode 100644 index 000000000..0bceff14e --- /dev/null +++ b/docs/JUPYTER.md @@ -0,0 +1,290 @@ +# Jupyter kernels + +`mcp-console serve --jupyter-kernel NAME` runs an installed local Jupyter kernel instead of Console's built-in worker. +The MCP client submits native source through `send(code=...)` and keeps the same polling, stdin, interrupt, and restart controls. +This mode supports macOS and Linux. +Windows is rejected at launch; its inherited handles, interrupt events, and process retirement need a separate adapter implementation. + +The adapter uses published `jupyter_client` and the selected kernelspec. +It does not bundle, patch, or build the kernel. +The tested releases are ark 0.1.252, ipykernel 7.4.0, jupyter_client 8.10.0, and pyzmq 27.2.0, with Python 3.13.16 and R 4.6.1 on macOS arm64 and Linux x86_64. +Ark 0.1.252 is a published prerelease. +Other standard local kernelspecs use the same protocol, but have not been validated here. + +## Start ipykernel + +Create a Python 3.11+ environment containing the client and kernel: + +```sh +uv venv .venv-jupyter --python 3.13 +uv pip install --python .venv-jupyter/bin/python \ + 'jupyter_client==8.10.0' 'ipykernel==7.4.0' +``` + +Select that interpreter explicitly for the adapter: + +```sh +mcp-console serve --no-config \ + --jupyter-python "$PWD/.venv-jupyter/bin/python" \ + --jupyter-kernel python3 +``` + +The `python3` kernelspec provided by ipykernel uses that environment, unless a higher-priority kernelspec with the same name overrides it. +Inspect the resolved specification with `.venv-jupyter/bin/jupyter kernelspec list --json` when an existing Jupyter installation also defines `python3`. +This example uses the default read-only sandbox, including private kernel communication and temporary storage. +Host networking and ordinary workspace writes remain blocked. +Unless explicitly selected in the environment, IPython's profile/history and Matplotlib's cache use the worker's disposable temporary storage. +Install analysis libraries in the kernel environment before launch. +Console's optional `jupyter` Python extra installs `jupyter_client`, but not ipykernel, ark, R, or reticulate. + +The MCP client can then submit: + +```json +{ "code": "answer = 40\nanswer + 2" } +``` + +The next call can reuse `answer`. +Use `code` for native Python in this mode; `python`, `r`, and `sql` are hidden and rejected in this mode. + +The Python clients accept the same field and can pass the kernel selection through their launch arguments: + +```python +from mcp_console import MCPConsole + +with MCPConsole( + args=[ + "serve", + "--no-config", + "--jupyter-python", + "/absolute/path/to/.venv-jupyter/bin/python", + "--jupyter-kernel", + "python3", + ] +) as console: + print(console.send(code="answer = 40\nanswer + 2")) +``` + +`AsyncMCPConsole.send()` also accepts `code`. +The Python framework adapters use the discovered tool schema. +The R `console_tool()` wrapper derives its input fields from that schema, but its constructor currently has no kernel-selection setting or extra server arguments. +Use an MCP client that accepts the `serve` flags to select Jupyter directly; the ordinary R constructor does not yet expose this launch mode. + +## Start ark + +Download the matching binary from the [published ark release](https://github.com/posit-dev/ark/releases/tag/0.1.252) and install R separately. +Use ark's `--install` command to register its kernel, or create `.jupyter/kernels/ark/kernel.json` explicitly: + +```json +{ + "argv": [ + "/absolute/path/to/ark", + "--connection_file", + "{connection_file}", + "--session-mode", + "notebook", + "--", + "--vanilla" + ], + "display_name": "Ark R", + "language": "R", + "env": { "RUST_LOG": "error" } +} +``` + +On Linux, ark's installer also configures the R library loader path; a manually written kernelspec must make the selected R installation's shared libraries discoverable. +Use the following launch with the manually written spec: + +```sh +JUPYTER_PATH="$PWD/.jupyter" \ +mcp-console serve --no-config \ + --jupyter-python "$PWD/.venv-jupyter/bin/python" \ + --jupyter-kernel ark +``` + +No network or filesystem grants are needed for kernel startup with the default sandbox. +Console automatically selects private communication appropriate to the platform; see [Private kernel communication](#private-kernel-communication). +`--no-sandbox` remains available when host permissions are intended. + +Submit native R through `code`: + +```json +{ "code": "answer <- 40L; answer + 2L" } +``` + +## Configuration and ownership + +| Option | Meaning | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `--jupyter-kernel NAME` | Select a locally discoverable kernelspec by name. | +| `--jupyter-python PATH` | Python 3.11+ with `jupyter_client`; defaults to `python3` on PATH. This is the adapter interpreter, not necessarily the kernel interpreter. | +| `--jupyter-transport auto` | Default: private Unix sockets on restricted macOS; private namespace TCP on restricted Linux; ordinary loopback TCP with host permissions. | +| `--jupyter-transport tcp` | Authenticated loopback TCP using the standard connection file. | +| `--jupyter-transport ipc` | Unix-domain sockets in private temporary storage; requires kernel IPC support. | + +Kernel selection is currently a `serve` launch option, not a YAML setting. +Built-in `startup`, `python`, `r`, and `languages` settings conflict with this mode rather than silently changing meaning. +Use kernelspec `argv` and `env` entries for kernel initialization and environment selection. +Kernelspec discovery follows `jupyter_client`, including `JUPYTER_PATH`. +Arguments support `{connection_file}`, `{resource_dir}`, `{prefix}`, and tilde expansion; environment values support Jupyter's environment-variable substitution. +Kernelspecs are trusted launch configuration with executable and environment access. +Remote kernels, existing connection files, custom provisioners, and daemonizing launchers are outside this mode's contract. + +Console owns the adapter; the adapter owns and reaps its direct kernel child. +It keeps the child in the existing process group instead of using `KernelManager`'s detached-session launcher. +Normal shutdown requests kernel shutdown, then terminates or kills an unresponsive child within the relay's retirement budget. +Kernel death wakes an idle or evaluating adapter and fails that worker; restart creates a fresh kernel and fresh connection credentials. +Readiness requires a matching kernel-information reply and IOPub status within 30 seconds; a missing heartbeat during startup does not imply process death. +Replies and idle messages remain correlated across readiness retries, so a delayed channel does not invalidate an earlier request. +Readiness forwards received startup text and errors through process stdout/stderr before announcing the worker ready. +Messages published before subscription and startup rich displays are not guaranteed to be captured. +Interrupting startup retires the acquired child and discards the pending source before evaluation. +Native sandbox supervision retains ownership of descendants. +Outside a sandbox, neither arbitrary kernel subprocesses nor an adapter killed externally with `SIGKILL` have a general descendant-cleanup guarantee. + +## Assessment + +Ark delegates R execution and graphics to a published kernel. +Standalone ipykernel provides Python expression display, tracebacks, magics, and async execution. +The private transports make these capabilities usable under Console's default read-only sandbox on macOS and Linux. +The main benefit is reuse of the kernels' language behavior and ecosystem; Console still owns MCP translation, sandbox policy, and lifecycle coordination. + +The built-in worker should remain the default while managed dependencies, SQL sessions, Windows support, and the existing runtime contracts matter. +This implementation provides a practical alternative for preinstalled R/Python environments, but replacing the default would remove those features and add installation, compatibility, output-delivery, and operational costs described below. +It also increases the number of externally selected components; it does not currently produce a smaller Console distribution. + +## Differences from the built-in worker + +| Area | Jupyter behavior and limitation | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Languages | One kernel's native language per session. The adapter does not combine kernels or expose a cross-language bridge. | +| R/Python objects | Console's custom cross-language helpers and conversion behavior are not installed. Standalone ipykernel has no automatic R bridge. | +| Packages | No host-side preparation, automatic missing-import installation, live dependency activation, version selection, or managed DuckDB extensions. `requirements={action: "get"}` reports an empty Console declaration, not an installed-package inventory. Mutation requests fail before code, input, or control effects. | +| SQL | No `sql` source field, selected connection registry, automatic data-frame registration, or retained last-query result. Installed language packages can execute SQL explicitly. | +| Startup | Lazy kernel launch on the first operation requiring it. Kernelspec launch failures and a 30-second readiness timeout fail the worker. Built-in startup scripts and interpreter discovery do not run. `warmup` does not select or exercise this mode; a configured client's background warmup may only validate its MCP handshake. | +| Text and errors | Kernel streams and tracebacks are preserved. Ordinary language errors complete the cell without an MCP tool error, as in the built-in worker. Formatting, ANSI escapes, warnings, execution counts, and traceback frames differ by kernel/version. | +| Rich output | PNG, JPEG, GIF, and WebP are forwarded. Otherwise the kernel's `text/plain` representation is used. HTML, SVG, PDF, JavaScript, widgets, comms, debugger/UI integrations, and shell payloads are not implemented. Displays without a supported representation produce an omission notice. | +| Display updates | `update_display_data` appends a marked update; `clear_output` adds a notice and retains earlier output. Previously delivered MCP content cannot be replaced. There is no persistent notebook display-ID model. | +| Input | Jupyter `input_request` receives one newline-terminated UTF-8 reply. Partial lines wait; extra lines remain queued. Direct kernel reads from OS stdin see EOF. Password prompts use the same recorded stdin interface; it is not a secret-input channel. | +| Interrupt | The adapter uses the kernelspec's signal or control-message interrupt mode. Kernel and native-library behavior determine responsiveness. | +| Ordering | Completion waits for the matching shell reply and IOPub idle, independent of their arrival order. Late output remains visible, but output from background work may arrive after a cell or during another cell; Jupyter does not supply a universal cross-channel chronology. | +| Output volume | Console's response and recording limits still apply after translation. The adapter bounds its event queue by count, but Jupyter messages, ZeroMQ queues, MIME bundles, and queued stdin have additional allocations. This is not an end-to-end bounded-memory guarantee. | +| Platforms | Native ark and ipykernel were tested on macOS arm64 and Linux x86_64, including the default read-only sandbox. Native Windows is unavailable. Other kernels require their own validation. | + +Jupyter IOPub uses ZeroMQ publish/subscribe delivery. +[ZeroMQ can drop published messages](https://libzmq.readthedocs.io/en/latest/zmq_socket.html#zmq-pub) when a subscriber's queue reaches its high-water limit; the adapter has no replay or delivery acknowledgment mechanism. +Consequently, a kernel that floods output can lose output before Console records it, and a lost terminal status can leave the operation pending until restart. +The acceptance tests do not establish lossless delivery under sustained output overload. + +## Private kernel communication + +The default `auto` transport allows the adapter and kernel to communicate without granting access to host services or external networks. +Console adds the required private communication policy only to Jupyter worker launches. +The built-in worker and standalone sandbox commands retain their existing permissions. + +On macOS, the kernel uses filesystem Unix sockets beneath the native runner's private temporary directory. +A Seatbelt extension permits binding and connecting only beneath that directory; TCP and host Unix sockets remain denied. +The runner supplies the directory to the policy, rather than accepting its location from the workload environment. +The adapter creates local aliases for published ark's `prefix:port` naming and Jupyter's `prefix-port` naming, without changing either kernel. +Private IPC requires `sandbox.tempdir: managed`, the default. +Unix socket paths have OS length limits; unusually long temporary-directory paths can prevent startup. +Published ipykernel also attempts a TCP socket for forwarding output from forked children, even when its Jupyter channels use IPC. +macOS denies that optional socket; ipykernel warns and disables that forwarding. +Do not assume that Python output from forked children is captured. +Its shutdown may also print a `psutil` permission traceback while enumerating processes; Console and the native supervisor still retire the worker. + +On Linux, the pinned runner's restricted syscall policy cannot selectively permit filesystem Unix sockets. +Console instead requests its existing managed-network path with a deny-all proxy, which creates a private network namespace. +The kernel and adapter communicate over that namespace's loopback TCP. +Host loopback, external routes, and host Unix sockets remain inaccessible; the proxy permits no destinations, SOCKS connections, or upstream proxy. +This adds native proxy/namespace setup and processes even though the session cannot access the network. +It does not grant `allow_local_binding` on the host or change filesystem permissions. +Explicitly configured network permissions retain their selected behavior. + +The standard `python -m ipykernel_launcher` kernelspec receives a Console-owned bootstrap inside that namespace. +Published libzmq 4.3.5 enumerates interfaces when binding a numeric TCP address and aborts when the sandbox denies its `AF_NETLINK` socket. +The bootstrap uses pyzmq's socket subclass hook to bind to the namespace's interfaces without that enumeration. +It runs only after native namespace isolation is selected; it is never used with host networking or `--no-sandbox`. +Custom kernel launchers keep their original command and may need their own compatibility handling. +The published Jupyter client's interface-discovery warning is retained. + +TCP connections use private credentials and HMAC authentication without encryption. +The default Linux namespace supplies isolation from host networking; it does not make the kernel and adapter separate trust domains. +Kernel startup warnings remain visible, including ipykernel's TCP warning. +Restart discards the kernel's private temporary files and default IPython history along with interpreter state. +Use explicit writable storage if history must outlive the worker. +Matplotlib's default font cache is also disposable, so the first plot after restart may rebuild it and take substantially longer than a warm plot. + +## History and exports + +Console retains translated text, images, source, input, and lifecycle evidence using its existing recording machinery. +Generic `code` is saved as text. +Jupyter session QMDs identify the kernelspec, contain non-executable source fences, and set `execute.eval: false`. +They are source archives with no promise of IR or Quarto replay. +They do not lock kernel versions, interpreter versions, environments, or external inputs. + +Native ipykernel also maintains IPython history when its configured profile is writable. +IPython adds its own namespace entries and in-memory output cache, so displayed results can remain referenced after a cell completes. +That cache is separate from Console's response and recording limits and can retain large objects. +Other kernels own their own history behavior. +Console does not currently expose Jupyter `history_request`, export `.ipynb`, retain complete MIME bundles/raw Jupyter messages, or save widget/comm state. +Execution counts and display metadata, including display IDs, are not retained as structured fields. +A future notebook exporter could use source, execution replies, and display messages directly. +The current archives are not live-session checkpoints. + +## Performance and distribution + +The extra Python adapter and ZeroMQ channels add another protocol hop between Console and the interpreter. +Messages incur Jupyter framing and authentication, then translation into Console's existing JSON sideband protocol. +Base64 images pass through both protocols; the adapter does not eliminate the encoding and recording costs of the built-in worker. +Kernel startup also includes its own history, display, debugger, event-loop, and import machinery. +For long computations these costs may be small relative to execution; short cells and large MIME output are more sensitive. +The reproducible [Jupyter benchmark](../benchmarks/jupyter.py) compares the completed MCP path with matched trivial assignments and records startup, warm latency, and process RSS. +Its results are workload- and host-specific; kernel-only timings are not substitutes for an end-to-end comparison. +The [initial October 23 comparison](benchmarks/jupyter.md) records all samples and limitations: warm medians were 3.10–3.11 ms for the built-in worker, 5.50 ms for ipykernel, and 4.27 ms for ark. +Standalone kernels started sooner in that setup. +Startup remains sensitive to host load: a concurrent macOS test run exceeded ark's 30-second readiness budget. +Direct ark probes reproduced the slowdown outside Console; samples showed its R-discovery launcher stalled at the process loader before entering R. +This observation does not establish a kernel defect or a general startup bound. +The runtime deadline remains unchanged; the final macOS kernel verification uses limited test concurrency. + +No new Rust dependencies or kernel binaries are linked into Console for this mode. +The adapter is embedded Python source plus Rust launch/schema glue. +The built-in worker remains compiled in; this mode provides no smaller Jupyter-only executable and removes none of the existing runtime dependencies. +The optional Python extra moves Jupyter client dependencies into the selected Python environment; it does not make the executable a standalone Jupyter distribution. + +Measured external footprints on this macOS arm64 host: + +| Component | Size | +| ---------------------------------------- | --------------------: | +| Published ark arm64 ZIP | 8,456,517 bytes | +| Unpacked ark executable | 24,592,144 bytes | +| `jupyter_client` 8.10.0 and dependencies | 12,448 KiB allocated | +| ipykernel + client + NumPy | 97,876 KiB allocated | +| The same environment plus matplotlib | 162,180 KiB allocated | + +Directory figures are `du -sk` observations of temporary installations, not wheel download sizes or incremental physical disk use. +The Python environment rows overlap: the ipykernel and matplotlib rows already include the client dependencies and should not be summed. +They exclude Python, R, user data/packages, download caches, and later history/bytecode caches; uv may share cached files. +Ark's release also distributes separate macOS x64/universal, Linux arm64/x64, and Windows arm64/x64 archives. +Bundling these components would add platform selection, native pyzmq libraries, interpreter/R installations, notices/licenses, updates, and compatibility testing. +Keeping them external makes installation less self-contained and requires the caller to select mutually compatible releases. + +## Validation and upstream contracts + +Public acceptance cases exercise native ipykernel/ark evaluation, persistence, language errors and recovery, image output, prompted input, interruption, restart, unexpected kernel exit, process retirement, large cells, transport selection, and source archives. +They require explicitly supplied published installations: + +```sh +MCP_CONSOLE_TEST_JUPYTER_PYTHON=/absolute/path/to/python \ +MCP_CONSOLE_TEST_ARK=/absolute/path/to/ark \ +scripts/test client_server/jupyter/test_python client_server/jupyter/test_ark \ + client_server/jupyter/test_macos_sandbox \ + client_server/jupyter/test_linux_sandbox +``` + +Missing capabilities are reported as skips. +The ordinary final gate does not substitute for these optional kernel cases or validation on another platform. + +The [Jupyter messaging specification](https://jupyter-client.readthedocs.io/en/stable/messaging.html) owns the shell, IOPub, stdin, control, history, and display contracts. +The [kernelspec documentation](https://jupyter-client.readthedocs.io/en/stable/kernels.html) describes local launch metadata. +Relevant published implementation details include [ark's endpoint naming](https://github.com/posit-dev/ark/blob/0.1.252/crates/amalthea/src/connection_file.rs) and [jupyter_client's process launcher](https://github.com/jupyter/jupyter_client/blob/v8.10.0/jupyter_client/launcher.py). diff --git a/docs/README.md b/docs/README.md index 8c959aec4..90cd31bfc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ These guides introduce Console through a session with your agent: | Install Console or connect an MCP client | [Install and connect](CLIENTS.md) | | Use Console from R | [Use Console from R](R.md) | | Use Console from Python | [Use Console from Python](PYTHON.md) | +| Use an installed Jupyter kernel | [Jupyter kernels](JUPYTER.md) | | Explore data across R, Python, and SQL | [Sessions and languages](BUILTIN_RUNTIME.md) | | Use packages or an existing Python environment | [Packages and environments](REQUIREMENTS.md) | | Give the agent access to files or a service | [Sandbox and trust](SANDBOX.md) | diff --git a/docs/RECORDING.md b/docs/RECORDING.md index 378aa6cae..1b2cb48ed 100644 --- a/docs/RECORDING.md +++ b/docs/RECORDING.md @@ -38,6 +38,9 @@ Reference: [Retrieve the last SQL result](RUNTIME_REFERENCE.md#retrieve-the-last ## Turn the code into a report {#render-a-reviewed-copy} +Jupyter mode produces non-executable source archives with the selected kernelspec recorded in metadata. +See [Jupyter history and exports](JUPYTER.md#history-and-exports) for their limitations; the IR rendering workflow below applies to the built-in worker. + Each `sessions/session-NNN.qmd` is a starting point for a report, **not a replay or checkpoint**. It contains the R, Python, and SQL source associated with that worker, including failed evaluations, rejected calls, and source submitted alongside a control. Restarts and automatic worker replacement begin a separate document; an interrupt that preserves the worker stays in the same document. diff --git a/docs/SANDBOX_CONFIGURATION.md b/docs/SANDBOX_CONFIGURATION.md index 5c52b2e24..f1883aa9f 100644 --- a/docs/SANDBOX_CONFIGURATION.md +++ b/docs/SANDBOX_CONFIGURATION.md @@ -183,7 +183,9 @@ The value of `sandbox.network` or `resolver.sandbox.network` selects the network | `network: enabled` | Host networking; filesystem enforcement remains. | | `network: {proxy: {…}}` | Restricted native networking with a managed proxy. | -Worker networking defaults to restricted without a proxy. +Built-in worker networking defaults to restricted without a proxy. +The optional [Jupyter worker](JUPYTER.md#private-kernel-communication) adds private kernel communication automatically: temporary-directory Unix sockets on macOS, or isolated-namespace loopback with a deny-all proxy on Linux. +These Jupyter defaults do not permit host services or external destinations. Resolver networking defaults to its [download proxy](PACKAGE_CONFIGURATION.md#expanded-public-default). Explicit scalar values also remove that generated resolver proxy. diff --git a/docs/WORKER_PROTOCOL.md b/docs/WORKER_PROTOCOL.md index 4f8084ef3..5cd4c7040 100644 --- a/docs/WORKER_PROTOCOL.md +++ b/docs/WORKER_PROTOCOL.md @@ -5,6 +5,8 @@ It evolves in lockstep without an independent version. The `--worker` and `--relay` launch overrides are internal testing hooks, not part of Console's public API. Test worker fixtures must match the current build's protocol. +The public `--jupyter-kernel` option launches a Console-owned adapter that implements this interface and connects to a kernel through the standard Jupyter protocol; kernels do not implement Console's private messages. +See [Jupyter kernels](JUPYTER.md) for setup and supported behavior. [`src/worker_protocol.rs`](../src/worker_protocol.rs) defines the exact message objects. [`src/jsonl.rs`](../src/jsonl.rs), [`src/sideband.rs`](../src/sideband.rs), and the boundary tests define framing and endpoints. @@ -176,6 +178,22 @@ The relay can forcibly terminate and reap the direct child after grace; descenda Outside intentional retirement, unexpected sideband EOF or worker exit, including status zero, fails the generation. Closure and direct-worker outcome do not prove native cleanup. +## Jupyter adapter + +The adapter maps a public `code` cell to `evaluate` with `language: "kernel"` and forwards its source to the selected kernel. +Dependency changes are rejected by the server before dispatch; the adapter implements no preparation callbacks. + +`ready` follows Jupyter's kernel and IOPub readiness handshake. +The active execution's `busy` status produces `cell_started`. +Both its shell `execute_reply` and IOPub `idle` status must arrive before `completed`, so a shell reply cannot discard pending IOPub results. +Kernel language errors remain diagnostic output followed by completion. +Jupyter input requests consume newline-terminated strings from the generation's queued stdin; direct kernel-process stdin is unavailable. + +The relay signals and reaps the adapter, which owns the kernel child and translates interruption according to the kernelspec's interrupt mode. +Adapter shutdown requests kernel shutdown, then terminates and reaps an unresponsive kernel within the relay's retirement allowance. +Kernel exit ends the adapter even while idle. +The runner's descendant-retirement barrier still governs sandboxed replacement; adapter exit alone is not that receipt. + ## Test worker fixtures {#custom-worker-conformance} Test worker fixtures have no built-in defaults or managed Python preparation. diff --git a/docs/_quarto.yml b/docs/_quarto.yml index bc160b2a2..24350bd41 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -71,6 +71,7 @@ website: - section: Other tasks contents: - SANDBOX_COMMAND.md + - JUPYTER.md - id: build title: Build style: docked @@ -150,6 +151,7 @@ website: contents: - RESOLVER_RELEASE_AUDIT.md - benchmarks/transcript-concurrency.md + - benchmarks/jupyter.md - href: https://github.com/posit-dev/mcp-console/blob/main/RELEASE.md text: Release procedure page-footer: diff --git a/docs/benchmarks/jupyter.md b/docs/benchmarks/jupyter.md new file mode 100644 index 000000000..ef05d6c77 --- /dev/null +++ b/docs/benchmarks/jupyter.md @@ -0,0 +1,100 @@ +# Jupyter worker comparison + +The initial comparison was measured on October 23, 2026, on an Apple M4 Max with 16 logical CPUs and 128 GiB RAM, running macOS 26.7.1 arm64. +These measurements compare Console's built-in worker with the optional [Jupyter mode](../JUPYTER.md). +They describe this host, these installed releases, and trivial cells; they do not establish general performance parity. +The results retained here are the standalone-kernel subset of an experiment measured before this branch was split; they have not been remeasured on the split branch. + +## Method and provenance + +The initial comparison's implementation corresponds to commit `79b55739d12592fb1dde0e2858325bad0056bac3`. +The measured release executable is 12,375,056 bytes, with SHA-256 `8aa187a83b1b2764748c667867bdf4ae122462e42e4179c1fdf7075c8d452812`. +Its hash was checked again after the public acceptance tests. +The [benchmark program](../../benchmarks/jupyter.py) drives the public MCP connection and saves [individual measurements](../../benchmarks/results/jupyter-macos-arm64-2026-10-23.json). +The saved results retain the four standalone variants from the original five-variant experiment. +Only executable paths are normalized; their timing, memory, process IDs, version data, and original implementation revision are retained. + +The installed releases were Python 3.13.16, R 4.6.1, jupyter_client 8.10.0, ipykernel 7.4.0, pyzmq 27.2.0, ark 0.1.252, and CRAN reticulate 1.47.0. +Ark is a published prerelease. +No development checkout of a kernel or reticulate was used. + +The original experiment launched five fresh Console sessions per variant, in round-robin order across all five variants, and ran the identical assignment `x = 1` once followed by 30 warm repetitions. +The first-result interval starts before the Console process is launched and ends after the first completed assignment, including MCP initialization and tool discovery. +Warm latency measures one complete subsequent `send` call. +Each variant therefore contributes five startup samples and 150 warm-cell samples. + +The initial comparison used direct execution with host permissions, ordinary Console recording, fresh temporary workspaces, and fresh Jupyter/IPython configuration directories. +The Python executable and published R library were selected explicitly; R resolution was disabled and `ir`/`uv` were absent from the child PATH. +No dependency downloads or package installation were timed. +Installed-runtime and filesystem caches remained warm across launches. +The host was not isolated from other activity; these are descriptive samples, not a controlled estimate of statistical significance. + +The built-in variants exposed only the measured language. +Built-in Python used the custom evaluator without initializing R; built-in R did not initialize Python. +The ipykernel variant used IPC, while ark used loopback TCP because the tested ark release does not use jupyter_client's IPC endpoint convention. + +## Observations + +| Variant | First result median (s) | First result range (s) | Warm median (ms) | Warm p95 (ms) | Median summed RSS (MiB) | +| --------------- | ----------------------: | ---------------------- | ---------------: | ------------: | ----------------------: | +| Built-in Python | 1.134 | 1.115–1.259 | 3.108 | 3.421 | 129.0 | +| ipykernel, IPC | 0.385 | 0.378–0.453 | 5.503 | 6.001 | 112.4 | +| Built-in R | 1.151 | 1.136–1.195 | 3.096 | 3.520 | 129.8 | +| ark, TCP | 0.894 | 0.873–0.902 | 4.270 | 4.799 | 163.6 | + +Warm p95 is the linearly interpolated 95th percentile of all 150 warm cells for that variant. +RSS is a `ps` snapshot after warm execution, summed over Console and its descendants, then summarized across the five launches. +It excludes the benchmark client, is not peak memory, and can count shared pages more than once. +All variants had four live processes in these snapshots: the built-in path retained its preparation process, while the Jupyter path had separate adapter and kernel processes alongside Console and its relay. +Equal process counts do not imply equal responsibilities or memory ownership. + +For this assignment, ipykernel's median warm call took about 1.77 times the built-in Python call; ark's R call took about 1.38 times the built-in R call. +The absolute differences were approximately 2.40 ms and 1.17 ms. +These comparisons measure dispatch overhead with very little language work. +They do not compare Python final-expression display. + +Standalone ipykernel and ark reached their first result sooner than the built-in worker in this setup. +Jupyter is therefore not uniformly slower to start: the paths do different discovery, preparation, initialization, and protocol work. +None of these timings include a cold installation, sandbox startup, SQL, package activation, large text or image transfer, concurrent load, or a long numerical workload. +The subsequently added automatic private sandbox transports are outside these measurements: macOS adds a private-directory Unix-socket policy, while Linux starts a native deny-all proxy and private network namespace. +The Linux path therefore has additional setup and process costs that this direct-execution comparison does not quantify. +Image serialization, output queues, history writes, and extra protocol copies need their own workload measurements before selecting a mode for those cases. + +## Executable and distribution costs + +The initial comparison's Console executable is about 11.80 MiB. +The feature adds embedded Python source and Rust configuration/protocol glue; it adds no Rust dependencies and embeds no ark or ipykernel binary. +The built-in worker remains available in the same executable. +There is no Jupyter-only build that removes its code or dependencies. +A comparative clean-base binary and distribution wheels were not built for this experiment, so the exact change in executable or wheel bytes is unmeasured. + +The optional `jupyter` extra installs the client dependencies into a Python environment. +Running an R or another non-Python kernel still requires a Python 3.11+ adapter environment. +ipykernel, ark, R, and analysis packages are separate installations. +The [external footprint measurements](../JUPYTER.md#performance-and-distribution) include a 24,592,144-byte unpacked ark executable and approximately 12 MiB of installed client dependencies before the kernel or language runtimes. +Ark's published executable includes language/debug services that this adapter does not expose. +The 8,456,517-byte macOS arm64 ark download excludes R; the universal macOS archive is 17,196,572 bytes. +The [published release assets](https://github.com/posit-dev/ark/releases/tag/0.1.252) also require separate operating-system and architecture selection. + +Bundling a self-contained Jupyter distribution would require Python, native pyzmq/ZeroMQ components, the chosen kernel, its language runtime, licenses/notices, and a larger compatibility/update matrix. +Keeping them external avoids adding that payload to Console's wheels, at the cost of more installation steps and environment-selection requirements. +The measured environment sizes overlap and exclude shared caches, interpreter installations, user packages, and later generated history/bytecode; they are not estimates of total download size or incremental physical disk use. + +## Reproduce + +Build and stage Console through the repository workflow, then select published installations. +The R library must include published reticulate for the built-in worker baseline: + +```sh +MCP_CONSOLE_TEST_JUPYTER_PYTHON=/absolute/path/to/python \ +MCP_CONSOLE_TEST_ARK=/absolute/path/to/ark \ +MCP_CONSOLE_TEST_RETICULATE_LIBRARY=/absolute/path/to/R/library \ +python3 benchmarks/jupyter.py --binary target/release/mcp-console \ + --variant builtin-python --variant ipykernel-ipc --variant builtin-r \ + --variant ark-tcp \ + --launches 5 --warm-cells 30 > jupyter-results.json +``` + +Check that every variant reports `status: "ok"` and has five samples before summarizing the results. +The script records failed variants rather than substituting another runtime. +For supported behavior, sandbox requirements, lost features, export limits, and unvalidated platforms, use the [Jupyter guide](../JUPYTER.md#differences-from-the-built-in-worker). diff --git a/pyproject.toml b/pyproject.toml index cfcd06384..347095be7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ anthropic = ["anthropic[mcp]>=1.15.0", "anyio>=4.15.1", "mcp>=2.3.0,<3"] chatlas = ["chatlas[mcp]>=0.23.0", "anyio>=4.15.1", "mcp>=2.3.0,<3"] client = ["anyio>=4.15.1", "mcp>=2.3.0,<3"] +jupyter = ["jupyter_client>=8.10,<9"] codex = ["openai-codex>=0.163.0"] openai = ["openai>=3.31.0", "anyio>=4.15.1", "mcp>=2.3.0,<3"] openai-agents = ["openai-agents>=0.24.0", "anyio>=4.15.1", "mcp>=2.3.0,<3"] diff --git a/python/mcp_console/_client.py b/python/mcp_console/_client.py index be9d7d56d..f4b3dfc0b 100644 --- a/python/mcp_console/_client.py +++ b/python/mcp_console/_client.py @@ -169,6 +169,7 @@ async def _call_send(self, arguments: Mapping[str, Any]) -> "CallToolResult": async def send( self, *, + code: str | None = None, r: str | None = None, python: str | None = None, sql: str | None = None, @@ -177,15 +178,18 @@ async def send( stdin: str | None = None, timeout_ms: int | None = None, ) -> str: - """Run or control the persistent R, Python, and SQL console. + """Run or control the persistent console. - Send at most one of r, python, or sql per call. + Send at most one of code, r, python, or sql per call. + The connected server's schema determines which source fields are available. If output ends in ``[running; poll with an empty send]``, call again without code or stdin until completion before submitting another cell. `console(...)` is an alias for `send(...)`. If tool configuration hides `timeout_ms`, poll without that argument. Args: + code: One complete cell in the selected Jupyter kernel's language. + Requires a server launched with `--jupyter-kernel`. r: One complete R cell. python: One complete Python cell. sql: One complete SQL cell on the selected connection. @@ -230,6 +234,7 @@ async def main(): ``` """ arguments = { + "code": code, "r": r, "python": python, "sql": sql, diff --git a/python/mcp_console/_sync.py b/python/mcp_console/_sync.py index 2820cab4e..8bcd77fdc 100644 --- a/python/mcp_console/_sync.py +++ b/python/mcp_console/_sync.py @@ -276,6 +276,7 @@ def _call_send(self, arguments: Mapping[str, Any]) -> "CallToolResult": def send( self, *, + code: str | None = None, r: str | None = None, python: str | None = None, sql: str | None = None, @@ -287,8 +288,9 @@ def send( """Run or control a cell on this connection and return its text output. The arguments and polling behavior match - [AsyncMCPConsole.send()](AsyncMCPConsole.html#send). Send one language - per call. If output ends in `[running; poll with an empty send]`, collect + [AsyncMCPConsole.send()](AsyncMCPConsole.html#send). Use `code` for the + selected Jupyter kernel's language. Send one source field per call. + If output ends in `[running; poll with an empty send]`, collect the remaining output with `console.send()` before submitting another cell. Images appear as MIME placeholders; MCP tool errors raise `RuntimeError`. `console(...)` is an alias for `send(...)`. @@ -296,6 +298,7 @@ def send( without that argument. """ arguments = { + "code": code, "r": r, "python": python, "sql": sql, diff --git a/scripts/validate_runtime_sources.py b/scripts/validate_runtime_sources.py index d6b5606ee..588b80275 100755 --- a/scripts/validate_runtime_sources.py +++ b/scripts/validate_runtime_sources.py @@ -19,6 +19,8 @@ SOURCE_ROOT = ROOT / "src" EXPECTED_SOURCES = { + "src/jupyter/namespace_ipykernel.py", + "src/jupyter/worker.py", "src/python/bridge.R", "src/python/initialize.R", "src/python/environment.py", diff --git a/src/cell.rs b/src/cell.rs index 6d211106f..38fc44140 100644 --- a/src/cell.rs +++ b/src/cell.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] pub(crate) enum Language { + Kernel, R, Python, Sql, @@ -19,6 +20,7 @@ pub(crate) const LANGUAGES_ENV: &str = "MCP_CONSOLE_LANGUAGES"; #[derive(Clone, Copy, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct Languages { + pub(crate) kernel: bool, pub(crate) r: bool, pub(crate) python: bool, pub(crate) sql: bool, @@ -26,11 +28,16 @@ pub(crate) struct Languages { impl Languages { pub(crate) fn fields(self) -> Vec<&'static str> { - [Language::R, Language::Python, Language::Sql] - .into_iter() - .filter(|language| self.enables(*language)) - .map(Self::field) - .collect() + [ + Language::Kernel, + Language::R, + Language::Python, + Language::Sql, + ] + .into_iter() + .filter(|language| self.enables(*language)) + .map(Self::field) + .collect() } pub(crate) fn cell_fields(self) -> String { @@ -68,6 +75,7 @@ impl Languages { pub(crate) fn all() -> Self { Self { + kernel: false, r: true, python: true, sql: true, @@ -75,16 +83,13 @@ impl Languages { } pub(crate) fn configure(self, command: &mut std::process::Command) { - let fields = [Language::R, Language::Python, Language::Sql] - .into_iter() - .filter(|language| self.enables(*language)) - .map(Self::field) - .collect::>(); + let fields = self.fields(); command.env(LANGUAGES_ENV, fields.join(",")); } pub(crate) fn enables(self, language: crate::cell::Language) -> bool { match language { + crate::cell::Language::Kernel => self.kernel, crate::cell::Language::R => self.r, crate::cell::Language::Python => self.python, crate::cell::Language::Sql => self.sql, @@ -93,6 +98,7 @@ impl Languages { pub(crate) fn field(language: crate::cell::Language) -> &'static str { match language { + crate::cell::Language::Kernel => "code", crate::cell::Language::R => "r", crate::cell::Language::Python => "python", crate::cell::Language::Sql => "sql", diff --git a/src/cli.rs b/src/cli.rs index ad9b187ca..d668f1f7b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -180,6 +180,18 @@ pub enum Command { #[arg(long, value_name = "PATH", conflicts_with = "no_sandbox")] writable_root: Vec, + /// Run an installed local Jupyter kernel instead of the built-in worker + #[arg(long, value_name = "NAME", conflicts_with_all = ["worker", "relay"])] + jupyter_kernel: Option, + + /// Python with jupyter_client installed (defaults to python3) + #[arg(long, value_name = "PATH", requires = "jupyter_kernel")] + jupyter_python: Option, + + /// Jupyter transport (auto selects a private sandbox transport) + #[arg(long, value_enum, requires = "jupyter_kernel")] + jupyter_transport: Option, + /// Replace the runtime worker during development #[arg(long, hide = true, value_name = "PATH")] worker: Option, diff --git a/src/jupyter.rs b/src/jupyter.rs new file mode 100644 index 000000000..18e3a75a7 --- /dev/null +++ b/src/jupyter.rs @@ -0,0 +1,101 @@ +//! Captured launch selection for the Jupyter protocol adapter. + +use std::path::PathBuf; + +#[cfg(target_os = "linux")] +mod linux; + +#[derive(Clone)] +pub(crate) struct Configuration { + pub(crate) kernel: String, + pub(crate) python: PathBuf, + pub(crate) transport: Transport, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, clap::ValueEnum)] +pub enum Transport { + #[default] + Auto, + Tcp, + Ipc, +} + +impl Transport { + pub(crate) fn name(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Tcp => "tcp", + Self::Ipc => "ipc", + } + } +} + +impl Configuration { + /// Grant only the private transport needed by the selected local kernel. + /// This never authorizes access to host services or external networks. + pub(crate) fn launch( + &self, + no_sandbox: bool, + settings: &mut crate::settings::SandboxSettings, + ) -> Result { + let restricted = + settings.get("network").and_then(serde_json::Value::as_str) == Some("restricted"); + let transport = match self.transport { + Transport::Auto if !no_sandbox && restricted && cfg!(target_os = "macos") => { + Transport::Ipc + } + Transport::Auto => Transport::Tcp, + transport => transport, + }; + if no_sandbox { + return Ok(transport); + } + #[cfg(target_os = "macos")] + if transport == Transport::Ipc { + if crate::settings::Tempdir::from_settings(settings) + != crate::settings::Tempdir::Managed + { + return Err("Jupyter private IPC requires sandbox.tempdir: managed".into()); + } + if settings + .get("filesystem") + .is_some_and(|filesystem| filesystem["kind"] == "unrestricted") + { + // Preserve inherited host access using the existing root-write + // representation. The runner can then protect its private root + // and supply RUNNER_PRIVATE_DATA for the IPC path rules. + settings.insert( + "filesystem".into(), + serde_json::json!({ + "kind": "restricted", + "entries": [{ + "path": {"type": "special", "value": {"kind": "root"}}, + "access": "write", + }], + }), + ); + } + let extension = settings + .entry("macos_seatbelt_profile_extension") + .or_insert_with(|| "".into()); + let mut policy = extension + .as_str() + .expect("captured Seatbelt extension") + .to_owned(); + policy.push_str(include_str!("jupyter/private_ipc.sbpl")); + *extension = policy.into(); + } + #[cfg(target_os = "linux")] + linux::configure(settings, transport)?; + Ok(transport) + } + + pub(crate) fn languages(&self) -> crate::cell::Languages { + crate::cell::Languages { + kernel: true, + r: false, + python: false, + sql: false, + } + } +} diff --git a/src/jupyter/linux.rs b/src/jupyter/linux.rs new file mode 100644 index 000000000..7537cd8a4 --- /dev/null +++ b/src/jupyter/linux.rs @@ -0,0 +1,46 @@ +//! Private kernel communication inside the runner's Linux network namespace. + +use super::Transport; +use crate::settings::SandboxSettings; +use serde_json::{Value, json}; + +pub(super) fn configure( + settings: &mut SandboxSettings, + transport: Transport, +) -> Result<(), String> { + if let Some(proxy) = settings.get("proxy") { + if transport == Transport::Ipc && proxy["dangerouslyAllowAllUnixSockets"] != true { + return Err("Jupyter IPC requires Unix-socket access unavailable under this Linux proxy policy; use tcp for communication inside the private network namespace".into()); + } + return Ok(()); + } + if settings.get("network").and_then(Value::as_str) == Some("enabled") { + return Ok(()); + } + match transport { + Transport::Auto => unreachable!("Jupyter transport must be resolved before sandbox setup"), + Transport::Ipc => { + return Err("Jupyter IPC is unavailable under the pinned Linux restricted-network policy; use auto or tcp for communication inside a private network namespace".into()); + } + Transport::Tcp => {} + } + // The pinned runner's restricted seccomp denies even local bind/connect. + // Its managed-network path permits TCP only inside a new network namespace. + // An empty proxy allowlist supplies that path without any host destination; + // standalone Unix sockets remain denied, so host services stay unreachable. + settings.insert( + "proxy".into(), + json!({ + "enabled": true, + "mode": "limited", + "enableSocks5": false, + "enableSocks5Udp": false, + "allowUpstreamProxy": false, + "domains": {}, + "unixSockets": {}, + "dangerouslyAllowAllUnixSockets": false, + "allowLocalBinding": false, + }), + ); + Ok(()) +} diff --git a/src/jupyter/namespace_ipykernel.py b/src/jupyter/namespace_ipykernel.py new file mode 100644 index 000000000..f4c4abbf7 --- /dev/null +++ b/src/jupyter/namespace_ipykernel.py @@ -0,0 +1,27 @@ +"""Run stock ipykernel inside the native runner's private Linux network namespace. + +libzmq 4.3.5 enumerates interfaces even for numeric TCP bind addresses. That +uses AF_NETLINK, which the pinned sandbox denies. A wildcard bind skips this +enumeration. It is safe here only because all interfaces belong to the private +namespace, with no host loopback or external route. Never use this on the host. +""" + +import runpy +from typing import Any + +import zmq + + +class NamespaceSocket(zmq.Socket): + def bind(self, addr: str | bytes) -> Any: + if isinstance(addr, bytes): + addr = addr.decode("utf-8") + if addr.startswith("tcp://127.0.0.1:"): + addr = "tcp://*:" + addr.removeprefix("tcp://127.0.0.1:") + return super().bind(addr) + + +zmq.Context._socket_class = NamespaceSocket + +if __name__ == "__main__": + runpy.run_module("ipykernel_launcher", run_name="__main__") diff --git a/src/jupyter/private_ipc.sbpl b/src/jupyter/private_ipc.sbpl new file mode 100644 index 000000000..2f630642f --- /dev/null +++ b/src/jupyter/private_ipc.sbpl @@ -0,0 +1,10 @@ + +; Jupyter kernels and their adapter share the runner's disposable storage. +; The runner supplies this parameter after creating that private directory. +; Path matching follows symlinks: a link to a host socket is not an exception. +; TCP, host Unix sockets, and host filesystem writes retain their base policy. +(allow system-socket (socket-domain AF_UNIX)) +(allow network-bind + (local unix-socket (subpath (param "RUNNER_PRIVATE_DATA")))) +(allow network-outbound + (remote unix-socket (subpath (param "RUNNER_PRIVATE_DATA")))) diff --git a/src/jupyter/worker.py b/src/jupyter/worker.py new file mode 100644 index 000000000..587d98163 --- /dev/null +++ b/src/jupyter/worker.py @@ -0,0 +1,536 @@ +"""Translate Console's private worker protocol to a local Jupyter kernel. + +The adapter owns the kernel process, but never evaluates submitted source itself. +Keep socket access on one event loop; bounded queues backpressure channel readers. +Kernel execution ends only after its shell reply AND matching IOPub idle message. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import secrets +import signal +import string +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from jupyter_client import AsyncKernelClient + + +def error_text(content: dict) -> str: + traceback = content.get("traceback", []) + return ( + "\n".join(traceback) + if traceback + else f"{content.get('ename', 'Error')}: {content.get('evalue', '')}" + ) + "\n" + + +@dataclass +class Execution: + request_id: str + idle: bool = False + reply: dict | None = None + error_seen: bool = False + + +class Adapter: + def __init__(self, options: argparse.Namespace) -> None: + self.options = options + self.read_fd = int(os.environ.pop("MCP_CONSOLE_SIDEBAND_READ_FD")) + self.write_fd = int(os.environ.pop("MCP_CONSOLE_SIDEBAND_WRITE_FD")) + os.set_inheritable(self.read_fd, False) + os.set_inheritable(self.write_fd, False) + self.events: asyncio.Queue[tuple[str, Any]] = asyncio.Queue(maxsize=64) + self.stopping = asyncio.Event() + self.interrupted = asyncio.Event() + self.child: asyncio.subprocess.Process | None = None + self.client: AsyncKernelClient | None = None + self.pending_reads: dict[tuple[AsyncKernelClient, str], asyncio.Task[dict]] = {} + self.current: Execution | None = None + self.started = False + self.input_pending = False + self.input_buffer = bytearray() + self.writer: asyncio.StreamWriter | None = None + self.channels_started = False + self.kernel_ready = False + + async def emit(self, kind: str, **content: Any) -> None: + assert self.writer is not None + self.writer.write((json.dumps({"kind": kind, **content}) + "\n").encode()) + await self.writer.drain() + + async def read_pipe(self, descriptor: int, commands: bool) -> None: + reader = asyncio.StreamReader() + transport, _ = await asyncio.get_running_loop().connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(reader), os.fdopen(descriptor, "rb") + ) + pending = bytearray() + try: + while True: + data = await reader.read(65536) + if not data: + if commands and pending: + raise ValueError("partial Console command at EOF") + self.stopping.set() + return + if commands: + pending.extend(data) + while b"\n" in pending: + line, _, rest = pending.partition(b"\n") + pending = bytearray(rest) + command = json.loads(line) + if command["kind"] == "shutdown": + self.stopping.set() + return + await self.events.put(("command", command)) + else: + await self.events.put(("input", data)) + finally: + transport.close() + + def pending_read( + self, client: AsyncKernelClient, channel: str + ) -> asyncio.Task[dict]: + # A ZeroMQ receive belongs to the generation, not its current consumer. + # Transfer the live read and any received message from readiness through + # dispatch; cancel it only when the generation retires. + key = (client, channel) + if key not in self.pending_reads: + receive = getattr(client, f"get_{channel}_msg") + self.pending_reads[key] = asyncio.create_task(receive()) + return self.pending_reads[key] + + async def receive_message(self, client: AsyncKernelClient, channel: str) -> dict: + message = await asyncio.shield(self.pending_read(client, channel)) + del self.pending_reads[(client, channel)] + return message + + async def read_channel(self, client: AsyncKernelClient, name: str) -> None: + while True: + message = await self.receive_message(client, name) + await self.events.put((name, message)) + + async def send_message( + self, client: AsyncKernelClient, channel: str, kind: str, **content: Any + ) -> str: + # Session.send shadows async sockets with synchronous sockets, bypassing + # pyzmq's pending-receive wakeups. Sign with Session, but keep both socket + # directions on the async API so sending cannot strand a queued reply. + message = client.session.msg(kind, content) + frames = client.session.serialize(message) + socket = getattr(client, f"{channel}_channel").socket + await socket.send_multipart( + frames, copy=max(map(len, frames)) < client.session.copy_threshold + ) + return message["header"]["msg_id"] + + async def interrupt(self) -> None: + while True: + await self.interrupted.wait() + self.interrupted.clear() + if not self.kernel_ready: + raise RuntimeError("Jupyter startup interrupted") + await self.events.put(("interrupt", None)) + + async def watch_child(self) -> None: + assert self.child is not None + status = await self.child.wait() + raise RuntimeError(f"Jupyter kernel exited with status {status}") + + def startup_output(self, message: dict) -> None: + # The private protocol accepts console events only after ready. Raw + # process streams retain startup diagnostics and initialization errors. + kind, content = message["msg_type"], message["content"] + if kind == "stream": + stream = sys.stderr if content["name"] == "stderr" else sys.stdout + stream.write(content["text"]) + stream.flush() + elif kind == "error": + sys.stderr.write(error_text(content)) + sys.stderr.flush() + + async def wait_ready(self, client: AsyncKernelClient) -> dict: + # A live child can take time to bind its heartbeat socket. Its process + # observer owns death detection; readiness requires both Jupyter channels. + # Retry because PUB/SUB can drop messages before subscription, retaining + # each request's progress: a late reply must not become stale on retry. + pending: set[str] = set() + replies: dict[str, dict] = {} + idle: set[str] = set() + receivers = { + self.pending_read(client, "shell"): "shell", + self.pending_read(client, "iopub"): "iopub", + } + loop = asyncio.get_running_loop() + retry = loop.time() + async with asyncio.timeout(30): + while True: + if loop.time() >= retry: + pending.add( + await self.send_message(client, "shell", "kernel_info_request") + ) + retry = loop.time() + 1 + ready, _ = await asyncio.wait( + receivers, + timeout=max(0, retry - loop.time()), + return_when=asyncio.FIRST_COMPLETED, + ) + for task in ready: + channel = receivers.pop(task) + message = await self.receive_message(client, channel) + request = message.get("parent_header", {}).get("msg_id") + kind = message["msg_type"] + if channel == "iopub": + self.startup_output(message) + if request in pending: + if channel == "shell" and kind == "kernel_info_reply": + replies[request] = message["content"] + elif ( + channel == "iopub" + and kind == "status" + and message["content"]["execution_state"] == "idle" + ): + idle.add(request) + receivers[self.pending_read(client, channel)] = channel + for request in replies.keys() & idle: + return replies[request] + + async def display(self, data: dict) -> None: + # MCP has no HTML/widget display surface. Prefer one supported image; + # otherwise preserve the kernel's plain-text representation. + for mime in ("image/png", "image/jpeg", "image/gif", "image/webp"): + if mime in data: + value = "".join(data[mime].split()) + # Ark publishes unpadded base64. Console's private image + # protocol requires standard padding; retain its validation. + value += "=" * (-len(value) % 4) + await self.emit("image", data=value, mime_type=mime) + return + text = data.get("text/plain") + if text is not None: + await self.emit( + "console_output", data=text + ("" if text.endswith("\n") else "\n") + ) + elif data: + await self.emit( + "console_diagnostic", + data="[Jupyter display omitted: " + ", ".join(sorted(data)) + "]\n", + ) + + async def error(self, content: dict) -> None: + await self.emit("console_diagnostic", data=error_text(content)) + + async def input_reply(self) -> None: + if not self.input_pending or b"\n" not in self.input_buffer: + return + assert self.client is not None + line, _, rest = self.input_buffer.partition(b"\n") + self.input_buffer = bytearray(rest) + await self.send_message( + self.client, + "stdin", + "input_reply", + value=line.decode("utf-8").removesuffix("\r"), + ) + self.input_pending = False + await self.emit("input_received") + + async def cancel_input(self) -> None: + if self.input_pending: + self.input_pending = False + await self.emit("input_cancelled") + + async def evaluate(self, command: dict) -> None: + assert self.current is None, "overlapping Jupyter execution" + assert command["kind"] == "evaluate", "unsupported Jupyter worker command" + assert command["language"] == "kernel", ( + "unsupported source language for this kernel" + ) + assert self.client is not None + self.started = False + request = await self.send_message( + self.client, + "shell", + "execute_request", + code=command["source"], + silent=False, + store_history=True, + user_expressions={}, + allow_stdin=True, + stop_on_error=True, + ) + self.current = Execution(request) + + async def dispatch(self) -> None: + assert self.client is not None and self.child is not None + while True: + channel, message = await self.events.get() + if channel == "command": + await self.evaluate(message) + elif channel == "input": + self.input_buffer.extend(message) + await self.input_reply() + elif channel == "interrupt": + await self.cancel_input() + if self.child.returncode is None: + if self.interrupt_mode == "message": + await self.send_message( + self.client, "control", "interrupt_request" + ) + else: + self.child.send_signal(signal.SIGINT) + else: + kind, content = message["msg_type"], message["content"] + parent = message.get("parent_header", {}).get("msg_id") + current = ( + self.current + if self.current and self.current.request_id == parent + else None + ) + if channel == "iopub": + if kind == "stream": + await self.emit( + "console_diagnostic" + if content["name"] == "stderr" + else "console_output", + data=content["text"], + ) + elif kind in { + "display_data", + "execute_result", + "update_display_data", + }: + if kind == "update_display_data": + await self.emit( + "console_diagnostic", data="[Jupyter display updated]\n" + ) + await self.display(content["data"]) + elif kind == "clear_output": + await self.emit( + "console_diagnostic", + data="[Jupyter clear_output; earlier output retained]\n", + ) + elif kind == "error": + await self.error(content) + if current is not None: + current.error_seen = True + elif current and kind == "status": + if content["execution_state"] == "busy" and not self.started: + self.started = True + await self.emit("cell_started") + elif content["execution_state"] == "idle": + current.idle = True + elif current and channel == "shell" and kind == "execute_reply": + current.reply = content + elif current and channel == "stdin" and kind == "input_request": + assert not self.input_pending, "overlapping Jupyter input requests" + self.input_pending = True + await self.emit("input_requested", prompt=content["prompt"]) + await self.input_reply() + if self.current and self.current.idle and self.current.reply is not None: + assert self.started, "kernel completed without publishing busy status" + await self.cancel_input() + reply = self.current.reply + if reply["status"] == "error" and not self.current.error_seen: + await self.error(reply) + elif reply["status"] == "aborted": + await self.emit( + "console_diagnostic", data="[Jupyter execution aborted]\n" + ) + await self.emit("completed") + self.current = None + + async def serve(self, directory: str) -> None: + import zmq.asyncio + from jupyter_client import AsyncKernelClient + from jupyter_client.connect import write_connection_file + from jupyter_client.kernelspec import KernelSpecManager + + spec = KernelSpecManager().get_kernel_spec(self.options.kernel) + provisioner = spec.metadata.get("kernel_provisioner", {}).get( + "provisioner_name", "local-provisioner" + ) + if provisioner != "local-provisioner": + raise ValueError("Jupyter mode supports local kernelspecs only") + self.interrupt_mode = spec.interrupt_mode + connection_file = str(Path(directory) / "connection.json") + _, connection = write_connection_file( + connection_file, + transport=self.options.transport, + ip="127.0.0.1" + if self.options.transport == "tcp" + else str(Path(directory) / "k"), + key=secrets.token_hex(32).encode(), + ) + if self.options.transport == "ipc": + # Published ark uses prefix:port where jupyter_client uses + # prefix-port. Both resolve inside this private directory. Kernels + # using the standard name unlink the alias when binding it. + for channel in ("shell", "iopub", "stdin", "control", "hb"): + port = connection[f"{channel}_port"] + Path(f"{connection['ip']}-{port}").symlink_to(f"k:{port}") + substitutions = { + "connection_file": connection_file, + "resource_dir": spec.resource_dir, + "prefix": sys.prefix, + } + command = [ + os.path.expanduser( + re.sub( + r"\{(connection_file|resource_dir|prefix)\}", + lambda m: substitutions[m[1]], + arg, + ) + ) + for arg in spec.argv + ] + if command[0] in { + "python", + f"python{sys.version_info.major}", + f"python{sys.version_info.major}.{sys.version_info.minor}", + }: + command[0] = sys.executable + if self.options.namespace_ipykernel and "-m" in command: + module = command.index("-m") + if command[module + 1 : module + 2] == ["ipykernel_launcher"]: + # Preserve interpreter options such as -Xfrozen_modules=off in + # the published installer. Other modules keep their own launch. + command[module : module + 2] = ["-c", self.options.namespace_ipykernel] + environment = os.environ.copy() + environment.update( + { + key: string.Template(value).safe_substitute(environment) + for key, value in spec.env.items() + } + ) + if spec.language.lower().startswith("python"): + environment.pop("PYTHONEXECUTABLE", None) + environment["JPY_PARENT_PID"] = str(os.getpid()) + # KernelManager's launcher unconditionally detaches POSIX sessions. Keep + # this child in Console's process tree and own its wait/termination here. + context = zmq.asyncio.Context() + launch = asyncio.create_task( + asyncio.create_subprocess_exec( + *command, stdin=asyncio.subprocess.DEVNULL, env=environment + ) + ) + try: + # Cancellation cannot discard a process acquired by a pending spawn. + try: + self.child = await asyncio.shield(launch) + except asyncio.CancelledError: + self.child = await launch + raise + self.client = AsyncKernelClient( + connection_file=connection_file, context=context + ) + self.client.load_connection_file() + self.client.start_channels(hb=False) + self.channels_started = True + async with asyncio.TaskGroup() as group: + group.create_task(self.watch_child()) + await self.wait_ready(self.client) + self.kernel_ready = True + await self.emit("ready") + for channel in ("shell", "iopub", "stdin"): + group.create_task(self.read_channel(self.client, channel)) + group.create_task(self.dispatch()) + finally: + # Fit inside the relay's one-second worker retirement grace. The + # child is reaped before the adapter closes its own sideband pipes. + if self.child is not None and self.child.returncode is None: + try: + if self.channels_started: + try: + async with asyncio.timeout(0.2): + await self.send_message( + self.client, + "control", + "shutdown_request", + restart=False, + ) + except TimeoutError: + pass + finally: + # A failed socket setup/shutdown must still retire the child. + for action in (None, self.child.terminate, self.child.kill): + if action is not None and self.child.returncode is None: + action() + try: + await asyncio.wait_for(self.child.wait(), timeout=0.2) + break + except TimeoutError: + continue + if self.child.returncode is None: + raise RuntimeError( + "Jupyter kernel did not retire after SIGKILL" + ) + for task in self.pending_reads.values(): + task.cancel() + await asyncio.gather(*self.pending_reads.values(), return_exceptions=True) + self.pending_reads.clear() + if self.client is not None: + self.client.stop_channels() + context.destroy(linger=0) + + async def run(self) -> None: + loop = asyncio.get_running_loop() + transport, protocol = await loop.connect_write_pipe( + asyncio.streams.FlowControlMixin, os.fdopen(self.write_fd, "wb") + ) + self.writer = asyncio.StreamWriter(transport, protocol, None, loop) + loop.add_signal_handler(signal.SIGINT, self.interrupted.set) + loop.add_signal_handler(signal.SIGTERM, self.stopping.set) + # Unix-domain socket names include the runner's private TMPDIR prefix. + with tempfile.TemporaryDirectory(prefix="j-") as directory: + if self.options.private_runtime: + # Set these before importing jupyter_client, which imports + # IPython's path discovery. Explicit selections remain intact. + os.environ.setdefault("IPYTHONDIR", str(Path(directory) / "ipython")) + os.environ.setdefault( + "MPLCONFIGDIR", str(Path(directory) / "matplotlib") + ) + tasks = [ + asyncio.create_task(self.read_pipe(self.read_fd, True)), + asyncio.create_task(self.read_pipe(os.dup(sys.stdin.fileno()), False)), + asyncio.create_task(self.interrupt()), + asyncio.create_task(self.serve(directory)), + asyncio.create_task(self.stopping.wait()), + ] + # Child death must wake an idle adapter too. The serve task installs + # its process observer immediately after acquiring the subprocess. + try: + done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in done: + task.result() + finally: + for task in tasks: + task.cancel() + outcomes = await asyncio.gather(*tasks, return_exceptions=True) + transport.close() + for outcome in outcomes: + if isinstance(outcome, Exception): + raise outcome + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--kernel", required=True) + parser.add_argument("--transport", choices=("tcp", "ipc"), default="tcp") + parser.add_argument("--private-runtime", action="store_true") + parser.add_argument("--namespace-ipykernel") + options = parser.parse_args() + asyncio.run(Adapter(options).run()) + + +if __name__ == "__main__": + main() diff --git a/src/main.rs b/src/main.rs index 928cdc460..bbf8ff25c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ mod console_paths; mod input_watch; #[cfg(any(unix, windows))] mod jsonl; +mod jupyter; mod local_runtime; #[cfg(unix)] mod process_descriptors; @@ -127,6 +128,9 @@ fn main() -> ExitCode { cli::Command::Serve { worker, relay, + jupyter_kernel, + jupyter_python, + jupyter_transport, no_sandbox, writable_root, overrides: command_overrides, @@ -134,7 +138,19 @@ fn main() -> ExitCode { if let Err(error) = overrides.extend(command_overrides) { return exit_with_error(error); } - match run_server(worker, relay, no_sandbox, writable_root, &overrides) { + let jupyter = jupyter_kernel.map(|kernel| jupyter::Configuration { + kernel, + python: jupyter_python.unwrap_or_else(|| "python3".into()), + transport: jupyter_transport.unwrap_or_default(), + }); + match run_server( + worker, + relay, + jupyter, + no_sandbox, + writable_root, + &overrides, + ) { Ok(()) => ExitCode::SUCCESS, Err(error) => exit_with_error(error), } @@ -238,6 +254,7 @@ fn resolve_configuration( fn run_server( worker: Option, relay: Option, + jupyter: Option, no_sandbox: bool, writable_roots: Vec, overrides: &cli::ConfigOverrides, @@ -257,10 +274,35 @@ fn run_server( resolver, sandbox_requested, resolver_sandbox_requested, - } = settings::discover(&directory, overrides)?; + } = settings::discover( + &directory, + overrides, + match &jupyter { + Some(jupyter) => settings::ToolInterface::Jupyter(jupyter.languages()), + None => settings::ToolInterface::Configured, + }, + )?; if no_sandbox && (sandbox_requested || resolver_sandbox_requested) { return Err("explicit sandbox or resolver.sandbox permissions require sandboxing; remove them when using --no-sandbox".into()); } + if let Some(jupyter) = &jupyter { + if jupyter.kernel.is_empty() { + return Err("--jupyter-kernel requires a nonempty kernelspec name".into()); + } + for (name, configured) in [ + ("startup", startup.is_some()), + ("python", python.is_some()), + ("r", r.is_some()), + ("languages", languages.is_some()), + ] { + if configured { + return Err(format!("{name} settings cannot be combined with --jupyter-kernel; configure the installed kernelspec instead").into()); + } + } + if cfg!(windows) { + return Err("Jupyter workers currently require macOS or Linux".into()); + } + } if startup.is_some() && (worker.is_some() || relay.is_some()) { return Err("startup requires the built-in worker and relay".into()); } @@ -298,6 +340,7 @@ fn run_server( let result = runtime.block_on(server::run( worker, relay, + jupyter, sandbox, python, r.unwrap_or_default(), diff --git a/src/sandbox.rs b/src/sandbox.rs index e0ab45e83..77dec140b 100644 --- a/src/sandbox.rs +++ b/src/sandbox.rs @@ -24,7 +24,11 @@ fn capture_settings( } = if let Some(name) = config_env { crate::settings::from_configuration_environment(&workspace, name, overrides)? } else { - crate::settings::discover(&workspace, overrides)? + crate::settings::discover( + &workspace, + overrides, + crate::settings::ToolInterface::Configured, + )? }; if let Some(name) = config_env { // The selected document is launch input, including when its environment diff --git a/src/server.rs b/src/server.rs index 05716b95b..a3e6e9113 100644 --- a/src/server.rs +++ b/src/server.rs @@ -51,6 +51,7 @@ impl ConsoleServer { input_closed: InputClosed, worker: Option, relay: Option, + jupyter: Option, sandbox: SandboxConfiguration, python: Option, r: crate::settings::R, @@ -59,9 +60,10 @@ impl ConsoleServer { tool_settings: crate::settings::Tool, ) -> Result { let recording_directory = std::env::current_dir(); - let (languages, language_setting) = match visibility { - Some(languages) => (languages, "languages"), - None => (Languages::from_environment()?, LANGUAGES_ENV), + let (languages, language_setting) = match (&jupyter, visibility) { + (Some(jupyter), _) => (jupyter.languages(), "--jupyter-kernel"), + (None, Some(languages)) => (languages, "languages"), + (None, None) => (Languages::from_environment()?, LANGUAGES_ENV), }; if worker.is_none() && relay.is_some() { return Err("a custom relay requires a custom worker".into()); @@ -69,7 +71,7 @@ impl ConsoleServer { // Presentation has no dependency on the client or its discovered capabilities. let (tool_router, instructions) = Self::configured_presentation( languages, - worker.is_none(), + worker.is_none() && jupyter.is_none(), python.as_ref(), &sandbox.policy, sandbox.no_sandbox, @@ -88,7 +90,7 @@ impl ConsoleServer { .map_err(|error| std::io::Error::new(error.kind(), error.to_string())), ), }); - let prelaunch = worker.is_none(); + let prelaunch = worker.is_none() && jupyter.is_none(); let uses_client_profile = sandbox .profile .as_ref() @@ -107,7 +109,13 @@ impl ConsoleServer { } else { sandbox.clone() }; - let configuration = if let Some(program) = worker.clone() { + let configuration = if let Some(selection) = &jupyter { + crate::worker_client::ClientConfiguration::jupyter( + selection, + sandbox.no_sandbox, + sandbox.policy, + )? + } else if let Some(program) = worker.clone() { crate::worker_client::ClientConfiguration::new( program, relay.clone(), @@ -134,8 +142,9 @@ impl ConsoleServer { .map_err(|error| std::io::Error::new(error.kind(), error.to_string())), configuration.dynamic_resolution(), configuration.python_preparation(), - !configuration.python_only(), + jupyter.is_none() && !configuration.python_only(), configuration.startup_declaration(), + jupyter.as_ref().map(|selection| selection.kernel.clone()), ); Ok(startup::PreparedRuntime { configuration, @@ -169,6 +178,7 @@ impl ConsoleServer { Extension(delivery): Extension, Extension(started): Extension, Parameters(SendArguments { + code, r, python, sql, @@ -178,20 +188,24 @@ impl ConsoleServer { timeout_ms, }): Parameters, ) -> Result { - let cell = match (r, python, sql) { - (Some(source), None, None) => Some(crate::cell::Cell { + let cell = match (code, r, python, sql) { + (Some(source), None, None, None) => Some(crate::cell::Cell { + language: crate::cell::Language::Kernel, + source, + }), + (None, Some(source), None, None) => Some(crate::cell::Cell { language: crate::cell::Language::R, source, }), - (None, Some(source), None) => Some(crate::cell::Cell { + (None, None, Some(source), None) => Some(crate::cell::Cell { language: crate::cell::Language::Python, source, }), - (None, None, Some(source)) => Some(crate::cell::Cell { + (None, None, None, Some(source)) => Some(crate::cell::Cell { language: crate::cell::Language::Sql, source, }), - (None, None, None) => None, + (None, None, None, None) => None, _ => { return Err(format!( "only one of {} may be supplied", @@ -201,6 +215,9 @@ impl ConsoleServer { }; if let Some(requirements) = &requirements { use crate::worker_client::RequirementsAction::{Get, Reset}; + if self.languages.kernel && requirements.action != Get { + return Err("dependency preparation is unavailable with Jupyter kernels; install packages in the kernel environment before launch".into()); + } if matches!(requirements.action, Get | Reset) && (requirements.r.is_some() || requirements.python.is_some() @@ -546,6 +563,7 @@ impl ServerHandler for ConsoleServer { pub async fn run( worker: Option, relay: Option, + jupyter: Option, sandbox: SandboxConfiguration, python: Option, r: crate::settings::R, @@ -561,6 +579,7 @@ pub async fn run( input_closed.clone(), worker, relay, + jupyter, sandbox, python, r, diff --git a/src/server/arguments.rs b/src/server/arguments.rs index f0258ff43..c82420b7b 100644 --- a/src/server/arguments.rs +++ b/src/server/arguments.rs @@ -14,12 +14,16 @@ impl SendArguments { return Ok(()); }; for language in [ + crate::cell::Language::Kernel, crate::cell::Language::R, crate::cell::Language::Python, crate::cell::Language::Sql, ] { let field = crate::cell::Languages::field(language); if arguments.contains_key(field) && !languages.enables(language) { + if matches!(language, crate::cell::Language::Kernel) { + return Err("`code` cells require --jupyter-kernel".into()); + } return Err(if setting == crate::cell::LANGUAGES_ENV { format!("`{field}` cells are disabled by `{setting}`") } else { @@ -49,6 +53,8 @@ impl SendArguments { #[derive(Deserialize, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub(super) struct SendArguments { + /// Evaluate one complete cell in the selected Jupyter kernel's language. + pub(super) code: Option, #[schemars(description = super::presentation::r_description())] pub(super) r: Option, #[schemars(description = super::presentation::python_description())] diff --git a/src/server/presentation.rs b/src/server/presentation.rs index 1e7cabb86..45070bd08 100644 --- a/src/server/presentation.rs +++ b/src/server/presentation.rs @@ -183,8 +183,18 @@ pub(crate) fn parameter_schema(languages: Languages) -> Map { .expect("requirement property schema") .remove("default"); } + if languages.kernel { + let requirements = &mut properties["requirements"]; + let fields = requirements["properties"] + .as_object_mut() + .expect("requirements schema properties"); + fields.retain(|name, _| name == "action"); + fields["action"]["enum"] = serde_json::json!(["get"]); + requirements["required"] = serde_json::json!(["action"]); + } // Configured fields stay visible even when an interpreter is unavailable. for (field, enabled) in [ + ("code", languages.kernel), ("r", languages.r), ("python", languages.python), ("sql", languages.sql), @@ -210,6 +220,9 @@ impl Profile { } fn summary(&self) -> String { + if self.languages.kernel { + return "Persistent Jupyter kernel session. Send code in the selected kernel's language and reuse objects across calls.".into(); + } let names = if self.languages.r && self.languages.python && self.languages.sql { "R, Python, and SQL".to_string() } else { @@ -240,7 +253,9 @@ impl Profile { description.push_str("\n\nSend one complete "); description.push_str(&self.languages.cell_fields()); description.push_str(sections::SEND_WORKFLOW); - if self.builtin { + if self.languages.kernel { + description.push_str("\n\nThe installed kernel supplies evaluation, display, history, and packages. Console returns text and supported images; interactive displays are unavailable. Dependency preparation and Console's built-in SQL and language helpers are unavailable."); + } else if self.builtin { description.push_str(sections::DISPLAY); description.push_str("\n\n"); description.push_str(&self.language_guidance()); @@ -260,7 +275,13 @@ impl Profile { description.push(' '); if self.timeout { description.push_str(sections::POLLING); - description.push_str(sections::DEFAULTS); + description.push_str(if self.languages.kernel { + " Default: `timeout_ms=60000` (60 seconds)." + } else { + sections::DEFAULTS + }); + } else if self.languages.kernel { + description.push_str("If work is still running, poll with an empty `send`; do not resubmit the cell. Each call waits for output for up to 60 seconds and returns early when work finishes or needs input."); } else { description.push_str(sections::POLLING_DEFAULT_TIMEOUT); } @@ -271,6 +292,7 @@ impl Profile { policy, no_sandbox, self.uses_client_profile, + !self.languages.kernel, )); description } @@ -310,6 +332,15 @@ impl Profile { } fn configure_fields(&self, properties: &mut Map) { + if self.languages.kernel { + properties["stdin"]["description"] = "Answer Jupyter input requests with newline-terminated text. Input queues before a bundled cell; unread input can satisfy later requests. Restart discards queued input. Direct reads from the kernel process's stdin are unsupported.".into(); + properties["control"]["description"] = "Apply control before same-call input or code. interrupt requests kernel interruption; a following cell runs only if prior work stops. restart discards the kernel's live state and queued input, then sends bundled input and code to the replacement.".into(); + properties["timeout_ms"]["description"] = "Wait for output for up to this many milliseconds (default 60,000). Use 300000 for a long poll; it returns early when work finishes or needs input. 0 returns immediately. Expiry leaves execution running; poll with an empty send.".into(); + let requirements = &mut properties["requirements"]; + requirements["description"] = "Inspect Console's empty dependency declaration with action=get. Package installation and dependency changes are unavailable; use packages already installed in the kernel environment.".into(); + requirements["properties"]["action"]["description"] = "get returns the empty Console declaration, not an installed-package inventory. Use it alone without code, stdin, or control.".into(); + return; + } if self.builtin { for (field, description) in [ ("r", r_description_for(self.languages)), @@ -437,12 +468,21 @@ fn description_for_launch( policy: &SandboxSettings, no_sandbox: bool, uses_client_profile: bool, + preparation: bool, ) -> String { if uses_client_profile { - return "The sandbox base comes from Codex metadata on the first send, or the configured fallback (default: read_only). Missing metadata is silent; malformed metadata warns on server stderr. Explicit Console permissions apply to the selected base. Full access disables Console's sandbox. Reconnect to change client permissions. Package preparation has separate permissions unless sandboxing is disabled; use only trusted dependencies.".into(); + let mut description = "The sandbox base comes from Codex metadata on the first send, or the configured fallback (default: read_only). Missing metadata is silent; malformed metadata warns on server stderr. Explicit Console permissions apply to the selected base. Full access disables Console's sandbox. Reconnect to change client permissions.".to_string(); + if preparation { + description.push_str(" Package preparation has separate permissions unless sandboxing is disabled; use only trusted dependencies."); + } + return description; } if no_sandbox { - return "Evaluated code runs without a sandbox, with the server's permissions, including filesystem and network access. Package preparation may execute installation or build code; use only trusted dependencies.".into(); + let mut description = "Evaluated code runs without a sandbox, with the server's permissions, including filesystem and network access.".to_string(); + if preparation { + description.push_str(" Package preparation may execute installation or build code; use only trusted dependencies."); + } + return description; } let network_access = if policy.contains_key("proxy") { "can access the network subject to the launcher's proxy settings" @@ -489,9 +529,13 @@ fn description_for_launch( writable.join(", ") } ); - format!( - "Evaluated code can read host files subject to configured restrictions and {network_access}. {writable}. Package preparation may execute installation or build code with separate filesystem and network permissions; use only trusted dependencies." - ) + let mut description = format!( + "Evaluated code can read host files subject to configured restrictions and {network_access}. {writable}." + ); + if preparation { + description.push_str(" Package preparation may execute installation or build code with separate filesystem and network permissions; use only trusted dependencies."); + } + description } pub(super) fn stdin_description() -> String { diff --git a/src/settings.rs b/src/settings.rs index cb3c2c0e6..1f1c54860 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -169,7 +169,7 @@ struct Project { python: Option, #[serde(deserialize_with = "r_settings")] r: Option, - languages: Option>, + languages: Option>, #[serde(default = "inherit_by_default")] inherit_environment: bool, #[serde(deserialize_with = "environment")] @@ -180,6 +180,14 @@ struct Project { resolver: Resolver, } +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum BuiltinLanguage { + R, + Python, + Sql, +} + #[derive(Clone, Default, serde::Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct R { @@ -251,11 +259,17 @@ pub(crate) struct Captured { pub resolver_sandbox_requested: bool, } +pub(crate) enum ToolInterface { + Configured, + Jupyter(crate::cell::Languages), +} + pub fn discover( directory: &Path, overrides: &crate::cli::ConfigOverrides, + interface: ToolInterface, ) -> Result { - read(directory, overrides).map(|(captured, _)| captured) + read_for_tool(directory, overrides, interface).map(|(captured, _)| captured) } #[derive(Default, Deserialize)] @@ -268,6 +282,14 @@ struct AdapterOverrides { pub(crate) fn read( directory: &Path, overrides: &crate::cli::ConfigOverrides, +) -> Result<(Captured, Value), String> { + read_for_tool(directory, overrides, ToolInterface::Configured) +} + +fn read_for_tool( + directory: &Path, + overrides: &crate::cli::ConfigOverrides, + interface: ToolInterface, ) -> Result<(Captured, Value), String> { // The R adapter transports the same ordered overrides in a file so tool // prose and valid startup programs are not bounded by process argv limits. @@ -322,7 +344,7 @@ pub(crate) fn read( (_, true) => format!("configuration from {files}"), (_, false) => format!("configuration from {files} with CLI overrides"), }; - capture(directory, value, name) + capture(directory, value, name, interface) } pub fn from_configuration_environment( @@ -336,13 +358,14 @@ pub fn from_configuration_environment( if !overrides.values.is_empty() { source.push_str(" with CLI overrides"); } - capture(directory, Some(value), source).map(|(captured, _)| captured) + capture(directory, Some(value), source, ToolInterface::Configured).map(|(captured, _)| captured) } fn capture( directory: &Path, value: Option, name: String, + interface: ToolInterface, ) -> Result<(Captured, Value), String> { let configured = value.is_some(); let mut value = value.unwrap_or_else(|| serde_json::json!({})); @@ -397,9 +420,9 @@ fn capture( let mut languages = crate::cell::Languages::default(); for language in selected { match language { - crate::cell::Language::R => languages.r = true, - crate::cell::Language::Python => languages.python = true, - crate::cell::Language::Sql => languages.sql = true, + BuiltinLanguage::R => languages.r = true, + BuiltinLanguage::Python => languages.python = true, + BuiltinLanguage::Sql => languages.sql = true, } } Ok(languages) @@ -407,7 +430,10 @@ fn capture( .transpose()?; project .tool - .validate(languages) + .validate(match interface { + ToolInterface::Configured => languages, + ToolInterface::Jupyter(languages) => Some(languages), + }) .map_err(|error| format!("{name}: {error}"))?; let sandbox_requested = project.sandbox.is_some(); let profile = project diff --git a/src/transcript.rs b/src/transcript.rs index 90158b551..61398fc9f 100644 --- a/src/transcript.rs +++ b/src/transcript.rs @@ -88,6 +88,7 @@ struct TranscriptState { dynamic_resolution: bool, python_preparation: bool, r_available: bool, + jupyter_kernel: Option, startup_requirements: crate::worker_client::Declaration, metadata_known: bool, storage: Arc, @@ -142,6 +143,7 @@ impl Transcript { false, true, crate::worker_client::Declaration::default(), + None, ) } pub(crate) fn configured( @@ -150,6 +152,7 @@ impl Transcript { python_preparation: bool, r_available: bool, startup_requirements: crate::worker_client::Declaration, + jupyter_kernel: Option, ) -> Self { let storage = Arc::new(CaptureStorage::default()); Self( @@ -160,6 +163,7 @@ impl Transcript { dynamic_resolution, python_preparation, r_available, + jupyter_kernel, startup_requirements, metadata_known: true, storage: storage.clone(), @@ -172,7 +176,14 @@ impl Transcript { ) } pub(crate) fn pending(working_directory: std::io::Result) -> Self { - let value = Self::configured(working_directory, false, false, false, Default::default()); + let value = Self::configured( + working_directory, + false, + false, + false, + Default::default(), + None, + ); value.0.lock().expect("transcript lock").metadata_known = false; value } @@ -183,6 +194,7 @@ impl Transcript { state.dynamic_resolution = configuration.dynamic_resolution; state.python_preparation = configuration.python_preparation; state.r_available = configuration.r_available; + state.jupyter_kernel = configuration.jupyter_kernel.clone(); state.startup_requirements = configuration.startup_requirements.clone(); state.metadata_known = true; state.capture()?.append( @@ -190,6 +202,7 @@ impl Transcript { dynamic_resolution: configuration.dynamic_resolution, python_preparation: configuration.python_preparation, r_available: configuration.r_available, + jupyter_kernel: configuration.jupyter_kernel.as_deref(), startup_requirements: &configuration.startup_requirements, }, Utc::now(), @@ -248,7 +261,7 @@ impl Transcript { // finish() associates otherwise unclaimed source only if this same // worker still owns output; actual replacement changes that owner. let has_source = request.arguments.as_ref().is_some_and(|arguments| { - ["r", "python", "sql"] + ["code", "r", "python", "sql"] .iter() .any(|language| arguments.get(*language).is_some_and(Value::is_string)) }); @@ -442,6 +455,7 @@ impl ActiveTranscript { configuration.r_available, &configuration.startup_requirements, configuration.metadata_known, + configuration.jupyter_kernel.as_deref(), ) }); let (projections, pending_projection_failure) = match projections { diff --git a/src/transcript/event.rs b/src/transcript/event.rs index a35448187..5e9620ab1 100644 --- a/src/transcript/event.rs +++ b/src/transcript/event.rs @@ -92,6 +92,8 @@ pub(super) enum Event<'a> { dynamic_resolution: bool, python_preparation: bool, r_available: bool, + #[serde(skip_serializing_if = "Option::is_none")] + jupyter_kernel: Option<&'a str>, startup_requirements: &'a crate::worker_client::Declaration, }, StartupFailed { diff --git a/src/transcript/markdown.rs b/src/transcript/markdown.rs index af165d955..1a01f0036 100644 --- a/src/transcript/markdown.rs +++ b/src/transcript/markdown.rs @@ -48,6 +48,7 @@ struct Metadata { python_requirements: Vec, dynamic_resolution: bool, r_available: bool, + jupyter_kernel: Option, metadata_known: bool, } struct QuartoWriter { @@ -66,6 +67,7 @@ impl Writers { r_available: bool, requirements: &crate::worker_client::Declaration, metadata_known: bool, + jupyter_kernel: Option<&str>, ) -> Result { let markdown = super::create_private_file(&directory.join("transcript.md")) .map_err(|e| e.to_string())?; @@ -83,6 +85,7 @@ impl Writers { python_requirements: requirements.python.clone(), dynamic_resolution, r_available, + jupyter_kernel: jupyter_kernel.map(str::to_owned), metadata_known, }, ..Default::default() @@ -131,12 +134,14 @@ impl Documents { Event::EnvironmentDiscovered { dynamic_resolution, r_available, + jupyter_kernel, startup_requirements, .. } => { self.metadata.metadata_known = true; self.metadata.dynamic_resolution = *dynamic_resolution; self.metadata.r_available = *r_available; + self.metadata.jupyter_kernel = jupyter_kernel.map(str::to_owned); self.metadata.requirements( &serde_json::to_value(startup_requirements).map_err(|e| e.to_string())?, ); @@ -165,8 +170,13 @@ impl Documents { // association, never stdin or request metadata. let mut sources = Vec::new(); if let Some(arguments) = &request.arguments { - for language in ["r", "python", "sql"] { - if let Some(contents) = arguments.get(language).and_then(Value::as_str) { + for (field, language) in [ + ("code", "text"), + ("r", "r"), + ("python", "python"), + ("sql", "sql"), + ] { + if let Some(contents) = arguments.get(field).and_then(Value::as_str) { sources.push(QuartoSource { language, contents: contents.into(), @@ -284,20 +294,32 @@ impl QuartoWriter { title: MCP Console session ", ); - writeln!( - document, - "# From a working copy, run `ir render {filename}` in a prepared environment. + if self.metadata.jupyter_kernel.is_some() { + document.push_str( + "# Jupyter source archive; recreate the original kernel environment to replay.\n", + ); + } else { + writeln!( + document, + "# From a working copy, run `ir render {filename}` in a prepared environment. # IR rendering requires R on the render host, including for Python-only documents." - ) - .unwrap(); - if !self.metadata.metadata_known { + ) + .unwrap(); + } + if !self.metadata.metadata_known || self.metadata.jupyter_kernel.is_some() { document.push_str( "execute: eval: false ", ); } - if !self.metadata.metadata_known { + if let Some(kernel) = &self.metadata.jupyter_kernel { + document.push_str("mcp-console:\n jupyter-kernel: "); + document.push_str(&yaml_string(kernel)); + document.push_str("\n original-working-directory: "); + document.push_str(&yaml_string(&self.metadata.working_directory)); + document.push('\n'); + } else if !self.metadata.metadata_known { document.push_str( "mcp-console: environment: unknown @@ -348,12 +370,18 @@ title: MCP Console session ", ); for source in &self.sources { - push_quarto_fence( - &mut document, - &format!("{{{}}}", source.language), - source.language, - &source.contents, - ); + if self.metadata.jupyter_kernel.is_some() || source.language == "text" { + // A generic kernelspec need not have a Quarto execution engine. + // Preserve its source without claiming it can be replayed here. + push_fence(&mut document, source.language, &source.contents); + } else { + push_quarto_fence( + &mut document, + &format!("{{{}}}", source.language), + source.language, + &source.contents, + ); + } } document } @@ -701,6 +729,7 @@ fn submitted_source(request: &CallToolRequestParams) -> Option> { let arguments = request.arguments.as_ref()?; let mut source = None; for (field, label, language) in [ + ("code", "Jupyter", "text"), ("r", "R", "r"), ("python", "Python", "python"), ("sql", "SQL", "sql"), @@ -743,7 +772,7 @@ fn parse_send(request: &CallToolRequestParams) -> Option> { if arguments.keys().any(|key| { !matches!( key.as_str(), - "r" | "python" | "sql" | "stdin" | "control" | "requirements" | "timeout_ms" + "code" | "r" | "python" | "sql" | "stdin" | "control" | "requirements" | "timeout_ms" ) }) { return None; @@ -752,7 +781,7 @@ fn parse_send(request: &CallToolRequestParams) -> Option> { if source.is_none() && arguments .keys() - .any(|key| matches!(key.as_str(), "r" | "python" | "sql")) + .any(|key| matches!(key.as_str(), "code" | "r" | "python" | "sql")) { return None; } @@ -762,7 +791,7 @@ fn parse_send(request: &CallToolRequestParams) -> Option> { }; let options = arguments .iter() - .filter(|(key, _)| !matches!(key.as_str(), "r" | "python" | "sql" | "stdin")) + .filter(|(key, _)| !matches!(key.as_str(), "code" | "r" | "python" | "sql" | "stdin")) .map(|(key, value)| (key.clone(), value.clone())) .collect(); Some(Send { diff --git a/src/worker/coordinator.rs b/src/worker/coordinator.rs index 4c3236c10..5810d0184 100644 --- a/src/worker/coordinator.rs +++ b/src/worker/coordinator.rs @@ -304,6 +304,7 @@ fn evaluate_cell( r.begin_graphics()?; } let result = match cell.language { + Language::Kernel => Err("kernel cells require a Jupyter worker".to_string()), Language::R => r.evaluate_r(cell.source), Language::Python => python.evaluate(&cell.source), Language::Sql => sql.evaluate(&cell.source), diff --git a/src/worker_client/configuration.rs b/src/worker_client/configuration.rs index 6b848506e..77148a9de 100644 --- a/src/worker_client/configuration.rs +++ b/src/worker_client/configuration.rs @@ -91,6 +91,62 @@ impl ClientConfiguration { ) } + pub(crate) fn jupyter( + selection: &crate::jupyter::Configuration, + no_sandbox: bool, + mut sandbox_settings: crate::settings::SandboxSettings, + ) -> Result { + let transport = selection.launch(no_sandbox, &mut sandbox_settings)?; + let mut arguments: Vec = vec![ + "-u".into(), + "-c".into(), + include_str!("../jupyter/worker.py").into(), + "--kernel".into(), + selection.kernel.clone().into(), + "--transport".into(), + transport.name().into(), + ]; + if cfg!(target_os = "linux") + && !no_sandbox + && sandbox_settings + .get("proxy") + .and_then(|proxy| proxy.get("enabled")) + .and_then(serde_json::Value::as_bool) + == Some(true) + { + // The pinned runner's managed-network path always isolates the + // workload's network namespace. This bootstrap must never run with + // host interfaces: it changes numeric loopback binds to wildcard. + arguments.extend([ + "--namespace-ipykernel".into(), + include_str!("../jupyter/namespace_ipykernel.py").into(), + ]); + } + if !no_sandbox + && crate::settings::Tempdir::from_settings(&sandbox_settings) + == crate::settings::Tempdir::Managed + { + arguments.push("--private-runtime".into()); + } + Ok(Self::with_arguments( + selection.python.clone(), + arguments, + None, + no_sandbox, + sandbox_settings, + Environment { + startup: Default::default(), + local_runtime: None, + custom_worker: true, + duckdb_extensions: Default::default(), + duckdb_r_targets: Vec::new(), + python: None, + r: None, + r_resolver: RResolver::Disabled, + }, + )) + } + pub(crate) fn builtin( no_sandbox: bool, sandbox_settings: crate::settings::SandboxSettings, diff --git a/src/worker_client/environment/inspection.rs b/src/worker_client/environment/inspection.rs index 36a479822..916b30606 100644 --- a/src/worker_client/environment/inspection.rs +++ b/src/worker_client/environment/inspection.rs @@ -122,10 +122,10 @@ impl Environment { } pub(in crate::worker_client) fn runtime_r_requirements(&self) -> &'static [&'static str] { - if self.custom_worker { - super::super::CUSTOM_DUCKDB_R_REQUIREMENTS - } else if matches!(self.r_resolver, RResolver::Disabled) { + if matches!(self.r_resolver, RResolver::Disabled) { &[] + } else if self.custom_worker { + super::super::CUSTOM_DUCKDB_R_REQUIREMENTS } else { R_RUNTIME_REQUIREMENTS } diff --git a/tests/boundaries/_run.py b/tests/boundaries/_run.py index 6b1fec54e..46610d6a3 100644 --- a/tests/boundaries/_run.py +++ b/tests/boundaries/_run.py @@ -4,6 +4,7 @@ # "py-yaml12>=0.2.0", # "joblib", # "numpy", +# "pillow", # "anyio>=4.15.1", # "mcp==2.*,>=2.3.0", # "anthropic[mcp]>=1.15.0", diff --git a/tests/boundaries/cli/interface/test_jupyter.py b/tests/boundaries/cli/interface/test_jupyter.py new file mode 100644 index 000000000..257fb0482 --- /dev/null +++ b/tests/boundaries/cli/interface/test_jupyter.py @@ -0,0 +1,139 @@ +"""Public launch options select Jupyter without configuring the built-in worker.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from boundaries.cli.interface.test_help import record +from support.client import McpClient +from support.execution import DIRECT +from support.records import Transcript +from support.requirements import POSIX, requires +from support.suites import run_this_suite + + +def test_launch_options(binary: Path) -> Transcript: + help = record(binary, "serve", "--help") + for flag in ( + "--jupyter-kernel", + "--jupyter-python", + "--jupyter-transport", + ): + assert flag in help["stdout"], help + entries = [help] + for arguments in ( + ("--jupyter-python", "python3"), + ("--jupyter-transport", "ipc"), + ): + entry = record(binary, "serve", *arguments) + assert entry.get("exit_code") == 2, entry + assert "required arguments were not provided" in entry["stderr"], entry + assert "--jupyter-kernel " in entry["stderr"], entry + entries.append(entry) + for setting in ( + "startup={code: '1', language: python}", + "python={managed: {}}", + "r={resolution: disabled}", + "languages=[r]", + ): + entry = record( + binary, + "serve", + "--no-config", + "--no-sandbox", + "--jupyter-kernel", + "unused-kernel", + "-c", + setting, + ) + assert entry.get("exit_code") == 1, entry + assert "cannot be combined with --jupyter-kernel" in entry["stderr"], entry + entries.append(entry) + return entries + + +@requires(POSIX) +def test_jupyter_tool_configuration(binary: Path) -> Transcript: + with McpClient( + binary, + DIRECT.serve( + "--no-config", + "--jupyter-kernel", + "unused-kernel", + "-c", + "tool.parameters.code.description=Evaluate the selected Julia kernel.", + "-c", + "tool.parameters.requirements.parameters.action.schema={type: string, enum: [get]}", + ), + ) as client: + client.initialize_and_list_tools() + (tool,) = client.transcript[-1]["result"]["tools"] + properties = tool["inputSchema"]["properties"] + assert ( + properties["code"]["description"] == "Evaluate the selected Julia kernel." + ) + assert properties["requirements"]["properties"]["action"]["enum"] == ["get"] + return client.finish() + + +@requires(POSIX) +def test_jupyter_hides_code(binary: Path) -> Transcript: + with McpClient( + binary, + DIRECT.serve( + "--no-config", + "--jupyter-kernel", + "unused-kernel", + "-c", + "tool.parameters.code.enabled=false", + ), + ) as client: + client.initialize_and_list_tools() + (tool,) = client.transcript[-1]["result"]["tools"] + assert "code" not in tool["inputSchema"]["properties"] + result = client.send(code="must not start the unused kernel") + assert result["isError"], result + assert result["content"] == [ + {"type": "text", "text": "`code` is hidden by `tool.parameters`"} + ], result + return client.finish() + + +def test_jupyter_rejects_unavailable_tool_parameters(binary: Path) -> Transcript: + entries = [] + for field in ("r", "sql", "requirements.parameters.python", "cod"): + entry = record( + binary, + "serve", + "--no-config", + "--no-sandbox", + "--jupyter-kernel", + "unused-kernel", + "-c", + f"tool.parameters.{field}.description=Unavailable field.", + ) + assert entry.get("exit_code") == 1, entry + assert ( + f"tool.parameters.{field}: unknown or unavailable parameter" + in entry["stderr"] + ), entry + entries.append(entry) + entry = record( + binary, + "config", + "resolve", + "--no-config", + "-c", + "tool.parameters.code.description=Unavailable field.", + ) + assert entry.get("exit_code") == 1, entry + assert ( + "tool.parameters.code: unknown or unavailable parameter" in entry["stderr"] + ), entry + entries.append(entry) + return entries + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/integrations/test_python.py b/tests/boundaries/client_server/integrations/test_python.py index 236b0e428..a30b07f57 100755 --- a/tests/boundaries/client_server/integrations/test_python.py +++ b/tests/boundaries/client_server/integrations/test_python.py @@ -42,6 +42,55 @@ def options(binary: Path, execution: Execution) -> dict: } +def test_clients_forward_jupyter_kernel_source(binary: Path) -> Transcript: + source = 'answer = "café Ω"\nprint(answer)' + results = [] + + async def exercise(console) -> None: + assert set(console.send_tool.input_schema["properties"]) == {"code"} + for call in (console.send, console): + output = call(code=source) + assert (await output if inspect.isawaitable(output) else output) == source + + with tempfile.TemporaryDirectory() as directory: + for mode in ("sync", "async"): + root = Path(directory) / mode + root.mkdir() + settings = { + "command": sys.executable, + "args": ["-u", ROOT / "tests/fixtures/python_sync_server.py", "kernel"], + "server_parameters": { + "cwd": root, + "env": dict(os.environ, MCP_CONSOLE_HOME=str(root)), + }, + } + if mode == "sync": + with MCPConsole(**settings) as console: + asyncio.run(exercise(console)) + else: + + async def asynchronous() -> None: + async with AsyncMCPConsole(**settings) as console: + await exercise(console) + + asyncio.run(asynchronous()) + requests = [ + json.loads(line) + for line in (root / "requests.jsonl") + .read_text(encoding="utf-8") + .splitlines() + ] + calls = [ + request["params"] + for request in requests + if request["method"] == "tools/call" + ] + assert calls == [{"name": "send", "arguments": {"code": source}}] * 2 + assert (root / "closed").exists() + results.append({"client": mode, "calls": calls, "closed": True}) + return results + + def test_clients_and_adapters_preserve_hidden_timeout(binary: Path) -> Transcript: from agents.tool_context import ToolContext diff --git a/tests/boundaries/client_server/jupyter/_sandbox.py b/tests/boundaries/client_server/jupyter/_sandbox.py new file mode 100644 index 000000000..fcdb2ea1f --- /dev/null +++ b/tests/boundaries/client_server/jupyter/_sandbox.py @@ -0,0 +1,235 @@ +#!/usr/bin/env -S uv run --script + +import errno +import json +import os +import socket +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from support.assertions import last_tool_text +from support.client import McpClient +from support.execution import SANDBOXED +from support.jupyter import normalize_jupyter_paths +from support.normalization import code, normalize_recording +from support.records import Transcript + + +def default_sandbox_session( + binary: Path, + kernel: str, + *, + network_fixture: str, + network_output: str, + startup_output: str = "", + shutdown_output: str = "", + python_arguments: tuple[str, ...] = ("-Xfrozen_modules=off",), +) -> Transcript: + python = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"]) + assert python.is_absolute() and python.is_file(), python + probe = Path(__file__).resolve().parents[3] / "fixtures" / network_fixture + assert probe.is_file(), probe + with ( + tempfile.TemporaryDirectory(prefix="console-private-ipc-") as directory, + socket.socket(socket.AF_INET, socket.SOCK_STREAM) as tcp, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as unix, + ): + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + sentinel = root / "host-sentinel" + sentinel.write_text("original\n", encoding="utf-8") + tcp.bind(("127.0.0.1", 0)) + tcp.listen() + unix_path = root / "host.sock" + unix.bind(str(unix_path)) + unix.listen() + for listener in (tcp, unix): + # An outside-sandbox connection proves the endpoint is listening; + # denial inside the kernel must not come from an absent host listener. + with socket.socket(listener.family, socket.SOCK_STREAM) as baseline: + baseline.settimeout(1) + baseline.connect(listener.getsockname()) + accepted, _ = listener.accept() + accepted.close() + listener.setblocking(False) + + name = f"console-{kernel}" + specification = root / "jupyter" / "kernels" / name + specification.mkdir(parents=True) + if kernel == "ipykernel": + argv = [ + str(python), + *python_arguments, + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}", + ] + language = "python" + else: + ark = Path(os.environ["MCP_CONSOLE_TEST_ARK"]) + assert ark.is_absolute() and ark.is_file(), ark + argv = [ + str(ark), + "--connection_file", + "{connection_file}", + "--session-mode", + "notebook", + "--", + "--vanilla", + ] + language = "R" + (specification / "kernel.json").write_text( + json.dumps({"argv": argv, "display_name": name, "language": language}), + encoding="utf-8", + ) + environment = dict(os.environ) + for variable in ( + "IPYTHONDIR", + "JUPYTER_RUNTIME_DIR", + "JUPYTER_CONFIG_DIR", + "MPLCONFIGDIR", + ): + environment.pop(variable, None) + environment.update( + JUPYTER_PATH=str(root / "jupyter"), + RUST_LOG="error", + MCP_CONSOLE_TEST_JUPYTER_PYTHON=str(python), + MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE=str(probe), + MCP_CONSOLE_TEST_JUPYTER_HOST_FILE=str(sentinel), + MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO=str( + errno.EPERM if sys.platform == "darwin" else errno.EROFS + ), + MCP_CONSOLE_TEST_JUPYTER_ENDPOINTS=json.dumps( + {"tcp_port": tcp.getsockname()[1], "unix_path": str(unix_path)} + ), + ) + with McpClient( + binary, + SANDBOXED.serve( + "--no-config", + "--jupyter-kernel", + name, + "--jupyter-python", + str(python), + ), + environment, + current_directory=workspace, + ) as client: + client.transcript.append( + { + "scenario": f"The published {kernel} kernel works in the default read-only sandbox.", + "fixture": "No transport or sandbox permissions are configured. Host TCP and Unix listeners accept baseline connections outside the sandbox; a host file contains a protected sentinel.", + } + ) + client.initialize_and_list_tools() + for generation in range(2): + if generation: + result = client.send(control="restart") + assert not result.get("isError"), result + output = last_tool_text(client) + assert output == ( + shutdown_output + + "[worker stopped: in-memory state lost]\n[starting new worker]\n" + + startup_output + + "[idle]" + ), output + client.expect( + "False\n" if kernel == "ipykernel" else "[1] FALSE\n", + code='"answer" in globals()' + if kernel == "ipykernel" + else 'exists("answer", inherits = FALSE)', + ) + client.expect( + (startup_output if generation == 0 else "") + + ("42\n" if kernel == "ipykernel" else "[1] 42\n"), + code="answer = 42\nprint(answer)", + ) + if kernel == "ipykernel": + client.send( + # fmt: python + code=code(""" + import os + import tempfile + from pathlib import Path + + private = Path(tempfile.gettempdir()) / "kernel-permission-probe" + private.write_text("private") + assert private.read_text() == "private" + private.unlink() + print("private temp: writable") + host = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_HOST_FILE"]) + assert host.read_text() == "original\\n" + try: + host.write_text("changed") + except OSError as error: + assert error.errno == int(os.environ["MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO"]), error + print("host file:", error) + else: + raise AssertionError("host file unexpectedly writable") + """), + ) + else: + client.send( + # fmt: r + code=code(""" + private <- tempfile("kernel-permission-probe-") + writeLines("private", private) + stopifnot(identical(readLines(private), "private")) + unlink(private) + cat("private temp: writable\\n") + host <- Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_HOST_FILE") + stopifnot(identical(readLines(host), "original")) + failure <- tryCatch( + withCallingHandlers( + writeLines("changed", host), + warning = function(w) { + cat("host file warning: ", conditionMessage(w), "\\n", sep = "") + invokeRestart("muffleWarning") + } + ), + error = identity + ) + stopifnot(inherits(failure, "error")) + cat("host file error: ", conditionMessage(failure), "\\n", sep = "") + """), + ) + output = last_tool_text(client) + assert output.startswith("private temp: writable\n"), output + assert str(sentinel) in output, output + assert sentinel.read_text(encoding="utf-8") == "original\n" + if kernel == "ipykernel": + client.send( + # fmt: python + code=code(""" + exec(Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE"]).read_text()) + """), + ) + else: + client.send( + # fmt: r + code=code(""" + status <- system2( + Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_PYTHON"), + shQuote(Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE")) + ) + stopifnot(status == 0L) + """), + ) + output = last_tool_text(client) + assert output == network_output, output + for listener in (tcp, unix): + try: + accepted, _ = listener.accept() + except BlockingIOError: + pass + else: + accepted.close() + raise AssertionError( + "host listener accepted a kernel connection" + ) + return normalize_jupyter_paths(normalize_recording(client.finish(), root)) diff --git a/tests/boundaries/client_server/jupyter/test_ark.py b/tests/boundaries/client_server/jupyter/test_ark.py new file mode 100644 index 000000000..e2a336d6f --- /dev/null +++ b/tests/boundaries/client_server/jupyter/test_ark.py @@ -0,0 +1,58 @@ +#!/usr/bin/env -S uv run --script + +import base64 +import io +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from support.assertions import wait_for_evaluation_output +from support.jupyter import jupyter_client +from support.normalization import code +from support.records import Transcript +from support.requirements import ARK, JUPYTER, R, requires +from support.suites import run_this_suite + + +@requires(JUPYTER, ARK, R) +def test_persistent_r_errors_and_plot_output(binary: Path) -> Transcript: + from PIL import Image + + with jupyter_client(binary, "ark") as (client, _): + client.expect("[1] 42\n", code="answer <- 40; answer + 2") + wait_for_evaluation_output( + client, + lambda output: "intentional kernel error" in output, + "ark error reply", + expected_error=False, + completion_timeout_seconds=30, + code='stop("intentional kernel error")', + ) + client.expect("[1] 42\n", code="answer + 2") + result = client.send( + # fmt: r + code=code(""" + par(mar = rep(0, 4), bg = "white") + plot.new() + rect(0.25, 0.25, 0.75, 0.75, col = "red", border = NA) + """), + ) + assert not result.get("isError"), result + assert len(result["content"]) == 1, result + block = result["content"][0] + assert block.keys() == {"type", "data", "mimeType"}, block + assert block["type"] == "image" and block["mimeType"] == "image/png", block + data = base64.b64decode(block["data"], validate=True) + with Image.open(io.BytesIO(data)) as image: + assert image.format == "PNG", image.format + assert image.size == (800, 600), image.size + rgb = image.convert("RGB") + assert rgb.getpixel((400, 300)) == (255, 0, 0) + assert rgb.getpixel((0, 0)) == (255, 255, 255) + block["data"] = "" + return client.finish() + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/jupyter/test_linux_sandbox.py b/tests/boundaries/client_server/jupyter/test_linux_sandbox.py new file mode 100644 index 000000000..65c3ea8d0 --- /dev/null +++ b/tests/boundaries/client_server/jupyter/test_linux_sandbox.py @@ -0,0 +1,82 @@ +#!/usr/bin/env -S uv run --script + +import os +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from boundaries.client_server.jupyter._sandbox import default_sandbox_session +from support.records import Transcript +from support.requirements import ( + ARK, + JUPYTER, + LINUX_SANDBOX, + R, + requires, +) +from support.suites import run_this_suite + + +NETWORK_OUTPUT = ( + "host TCP: [Errno 111] Connection refused\n" + "external TCP: [Errno 101] Network is unreachable\n" + "host Unix: [Errno 1] Operation not permitted\n" + "external HTTP through proxy: denied (403)\n" + "external CONNECT through proxy: denied (403)\n" + "host CONNECT through proxy: denied (403)\n" +) +TCP_WARNING = ( + "[IPKernelApp] WARNING | Kernel is running over TCP without encryption. " + "All communication (including code and outputs) is sent in plain text and " + "is susceptible to eavesdropping. Use IPC transport or launch with kernel " + "manager-provisioned CurveZMQ keys to enable transport encryption.\n" +) + + +def linux_session(binary: Path, kernel: str) -> Transcript: + localinterfaces = subprocess.check_output( + [ + os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"], + "-c", + "import jupyter_client.localinterfaces; print(jupyter_client.localinterfaces.__file__)", + ], + text=True, + ).strip() + assert Path(localinterfaces).is_absolute() and Path(localinterfaces).is_file(), ( + localinterfaces + ) + warning = ( + f"{localinterfaces}:58: UserWarning: Unexpected error discovering local " + "network interfaces: [Errno 1] Operation not permitted\n" + " ret = f(**kwargs)\n" + ) + startup_output = warning + if kernel == "ipykernel": + startup_output += warning + TCP_WARNING + return default_sandbox_session( + binary, + kernel, + network_fixture="jupyter_sandbox_network_linux.py", + network_output=NETWORK_OUTPUT, + startup_output=startup_output, + # The published `python -m ipykernel install` adds this interpreter flag. + python_arguments=("-Xfrozen_modules=off",) if kernel == "ipykernel" else (), + ) + + +@requires(JUPYTER, LINUX_SANDBOX) +def test_ipykernel_private_network_preserves_default_sandbox( + binary: Path, +) -> Transcript: + return linux_session(binary, "ipykernel") + + +@requires(JUPYTER, ARK, R, LINUX_SANDBOX) +def test_ark_private_network_preserves_default_sandbox(binary: Path) -> Transcript: + return linux_session(binary, "ark") + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/jupyter/test_macos_sandbox.py b/tests/boundaries/client_server/jupyter/test_macos_sandbox.py new file mode 100644 index 000000000..b36a90ba8 --- /dev/null +++ b/tests/boundaries/client_server/jupyter/test_macos_sandbox.py @@ -0,0 +1,89 @@ +#!/usr/bin/env -S uv run --script + +import os +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from boundaries.client_server.jupyter._sandbox import default_sandbox_session +from support.records import Transcript +from support.requirements import ( + ARK, + JUPYTER, + MACOS_SANDBOX, + R, + requires, +) +from support.suites import run_this_suite + +NETWORK_OUTPUT = ( + "private Unix: roundtrip\n" + "host TCP: [Errno 1] Operation not permitted\n" + "external TCP: [Errno 1] Operation not permitted\n" + "host Unix: [Errno 1] Operation not permitted\n" + "host Unix through private symlink: [Errno 1] Operation not permitted\n" +) + + +@requires(JUPYTER, MACOS_SANDBOX) +def test_ipykernel_private_ipc_preserves_default_sandbox(binary: Path) -> Transcript: + iostream = subprocess.check_output( + [ + os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"], + "-c", + "import ipykernel.iostream; print(ipykernel.iostream.__file__)", + ], + text=True, + ).strip() + assert Path(iostream).is_absolute() and Path(iostream).is_file(), iostream + packages = str(Path(iostream).parent.parent) + return default_sandbox_session( + binary, + "ipykernel", + network_fixture="jupyter_sandbox_network.py", + network_output=NETWORK_OUTPUT, + startup_output=( + f"{iostream}:73: UserWarning: Couldn't bind IOPub Pipe to 127.0.0.1: " + "Operation not permitted (addr='tcp://127.0.0.1:*')\n" + "subprocess output will be unavailable.\n" + " self._setup_pipe_in()\n" + ), + shutdown_output=( + "[IPKernelApp] ERROR | Exception during subprocesses termination\n" + "Traceback (most recent call last):\n" + f' File "{packages}/ipykernel/kernelbase.py", line 1645, in _at_shutdown\n' + " await self._progressively_terminate_all_children()\n" + f' File "{packages}/ipykernel/kernelbase.py", line 1623, in _progressively_terminate_all_children\n' + " if not self._process_children():\n" + " ~~~~~~~~~~~~~~~~~~~~~~^^\n" + f' File "{packages}/ipykernel/kernelbase.py", line 1606, in _process_children\n' + " all_children = kernel_process.children(recursive=True)\n" + f' File "{packages}/psutil/__init__.py", line 978, in children\n' + " ppid_map = _ppid_map()\n" + f' File "{packages}/psutil/__init__.py", line 256, in _ppid_map\n' + " for pid in pids():\n" + " ~~~~^^\n" + f' File "{packages}/psutil/__init__.py", line 1485, in pids\n' + " ret = sorted(_psplatform.pids())\n" + " ~~~~~~~~~~~~~~~~^^\n" + f' File "{packages}/psutil/_psosx.py", line 320, in pids\n' + " ls = cext.pids()\n" + "PermissionError: [Errno 1] Operation not permitted (originated from sysctl() malloc 1/3)\n" + ), + ) + + +@requires(JUPYTER, ARK, R, MACOS_SANDBOX) +def test_ark_private_ipc_preserves_default_sandbox(binary: Path) -> Transcript: + return default_sandbox_session( + binary, + "ark", + network_fixture="jupyter_sandbox_network.py", + network_output=NETWORK_OUTPUT, + ) + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/jupyter/test_python.py b/tests/boundaries/client_server/jupyter/test_python.py new file mode 100644 index 000000000..86db3ffbf --- /dev/null +++ b/tests/boundaries/client_server/jupyter/test_python.py @@ -0,0 +1,395 @@ +#!/usr/bin/env -S uv run --script + +import base64 +import json +import sys +import tempfile +from contextlib import closing +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from support.assertions import ( + assert_result_content, + last_tool_text, + wait_for_evaluation_output, +) +from support.checkpoints import FifoCheckpoint, wait_for_path +from support.evidence import compact_text +from support.execution import SANDBOXED, Execution, executions +from support.jupyter import jupyter_client, normalize_jupyter_paths +from support.normalization import code, normalize_recording +from support.progress import phase_progress, without_elapsed +from support.processes import capture_process_identity, kill_processes, live_processes +from support.records import Transcript +from support.requirements import JUPYTER, POSIX, PROCESS_EVENTS, requires +from support.suites import run_this_suite + + +@requires(JUPYTER) +@executions(SANDBOXED) +def test_ipc_kernel_with_explicit_socket_permissions( + binary: Path, execution: Execution +) -> Transcript: + with jupyter_client(binary, execution=execution) as (client, _): + client.expect("42\n", code="6 * 7") + return client.finish() + + +@requires(JUPYTER) +def test_persistent_state_streams_errors_and_recovery(binary: Path) -> Transcript: + with jupyter_client(binary) as (client, workspace): + schema = client.transcript[-1]["result"]["tools"][0]["inputSchema"] + assert set(schema["properties"]) == { + "code", + "stdin", + "control", + "timeout_ms", + "requirements", + } + requirements = schema["properties"]["requirements"] + assert set(requirements["properties"]) == {"action"} + assert requirements["properties"]["action"]["enum"] == ["get"] + assert requirements["required"] == ["action"] + inspected = client.send(requirements={"action": "get"}) + state = inspected["structuredContent"] + assert json.loads(last_tool_text(client)) == state + assert state["requirements"] == { + "r": [], + "python": [], + "duckdb": [], + "python_version": [], + "exclude_newer": None, + }, state + assert state["runtime_requirements"] == {"r": [], "python": []}, state + client.expect("42\n", code="answer = 40\nanswer + 2") + large_source = "# " + "x" * (100 * 1024) + "\nanswer + 2" + client.expect("42\n", code=large_source) + client.transcript[-1]["send"]["code"] = compact_text(large_source, "x") + client.expect( + "stdout café\nstderr Ω\n", + # fmt: python + code=code(""" + import sys + + print("stdout café", flush=True) + print("stderr Ω", file=sys.stderr, flush=True) + """), + ) + wait_for_evaluation_output( + client, + lambda output: "ValueError: intentional kernel error" in output, + "ipykernel error reply", + expected_error=False, + completion_timeout_seconds=30, + code='raise ValueError("intentional kernel error")', + ) + client.expect("42\n", code="answer + 2") + for arguments in ( + {"r": "1 + 1"}, + {"python": "1 + 1"}, + {"sql": "SELECT 1"}, + {"requirements": {"python": ["requests"]}}, + ): + rejected = client.send(**arguments) + assert rejected["isError"] is True, rejected + client.expect("42\n", code="answer + 2") + transcript = client.finish() + (recording,) = (workspace / ".agents/console/sessions").iterdir() + (document,) = (recording / "sessions").glob("*.qmd") + quarto = document.read_text(encoding="utf-8") + assert " jupyter-kernel: console-ipykernel\n" in quarto, quarto + assert "execute:\n eval: false\n" in quarto, quarto + assert "\nir:\n" not in quarto and "ir render" not in quarto, quarto + assert "```text\nanswer = 40\nanswer + 2\n```" in quarto, quarto + assert large_source in quarto, quarto + assert "```{code}" not in quarto, quarto + transcript.append( + { + "recording": "Generic kernel source is retained in plain text Quarto fences." + } + ) + return transcript + + +@requires(JUPYTER) +def test_delayed_kernel_info_replies_survive_retries(binary: Path) -> Transcript: + with jupyter_client(binary, kernel_fixture="jupyter_delayed_info.py") as ( + client, + _, + ): + client.transcript[0]["scenario"] = ( + "A delayed readiness reply remains valid after another kernel_info request." + ) + client.transcript[0]["fixture"] += ( + " A published IPythonKernel subclass constructs every readiness reply immediately, " + "but delivers it only when the next request arrives. Normal IOPub idle messages " + "are delivered immediately; no sleeps or adjusted deadlines are used." + ) + client.expect("42\n", code="answer = 6 * 7\nanswer") + client.expect("42\n", code="answer") + return client.finish() + + +@requires(JUPYTER) +def test_tcp_startup_warning_preserves_output_and_state(binary: Path) -> Transcript: + with jupyter_client(binary, transport="tcp") as (client, _): + client.expect( + "[IPKernelApp] WARNING | Kernel is running over TCP without encryption. " + "All communication (including code and outputs) is sent in plain text and " + "is susceptible to eavesdropping. Use IPC transport or launch with kernel " + "manager-provisioned CurveZMQ keys to enable transport encryption.\n42\n", + code="answer = 40\nanswer + 2", + ) + client.expect("42\n", code="answer + 2") + return client.finish() + + +@requires(JUPYTER) +def test_rich_output_and_prompted_input(binary: Path) -> Transcript: + with jupyter_client(binary) as (client, _): + encoded = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8" + "/x8AAwMCAO+aSuoAAAAASUVORK5CYII=" + ) + client.send( + # fmt: python + code=code(f""" + import base64 + from IPython.display import Image, display + + display(Image(data=base64.b64decode("{encoded}"))) + """), + ) + assert_result_content( + client, + [base64.b64decode(encoded)], + image_reference="the submitted one-pixel PNG", + ) + wait_for_evaluation_output( + client, + '[input requested: "Name: "]\n[waiting for stdin]', + "ipykernel input request", + completion_timeout_seconds=30, + code='name = input("Name: ")\nprint("hello", name)', + ) + client.expect("hello Ada\n", stdin="Ada\n") + client.expect("'Ada'\n", code="name") + return client.finish() + + +@requires(JUPYTER) +def test_interrupt_and_restart_preserve_generation_ownership( + binary: Path, +) -> Transcript: + with jupyter_client(binary) as (client, workspace): + client.transcript[0]["fixture"] += ( + " A file marker proves that an interruptible Python evaluation reached its gate." + ) + client.expect("ready\n", code='answer = 42\nprint("ready")') + client.send( + # fmt: python + code=code(""" + from pathlib import Path + import threading + + try: + Path("interrupt-ready").touch() + threading.Event().wait() + except KeyboardInterrupt: + print("interrupted") + """), + timeout_ms=0, + ) + assert without_elapsed(last_tool_text(client)) == ( + "\n[running; poll with an empty send]" + ) + wait_for_path( + workspace / "interrupt-ready", + "ipykernel reached its interruptible gate", + client=client, + ) + client.expect("interrupted\n", control="interrupt") + client.expect("42\n", code="answer") + restarted = client.send(control="restart") + assert not restarted.get("isError"), restarted + assert last_tool_text(client) == ( + "[worker stopped: in-memory state lost]\n[starting new worker]\n[idle]" + ) + client.expect("False\n", code='"answer" in globals()') + client.expect("42\n", code="6 * 7") + return normalize_recording(client.finish(), workspace) + + +@requires(JUPYTER, POSIX, PROCESS_EVENTS) +def test_slow_kernel_startup_remains_pending_until_ready(binary: Path) -> Transcript: + identities = [] + try: + with ( + tempfile.TemporaryDirectory(prefix="jupyter-startup-gate-") as directory, + closing(FifoCheckpoint.create(Path(directory) / "startup.fifo")) as gate, + jupyter_client(binary, startup_gate=gate.path) as (client, workspace), + ): + client.transcript[0]["scenario"] = ( + "A live kernel remains in startup until its Jupyter channels are ready." + ) + client.transcript[0]["fixture"] += ( + " A FIFO holds the acquired kernel before ipykernel initialization across a 1.5-second public poll." + ) + try: + client.send(code="answer = 42\nprint(answer)", timeout_ms=0) + assert without_elapsed(last_tool_text(client)) == ( + "\n[running; poll with an empty send]" + ) + wait_for_path( + gate.path.with_suffix(".ready"), + "kernel process acquired before Jupyter readiness", + client=client, + ) + pid = int(gate.path.with_suffix(".pid").read_text(encoding="utf-8")) + identity = capture_process_identity(pid) + identities.append(identity) + client.send(timeout_ms=1_500) + assert without_elapsed(last_tool_text(client)) == ( + "\n[running; poll with an empty send]" + ) + assert live_processes((identity,)) == [pid], ( + "live startup process was mistaken for a failed kernel" + ) + gate.release() + client.expect("42\n") + client.expect("42\n", code="answer") + return normalize_jupyter_paths( + normalize_recording(client.finish(), workspace) + ) + finally: + gate.release() + finally: + kill_processes(identities) + + +@requires(JUPYTER, POSIX, PROCESS_EVENTS) +def test_interrupt_cancels_kernel_startup_and_reaps_process(binary: Path) -> Transcript: + identities = [] + try: + with ( + tempfile.TemporaryDirectory(prefix="jupyter-startup-gate-") as directory, + closing(FifoCheckpoint.create(Path(directory) / "startup.fifo")) as gate, + jupyter_client(binary, startup_gate=gate.path) as (client, workspace), + ): + client.transcript[0]["scenario"] = ( + "Interrupt cancels kernel startup before Jupyter readiness and discards the pending cell." + ) + client.transcript[0]["fixture"] += ( + " A Python launcher records its PID and blocks on a FIFO before starting the released ipykernel." + ) + try: + evaluation = client.start_send( + # fmt: python + code=code(""" + from pathlib import Path + + Path("must-not-run").write_text("unexpected execution") + """), + ) + wait_for_path( + gate.path.with_suffix(".ready"), + "kernel process acquired before Jupyter readiness", + client=client, + ) + pid = int(gate.path.with_suffix(".pid").read_text(encoding="utf-8")) + identity = capture_process_identity(pid) + identities.append(identity) + interrupt = client.start_send(control="interrupt", timeout_ms=3_000) + client.receive_many([evaluation, interrupt]) + result = evaluation["result"] + assert result["isError"] is True, result + output = result["content"][0]["text"] + assert "Jupyter startup interrupted" in output, output + assert interrupt["result"]["isError"] is False, interrupt + assert phase_progress(last_tool_text(client)) == "startup" + assert without_elapsed(last_tool_text(client)) == ( + "\n[running; poll with an empty send]" + ) + assert live_processes((identity,)) == [], ( + "interrupted startup leaked its kernel" + ) + assert not (workspace / "must-not-run").exists(), ( + "cancelled startup evaluated pending source" + ) + return normalize_jupyter_paths( + normalize_recording(client.finish(), workspace) + ) + finally: + # Release before transport teardown even on the old broken + # implementation, so the regression does not wait for readiness's timeout. + gate.release() + finally: + kill_processes(identities) + + +@requires(JUPYTER, PROCESS_EVENTS) +def test_kernel_crash_recovery_and_process_retirement(binary: Path) -> Transcript: + identities = [] + try: + with jupyter_client(binary) as (client, workspace): + client.transcript[0]["scenario"] = ( + "A kernel crash loses state; explicit restart recovers and reaps prior processes." + ) + client.send(code="import os\nprint(os.getpid())") + pid = int(last_tool_text(client).strip()) + original = capture_process_identity(pid) + identities.append(original) + client.transcript[-1]["result"]["content"][0]["text"] = ( + "\n" + ) + result = client.send(control="restart") + assert not result.get("isError"), result + assert last_tool_text(client).endswith("[starting new worker]\n[idle]") + assert live_processes((original,)) == [], ( + "normal restart leaked its old kernel" + ) + + client.send(code="import os\nprint(os.getpid())") + pid = int(last_tool_text(client).strip()) + crashing = capture_process_identity(pid) + identities.append(crashing) + assert crashing != original + client.transcript[-1]["result"]["content"][0]["text"] = ( + "\n" + ) + result = client.send(code="os._exit(7)") + assert result["isError"] is True, result + text = result["content"][0]["text"] + assert "Jupyter kernel exited with status 7" in text, text + assert live_processes((crashing,)) == [], "crashed kernel was not reaped" + result = client.send(control="restart") + assert not result.get("isError"), result + assert last_tool_text(client).endswith("[starting new worker]\n[idle]") + client.expect("42\n", code="6 * 7") + client.send(code="import os\nprint(os.getpid())") + pid = int(last_tool_text(client).strip()) + replacement = capture_process_identity(pid) + identities.append(replacement) + assert replacement != crashing + client.transcript[-1]["result"]["content"][0]["text"] = ( + "\n" + ) + transcript = normalize_jupyter_paths( + normalize_recording(client.finish(), workspace) + ) + assert live_processes(identities) == [], ( + "server shutdown leaked a kernel process" + ) + transcript.append( + { + "processes": "Restart, crash, and shutdown each retired the kernel process." + } + ) + return transcript + finally: + kill_processes(identities) + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/jupyter/test_sandbox_inheritance.py b/tests/boundaries/client_server/jupyter/test_sandbox_inheritance.py new file mode 100644 index 000000000..c8210b38a --- /dev/null +++ b/tests/boundaries/client_server/jupyter/test_sandbox_inheritance.py @@ -0,0 +1,115 @@ +#!/usr/bin/env -S uv run --script + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from support.assertions import last_tool_text +from support.client import McpClient +from support.execution import SANDBOXED +from support.jupyter import normalize_jupyter_paths +from support.normalization import code, normalize_recording +from support.records import Transcript +from support.requirements import JUPYTER, MACOS_SANDBOX, requires +from support.suites import run_this_suite + + +@requires(JUPYTER, MACOS_SANDBOX) +def test_unrestricted_filesystem_preserves_private_ipc(binary: Path) -> Transcript: + python = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"]) + iostream = subprocess.check_output( + [ + str(python), + "-c", + "import ipykernel.iostream; print(ipykernel.iostream.__file__)", + ], + text=True, + ).strip() + assert Path(iostream).is_absolute() and Path(iostream).is_file(), iostream + with tempfile.TemporaryDirectory(prefix="console-inherited-ipc-") as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + host = root / "host-file" + specification = root / "jupyter" / "kernels" / "console-python" + specification.mkdir(parents=True) + (specification / "kernel.json").write_text( + json.dumps( + { + "argv": [ + str(python), + "-Xfrozen_modules=off", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}", + ], + "display_name": "console-python", + "language": "python", + } + ) + ) + environment = dict( + os.environ, + JUPYTER_PATH=str(root / "jupyter"), + MCP_CONSOLE_TEST_HOST_FILE=str(host), + ) + with McpClient( + binary, + SANDBOXED.serve( + "--no-config", + "--jupyter-kernel", + "console-python", + "--jupyter-python", + str(python), + "-c", + "sandbox.profile=inherit_codex", + ), + environment, + current_directory=workspace, + ) as client: + client.initialize_and_list_tools() + client.request( + "tools/call", + name="send", + arguments={ + "timeout_ms": 60000, + # fmt: python + "code": code(""" + import os + from pathlib import Path + + host = Path(os.environ["MCP_CONSOLE_TEST_HOST_FILE"]) + _ = host.write_text("inherited write access") + print(host.read_text()) + """), + }, + _meta={ + "codex/sandbox-state": { + "permissionProfile": { + "type": "managed", + "file_system": {"type": "unrestricted"}, + "network": "restricted", + } + } + }, + ) + assert last_tool_text(client) == ( + f"{iostream}:73: UserWarning: Couldn't bind IOPub Pipe to 127.0.0.1: " + "Operation not permitted (addr='tcp://127.0.0.1:*')\n" + "subprocess output will be unavailable.\n" + " self._setup_pipe_in()\n" + "inherited write access\n" + ), last_tool_text(client) + assert host.read_text() == "inherited write access" + transcript = normalize_recording(client.finish(), root) + return normalize_jupyter_paths(transcript) + + +if __name__ == "__main__": + run_this_suite(__file__) diff --git a/tests/boundaries/client_server/server/test_tools.py b/tests/boundaries/client_server/server/test_tools.py index 8fa74e441..1ce36e731 100755 --- a/tests/boundaries/client_server/server/test_tools.py +++ b/tests/boundaries/client_server/server/test_tools.py @@ -135,6 +135,18 @@ def test_initializes_and_lists_tools( "configured-r-sql.yaml": {"languages": ("r", "sql")}, "r-only.yaml": {"bootstrap_languages": "r"}, } + if POSIX.available: + variants.update( + { + "jupyter.yaml": {"jupyter": True}, + } + ) + if execution == SANDBOXED: + variants.update( + { + "jupyter-writable.yaml": {"jupyter": True, "writable": True}, + } + ) if execution == SANDBOXED: variants.update( { @@ -213,6 +225,55 @@ def test_initializes_and_lists_tools( return TranscriptWithCompanions(baseline, companions) +def _jupyter_initialization( + binary: Path, execution: Execution, *, writable: bool +) -> Transcript: + # Discovery is lazy: this canonical schema needs neither installed kernels + # nor Jupyter packages, and never asks the selected worker to start. + environment = os.environ.copy() + environment.pop("MCP_CONSOLE_LANGUAGES", None) + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + with McpClient( + binary, + execution.serve( + "--no-config", + "--jupyter-kernel", + "unused-kernel", + "--jupyter-python", + sys.executable, + *( + ( + "--writable-root", + str(workspace), + "-c", + "sandbox.network={proxy: {mode: limited}, sockets: {unix_sockets: dangerously_allow_all}, allow_local_binding: true}", + ) + if writable + else () + ), + ), + environment, + current_directory=workspace, + record_in_project=False, + ) as client: + client.initialize_and_list_tools() + (send,) = client.transcript[-1]["result"]["tools"] + assert send["name"] == "send", send + assert set(send["inputSchema"]["properties"]) == { + "code", + "control", + "stdin", + "timeout_ms", + "requirements", + }, send + assert ( + client.transcript[0]["result"]["instructions"] + == (send["description"].split("\n\n", 1)[0]) + ) + return client.finish() + + def _initializes_and_lists_tools( binary: Path, execution: Execution, @@ -228,7 +289,10 @@ def _initializes_and_lists_tools( sandbox_profile: str | None = None, bootstrap_languages: str | None = None, languages: tuple[str, ...] | None = None, + jupyter: bool = False, ) -> Transcript: + if jupyter: + return _jupyter_initialization(binary, execution, writable=writable) environment = os.environ.copy() environment.pop("MCP_CONSOLE_LANGUAGES", None) if bootstrap_languages is not None: diff --git a/tests/fixtures/jupyter_delayed_info.py b/tests/fixtures/jupyter_delayed_info.py new file mode 100644 index 000000000..c46480460 --- /dev/null +++ b/tests/fixtures/jupyter_delayed_info.py @@ -0,0 +1,29 @@ +"""Delay each readiness reply until the next request, without wall-clock sleeps.""" + +from typing import Any + +from ipykernel.ipkernel import IPythonKernel +from ipykernel.kernelapp import IPKernelApp + + +class DelayedInfoKernel(IPythonKernel): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.pending_info: tuple[Any, Any, dict] | None = None + + async def kernel_info_request(self, stream: Any, ident: Any, parent: dict) -> None: + assert self.session is not None + reply = self.session.msg( + "kernel_info_reply", {"status": "ok", **self.kernel_info}, parent + ) + # Construct the reply before stock dispatch emits idle. Hold only its + # delivery: independent Shell and IOPub channels need not arrive together. + # A subsequent request releases it and deterministically exercises retry + # correlation without changing deadlines or sleeping in the fixture. + if self.pending_info is not None: + previous_stream, previous_ident, previous_reply = self.pending_info + self.session.send(previous_stream, previous_reply, ident=previous_ident) + self.pending_info = stream, ident, reply + + +IPKernelApp.launch_instance(kernel_class=DelayedInfoKernel) diff --git a/tests/fixtures/jupyter_sandbox_network.py b/tests/fixtures/jupyter_sandbox_network.py new file mode 100644 index 000000000..9b8174e76 --- /dev/null +++ b/tests/fixtures/jupyter_sandbox_network.py @@ -0,0 +1,45 @@ +"""Exercise ordinary host sockets from public kernel code in a macOS sandbox.""" + +import errno +import json +import os +import socket +import tempfile +from pathlib import Path + +endpoints = json.loads(os.environ["MCP_CONSOLE_TEST_JUPYTER_ENDPOINTS"]) +with tempfile.TemporaryDirectory(prefix="") as directory: + private = Path(directory) / "s" + with ( + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection, + ): + listener.bind(str(private)) + listener.listen() + connection.connect(str(private)) + accepted, _ = listener.accept() + with accepted: + message = b"private IPC" + connection.sendall(message) + with accepted.makefile("rb") as stream: + assert stream.read(len(message)) == message + print("private Unix: roundtrip", flush=True) + +host_alias = Path(tempfile.gettempdir()) / "alias" +host_alias.symlink_to(endpoints["unix_path"]) +for label, family, address in ( + ("host TCP", socket.AF_INET, ("127.0.0.1", endpoints["tcp_port"])), + ("external TCP", socket.AF_INET, ("198.51.100.1", 80)), + ("host Unix", socket.AF_UNIX, endpoints["unix_path"]), + ("host Unix through private symlink", socket.AF_UNIX, str(host_alias)), +): + try: + with socket.socket(family, socket.SOCK_STREAM) as connection: + connection.settimeout(1) + connection.connect(address) + except OSError as error: + assert error.errno in {errno.EPERM, errno.EACCES}, (label, error) + print(f"{label}: {error}", flush=True) + else: + raise AssertionError(f"{label} unexpectedly connected outside the sandbox") +host_alias.unlink() diff --git a/tests/fixtures/jupyter_sandbox_network_linux.py b/tests/fixtures/jupyter_sandbox_network_linux.py new file mode 100644 index 000000000..7853a55ec --- /dev/null +++ b/tests/fixtures/jupyter_sandbox_network_linux.py @@ -0,0 +1,45 @@ +"""Probe host isolation from a kernel in its private Linux network namespace.""" + +import errno +import http.client +import json +import os +import socket +from contextlib import closing +from urllib.parse import urlsplit + +endpoints = json.loads(os.environ["MCP_CONSOLE_TEST_JUPYTER_ENDPOINTS"]) +for label, family, address, expected in ( + ( + "host TCP", + socket.AF_INET, + ("127.0.0.1", endpoints["tcp_port"]), + errno.ECONNREFUSED, + ), + ("external TCP", socket.AF_INET, ("198.51.100.1", 80), errno.ENETUNREACH), + ("host Unix", socket.AF_UNIX, endpoints["unix_path"], errno.EPERM), +): + try: + with socket.socket(family, socket.SOCK_STREAM) as connection: + connection.settimeout(1) + connection.connect(address) + except OSError as error: + assert error.errno == expected, (label, error) + print(f"{label}: {error}", flush=True) + else: + raise AssertionError(f"{label} unexpectedly connected outside the sandbox") + +proxy = urlsplit(os.environ["HTTP_PROXY"]) +assert proxy.hostname == "127.0.0.1", proxy +for label, method, destination in ( + ("external HTTP", "GET", "http://198.51.100.1/"), + ("external CONNECT", "CONNECT", "198.51.100.1:80"), + ("host CONNECT", "CONNECT", f"127.0.0.1:{endpoints['tcp_port']}"), +): + with closing( + http.client.HTTPConnection(proxy.hostname, proxy.port, timeout=2) + ) as connection: + connection.request(method, destination) + response = connection.getresponse() + assert response.status == 403, (label, response.status, response.read()) + print(f"{label} through proxy: denied (403)", flush=True) diff --git a/tests/fixtures/jupyter_startup.py b/tests/fixtures/jupyter_startup.py new file mode 100644 index 000000000..64aba986a --- /dev/null +++ b/tests/fixtures/jupyter_startup.py @@ -0,0 +1,12 @@ +"""Hold a real ipykernel process before its Jupyter readiness handshake.""" + +import os +import runpy +from pathlib import Path + +gate = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_STARTUP_GATE"]) +with gate.open("rb", buffering=0) as release: + gate.with_suffix(".pid").write_text(str(os.getpid()), encoding="utf-8") + gate.with_suffix(".ready").touch() + assert release.read(1) == b"1", "startup gate closed without release" +runpy.run_module("ipykernel_launcher", run_name="__main__", alter_sys=True) diff --git a/tests/fixtures/python_sync_server.py b/tests/fixtures/python_sync_server.py index 36bc9f76e..71cdb4a38 100644 --- a/tests/fixtures/python_sync_server.py +++ b/tests/fixtures/python_sync_server.py @@ -61,7 +61,11 @@ def main() -> None: "description": "Fixture send", "inputSchema": { "type": "object", - "properties": {"python": {"type": "string"}}, + "properties": { + "code" if mode == "kernel" else "python": { + "type": "string" + } + }, }, } ] @@ -79,7 +83,7 @@ def main() -> None: reply(request, output("missing request cancellation")) continue arguments = request["params"]["arguments"] - source = arguments.get("python", "") + source = arguments.get("code" if mode == "kernel" else "python", "") if source == "cancel wait": abandoned = request["id"] encoded = ( diff --git a/tests/snapshots/cli/interface/test_help/help.yaml b/tests/snapshots/cli/interface/test_help/help.yaml index 2a355caea..8aa02b39c 100644 --- a/tests/snapshots/cli/interface/test_help/help.yaml +++ b/tests/snapshots/cli/interface/test_help/help.yaml @@ -292,6 +292,17 @@ stdout: | --writable-root Allow writes to an additional path (temporary launch option) + --jupyter-kernel + Run an installed local Jupyter kernel instead of the built-in worker + + --jupyter-python + Python with jupyter_client installed (defaults to python3) + + --jupyter-transport + Jupyter transport (auto selects a private sandbox transport) + + [possible values: auto, tcp, ipc] + -h, --help Print help (see a summary with '-h') --- diff --git a/tests/snapshots/cli/interface/test_jupyter/jupyter_hides_code.yaml b/tests/snapshots/cli/interface/test_jupyter/jupyter_hides_code.yaml new file mode 100644 index 000000000..21a69df10 --- /dev/null +++ b/tests/snapshots/cli/interface/test_jupyter/jupyter_hides_code.yaml @@ -0,0 +1,120 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + Evaluated code runs without a sandbox, with the server's permissions, + including filesystem and network access. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action +--- +send: + code: must not start the unused kernel +result: + content: + - type: text + text: "`code` is hidden by `tool.parameters`" + isError: true diff --git a/tests/snapshots/cli/interface/test_jupyter/jupyter_rejects_unavailable_tool_parameters.yaml b/tests/snapshots/cli/interface/test_jupyter/jupyter_rejects_unavailable_tool_parameters.yaml new file mode 100644 index 000000000..817f5ec33 --- /dev/null +++ b/tests/snapshots/cli/interface/test_jupyter/jupyter_rejects_unavailable_tool_parameters.yaml @@ -0,0 +1,48 @@ +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c tool.parameters.r.description=Unavailable field. +exit_code: 1 +stdout: "" +stderr: > + configuration with CLI overrides: tool.parameters.r: unknown or + unavailable parameter +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c tool.parameters.sql.description=Unavailable field. +exit_code: 1 +stdout: "" +stderr: > + configuration with CLI overrides: tool.parameters.sql: unknown or + unavailable parameter +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c + tool.parameters.requirements.parameters.python.description=Unavailable + field. +exit_code: 1 +stdout: "" +stderr: > + configuration with CLI overrides: + tool.parameters.requirements.parameters.python: unknown or unavailable + parameter +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c tool.parameters.cod.description=Unavailable field. +exit_code: 1 +stdout: "" +stderr: > + configuration with CLI overrides: tool.parameters.cod: unknown or + unavailable parameter +--- +command: >- + mcp-console config resolve --no-config -c + tool.parameters.code.description=Unavailable field. +exit_code: 1 +stdout: "" +stderr: > + configuration with CLI overrides: tool.parameters.code: unknown or + unavailable parameter diff --git a/tests/snapshots/cli/interface/test_jupyter/jupyter_tool_configuration.yaml b/tests/snapshots/cli/interface/test_jupyter/jupyter_tool_configuration.yaml new file mode 100644 index 000000000..b21d0620f --- /dev/null +++ b/tests/snapshots/cli/interface/test_jupyter/jupyter_tool_configuration.yaml @@ -0,0 +1,117 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + Evaluated code runs without a sandbox, with the server's permissions, + including filesystem and network access. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + code: + description: Evaluate the selected Julia kernel. + type: + - string + - "null" + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action diff --git a/tests/snapshots/cli/interface/test_jupyter/launch_options.yaml b/tests/snapshots/cli/interface/test_jupyter/launch_options.yaml new file mode 100644 index 000000000..49d9ecba2 --- /dev/null +++ b/tests/snapshots/cli/interface/test_jupyter/launch_options.yaml @@ -0,0 +1,115 @@ +--- +command: mcp-console serve --help +stdout: | + Run the MCP server over standard input and output + + Usage: mcp-console serve [OPTIONS] + + Options: + --no-project-config + Skip launch-directory project configuration; still load global + configuration and apply -c overrides. + Automatic project configuration is trusted launcher input: it can + select executables, child environments, and sandbox permissions. + Trusting sandboxed code is not equivalent to trusting it to define the + sandbox. + Global settings supply defaults, not a mandatory security ceiling; + projects may override them. Integrations opening unfamiliar projects + should use this flag until they authorize project configuration. + + --no-global-config + Skip global configuration; still load project configuration and apply + -c overrides + + --no-config + Skip global and project configuration; still apply --default-config + and -c settings + + --config-file + Use only this YAML or JSON configuration file, then apply -c overrides + + --default-config + Supply defaults below configuration files; repeat for ordered dotted + KEY=VALUE assignments + + -c, --config + Override layered global and project configuration; repeat for ordered + dotted KEY=VALUE assignments + + --no-sandbox + Run evaluated code and dependency resolution with host permissions + + --writable-root + Allow writes to an additional path (temporary launch option) + + --jupyter-kernel + Run an installed local Jupyter kernel instead of the built-in worker + + --jupyter-python + Python with jupyter_client installed (defaults to python3) + + --jupyter-transport + Jupyter transport (auto selects a private sandbox transport) + + [possible values: auto, tcp, ipc] + + -h, --help + Print help (see a summary with '-h') +--- +command: mcp-console serve --jupyter-python python3 +exit_code: 2 +stdout: "" +stderr: | + error: the following required arguments were not provided: + --jupyter-kernel + + Usage: mcp-console serve --jupyter-kernel --jupyter-python + + For more information, try '--help'. +--- +command: mcp-console serve --jupyter-transport ipc +exit_code: 2 +stdout: "" +stderr: | + error: the following required arguments were not provided: + --jupyter-kernel + + Usage: mcp-console serve --jupyter-kernel --jupyter-transport + + For more information, try '--help'. +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c startup={code: '1', language: python} +exit_code: 1 +stdout: "" +stderr: > + startup settings cannot be combined with --jupyter-kernel; configure the + installed kernelspec instead +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c python={managed: {}} +exit_code: 1 +stdout: "" +stderr: > + python settings cannot be combined with --jupyter-kernel; configure the + installed kernelspec instead +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c r={resolution: disabled} +exit_code: 1 +stdout: "" +stderr: > + r settings cannot be combined with --jupyter-kernel; configure the + installed kernelspec instead +--- +command: >- + mcp-console serve --no-config --no-sandbox --jupyter-kernel + unused-kernel -c languages=[r] +exit_code: 1 +stdout: "" +stderr: > + languages settings cannot be combined with --jupyter-kernel; configure + the installed kernelspec instead diff --git a/tests/snapshots/client_server/integrations/test_python/clients_forward_jupyter_kernel_source.yaml b/tests/snapshots/client_server/integrations/test_python/clients_forward_jupyter_kernel_source.yaml new file mode 100644 index 000000000..885e2d23b --- /dev/null +++ b/tests/snapshots/client_server/integrations/test_python/clients_forward_jupyter_kernel_source.yaml @@ -0,0 +1,28 @@ +--- +client: sync +calls: + - name: send + arguments: + code: |- + answer = "café Ω" + print(answer) + - name: send + arguments: + code: |- + answer = "café Ω" + print(answer) +closed: true +--- +client: async +calls: + - name: send + arguments: + code: |- + answer = "café Ω" + print(answer) + - name: send + arguments: + code: |- + answer = "café Ω" + print(answer) +closed: true diff --git a/tests/snapshots/client_server/jupyter/test_ark/persistent_r_errors_and_plot_output.yaml b/tests/snapshots/client_server/jupyter/test_ark/persistent_r_errors_and_plot_output.yaml new file mode 100644 index 000000000..d8b0a6d62 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_ark/persistent_r_errors_and_plot_output.yaml @@ -0,0 +1,47 @@ +--- +scenario: A published ark kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + TCP transport. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: answer <- 40; answer + 2 +result: + content: + - type: text + text: | + [1] 42 + isError: false +--- +send: + code: stop("intentional kernel error") +result: + content: + - type: text + text: | + Error: + ! intentional kernel error + isError: false +--- +send: + code: answer + 2 +result: + content: + - type: text + text: | + [1] 42 + isError: false +--- +send: + code: | + par(mar = rep(0, 4), bg = "white") + plot.new() + rect(0.25, 0.25, 0.75, 0.75, col = "red", border = NA) +result: + content: + - type: image + data: "" + mimeType: image/png + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_linux_sandbox/ark_private_network_preserves_default_sandbox.yaml b/tests/snapshots/client_server/jupyter/test_linux_sandbox/ark_private_network_preserves_default_sandbox.yaml new file mode 100644 index 000000000..97a8a8eb4 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_linux_sandbox/ark_private_network_preserves_default_sandbox.yaml @@ -0,0 +1,152 @@ +--- +scenario: The published ark kernel works in the default read-only sandbox. +fixture: >- + No transport or sandbox permissions are configured. Host TCP and Unix + listeners accept baseline connections outside the sandbox; a host file + contains a protected sentinel. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + [1] 42 + isError: false +--- +send: + code: | + private <- tempfile("kernel-permission-probe-") + writeLines("private", private) + stopifnot(identical(readLines(private), "private")) + unlink(private) + cat("private temp: writable\n") + host <- Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_HOST_FILE") + stopifnot(identical(readLines(host), "original")) + failure <- tryCatch( + withCallingHandlers( + writeLines("changed", host), + warning = function(w) { + cat("host file warning: ", conditionMessage(w), "\n", sep = "") + invokeRestart("muffleWarning") + } + ), + error = identity + ) + stopifnot(inherits(failure, "error")) + cat("host file error: ", conditionMessage(failure), "\n", sep = "") +result: + content: + - type: text + text: | + private temp: writable + host file warning: cannot open file '/host-sentinel': Read-only file system + host file error: cannot open the connection + isError: false +--- +send: + code: | + status <- system2( + Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_PYTHON"), + shQuote(Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE")) + ) + stopifnot(status == 0L) +result: + content: + - type: text + text: | + host TCP: [Errno 111] Connection refused + external TCP: [Errno 101] Network is unreachable + host Unix: [Errno 1] Operation not permitted + external HTTP through proxy: denied (403) + external CONNECT through proxy: denied (403) + host CONNECT through proxy: denied (403) + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + [idle] + isError: false +--- +send: + code: exists("answer", inherits = FALSE) +result: + content: + - type: text + text: | + [1] FALSE + isError: false +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + [1] 42 + isError: false +--- +send: + code: | + private <- tempfile("kernel-permission-probe-") + writeLines("private", private) + stopifnot(identical(readLines(private), "private")) + unlink(private) + cat("private temp: writable\n") + host <- Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_HOST_FILE") + stopifnot(identical(readLines(host), "original")) + failure <- tryCatch( + withCallingHandlers( + writeLines("changed", host), + warning = function(w) { + cat("host file warning: ", conditionMessage(w), "\n", sep = "") + invokeRestart("muffleWarning") + } + ), + error = identity + ) + stopifnot(inherits(failure, "error")) + cat("host file error: ", conditionMessage(failure), "\n", sep = "") +result: + content: + - type: text + text: | + private temp: writable + host file warning: cannot open file '/host-sentinel': Read-only file system + host file error: cannot open the connection + isError: false +--- +send: + code: | + status <- system2( + Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_PYTHON"), + shQuote(Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE")) + ) + stopifnot(status == 0L) +result: + content: + - type: text + text: | + host TCP: [Errno 111] Connection refused + external TCP: [Errno 101] Network is unreachable + host Unix: [Errno 1] Operation not permitted + external HTTP through proxy: denied (403) + external CONNECT through proxy: denied (403) + host CONNECT through proxy: denied (403) + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_linux_sandbox/ipykernel_private_network_preserves_default_sandbox.yaml b/tests/snapshots/client_server/jupyter/test_linux_sandbox/ipykernel_private_network_preserves_default_sandbox.yaml new file mode 100644 index 000000000..d747b0fb0 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_linux_sandbox/ipykernel_private_network_preserves_default_sandbox.yaml @@ -0,0 +1,146 @@ +--- +scenario: The published ipykernel kernel works in the default read-only sandbox. +fixture: >- + No transport or sandbox permissions are configured. Host TCP and Unix + listeners accept baseline connections outside the sandbox; a host file + contains a protected sentinel. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + [IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption. + 42 + isError: false +--- +send: + code: | + import os + import tempfile + from pathlib import Path + + private = Path(tempfile.gettempdir()) / "kernel-permission-probe" + private.write_text("private") + assert private.read_text() == "private" + private.unlink() + print("private temp: writable") + host = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_HOST_FILE"]) + assert host.read_text() == "original\n" + try: + host.write_text("changed") + except OSError as error: + assert error.errno == int(os.environ["MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO"]), error + print("host file:", error) + else: + raise AssertionError("host file unexpectedly writable") +result: + content: + - type: text + text: | + private temp: writable + host file: [Errno 30] Read-only file system: '/host-sentinel' + isError: false +--- +send: + code: | + exec(Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE"]).read_text()) +result: + content: + - type: text + text: | + host TCP: [Errno 111] Connection refused + external TCP: [Errno 101] Network is unreachable + host Unix: [Errno 1] Operation not permitted + external HTTP through proxy: denied (403) + external CONNECT through proxy: denied (403) + host CONNECT through proxy: denied (403) + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + /jupyter_client/localinterfaces.py:58: UserWarning: Unexpected error discovering local network interfaces: [Errno 1] Operation not permitted + ret = f(**kwargs) + [IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption. + [idle] + isError: false +--- +send: + code: '"answer" in globals()' +result: + content: + - type: text + text: | + False + isError: false +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: | + import os + import tempfile + from pathlib import Path + + private = Path(tempfile.gettempdir()) / "kernel-permission-probe" + private.write_text("private") + assert private.read_text() == "private" + private.unlink() + print("private temp: writable") + host = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_HOST_FILE"]) + assert host.read_text() == "original\n" + try: + host.write_text("changed") + except OSError as error: + assert error.errno == int(os.environ["MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO"]), error + print("host file:", error) + else: + raise AssertionError("host file unexpectedly writable") +result: + content: + - type: text + text: | + private temp: writable + host file: [Errno 30] Read-only file system: '/host-sentinel' + isError: false +--- +send: + code: | + exec(Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE"]).read_text()) +result: + content: + - type: text + text: | + host TCP: [Errno 111] Connection refused + external TCP: [Errno 101] Network is unreachable + host Unix: [Errno 1] Operation not permitted + external HTTP through proxy: denied (403) + external CONNECT through proxy: denied (403) + host CONNECT through proxy: denied (403) + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_macos_sandbox/ark_private_ipc_preserves_default_sandbox.yaml b/tests/snapshots/client_server/jupyter/test_macos_sandbox/ark_private_ipc_preserves_default_sandbox.yaml new file mode 100644 index 000000000..0ef2e13fd --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_macos_sandbox/ark_private_ipc_preserves_default_sandbox.yaml @@ -0,0 +1,146 @@ +--- +scenario: The published ark kernel works in the default read-only sandbox. +fixture: >- + No transport or sandbox permissions are configured. Host TCP and Unix + listeners accept baseline connections outside the sandbox; a host file + contains a protected sentinel. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + [1] 42 + isError: false +--- +send: + code: | + private <- tempfile("kernel-permission-probe-") + writeLines("private", private) + stopifnot(identical(readLines(private), "private")) + unlink(private) + cat("private temp: writable\n") + host <- Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_HOST_FILE") + stopifnot(identical(readLines(host), "original")) + failure <- tryCatch( + withCallingHandlers( + writeLines("changed", host), + warning = function(w) { + cat("host file warning: ", conditionMessage(w), "\n", sep = "") + invokeRestart("muffleWarning") + } + ), + error = identity + ) + stopifnot(inherits(failure, "error")) + cat("host file error: ", conditionMessage(failure), "\n", sep = "") +result: + content: + - type: text + text: | + private temp: writable + host file warning: cannot open file '/host-sentinel': Operation not permitted + host file error: cannot open the connection + isError: false +--- +send: + code: | + status <- system2( + Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_PYTHON"), + shQuote(Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE")) + ) + stopifnot(status == 0L) +result: + content: + - type: text + text: | + private Unix: roundtrip + host TCP: [Errno 1] Operation not permitted + external TCP: [Errno 1] Operation not permitted + host Unix: [Errno 1] Operation not permitted + host Unix through private symlink: [Errno 1] Operation not permitted + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + [idle] + isError: false +--- +send: + code: exists("answer", inherits = FALSE) +result: + content: + - type: text + text: | + [1] FALSE + isError: false +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + [1] 42 + isError: false +--- +send: + code: | + private <- tempfile("kernel-permission-probe-") + writeLines("private", private) + stopifnot(identical(readLines(private), "private")) + unlink(private) + cat("private temp: writable\n") + host <- Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_HOST_FILE") + stopifnot(identical(readLines(host), "original")) + failure <- tryCatch( + withCallingHandlers( + writeLines("changed", host), + warning = function(w) { + cat("host file warning: ", conditionMessage(w), "\n", sep = "") + invokeRestart("muffleWarning") + } + ), + error = identity + ) + stopifnot(inherits(failure, "error")) + cat("host file error: ", conditionMessage(failure), "\n", sep = "") +result: + content: + - type: text + text: | + private temp: writable + host file warning: cannot open file '/host-sentinel': Operation not permitted + host file error: cannot open the connection + isError: false +--- +send: + code: | + status <- system2( + Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_PYTHON"), + shQuote(Sys.getenv("MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE")) + ) + stopifnot(status == 0L) +result: + content: + - type: text + text: | + private Unix: roundtrip + host TCP: [Errno 1] Operation not permitted + external TCP: [Errno 1] Operation not permitted + host Unix: [Errno 1] Operation not permitted + host Unix through private symlink: [Errno 1] Operation not permitted + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_macos_sandbox/ipykernel_private_ipc_preserves_default_sandbox.yaml b/tests/snapshots/client_server/jupyter/test_macos_sandbox/ipykernel_private_ipc_preserves_default_sandbox.yaml new file mode 100644 index 000000000..910eca970 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_macos_sandbox/ipykernel_private_ipc_preserves_default_sandbox.yaml @@ -0,0 +1,160 @@ +--- +scenario: The published ipykernel kernel works in the default read-only sandbox. +fixture: >- + No transport or sandbox permissions are configured. Host TCP and Unix + listeners accept baseline connections outside the sandbox; a host file + contains a protected sentinel. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + /ipykernel/iostream.py:73: UserWarning: Couldn't bind IOPub Pipe to 127.0.0.1: Operation not permitted (addr='tcp://127.0.0.1:*') + subprocess output will be unavailable. + self._setup_pipe_in() + 42 + isError: false +--- +send: + code: | + import os + import tempfile + from pathlib import Path + + private = Path(tempfile.gettempdir()) / "kernel-permission-probe" + private.write_text("private") + assert private.read_text() == "private" + private.unlink() + print("private temp: writable") + host = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_HOST_FILE"]) + assert host.read_text() == "original\n" + try: + host.write_text("changed") + except OSError as error: + assert error.errno == int(os.environ["MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO"]), error + print("host file:", error) + else: + raise AssertionError("host file unexpectedly writable") +result: + content: + - type: text + text: | + private temp: writable + host file: [Errno 1] Operation not permitted: '/host-sentinel' + isError: false +--- +send: + code: | + exec(Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE"]).read_text()) +result: + content: + - type: text + text: | + private Unix: roundtrip + host TCP: [Errno 1] Operation not permitted + external TCP: [Errno 1] Operation not permitted + host Unix: [Errno 1] Operation not permitted + host Unix through private symlink: [Errno 1] Operation not permitted + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [IPKernelApp] ERROR | Exception during subprocesses termination + Traceback (most recent call last): + File "/ipykernel/kernelbase.py", line 1645, in _at_shutdown + await self._progressively_terminate_all_children() + File "/ipykernel/kernelbase.py", line 1623, in _progressively_terminate_all_children + if not self._process_children(): + ~~~~~~~~~~~~~~~~~~~~~~^^ + File "/ipykernel/kernelbase.py", line 1606, in _process_children + all_children = kernel_process.children(recursive=True) + File "/psutil/__init__.py", line 978, in children + ppid_map = _ppid_map() + File "/psutil/__init__.py", line 256, in _ppid_map + for pid in pids(): + ~~~~^^ + File "/psutil/__init__.py", line 1485, in pids + ret = sorted(_psplatform.pids()) + ~~~~~~~~~~~~~~~~^^ + File "/psutil/_psosx.py", line 320, in pids + ls = cext.pids() + PermissionError: [Errno 1] Operation not permitted (originated from sysctl() malloc 1/3) + [worker stopped: in-memory state lost] + [starting new worker] + /ipykernel/iostream.py:73: UserWarning: Couldn't bind IOPub Pipe to 127.0.0.1: Operation not permitted (addr='tcp://127.0.0.1:*') + subprocess output will be unavailable. + self._setup_pipe_in() + [idle] + isError: false +--- +send: + code: '"answer" in globals()' +result: + content: + - type: text + text: | + False + isError: false +--- +send: + code: |- + answer = 42 + print(answer) +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: | + import os + import tempfile + from pathlib import Path + + private = Path(tempfile.gettempdir()) / "kernel-permission-probe" + private.write_text("private") + assert private.read_text() == "private" + private.unlink() + print("private temp: writable") + host = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_HOST_FILE"]) + assert host.read_text() == "original\n" + try: + host.write_text("changed") + except OSError as error: + assert error.errno == int(os.environ["MCP_CONSOLE_TEST_JUPYTER_FILE_ERRNO"]), error + print("host file:", error) + else: + raise AssertionError("host file unexpectedly writable") +result: + content: + - type: text + text: | + private temp: writable + host file: [Errno 1] Operation not permitted: '/host-sentinel' + isError: false +--- +send: + code: | + exec(Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_NETWORK_PROBE"]).read_text()) +result: + content: + - type: text + text: | + private Unix: roundtrip + host TCP: [Errno 1] Operation not permitted + external TCP: [Errno 1] Operation not permitted + host Unix: [Errno 1] Operation not permitted + host Unix through private symlink: [Errno 1] Operation not permitted + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/delayed_kernel_info_replies_survive_retries.yaml b/tests/snapshots/client_server/jupyter/test_python/delayed_kernel_info_replies_survive_retries.yaml new file mode 100644 index 000000000..5627e9995 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/delayed_kernel_info_replies_survive_retries.yaml @@ -0,0 +1,32 @@ +--- +scenario: >- + A delayed readiness reply remains valid after another kernel_info + request. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. A published IPythonKernel subclass constructs every + readiness reply immediately, but delivers it only when the next request + arrives. Normal IOPub idle messages are delivered immediately; no sleeps + or adjusted deadlines are used. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: |- + answer = 6 * 7 + answer +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: answer +result: + content: + - type: text + text: | + 42 + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/interrupt_and_restart_preserve_generation_ownership.yaml b/tests/snapshots/client_server/jupyter/test_python/interrupt_and_restart_preserve_generation_ownership.yaml new file mode 100644 index 000000000..c5eacb99b --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/interrupt_and_restart_preserve_generation_ownership.yaml @@ -0,0 +1,86 @@ +--- +scenario: A published ipykernel kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. A file marker proves that an interruptible Python + evaluation reached its gate. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: |- + answer = 42 + print("ready") +result: + content: + - type: text + text: | + ready + isError: false +--- +send: + code: | + from pathlib import Path + import threading + + try: + Path("interrupt-ready").touch() + threading.Event().wait() + except KeyboardInterrupt: + print("interrupted") + timeout_ms: 0 +result: + content: + - type: text + text: |- + + [elapsed: s since admission; no new output] + [running; poll with an empty send] + isError: false +--- +send: + control: interrupt +result: + content: + - type: text + text: | + interrupted + isError: false +--- +send: + code: answer +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + [idle] + isError: false +--- +send: + code: '"answer" in globals()' +result: + content: + - type: text + text: | + False + isError: false +--- +send: + code: 6 * 7 +result: + content: + - type: text + text: | + 42 + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/interrupt_cancels_kernel_startup_and_reaps_process.yaml b/tests/snapshots/client_server/jupyter/test_python/interrupt_cancels_kernel_startup_and_reaps_process.yaml new file mode 100644 index 000000000..048ed2748 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/interrupt_cancels_kernel_startup_and_reaps_process.yaml @@ -0,0 +1,59 @@ +--- +scenario: >- + Interrupt cancels kernel startup before Jupyter readiness and discards + the pending cell. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. A Python launcher records its PID and blocks on a FIFO + before starting the released ipykernel. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: | + from pathlib import Path + + Path("must-not-run").write_text("unexpected execution") +result: + content: + - type: text + text: |- + Traceback (most recent call last): + File "", line 536, in + main() + ~~~~^^ + File "", line 532, in main + asyncio.run(Adapter(options).run()) + ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^ + File "/asyncio/runners.py", line 196, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "/asyncio/runners.py", line 119, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "/asyncio/base_events.py", line 726, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "", line 522, in run + raise outcome + File "", line 514, in run + task.result() + ~~~~~~~~~~~^^ + File "", line 141, in interrupt + raise RuntimeError("Jupyter startup interrupted") + RuntimeError: Jupyter startup interrupted + [worker sideband read failed: worker sideband closed] + [worker exited with status 1] + isError: true +--- +send: + control: interrupt + timeout_ms: 3000 +result: + content: + - type: text + text: |- + + [elapsed: s since admission; phase: startup] + [running; poll with an empty send] + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/ipc_kernel_with_explicit_socket_permissions.yaml b/tests/snapshots/client_server/jupyter/test_python/ipc_kernel_with_explicit_socket_permissions.yaml new file mode 100644 index 000000000..17b488002 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/ipc_kernel_with_explicit_socket_permissions.yaml @@ -0,0 +1,16 @@ +--- +scenario: A published ipykernel kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. +--- +!same-as jupyter MCP initialization for this execution mode +--- +send: + code: 6 * 7 +result: + content: + - type: text + text: | + 42 + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/kernel_crash_recovery_and_process_retirement.yaml b/tests/snapshots/client_server/jupyter/test_python/kernel_crash_recovery_and_process_retirement.yaml new file mode 100644 index 000000000..d66617578 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/kernel_crash_recovery_and_process_retirement.yaml @@ -0,0 +1,126 @@ +--- +scenario: >- + A kernel crash loses state; explicit restart recovers and reaps prior + processes. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: |- + import os + print(os.getpid()) +result: + content: + - type: text + text: | + + isError: false +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + [idle] + isError: false +--- +send: + code: |- + import os + print(os.getpid()) +result: + content: + - type: text + text: | + + isError: false +--- +send: + code: os._exit(7) +result: + content: + - type: text + text: |2- + + Exception Group Traceback (most recent call last): + | File "", line 536, in + | main() + | ~~~~^^ + | File "", line 532, in main + | asyncio.run(Adapter(options).run()) + | ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/asyncio/runners.py", line 196, in run + | return runner.run(main) + | ~~~~~~~~~~^^^^^^ + | File "/asyncio/runners.py", line 119, in run + | return self._loop.run_until_complete(task) + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + | File "/asyncio/base_events.py", line 726, in run_until_complete + | return future.result() + | ~~~~~~~~~~~~~^^ + | File "", line 522, in run + | raise outcome + | File "", line 514, in run + | task.result() + | ~~~~~~~~~~~^^ + | File "", line 439, in serve + | async with asyncio.TaskGroup() as group: + | ~~~~~~~~~~~~~~~~~^^ + | File "/asyncio/taskgroups.py", line 71, in __aexit__ + | return await self._aexit(et, exc) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + | File "/asyncio/taskgroups.py", line 173, in _aexit + | raise BaseExceptionGroup( + | ...<2 lines>... + | ) from None + | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "", line 147, in watch_child + | raise RuntimeError(f"Jupyter kernel exited with status {status}") + | RuntimeError: Jupyter kernel exited with status 7 + +------------------------------------ + [worker sideband read failed: worker sideband closed] + [worker exited with status 1] + [worker stopped: in-memory state lost] + [starting new worker] + [idle] + isError: true +--- +send: + control: restart +result: + content: + - type: text + text: |- + [worker stopped: in-memory state lost] + [starting new worker] + [idle] + isError: false +--- +send: + code: 6 * 7 +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: |- + import os + print(os.getpid()) +result: + content: + - type: text + text: | + + isError: false +--- +processes: Restart, crash, and shutdown each retired the kernel process. diff --git a/tests/snapshots/client_server/jupyter/test_python/persistent_state_streams_errors_and_recovery.yaml b/tests/snapshots/client_server/jupyter/test_python/persistent_state_streams_errors_and_recovery.yaml new file mode 100644 index 000000000..96311b492 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/persistent_state_streams_errors_and_recovery.yaml @@ -0,0 +1,153 @@ +--- +scenario: A published ipykernel kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + requirements: + action: get +result: + content: + - type: text + text: |- + { + "requirements": { + "r": [], + "python": [], + "duckdb": [], + "python_version": [], + "exclude_newer": null + }, + "prepared": false, + "runtime_requirements": { + "r": [], + "python": [] + } + } + structuredContent: + requirements: + r: [] + python: [] + duckdb: [] + python_version: [] + exclude_newer: ~ + prepared: false + runtime_requirements: + r: [] + python: [] + isError: false +--- +send: + code: |- + answer = 40 + answer + 2 +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: + text_bytes: 102413 + concat: + - "# " + - repeat: x + count: 102400 + - |- + + answer + 2 +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: | + import sys + + print("stdout café", flush=True) + print("stderr Ω", file=sys.stderr, flush=True) +result: + content: + - type: text + text: | + stdout café + stderr Ω + isError: false +--- +send: + code: raise ValueError("intentional kernel error") +result: + content: + - type: text + text: | + --------------------------------------------------------------------------- + ValueError Traceback (most recent call last) + Cell In[4], line 1 + ----> 1 raise ValueError("intentional kernel error") + + ValueError: intentional kernel error + isError: false +--- +send: + code: answer + 2 +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + r: 1 + 1 +result: + content: + - type: text + text: "`r` source fields are hidden by `--jupyter-kernel`" + isError: true +--- +send: + python: 1 + 1 +result: + content: + - type: text + text: "`python` source fields are hidden by `--jupyter-kernel`" + isError: true +--- +send: + sql: SELECT 1 +result: + content: + - type: text + text: "`sql` source fields are hidden by `--jupyter-kernel`" + isError: true +--- +send: + requirements: + python: + - requests +result: + content: + - type: text + text: >- + dependency preparation is unavailable with Jupyter kernels; install + packages in the kernel environment before launch + isError: true +--- +send: + code: answer + 2 +result: + content: + - type: text + text: | + 42 + isError: false +--- +recording: Generic kernel source is retained in plain text Quarto fences. diff --git a/tests/snapshots/client_server/jupyter/test_python/rich_output_and_prompted_input.yaml b/tests/snapshots/client_server/jupyter/test_python/rich_output_and_prompted_input.yaml new file mode 100644 index 000000000..4e7d3cd51 --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/rich_output_and_prompted_input.yaml @@ -0,0 +1,51 @@ +--- +scenario: A published ipykernel kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: | + import base64 + from IPython.display import Image, display + + display(Image(data=base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aSuoAAAAASUVORK5CYII="))) +result: + content: + - type: image + data: + mimeType: image/png + isError: false +--- +send: + code: |- + name = input("Name: ") + print("hello", name) +result: + content: + - type: text + text: |- + [input requested: "Name: "] + [waiting for stdin] + isError: false +--- +send: + stdin: | + Ada +result: + content: + - type: text + text: | + hello Ada + isError: false +--- +send: + code: name +result: + content: + - type: text + text: | + 'Ada' + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/slow_kernel_startup_remains_pending_until_ready.yaml b/tests/snapshots/client_server/jupyter/test_python/slow_kernel_startup_remains_pending_until_ready.yaml new file mode 100644 index 000000000..a9b052a7d --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/slow_kernel_startup_remains_pending_until_ready.yaml @@ -0,0 +1,50 @@ +--- +scenario: A live kernel remains in startup until its Jupyter channels are ready. +fixture: >- + An isolated kernelspec selects the test host's installed release using + IPC transport. A FIFO holds the acquired kernel before ipykernel + initialization across a 1.5-second public poll. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: |- + answer = 42 + print(answer) + timeout_ms: 0 +result: + content: + - type: text + text: |- + + [elapsed: s since admission; no new output; phase: startup] + [running; poll with an empty send] + isError: false +--- +send: + timeout_ms: 1500 +result: + content: + - type: text + text: |- + + [elapsed: s since admission; no new output; phase: startup] + [running; poll with an empty send] + isError: false +--- +send: {} +result: + content: + - type: text + text: | + 42 + isError: false +--- +send: + code: answer +result: + content: + - type: text + text: | + 42 + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_python/tcp_startup_warning_preserves_output_and_state.yaml b/tests/snapshots/client_server/jupyter/test_python/tcp_startup_warning_preserves_output_and_state.yaml new file mode 100644 index 000000000..b83934eeb --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_python/tcp_startup_warning_preserves_output_and_state.yaml @@ -0,0 +1,28 @@ +--- +scenario: A published ipykernel kernel executes persistent cells. +fixture: >- + An isolated kernelspec selects the test host's installed release using + TCP transport. +--- +!same-as tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml +--- +send: + code: |- + answer = 40 + answer + 2 +result: + content: + - type: text + text: | + [IPKernelApp] WARNING | Kernel is running over TCP without encryption. All communication (including code and outputs) is sent in plain text and is susceptible to eavesdropping. Use IPC transport or launch with kernel manager-provisioned CurveZMQ keys to enable transport encryption. + 42 + isError: false +--- +send: + code: answer + 2 +result: + content: + - type: text + text: | + 42 + isError: false diff --git a/tests/snapshots/client_server/jupyter/test_sandbox_inheritance/unrestricted_filesystem_preserves_private_ipc.yaml b/tests/snapshots/client_server/jupyter/test_sandbox_inheritance/unrestricted_filesystem_preserves_private_ipc.yaml new file mode 100644 index 000000000..43634145f --- /dev/null +++ b/tests/snapshots/client_server/jupyter/test_sandbox_inheritance/unrestricted_filesystem_preserves_private_ipc.yaml @@ -0,0 +1,155 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + experimental: + codex/sandbox-state: {} + codex/sandbox-state-meta: {} + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + The sandbox base comes from Codex metadata on the first send, or the + configured fallback (default: read_only). Missing metadata is silent; + malformed metadata warns on server stderr. Explicit Console permissions + apply to the selected base. Full access disables Console's sandbox. + Reconnect to change client permissions. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + code: + description: >- + Evaluate one complete cell in the selected Jupyter kernel's + language. + type: + - string + - "null" + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action +--- +input: + method: tools/call + params: + name: send + arguments: + timeout_ms: 60000 + code: | + import os + from pathlib import Path + + host = Path(os.environ["MCP_CONSOLE_TEST_HOST_FILE"]) + _ = host.write_text("inherited write access") + print(host.read_text()) + _meta: + codex/sandbox-state: + permissionProfile: + type: managed + file_system: + type: unrestricted + network: restricted +result: + content: + - type: text + text: | + /ipykernel/iostream.py:73: UserWarning: Couldn't bind IOPub Pipe to 127.0.0.1: Operation not permitted (addr='tcp://127.0.0.1:*') + subprocess output will be unavailable. + self._setup_pipe_in() + inherited write access + isError: false diff --git a/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter-writable.yaml b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter-writable.yaml new file mode 100644 index 000000000..38aaea406 --- /dev/null +++ b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter-writable.yaml @@ -0,0 +1,121 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + Evaluated code can read host files subject to configured restrictions + and can access the network subject to the launcher's proxy settings. + Writable locations: private `TMPDIR`, `""`; subject to + more specific read/deny rules. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + code: + description: >- + Evaluate one complete cell in the selected Jupyter kernel's + language. + type: + - string + - "null" + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action diff --git a/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml new file mode 100644 index 000000000..a35601d1c --- /dev/null +++ b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.direct.yaml @@ -0,0 +1,119 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + Evaluated code runs without a sandbox, with the server's permissions, + including filesystem and network access. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + code: + description: >- + Evaluate one complete cell in the selected Jupyter kernel's + language. + type: + - string + - "null" + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action diff --git a/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml new file mode 100644 index 000000000..1cf51d097 --- /dev/null +++ b/tests/snapshots/client_server/server/test_tools/initializes_and_lists_tools.jupyter.yaml @@ -0,0 +1,120 @@ +--- +input: + method: initialize + params: + protocolVersion: 2025-11-25 + capabilities: {} + clientInfo: + name: acceptance-test + version: 1.0.0 +result: + protocolVersion: 2025-11-25 + capabilities: + tools: {} + serverInfo: + name: mcp-console + version: 0.0.6 + instructions: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. +--- +input: + method: notifications/initialized +--- +input: + method: tools/list +result: + tools: + - name: send + description: >- + Persistent Jupyter kernel session. Send code in the selected kernel's + language and reuse objects across calls. + + + Send one complete `code` cell per call. + + + The installed kernel supplies evaluation, display, history, and + packages. Console returns text and supported images; interactive + displays are unavailable. Dependency preparation and Console's built-in + SQL and language helpers are unavailable. + + + Run one cell at a time. Wait for its `send` to return before another + call with code. You can interrupt a pending call with + `send(control="interrupt")`. An error can leave earlier changes in + place. If work is still running, poll with an empty `send`; do not + resubmit the cell. When waiting for completion, use `timeout_ms=300000` + for a long poll; it returns early when work finishes or needs input. + Avoid repeated short polls such as `timeout_ms=1000`. Default: + `timeout_ms=60000` (60 seconds). + + + Text results, including notices, have an 8 KiB limit, retaining the + beginning and latest tail; images have separate limits. Response notices + report state and omitted-output locations. + + + Evaluated code can read host files subject to configured restrictions + and cannot directly access the network. Writable locations: private + `TMPDIR`; subject to more specific read/deny rules. + inputSchema: + $schema: https://json-schema.org/draft/2020-12/schema + additionalProperties: false + type: object + properties: + code: + description: >- + Evaluate one complete cell in the selected Jupyter kernel's + language. + type: + - string + - "null" + timeout_ms: + description: >- + Wait for output for up to this many milliseconds (default 60,000). Use + 300000 for a long poll; it returns early when work finishes or needs + input. 0 returns immediately. Expiry leaves execution running; poll with + an empty send. + type: integer + format: uint64 + minimum: 0 + default: 60000 + control: + description: >- + Apply control before same-call input or code. interrupt requests kernel + interruption; a following cell runs only if prior work stops. restart + discards the kernel's live state and queued input, then sends bundled + input and code to the replacement. + type: string + enum: + - interrupt + - restart + stdin: + description: >- + Answer Jupyter input requests with newline-terminated text. Input queues + before a bundled cell; unread input can satisfy later requests. Restart + discards queued input. Direct reads from the kernel process's stdin are + unsupported. + type: + - string + - "null" + requirements: + description: >- + Inspect Console's empty dependency declaration with action=get. Package + installation and dependency changes are unavailable; use packages + already installed in the kernel environment. + type: + - object + - "null" + properties: + action: + description: >- + get returns the empty Console declaration, not an installed-package + inventory. Use it alone without code, stdin, or control. + type: string + enum: + - get + additionalProperties: false + required: + - action diff --git a/tests/support/jupyter.py b/tests/support/jupyter.py new file mode 100644 index 000000000..7c4653a7c --- /dev/null +++ b/tests/support/jupyter.py @@ -0,0 +1,169 @@ +"""Published Jupyter kernels selected explicitly by the test host.""" + +import json +import os +import subprocess +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from support.client import McpClient +from support.execution import DIRECT, Execution +from support.records import Transcript + + +def normalize_jupyter_paths(transcript: Transcript) -> Transcript: + """Normalize the selected interpreter's location after full output assertions.""" + python = os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"] + locations = json.loads( + subprocess.check_output( + [ + python, + "-c", + "import json, sysconfig; print(json.dumps({name: sysconfig.get_path(name) for name in ('stdlib', 'purelib')}))", + ], + text=True, + ) + ) + assert all( + Path(path).is_absolute() and Path(path).is_dir() for path in locations.values() + ), locations + for entry in transcript: + for block in entry.get("result", {}).get("content", []): + if block["type"] == "text": + for name in ("purelib", "stdlib"): + label = "packages" if name == "purelib" else "stdlib" + paths = {locations[name], str(Path(locations[name]).resolve())} + paths.update( + "~/" + str(Path(path).relative_to(Path.home())) + for path in tuple(paths) + if Path(path).is_relative_to(Path.home()) + ) + for path in sorted(paths, key=len, reverse=True): + block["text"] = block["text"].replace( + path + os.sep, f"/" + ) + return transcript + + +@contextmanager +def jupyter_client( + binary: Path, + kernel: str = "ipykernel", + *, + transport: str | None = None, + execution: Execution = DIRECT, + startup_gate: Path | None = None, + kernel_fixture: str | None = None, +) -> Iterator[tuple[McpClient, Path]]: + assert kernel in {"ipykernel", "ark"} + assert startup_gate is None or kernel == "ipykernel" + assert kernel_fixture is None or (kernel == "ipykernel" and startup_gate is None) + transport = transport or ("ipc" if kernel == "ipykernel" else "tcp") + assert transport in {"ipc", "tcp"} + python = Path(os.environ["MCP_CONSOLE_TEST_JUPYTER_PYTHON"]) + assert python.is_absolute() and python.is_file(), python + with tempfile.TemporaryDirectory(prefix="console-jupyter-") as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + name = f"console-{kernel}" + specification = root / "jupyter" / "kernels" / name + specification.mkdir(parents=True) + environment = dict(os.environ) + environment.update( + JUPYTER_PATH=str(root / "jupyter"), + JUPYTER_CONFIG_DIR=str(root / "jupyter-config"), + IPYTHONDIR=str(root / "ipython"), + JUPYTER_RUNTIME_DIR=str(root / "jupyter-runtime"), + ) + if kernel == "ipykernel": + # Match the published installer's default: permit debugging stdlib + # modules instead of triggering debugpy's frozen-module warning. + argv = [ + str(python), + "-Xfrozen_modules=off", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}", + ] + if startup_gate is not None: + launcher = ( + Path(__file__).resolve().parents[1] / "fixtures/jupyter_startup.py" + ) + argv = [ + str(python), + "-Xfrozen_modules=off", + str(launcher), + "-f", + "{connection_file}", + ] + environment["MCP_CONSOLE_TEST_JUPYTER_STARTUP_GATE"] = str(startup_gate) + if kernel_fixture is not None: + launcher = ( + Path(__file__).resolve().parents[1] / "fixtures" / kernel_fixture + ) + assert launcher.is_file(), launcher + argv = [ + str(python), + "-Xfrozen_modules=off", + str(launcher), + "-f", + "{connection_file}", + ] + language = "python" + else: + ark = Path(os.environ["MCP_CONSOLE_TEST_ARK"]) + assert ark.is_absolute() and ark.is_file(), ark + argv = [ + str(ark), + "--connection_file", + "{connection_file}", + "--session-mode", + "notebook", + "--", + "--vanilla", + ] + language = "R" + environment["RUST_LOG"] = "error" + (specification / "kernel.json").write_text( + json.dumps({"argv": argv, "display_name": name, "language": language}), + encoding="utf-8", + ) + with McpClient( + binary, + execution.serve( + "--no-config", + "--jupyter-kernel", + name, + "--jupyter-python", + str(python), + "--jupyter-transport", + transport, + *( + ( + "--writable-root", + str(root), + "-c", + "sandbox.network={proxy: {mode: limited}, sockets: {unix_sockets: dangerously_allow_all}, allow_local_binding: true}", + ) + if execution.name == "sandbox" + else () + ), + ), + environment, + current_directory=workspace, + ) as client: + client.transcript.append( + { + "scenario": f"A published {kernel} kernel executes persistent cells.", + "fixture": ( + "An isolated kernelspec selects the test host's installed release " + f"using {transport.upper()} transport." + ), + } + ) + client.initialize_and_list_tools() + yield client, workspace diff --git a/tests/support/requirements.py b/tests/support/requirements.py index df07d81a6..20b795098 100644 --- a/tests/support/requirements.py +++ b/tests/support/requirements.py @@ -133,6 +133,16 @@ class Requirement: ) SQL = Requirement("SQL", WORKER.available, "requires a supported built-in SQL worker") +JUPYTER = Requirement( + "Jupyter kernel dependencies", + POSIX.available and bool(os.environ.get("MCP_CONSOLE_TEST_JUPYTER_PYTHON")), + "requires POSIX and MCP_CONSOLE_TEST_JUPYTER_PYTHON with jupyter_client and ipykernel", +) +ARK = Requirement( + "ark kernel", + bool(os.environ.get("MCP_CONSOLE_TEST_ARK")), + "requires MCP_CONSOLE_TEST_ARK selecting a published ark executable", +) R_EVENT_LOOP = Requirement( "R event loop", WORKER.available, From 911d2f28fd53877920a9a11488fb1fbf11f34443 Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 09:39:34 -0400 Subject: [PATCH 2/7] Pin native Windows SQL interrupt fixtures to the dated R repository Apply the existing test-only DuckDB snapshot to direct and MXC SQL scenarios. Assert the selected version before the long query so cold caches cannot silently introduce the known Arrow cancellation limitation. Preserve production defaults, interruption deadlines, and exact results. --- docs/DEVELOPMENT.md | 2 ++ tests/windows.py | 13 +++++++++---- tests/windows_sandbox.py | 7 ++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 8d971f472..c3fc01d7d 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -89,6 +89,8 @@ Never hand-edit snapshots. Four SQL cases use the `2026-10-22` R repository snapshot through `r_duckdb_snapshot_environment()` in [shared R support](../tests/support/r.py): `recovers_from_sql_errors`, `interrupts_running_sql_query`, `gets_selects_and_resets_the_active_native_connection`, and `managed_r_defaults`. This retains DuckDB 1.5.6 binaries for exact error transcripts and the native interrupt fixture; the cases prepare only runtime infrastructure, adding RSQLite when needed. Other SQL cases continue to resolve current packages. +Native Windows `WindowsConsole.test_sql_with_r` and `WindowsSandbox.test_network_enabled_sql` use the same snapshot for their shared R DuckDB interrupt scenario. +The shared and Windows interrupt scenarios assert DuckDB 1.5.6 before starting the long query, so a lost pin fails before cancellation is exercised. Before advancing the date, verify the complete error diagnostics and interrupt completion; [RT-5](TODO.md#runtime-and-resource-limits) tracks DuckDB 2.0.0's Arrow cancellation limitation. The shared selector syntax works on every platform. diff --git a/tests/windows.py b/tests/windows.py index 0493856a2..fa9f188ae 100644 --- a/tests/windows.py +++ b/tests/windows.py @@ -28,7 +28,7 @@ from support.checkpoints import wait_for_path from support.installation import native_console from support.normalization import code -from support.r import r_test_environment +from support.r import r_duckdb_snapshot_environment, r_test_environment from windows_distribution import NATIVE_SUFFIXES, inspect_artifacts from windows_cargo import WindowsCargo # noqa: F401 -- include build acceptance @@ -337,6 +337,7 @@ def exercise_r_sql(session: Session) -> None: "R connection ready", # fmt: r r=code(""" + stopifnot(packageVersion("duckdb") == "1.5.6") retained_pid <- Sys.getpid() managed <- .console$sql_connection() stopifnot(DBI::dbGetQuery(managed, "SELECT * FROM retained")$answer == 1234567) @@ -1602,7 +1603,11 @@ def test_sql_without_r(self): session.expect("1234567", control="restart", sql="SELECT 1234567 AS answer") def test_sql_with_r(self): - session = self.session() + session = self.session( + environment=dict( + r_duckdb_snapshot_environment(), RETICULATE_PYTHON=sys.executable + ) + ) exercise_r_sql(session) def test_startup_without_processor_architecture(self): @@ -2087,8 +2092,8 @@ def test_python_inspection_descendants_are_retired(self): ) session.close() - def session(self, *, defer_bootstrap=False): - session = Session(defer_bootstrap=defer_bootstrap) + def session(self, *, environment=None, defer_bootstrap=False): + session = Session(environment, defer_bootstrap=defer_bootstrap) self.addCleanup(session.close) session.initialize() return session diff --git a/tests/windows_sandbox.py b/tests/windows_sandbox.py index ee2b2baea..c777cdeab 100644 --- a/tests/windows_sandbox.py +++ b/tests/windows_sandbox.py @@ -20,6 +20,7 @@ from support.installation import native_console from support.normalization import code from support.public_configuration import INVALID_CONFIGURATIONS +from support.r import r_duckdb_snapshot_environment from support.windows_mxc import requires_mxc from support.windows_firewall import application_rules @@ -361,7 +362,11 @@ def console_cache_paths(self, *, profile_fallback: bool): def test_network_enabled_sql(self): from windows import exercise_r_sql - session = worker_session(sandbox=True, overrides=['sandbox.network="enabled"']) + session = worker_session( + r_duckdb_snapshot_environment(), + sandbox=True, + overrides=['sandbox.network="enabled"'], + ) try: session.initialize() exercise_r_sql(session) From 0257b7adb4caf6b9d71da444b4459679b80c2076 Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 11:08:44 -0400 Subject: [PATCH 3/7] Collect replacement startup completion before asserting output Queue stdin while the replacement cell is held at its checkpoint, then release it and collect the complete result. Preserve the exact output and idle-state assertions without assuming one response completes the cell. --- .../client_server/lifecycle/test_startup.py | 8 ++++++-- .../restart_replaces_first_use_cell_and_stdin.yaml | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/boundaries/client_server/lifecycle/test_startup.py b/tests/boundaries/client_server/lifecycle/test_startup.py index d60cbfaf0..37ea0d052 100755 --- a/tests/boundaries/client_server/lifecycle/test_startup.py +++ b/tests/boundaries/client_server/lifecycle/test_startup.py @@ -556,9 +556,13 @@ def test_restart_replaces_first_use_cell_and_stdin( client.send(timeout_ms=0) assert without_elapsed(last_tool_text(client)) == RUNNING assert "phase:" not in last_tool_text(client) + # Queue input while the replacement is still held so this response + # cannot accidentally stand in for its eventual completion receipt. + client.send(stdin="replacement still usable\n", timeout_ms=0) + assert without_elapsed(last_tool_text(client)) == RUNNING + assert "phase:" not in last_tool_text(client) new_cell_release.release() - client.send(stdin="replacement still usable\n") - assert last_tool_text(client) == ( + client.expect( "[input requested: \"replacement> \"]\n'replacement still usable'\n" ) assert "phase:" not in last_tool_text(client) diff --git a/tests/snapshots/client_server/lifecycle/test_startup/restart_replaces_first_use_cell_and_stdin.yaml b/tests/snapshots/client_server/lifecycle/test_startup/restart_replaces_first_use_cell_and_stdin.yaml index ad3fab6ce..f2b3c3c51 100644 --- a/tests/snapshots/client_server/lifecycle/test_startup/restart_replaces_first_use_cell_and_stdin.yaml +++ b/tests/snapshots/client_server/lifecycle/test_startup/restart_replaces_first_use_cell_and_stdin.yaml @@ -72,6 +72,17 @@ result: send: stdin: | replacement still usable + timeout_ms: 0 +result: + content: + - type: text + text: |- + + [elapsed: s since admission; no new output] + [running; poll with an empty send] + isError: false +--- +send: {} result: content: - type: text From a7b0de4c1343efd0d35b48bb3830919f0f4df76d Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 11:14:29 -0400 Subject: [PATCH 4/7] Cover the Jupyter guide in website navigation checks Require the new guide in the task table and Guides sidebar, and retain the explicit allowlist of supported task destinations. --- tests/website.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/website.py b/tests/website.py index 5fc717e05..efd8f87bd 100644 --- a/tests/website.py +++ b/tests/website.py @@ -352,6 +352,7 @@ def test_guides_keep_shared_navigation(self) -> None: "requirements/index.html", "sandbox/index.html", "recording/index.html", + "jupyter/index.html", "sandbox_command/index.html", ) navigation = [] @@ -641,6 +642,7 @@ def test_documentation_links_to_learning_and_tasks(self) -> None: "build_on_console/index.html", "send_operations/index.html", "recording/index.html", + "jupyter/index.html", ): self.assertIn(target, links) runtime = self.pages[self.site / "runtime_reference/index.html"] @@ -659,6 +661,7 @@ def test_documentation_links_to_learning_and_tasks(self) -> None: "build_on_console", "send_operations", "sandbox_command", + "jupyter", }, ) From b78918690180a6cd9cda61755e0fe29bf5092b3d Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 19:29:53 -0400 Subject: [PATCH 5/7] Preserve controlled completion markers in transcript collectors Keep the required completion marker when a control-and-cell response is split across polls. Exercise the split with a gated native R startup cell and select the same contract in related restart continuations. --- tests/boundaries/AUTHORING.md | 2 + .../client_server/lifecycle/test_startup.py | 1 + .../client_server/python/test_without_r.py | 1 + .../client_server/r/test_native_startup.py | 84 +++++++++++++++---- .../server/test_runtime_bootstrap.py | 1 + ...containment_and_preparation_isolation.yaml | 17 +++- tests/support/assertions.py | 8 +- tests/support/client.py | 7 ++ 8 files changed, 105 insertions(+), 16 deletions(-) diff --git a/tests/boundaries/AUTHORING.md b/tests/boundaries/AUTHORING.md index e3e5573d2..682d681b9 100644 --- a/tests/boundaries/AUTHORING.md +++ b/tests/boundaries/AUTHORING.md @@ -84,6 +84,8 @@ Use `read_relay_ready()` for the ordered `worker_started` and `ready` handshake, `wait_for_evaluation_output()` submits once, accumulates every text delta, and bounds transport receives with its completion budget. `client.expect()` uses that collector with the client's existing response budget and checks exact successful output (`[done]` by default). +Combined control-and-cell calls retain their final `[done]` marker even when earlier polls took all output. +When collecting such a call directly, including a later stdin or empty-poll continuation, pass `controlled_completion=True` to `wait_for_evaluation_output()`; `client.expect()` selects this for its own combined submissions. Use `client.expect(None, ...)` when asserting the collected content separately, such as SQL preview row, column, and byte limits. An initial `timeout_ms=0` exercises polling without making query speed part of those assertions. Give cold preparation an explicit longer budget. diff --git a/tests/boundaries/client_server/lifecycle/test_startup.py b/tests/boundaries/client_server/lifecycle/test_startup.py index 37ea0d052..ca7492676 100755 --- a/tests/boundaries/client_server/lifecycle/test_startup.py +++ b/tests/boundaries/client_server/lifecycle/test_startup.py @@ -526,6 +526,7 @@ def test_restart_replaces_first_use_cell_and_stdin( client, "replacement only\n[done]", "replacement managed input", + controlled_completion=True, stdin="\n", ) assert last_tool_text(client).count("replacement only\n") == 1, ( diff --git a/tests/boundaries/client_server/python/test_without_r.py b/tests/boundaries/client_server/python/test_without_r.py index d0f264390..e45a91a24 100644 --- a/tests/boundaries/client_server/python/test_without_r.py +++ b/tests/boundaries/client_server/python/test_without_r.py @@ -706,6 +706,7 @@ def test_prepares_managed_python_at_startup_and_restart( client, lambda output: "replacement input\n" in output, "replacement Python input", + controlled_completion=True, completion_timeout_seconds=client.response_timeout, control="restart", requirements={"python": ["more-itertools"]}, diff --git a/tests/boundaries/client_server/r/test_native_startup.py b/tests/boundaries/client_server/r/test_native_startup.py index daf825cf9..404eb08e1 100644 --- a/tests/boundaries/client_server/r/test_native_startup.py +++ b/tests/boundaries/client_server/r/test_native_startup.py @@ -17,6 +17,7 @@ from support.snapshots import execution_snapshots from support.normalization import code from support.records import Transcript +from support.progress import RUNNING from support.requirements import POSIX, R, SANDBOX, requires from support.r import r_test_environment from support.resolvers import bare_runtime_environment @@ -588,13 +589,23 @@ def test_interrupted_profile_is_not_replayed( @requires(R, SANDBOX) def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcript: - with tempfile.TemporaryDirectory() as temporary: + with tempfile.TemporaryDirectory() as temporary, ExitStack() as resources: root = Path(temporary).resolve() workspace = root / "workspace" workspace.mkdir() environment = startup_environment(workspace, managed=True) environment["IR_REFRESH"] = "1" environment["CONSOLE_FORBIDDEN_STARTUP_WRITE"] = str(root / "forbidden") + reached = resources.enter_context( + closing(FifoCheckpoint.create(workspace / "cell-reached")) + ) + release = resources.enter_context( + closing(FifoCheckpoint.create(workspace / "cell-release")) + ) + environment.update( + MCP_CONSOLE_TEST_CELL_REACHED=str(reached.path), + MCP_CONSOLE_TEST_CELL_RELEASE=str(release.path), + ) # fmt: r profile = code(r""" cat("attempt\n", file = "attempts", append = TRUE) @@ -612,28 +623,73 @@ def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcri cat("native profile contained\n") """) (workspace / ".Rprofile").write_text(profile) - with McpClient( - binary, - writable_arguments(SANDBOXED, workspace), - environment, - current_directory=workspace, - use_r_startup_files=True, - ) as client: + with ( + McpClient( + binary, + writable_arguments(SANDBOXED, workspace), + environment, + current_directory=workspace, + use_r_startup_files=True, + ) as client, + ExitStack() as cleanup, + ): + cleanup.callback(release.release) client.initialize_and_list_tools() client.expect("native profile contained\n", r="invisible(NULL)") assert not (workspace / "home/.duckdb/extensions").exists() assert (workspace / "attempts").read_text().splitlines() == ["attempt"] assert not (root / "forbidden").exists() - output = wait_for_evaluation_output( + # Keep the replacement cell open after its native profile finishes, + # so profile output and controlled completion use separate replies. + pending = client.start_send( + control="restart", + requirements={"action": "set", "r": ["MASS"]}, + # fmt: r + r=code(r""" + reached <- fifo( + Sys.getenv("MCP_CONSOLE_TEST_CELL_REACHED"), + "wb", + blocking = TRUE + ) + writeBin(charToRaw("1"), reached) + close(reached) + release <- fifo( + Sys.getenv("MCP_CONSOLE_TEST_CELL_RELEASE"), + "rb", + blocking = TRUE + ) + invisible(readBin(release, "raw", 1L)) + close(release) + """), + timeout_ms=0, + ) + reached.wait( + "replacement cell entered after native startup", + timeout=client.response_timeout, + client=client, + ) + client.receive(pending) + prefix = ( + "[worker stopped: in-memory state lost]\n" + "[starting new worker]\n" + "native profile contained\n" + ) + wait_for_evaluation_output( client, - None, + prefix + RUNNING, + "replacement native profile output before completion", + completion_timeout_seconds=client.response_timeout, + received=pending, + ) + release.release() + wait_for_evaluation_output( + client, + prefix + "[done]", "explicit restart and host preparation", + controlled_completion=True, completion_timeout_seconds=client.response_timeout, - control="restart", - requirements={"action": "set", "r": ["MASS"]}, - r="invisible(NULL)", + received=pending, ) - assert "native profile contained" in output, output assert (workspace / "attempts").read_text().splitlines() == [ "attempt", "attempt", diff --git a/tests/boundaries/client_server/server/test_runtime_bootstrap.py b/tests/boundaries/client_server/server/test_runtime_bootstrap.py index d977c1dbe..3094cf1d6 100644 --- a/tests/boundaries/client_server/server/test_runtime_bootstrap.py +++ b/tests/boundaries/client_server/server/test_runtime_bootstrap.py @@ -241,6 +241,7 @@ def test_restart_retires_bootstrap_before_new_cell( client, "[input requested: \"startup> \"]\n'new generation'\n[done]", "replacement cell input", + controlled_completion=True, ) return client.finish()[3:] diff --git a/tests/snapshots/client_server/r/test_native_startup/startup_containment_and_preparation_isolation.yaml b/tests/snapshots/client_server/r/test_native_startup/startup_containment_and_preparation_isolation.yaml index 58cd5e302..a0f444738 100644 --- a/tests/snapshots/client_server/r/test_native_startup/startup_containment_and_preparation_isolation.yaml +++ b/tests/snapshots/client_server/r/test_native_startup/startup_containment_and_preparation_isolation.yaml @@ -16,7 +16,22 @@ send: action: set r: - MASS - r: invisible(NULL) + r: | + reached <- fifo( + Sys.getenv("MCP_CONSOLE_TEST_CELL_REACHED"), + "wb", + blocking = TRUE + ) + writeBin(charToRaw("1"), reached) + close(reached) + release <- fifo( + Sys.getenv("MCP_CONSOLE_TEST_CELL_RELEASE"), + "rb", + blocking = TRUE + ) + invisible(readBin(release, "raw", 1L)) + close(release) + timeout_ms: 0 result: content: - type: text diff --git a/tests/support/assertions.py b/tests/support/assertions.py index 0ca01d8b6..42e551f63 100644 --- a/tests/support/assertions.py +++ b/tests/support/assertions.py @@ -202,6 +202,7 @@ def wait_for_evaluation_output( description: str, *, expected_error: bool | None = False, + controlled_completion: bool = False, completion_timeout_seconds: float = 3, initial_cuts: tuple[str, ...] = (), output_cuts: list[str] | None = None, @@ -258,7 +259,12 @@ def wait_for_evaluation_output( cuts.append(collected[len("".join(cuts)) :]) break else: - if output != "[done]" or not collected: + # A combined control-and-cell call always reports completion. Its + # marker remains meaningful when an earlier poll took all output. + if output == "[done]" and controlled_completion and collected: + if not collected.endswith("\n"): + output = "\n" + output + if output != "[done]" or not collected or controlled_completion: collected += output cuts.append(output) assert expected is None or ( diff --git a/tests/support/client.py b/tests/support/client.py index 9073f1418..e2c22566c 100644 --- a/tests/support/client.py +++ b/tests/support/client.py @@ -219,6 +219,13 @@ def expect(self, expected: str | None = "[done]", **arguments: Any) -> ToolResul self, expected, "expected send output", + controlled_completion=( + arguments.get("control") is not None + and any( + arguments.get(language) is not None + for language in ("r", "python", "sql") + ) + ), completion_timeout_seconds=self.response_timeout, **arguments, ) From 121bb746c0e38e5ac0f80c4b22c9d5477dbe20cb Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 19:40:54 -0400 Subject: [PATCH 6/7] Bound startup completion gates by one response deadline --- tests/boundaries/client_server/r/test_native_startup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/boundaries/client_server/r/test_native_startup.py b/tests/boundaries/client_server/r/test_native_startup.py index 404eb08e1..82953bf68 100644 --- a/tests/boundaries/client_server/r/test_native_startup.py +++ b/tests/boundaries/client_server/r/test_native_startup.py @@ -641,6 +641,7 @@ def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcri assert not (root / "forbidden").exists() # Keep the replacement cell open after its native profile finishes, # so profile output and controlled completion use separate replies. + deadline = client.response_deadline() pending = client.start_send( control="restart", requirements={"action": "set", "r": ["MASS"]}, @@ -665,7 +666,7 @@ def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcri ) reached.wait( "replacement cell entered after native startup", - timeout=client.response_timeout, + timeout=max(0, deadline - time.monotonic()), client=client, ) client.receive(pending) @@ -678,7 +679,7 @@ def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcri client, prefix + RUNNING, "replacement native profile output before completion", - completion_timeout_seconds=client.response_timeout, + completion_timeout_seconds=max(0, deadline - time.monotonic()), received=pending, ) release.release() @@ -687,7 +688,7 @@ def test_startup_containment_and_preparation_isolation(binary: Path) -> Transcri prefix + "[done]", "explicit restart and host preparation", controlled_completion=True, - completion_timeout_seconds=client.response_timeout, + completion_timeout_seconds=max(0, deadline - time.monotonic()), received=pending, ) assert (workspace / "attempts").read_text().splitlines() == [ From 5af6ba46c79beaf1e6bc10ae75d0fdc7d026f06a Mon Sep 17 00:00:00 2001 From: Tomasz Kalinowski Date: Mon, 26 Oct 2026 19:50:51 -0400 Subject: [PATCH 7/7] Preserve controlled completion for Jupyter source cells --- tests/support/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/client.py b/tests/support/client.py index e2c22566c..87ffcf04f 100644 --- a/tests/support/client.py +++ b/tests/support/client.py @@ -223,7 +223,7 @@ def expect(self, expected: str | None = "[done]", **arguments: Any) -> ToolResul arguments.get("control") is not None and any( arguments.get(language) is not None - for language in ("r", "python", "sql") + for language in ("r", "python", "sql", "code") ) ), completion_timeout_seconds=self.response_timeout,