From 7535bcef89529ad25d4bf5ba323130f9acb3558a Mon Sep 17 00:00:00 2001 From: JasonBuildAI Date: Sun, 11 Oct 2026 18:23:28 +0800 Subject: [PATCH] fix(extensions): transport the isolated view validator payload as UTF-8 bytes `load_presentation_view_validator` launches the extension-declared helper as `python -I -c ` inside the extension's runtime interpreter. `-I` ignores `PYTHONIOENCODING`/`PYTHONUTF8`, so the helper's `sys.stdin` and `sys.stdout` fall back to the host locale codec while the transport owner writes the request as UTF-8 bytes and decodes the response as UTF-8. On a zh-CN Windows host (cp936) a view with non-ASCII text decodes into mojibake with lone surrogates; the parent then raises `UnicodeDecodeError` when the surrogates are re-encoded, or `UnicodeEncodeError` in the envelope hash and persisted projection when the response survives. Decode and encode the JSON payload explicitly as UTF-8 bytes in the generated helper and document the isolation contract. Signed-off-by: JasonBuildAI --- docs/reference/extensions.md | 8 ++ loopx/extensions/presentation.py | 10 +- .../test_isolated_view_validator_utf8.py | 102 ++++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 tests/extensions/test_isolated_view_validator_utf8.py diff --git a/docs/reference/extensions.md b/docs/reference/extensions.md index e3e68b5bef..29d9abc683 100644 --- a/docs/reference/extensions.md +++ b/docs/reference/extensions.md @@ -594,6 +594,14 @@ into core. Dashboard's generic status parser consumes only that compact contract. A built-in renderer may separately own a provider view schema, as the Finance renderer does for `decision_research_dashboard_v0`. +The validator executes inside the active runtime interpreter - the extension's +own environment, not the LoopX process - and LoopX launches it isolated +(`python -I`), so `PYTHONPATH`, `PYTHONIOENCODING`, and `PYTHONUTF8` are ignored +there. It must be importable from that environment alone, and the request and +response cross stdio as UTF-8 bytes: the generated helper decodes and encodes +them explicitly instead of through the locale text streams, which would corrupt +non-ASCII view text on a host whose locale is not UTF-8. + The finance value-discovery extension declares the first such view, `decision_research_dashboard_v0`, a read-only decision-research view: diff --git a/loopx/extensions/presentation.py b/loopx/extensions/presentation.py index 508b8de0c1..d5c61ca803 100644 --- a/loopx/extensions/presentation.py +++ b/loopx/extensions/presentation.py @@ -89,7 +89,11 @@ import json import sys -request = json.load(sys.stdin) +# The transport owner writes and reads UTF-8 bytes, and this helper runs under +# `python -I`, which ignores PYTHONIOENCODING/PYTHONUTF8: its text streams fall +# back to the host locale codec (`cp936` on a zh-CN Windows host), so the JSON +# payload must be decoded and encoded as bytes here instead. +request = json.loads(sys.stdin.buffer.read().decode("utf-8")) reference = request["reference"] module_name, attribute_name = reference.split(":", 1) try: @@ -111,7 +115,9 @@ "status": "rejected", "error": str(exc)[:1000], } -json.dump(response, sys.stdout, ensure_ascii=False, separators=(",", ":")) +sys.stdout.buffer.write( + json.dumps(response, ensure_ascii=False, separators=(",", ":")).encode("utf-8") +) """ diff --git a/tests/extensions/test_isolated_view_validator_utf8.py b/tests/extensions/test_isolated_view_validator_utf8.py new file mode 100644 index 0000000000..dd76b5fc48 --- /dev/null +++ b/tests/extensions/test_isolated_view_validator_utf8.py @@ -0,0 +1,102 @@ +"""The isolated presentation view validator must transport UTF-8 bytes. + +`load_presentation_view_validator` runs a non-core, extension-declared +`view_validator` inside the extension's own runtime interpreter, launched as +`python -I -c `. `-I` isolates that interpreter from `PYTHONPATH` and +from `PYTHONIOENCODING`/`PYTHONUTF8`, so its `sys.stdin`/`sys.stdout` fall back +to the host locale codec - `cp936` on a zh-CN Windows host - while the transport +owner writes the request as UTF-8 bytes (`ensure_ascii=False` then +`.encode("utf-8")`) and decodes the response as UTF-8. A view that contains +non-ASCII text (a Chinese headline, for example) then decodes into mojibake and +lone surrogates, and those surrogate escapes survive the JSON round trip: the +returned view no longer matches what the provider published, and persisting it +fails, because the envelope hash and the projection file are serialized with +`ensure_ascii=False` and written as UTF-8. + +`tests/extensions/test_extension_provider_stdio_utf8.py` guards the provider +entrypoints, which pin their own stdio. The generated helper cannot be edited +by the extension, so it must not depend on the text streams at all. +""" + +from __future__ import annotations + +import io +import json +import sys +from typing import Any + +from loopx.extensions.presentation import ( + _ISOLATED_VIEW_VALIDATOR, + load_presentation_view_validator, +) +from loopx.extensions.readiness import ResolvedRuntimeEntrypoint + +LOCALE_CODEC = "gbk" +VIEW = { + "headline": "中文标题，带标点。", + "detail": "研究结论：证据不足。", +} +# `builtins.str` is a minimal echo validator with no extension dependency; the +# string form keeps the non-ASCII text in the response as well as the request. +ECHO_REFERENCE = "builtins:str" + + +def _transport_request(view: Any) -> bytes: + """The exact request bytes `invoke_runtime_validator` writes to the child.""" + + return json.dumps( + {"operation": "validate", "reference": ECHO_REFERENCE, "view": view}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + + +def _run_helper_behind_locale_streams(request: bytes) -> dict[str, Any]: + """Run the generated helper with the text streams a cp936 child would get. + + The transport hands the child UTF-8 bytes and reads UTF-8 bytes back, so + only the helper's own text layer links locale to payload. Substituting a + locale-coded `sys.stdin`/`sys.stdout` pair reproduces the affected host on + any test host. + """ + + stdin = io.TextIOWrapper( + io.BytesIO(request), encoding=LOCALE_CODEC, errors="surrogateescape" + ) + stdout_buffer = io.BytesIO() + stdout = io.TextIOWrapper(stdout_buffer, encoding=LOCALE_CODEC, errors="strict") + previous_stdin, previous_stdout = sys.stdin, sys.stdout + try: + sys.stdin, sys.stdout = stdin, stdout + exec( + compile(_ISOLATED_VIEW_VALIDATOR, "", "exec"), + {"__name__": "__main__"}, + ) + stdout.flush() + finally: + sys.stdin, sys.stdout = previous_stdin, previous_stdout + return json.loads(stdout_buffer.getvalue().decode("utf-8")) + + +def test_generated_helper_transports_utf8_bytes_behind_locale_streams() -> None: + """The helper's JSON transport must not depend on the locale text streams.""" + + response = _run_helper_behind_locale_streams(_transport_request(VIEW)) + + assert response == {"ok": True, "view": str(VIEW)} + + +def test_isolated_validator_round_trips_non_ascii_view_in_runtime() -> None: + """A real isolated child behind the production transport keeps the view.""" + + entrypoint = ResolvedRuntimeEntrypoint( + argv_prefix=(sys.executable,), + identity="test-runtime", + python_executable=sys.executable, + ) + validator = load_presentation_view_validator( + {"view_validator": ECHO_REFERENCE}, + runtime_entrypoint=entrypoint, + ) + + assert validator(VIEW) == str(VIEW) \ No newline at end of file