feat: Emit Unicode CLI JSON on UTF streams

src/semble/cli.py

@@ -1,5 +1,6 @@
 import argparse
 import asyncio
+import codecs
 import io
 import json
 import logging
@@ -166,7 +167,10 @@ def _load_index(paths: list[str], content: list[ContentType]) -> tuple[SembleInd
 def _print_results(out: dict, output_format: str) -> None:
     """Print a format_results() payload as JSON or as human-readable text."""
     if output_format == "json":
-        print(json.dumps(out))
+        # Legacy encodings may reject or alter non-ASCII characters, so keep their JSON ASCII-only.
+        encoding = getattr(sys.stdout, "encoding", "") or ""
+        ensure_ascii = not encoding or not codecs.lookup(encoding).name.startswith("utf")
+        print(json.dumps(out, ensure_ascii=ensure_ascii))
     elif "error" in out:
         print(out["error"])
     else:

tests/test_cli.py

@@ -1,16 +1,18 @@
 import asyncio
 import hashlib
+import io
 import json
 import signal
 import sys
 import warnings
 from importlib.resources import files
 from pathlib import Path
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
-from semble.cli import _cli_main, _maybe_save_index, _run_clear, main
+from semble.cli import _cli_main, _maybe_save_index, _print_results, _run_clear, main
 from semble.types import ContentType, SearchResult
 from semble.version import __version__
 from tests.conftest import make_chunk
@@ -154,6 +156,32 @@ def test_main_dispatches_to_cli(
     assert "query text" in capsys.readouterr().out
 
 
+@pytest.mark.parametrize(
+    ("encoding", "expected"),
+    [
+        ("utf-8", '{"query": "Hi, Chào, 你好", "results": []}\n'),
+        ("utf-16", '{"query": "Hi, Chào, 你好", "results": []}\n'),
+        ("cp65001", '{"query": "Hi, Chào, 你好", "results": []}\n'),
+        ("cp1251", '{"query": "Hi, Ch\\u00e0o, \\u4f60\\u597d", "results": []}\n'),
+        (None, '{"query": "Hi, Ch\\u00e0o, \\u4f60\\u597d", "results": []}\n'),
+        ("", '{"query": "Hi, Ch\\u00e0o, \\u4f60\\u597d", "results": []}\n'),
+    ],
+)
+def test_print_results_json_respects_stdout_encoding(
+    encoding: str | None,
+    expected: str,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """JSON uses literal Unicode only when stdout declares a UTF encoding."""
+    with io.StringIO() as output, monkeypatch.context() as streams:
+        stdout = SimpleNamespace(write=output.write, flush=output.flush)
+        if encoding != "":
+            stdout.encoding = encoding
+        streams.setattr(sys, "stdout", stdout)
+        _print_results({"query": "Hi, Chào, 你好", "results": []}, "json")
+        assert output.getvalue() == expected
+
+
 @pytest.mark.parametrize(
     ("argv", "expected_stdout", "expect_system_exit"),
     [