From 976c320389a539dd92a27cfc636451a85ae1b454 Mon Sep 17 00:00:00 2001 From: Matlan Date: Thu, 15 Oct 2026 23:05:02 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Native=20image=20generation:=20a=20prompt?= =?UTF-8?q?=20with=20non-ASCII=20letters=20such=20as=20=C3=9F=20reaches=20?= =?UTF-8?q?stable-diffusion.cpp=20intact=20on=20Windows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stable-diffusion.cpp worker sets its C runtime LC_CTYPE to C on Windows: its CLIP tokenizer lowercases prompt bytes with tolower, which under a code-page locale turns UTF-8 into invalid bytes. Co-Authored-By: Claude Opus 5.5 --- localm/media/sdcpp/_child.py | 12 ++++++ tests/test_sdcpp_child_locale.py | 71 ++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 tests/test_sdcpp_child_locale.py diff --git a/localm/media/sdcpp/_child.py b/localm/media/sdcpp/_child.py index c2a39815d..0455472bc 100644 --- a/localm/media/sdcpp/_child.py +++ b/localm/media/sdcpp/_child.py @@ -110,6 +110,17 @@ def _prepare_environment(runtime_dir: Path) -> None: os.environ.pop(_HOST_VIS_VAR, None) +def _use_c_ctype_locale() -> None: + """On Windows, set this process's C runtime LC_CTYPE to "C", so the byte-wise + tolower in stable-diffusion.cpp's CLIP tokenizer leaves UTF-8 prompt bytes + unchanged. Other platforms are left as they are. + See test_the_worker_keeps_utf8_prompt_bytes_through_tolower.""" + if sys.platform != "win32": + return + import locale + locale.setlocale(locale.LC_CTYPE, "C") + + def _set_rocblas_tensile(extra_dirs) -> None: if os.environ.get("ROCBLAS_TENSILE_LIBPATH"): return @@ -648,6 +659,7 @@ def worker_main(req_q, resp_q, cancel_event, crash_trace_path=None) -> None: install_parent_death_watchdog() ignore_interrupt_signals() suppress_native_error_dialogs() + _use_c_ctype_locale() log_cb, progress_cb, _State.preview_cb = _make_callbacks(resp_q) callbacks_set = False diff --git a/tests/test_sdcpp_child_locale.py b/tests/test_sdcpp_child_locale.py new file mode 100644 index 000000000..6815244cd --- /dev/null +++ b/tests/test_sdcpp_child_locale.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""The stable-diffusion.cpp worker keeps non-ASCII prompt bytes intact through +the native tokenizer's byte-wise tolower.""" + +import subprocess +import sys +import textwrap + +import pytest + +from localm.media.sdcpp import _child + +_PROMPT = "Ein weißer Fuchs läuft über einen vereisten See" + +_CRT_PROBE = textwrap.dedent(""" + import ctypes, locale, sys + from localm.media.sdcpp import _child + crt = ctypes.CDLL("ucrtbase") + raw = sys.argv[1].encode("utf-8") + + def lowered(): + return bytes(crt.tolower(b) & 0xFF for b in raw) + + locale.setlocale(locale.LC_CTYPE, ".1252") + before = lowered() + _child._use_c_ctype_locale() + after = lowered() + print(before.hex()) + print(after.hex()) +""") + + +@pytest.mark.skipif(sys.platform != "win32", reason="the code-page C runtime locale is Windows-only") +def test_the_worker_keeps_utf8_prompt_bytes_through_tolower(): + out = subprocess.run([sys.executable, "-c", _CRT_PROBE, _PROMPT], + capture_output=True, text=True, timeout=60, check=True) + before_hex, after_hex = out.stdout.split() + before, after = bytes.fromhex(before_hex), bytes.fromhex(after_hex) + with pytest.raises(UnicodeDecodeError): + before.decode("utf-8") + assert after.decode("utf-8") == _PROMPT.lower() + + +def test_the_worker_entry_point_sets_the_locale_before_serving(monkeypatch): + calls = [] + monkeypatch.setattr(_child, "_use_c_ctype_locale", lambda: calls.append("locale")) + monkeypatch.setattr(_child, "_arm_crash_trace", lambda path: None) + import localm._mp_spawn as spawn + import localm.debuglog as debuglog + for name in ("install_parent_death_watchdog", "ignore_interrupt_signals", + "suppress_native_error_dialogs"): + monkeypatch.setattr(spawn, name, lambda: None) + monkeypatch.setattr(debuglog, "attach_child_logging", lambda: None) + + class _Queue: + def get(self): + calls.append("get") + return None + + import threading + _child.worker_main(_Queue(), _Queue(), threading.Event()) + assert calls[:2] == ["locale", "get"] + + +def test_other_platforms_leave_the_locale_alone(monkeypatch): + import locale + seen = [] + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(locale, "setlocale", lambda *a: seen.append(a)) + _child._use_c_ctype_locale() + assert seen == [] From a6774e21ffb6ee1c8272d2b65ae4d6fb07dcf07b Mon Sep 17 00:00:00 2001 From: Matlan Date: Thu, 15 Oct 2026 23:20:16 +0200 Subject: [PATCH 2/2] Changelog: native image prompts with non-ASCII letters Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab08b7975..c54615fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ permanent public record of what shipped and are never rewritten; the in-progress ## [Unreleased] +### Fixed + +- **Image prompts with letters such as ß work with the native image backend on Windows.** A prompt + containing ß failed with "failed to encode prompt", and letters such as ä, ö and ü reached the + model garbled; the prompt now reaches stable-diffusion.cpp unchanged. + ## [0.3.0] - 2026-10-14 ### Added