diff --git a/CHANGELOG.md b/CHANGELOG.md index ab08b7975..c54615fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,12 @@ permanent public record of what shipped and are never rewritten; the in-progress ## [Unreleased] +### Fixed + +- **Image prompts with letters such as ß work with the native image backend on Windows.** A prompt + containing ß failed with "failed to encode prompt", and letters such as ä, ö and ü reached the + model garbled; the prompt now reaches stable-diffusion.cpp unchanged. + ## [0.3.0] - 2026-10-14 ### Added diff --git a/localm/media/sdcpp/_child.py b/localm/media/sdcpp/_child.py index c2a39815d..0455472bc 100644 --- a/localm/media/sdcpp/_child.py +++ b/localm/media/sdcpp/_child.py @@ -110,6 +110,17 @@ def _prepare_environment(runtime_dir: Path) -> None: os.environ.pop(_HOST_VIS_VAR, None) +def _use_c_ctype_locale() -> None: + """On Windows, set this process's C runtime LC_CTYPE to "C", so the byte-wise + tolower in stable-diffusion.cpp's CLIP tokenizer leaves UTF-8 prompt bytes + unchanged. Other platforms are left as they are. + See test_the_worker_keeps_utf8_prompt_bytes_through_tolower.""" + if sys.platform != "win32": + return + import locale + locale.setlocale(locale.LC_CTYPE, "C") + + def _set_rocblas_tensile(extra_dirs) -> None: if os.environ.get("ROCBLAS_TENSILE_LIBPATH"): return @@ -648,6 +659,7 @@ def worker_main(req_q, resp_q, cancel_event, crash_trace_path=None) -> None: install_parent_death_watchdog() ignore_interrupt_signals() suppress_native_error_dialogs() + _use_c_ctype_locale() log_cb, progress_cb, _State.preview_cb = _make_callbacks(resp_q) callbacks_set = False diff --git a/tests/test_sdcpp_child_locale.py b/tests/test_sdcpp_child_locale.py new file mode 100644 index 000000000..6815244cd --- /dev/null +++ b/tests/test_sdcpp_child_locale.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""The stable-diffusion.cpp worker keeps non-ASCII prompt bytes intact through +the native tokenizer's byte-wise tolower.""" + +import subprocess +import sys +import textwrap + +import pytest + +from localm.media.sdcpp import _child + +_PROMPT = "Ein weißer Fuchs läuft über einen vereisten See" + +_CRT_PROBE = textwrap.dedent(""" + import ctypes, locale, sys + from localm.media.sdcpp import _child + crt = ctypes.CDLL("ucrtbase") + raw = sys.argv[1].encode("utf-8") + + def lowered(): + return bytes(crt.tolower(b) & 0xFF for b in raw) + + locale.setlocale(locale.LC_CTYPE, ".1252") + before = lowered() + _child._use_c_ctype_locale() + after = lowered() + print(before.hex()) + print(after.hex()) +""") + + +@pytest.mark.skipif(sys.platform != "win32", reason="the code-page C runtime locale is Windows-only") +def test_the_worker_keeps_utf8_prompt_bytes_through_tolower(): + out = subprocess.run([sys.executable, "-c", _CRT_PROBE, _PROMPT], + capture_output=True, text=True, timeout=60, check=True) + before_hex, after_hex = out.stdout.split() + before, after = bytes.fromhex(before_hex), bytes.fromhex(after_hex) + with pytest.raises(UnicodeDecodeError): + before.decode("utf-8") + assert after.decode("utf-8") == _PROMPT.lower() + + +def test_the_worker_entry_point_sets_the_locale_before_serving(monkeypatch): + calls = [] + monkeypatch.setattr(_child, "_use_c_ctype_locale", lambda: calls.append("locale")) + monkeypatch.setattr(_child, "_arm_crash_trace", lambda path: None) + import localm._mp_spawn as spawn + import localm.debuglog as debuglog + for name in ("install_parent_death_watchdog", "ignore_interrupt_signals", + "suppress_native_error_dialogs"): + monkeypatch.setattr(spawn, name, lambda: None) + monkeypatch.setattr(debuglog, "attach_child_logging", lambda: None) + + class _Queue: + def get(self): + calls.append("get") + return None + + import threading + _child.worker_main(_Queue(), _Queue(), threading.Event()) + assert calls[:2] == ["locale", "get"] + + +def test_other_platforms_leave_the_locale_alone(monkeypatch): + import locale + seen = [] + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(locale, "setlocale", lambda *a: seen.append(a)) + _child._use_c_ctype_locale() + assert seen == []