From 7aa973caf30bb23875f45f33fc405afe10bfc0f9 Mon Sep 17 00:00:00 2001 From: lihujun Date: Fri, 18 Sep 2026 14:50:17 +0800 Subject: [PATCH 1/3] Fix Unicode input fallback when clipboard writes are denied --- artemis/clients/screen_client_factory.py | 14 ++- artemis/clients/ui_automator_client.py | 3 +- artemis/drivers/android/adb_driver.py | 23 ++-- .../clients/test_screen_client_factory.py | 46 ++++++++ tests/unit/test_android_text_input.py | 104 ++++++++++++++++++ 5 files changed, 175 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_android_text_input.py diff --git a/artemis/clients/screen_client_factory.py b/artemis/clients/screen_client_factory.py index 311f2629..93149d12 100644 --- a/artemis/clients/screen_client_factory.py +++ b/artemis/clients/screen_client_factory.py @@ -300,8 +300,18 @@ def get_screenshot_base64(self) -> str | None: def set_clipboard(self, text: str) -> bool: return self._call("set_clipboard", text) - def send_text(self, text: str) -> Any: - return self._call("send_text", text) + def send_text(self, text: str) -> bool: + result = self._call("send_text", text) + if result is False and self._active_backend == "helper": + # A healthy helper may observe a field but be unable to set its text. + # Try IME input without marking the whole helper service unavailable. + self._ensure_device_online() + try: + return self.uiautomator.send_text(text) + finally: + # UiAutomation can unbind the helper even when input fails. + self._set_active("uiautomator", "Helper rejected direct text input") + return result def clear_text(self) -> bool: return self._call("clear_text") diff --git a/artemis/clients/ui_automator_client.py b/artemis/clients/ui_automator_client.py index 12a73fb0..d3eb3da6 100644 --- a/artemis/clients/ui_automator_client.py +++ b/artemis/clients/ui_automator_client.py @@ -324,7 +324,7 @@ def press_key(self, key: str): device = self._ensure_connected() return device.press(key=key) - def send_text(self, text: str) -> None: + def send_text(self, text: str) -> bool: """Send text input to the device using FastInputIME. This method supports special characters (e.g., 'ö') that ADB shell @@ -343,6 +343,7 @@ def send_text(self, text: str) -> None: # Give FastInputIME time to process the broadcast and commit text # before switching it off and killing it. time.sleep(0.5) + return True finally: device.set_fastinput_ime(False) diff --git a/artemis/drivers/android/adb_driver.py b/artemis/drivers/android/adb_driver.py index 4a543810..639e54f8 100644 --- a/artemis/drivers/android/adb_driver.py +++ b/artemis/drivers/android/adb_driver.py @@ -325,21 +325,15 @@ async def input_text(self, text: str, clear_existing: bool = True) -> bool: # Normalize literal escaped newlines from LLM / tool call serialization norm_text = text.replace(r"\r\n", "\n").replace(r"\n", "\n").replace(r"\r", "\n") - # 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets) + # Prefer direct input. A background clipboard write can be silently + # denied by Android even when the helper reports success. if self._ui_adb_client: try: - set_clip_ok = False - if hasattr(self._ui_adb_client, "set_clipboard"): - set_clip_ok = self._ui_adb_client.set_clipboard(norm_text) - elif hasattr(self._ui_adb_client, "_device") and self._ui_adb_client._device: - self._ui_adb_client._device.set_clipboard(norm_text) - set_clip_ok = True - - if set_clip_ok: - await asyncio.to_thread(self.device.shell, "input keyevent 279") + result = self._ui_adb_client.send_text(norm_text) + if result is True: return True except Exception as e: - logger.debug(f"Clipboard paste fallback to ADB input: {e}") + logger.debug(f"Direct text input failed, trying ADBKeyboard: {e}") # 2. Tier 2: Check if ADBKeyboard is currently active try: @@ -355,7 +349,12 @@ async def input_text(self, text: str, clear_existing: bool = True) -> bool: # ADBKeyboard probe/broadcast failed; fall through to native input. logger.debug(f"ADBKeyboard IME path failed, falling back to ADB input: {e}") - # 3. Tier 3: Universal Native ADB input text fallback + # Native adb input text cannot reliably enter Unicode characters. + if not norm_text.isascii(): + logger.warning("Unicode input failed: no supported input channel succeeded") + return False + + # 3. Tier 3: Native ADB input text fallback for ASCII lines = norm_text.split("\n") for i, line in enumerate(lines): if i > 0: diff --git a/tests/unit/clients/test_screen_client_factory.py b/tests/unit/clients/test_screen_client_factory.py index 836af3b9..474d291a 100644 --- a/tests/unit/clients/test_screen_client_factory.py +++ b/tests/unit/clients/test_screen_client_factory.py @@ -225,3 +225,49 @@ def test_disconnect_stops_the_uiautomator_server_it_started(composite): client.connect() client.disconnect() u2.disconnect.assert_called_once_with(stop_server=True) + + +def test_rejected_helper_text_uses_uiautomator(composite): + client, helper, u2, _, _ = composite + helper.send_text.return_value = False + u2.send_text.return_value = True + assert client.send_text("你好") is True + u2.send_text.assert_called_once_with("你好") + assert client.active_backend == "uiautomator" + + +def test_successful_helper_text_is_not_duplicated(composite): + client, helper, u2, _, _ = composite + helper.send_text.return_value = True + assert client.send_text("你好") is True + u2.send_text.assert_not_called() + + +def test_rejected_uiautomator_text_is_not_retried(composite): + client, helper, u2, _, _ = composite + helper.send_text.side_effect = RuntimeError("unavailable") + u2.send_text.return_value = False + assert client.send_text("你好") is False + u2.send_text.assert_called_once_with("你好") + + +def test_failed_ime_fallback_releases_uiautomation_before_helper_retry(composite): + client, helper, u2, _, _ = composite + helper.send_text.return_value = False + u2.send_text.side_effect = RuntimeError("IME unavailable") + with pytest.raises(RuntimeError, match="IME unavailable"): + client.send_text("你好") + assert client.active_backend == "uiautomator" + helper.get_hierarchy.return_value = "" + assert client.get_hierarchy() == "" + u2.stop_server.assert_called_once() + assert client.active_backend == "helper" + + +def test_rejected_text_does_not_start_fallback_on_offline_device(composite): + client, helper, _, factory, clock = composite + helper.send_text.return_value = False + clock["state"] = "offline" + with pytest.raises(DeviceOfflineError): + client.send_text("你好") + factory.assert_not_called() diff --git a/tests/unit/test_android_text_input.py b/tests/unit/test_android_text_input.py new file mode 100644 index 00000000..02eab8ea --- /dev/null +++ b/tests/unit/test_android_text_input.py @@ -0,0 +1,104 @@ +"""Regression coverage for rejected clipboard writes and Unicode input fallback.""" + +import base64 +from unittest.mock import MagicMock + +import pytest + +from artemis.clients.ui_automator_client import UIAutomatorClient +from artemis.drivers.android.adb_driver import AndroidAdbDriver + + +@pytest.fixture +def input_path(): + device = MagicMock() + device.shell.return_value = "com.example/.Keyboard" + client = MagicMock() + client.send_text.return_value = True + # The real helper can claim clipboard success despite a denied write. + client.set_clipboard.return_value = True + adb = MagicMock() + adb.device.return_value = device + action = AndroidAdbDriver("test", adb, client).input_text + return action, device, client + + +@pytest.mark.asyncio +async def test_direct_unicode_input_does_not_paste_or_duplicate(input_path): + action, device, client = input_path + assert await action("héllo\n你好", clear_existing=False) is True + client.send_text.assert_called_once_with("héllo\n你好") + client.set_clipboard.assert_not_called() + assert [c.args[0] for c in device.shell.call_args_list] == ["input keyevent 123"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [False, RuntimeError("unsupported field")]) +async def test_failed_direct_input_falls_back_to_adbkeyboard(input_path, failure): + action, device, client = input_path + if isinstance(failure, Exception): + client.send_text.side_effect = failure + else: + client.send_text.return_value = failure + device.shell.return_value = "com.android.adbkeyboard/.AdbIME" + assert await action("你好", clear_existing=False) is True + payload = base64.b64encode("你好".encode()).decode() + device.shell.assert_any_call(f"am broadcast -a ADB_INPUT_B64 --es msg '{payload}'") + client.set_clipboard.assert_not_called() + + +@pytest.mark.asyncio +async def test_unicode_without_supported_channel_reports_failure(input_path): + action, device, client = input_path + client.send_text.return_value = False + assert await action("你好", clear_existing=False) is False + assert not any(c.args[0].startswith("input text ") for c in device.shell.call_args_list) + client.set_clipboard.assert_not_called() + + +def test_uiautomator_send_text_returns_success_and_restores_ime(monkeypatch): + device = MagicMock() + monkeypatch.setattr(UIAutomatorClient, "_ensure_connected", lambda self: device) + monkeypatch.setattr("artemis.clients.ui_automator_client.time.sleep", lambda _: None) + client = object.__new__(UIAutomatorClient) + assert client.send_text("你好") is True + device.send_keys.assert_called_once_with("你好") + assert [c.args for c in device.set_fastinput_ime.call_args_list] == [(True,), (False,)] + + +@pytest.mark.asyncio +async def test_ascii_fallback_still_inputs_text(input_path): + action, device, client = input_path + client.send_text.return_value = False + assert await action("hello world", clear_existing=False) is True + device.shell.assert_any_call("input text hello%sworld") + + +@pytest.mark.asyncio +async def test_driver_normalizes_escaped_newlines_before_direct_input(): + client = MagicMock() + client.send_text.return_value = True + driver = AndroidAdbDriver("test", MagicMock(), client) + assert await driver.input_text(r"你好\n世界") is True + client.send_text.assert_called_once_with("你好\n世界") + + +def test_uiautomator_send_text_restores_ime_on_error(monkeypatch): + device = MagicMock() + device.send_keys.side_effect = RuntimeError("input failed") + monkeypatch.setattr(UIAutomatorClient, "_ensure_connected", lambda self: device) + monkeypatch.setattr("artemis.clients.ui_automator_client.time.sleep", lambda _: None) + with pytest.raises(RuntimeError, match="input failed"): + object.__new__(UIAutomatorClient).send_text("你好") + device.set_fastinput_ime.assert_called_with(False) + + +@pytest.mark.asyncio +async def test_unicode_without_client_or_adbkeyboard_fails(): + device = MagicMock() + device.shell.return_value = "com.example/.Keyboard" + adb = MagicMock() + adb.device.return_value = device + action = AndroidAdbDriver("test", adb).input_text + assert await action("你好", clear_existing=False) is False + assert not any(c.args[0].startswith("input text ") for c in device.shell.call_args_list) From eb52e2f625cf2658bb595f610565955ba0def343 Mon Sep 17 00:00:00 2001 From: lihujun Date: Fri, 18 Sep 2026 15:01:02 +0800 Subject: [PATCH 2/3] Preserve literal Unicode goals in Flash prompts --- artemis/agents/flash/flash_runner.md | 6 ++++++ artemis/agents/flash/runner.py | 6 +++++- tests/unit/agents/test_flash_runner.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 3c21cf8d..9e259924 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -3,6 +3,12 @@ You are an autonomous and highly efficient Android Device Execution Agent. Your **Objective: {{ goal }}** +The same original objective as an ASCII-escaped JSON string (an exact character reference): +```json +{{ goal_json }} +``` +When entering user-provided text, copy the requested text exactly from the original objective. Preserve emoji, variation selectors, skin tones, joiners, punctuation and whitespace; do not substitute a visually similar character or reinterpret a symbol by its name. JSON Unicode escapes may be used in tool arguments; they decode to the actual characters, not literal backslash text. Before reporting completion, compare the field's contents with the original requested text, not just with your previous tool arguments. + --- # 1. COGNITIVE PROTOCOL diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 936f6065..ec9a3160 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -296,7 +296,11 @@ def _render_system_prompt(self, tools_declaration: list) -> str: prompt_path = Path(__file__).parent / "flash_runner.md" prompt_template = prompt_path.read_text(encoding="utf-8") available_tools = frozenset(t.name for t in tools_declaration) - return Template(prompt_template).render(goal=self.goal, available_tools=available_tools) + return Template(prompt_template).render( + goal=self.goal, + goal_json=json.dumps(self.goal, ensure_ascii=True), + available_tools=available_tools, + ) # ------------------------------------------------------------------ # Per-turn helpers (observe / think) diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index f188b5e1..d1b153e2 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -556,3 +556,19 @@ async def test_final_report_persists_native_thinking(mock_context): assert kwargs["operator_raw_thinking"] == "final text" assert kwargs["operator_native_thinking"] == "native summary" assert isinstance(messages[-1], ToolMessage) + + +@pytest.mark.parametrize('text', ['你好中文英😅', '👍🏽👩‍💻🇨🇳❤️', r'路径\n"原文"']) +def test_prompt_preserves_literal_unicode_goal_as_json(mock_context, text): + import json + + goal = '在当前输入框输入：' + text + with patch('artemis.controllers.unified_controller.get_driver'): + runner = FlashRunner(mock_context, goal=goal) + prompt = runner._render_system_prompt(runner._get_tools()) + # A second, ASCII-only representation makes the exact code points available + # even when the model misreads an emoji glyph (😅 was changed to ㊅ in a trace). + encoded = json.dumps(goal, ensure_ascii=True) + assert encoded in prompt + assert json.loads(encoded) == goal + assert 'original objective' in prompt From 3446fd117359c4684b66a759618c84bf2d4ad61a Mon Sep 17 00:00:00 2001 From: lihujun101 <34243386+lihujun101@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:15:06 +0800 Subject: [PATCH 3/3] fix: resolve visual summarizer from agent model configuration --- artemis/agents/flash/summarizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 56c5e098..e844d9c3 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -168,7 +168,7 @@ def __init__( if model_name: self._llm = get_google_llm(model_name=target_model, temperature=0.0) else: - self._llm = get_llm(ctx, name="summarizer", is_utils=True) + self._llm = get_llm(ctx, name="summarizer") except Exception: self._llm = get_google_llm(model_name=target_model, temperature=0.0) try: