diff --git a/crates/ember-app/src/main.rs b/crates/ember-app/src/main.rs index ff811aa..1d1093d 100644 --- a/crates/ember-app/src/main.rs +++ b/crates/ember-app/src/main.rs @@ -1012,6 +1012,8 @@ impl ApplicationHandler for App { let mut win = WindowState::new(renderer, tree); win.px = px; + win.ax = + ember_platform::accessibility::AxTextArea::install(&window, shared.wake.clone()); if !win.spawn_session( &mut shared, session, @@ -1767,6 +1769,7 @@ impl ApplicationHandler for App { win.render_ema_ms ))); } + win.refresh_ax(); let t = Instant::now(); match win.renderer.render() { // A drawable came through — the surface is ground truth, so @@ -1806,6 +1809,25 @@ impl ApplicationHandler for App { return; }; + // Accessibility clients (dictation tools) reading or writing a + // window's terminal view: an insert pastes into that window's focused + // pane, exactly like Cmd+V; a first read starts its text updates. + for req in ember_platform::accessibility::drain() { + use ember_platform::accessibility::AxRequest; + let (AxRequest::Insert { view, .. } | AxRequest::Refresh { view }) = &req; + let Some(w) = self + .windows + .values() + .find(|w| w.ax.as_ref().is_some_and(|ax| ax.view() == *view)) + else { + continue; + }; + match req { + AxRequest::Insert { text, .. } => w.paste_into_focused(shared, &text), + AxRequest::Refresh { .. } => w.refresh_ax(), + } + } + // Poll every session's pixel lane, routing each one's deltas to the // window that actually owns it (`session_window`) — "drain only the // focused window" (the pre-multi-window shape of this function) would @@ -2870,6 +2892,7 @@ fn open_window( let mut win = WindowState::new(renderer, tree); win.px = px; + win.ax = ember_platform::accessibility::AxTextArea::install(&window, shared.wake.clone()); win.sync_layout(shared); win.apply_appearance(shared); win.renderer.window().request_redraw(); diff --git a/crates/ember-app/src/window_state.rs b/crates/ember-app/src/window_state.rs index 64f624e..76a4f34 100644 --- a/crates/ember-app/src/window_state.rs +++ b/crates/ember-app/src/window_state.rs @@ -477,6 +477,9 @@ pub(crate) struct WindowState { /// IME composition in progress (preedit text); non-empty = composing, /// during which raw key events are suppressed (they belong to the IME). pub(crate) ime_preedit: String, + /// This window's view registered as an accessibility text area (macOS), + /// so dictation tools can insert into it; `None` elsewhere. + pub(crate) ax: Option, /// Command palette (Cmd+Shift+P): open flag, query, selected row index /// (into the CURRENT filtered list). pub(crate) palette_open: bool, @@ -853,6 +856,7 @@ impl WindowState { search_query: String::new(), search_count: None, ime_preedit: String::new(), + ax: None, palette_open: false, palette_query: String::new(), palette_sel: 0, @@ -1581,6 +1585,21 @@ impl WindowState { self.send_to_focused(shared, bracket_paste(text, bracketed)); } + /// Publish the focused pane's screen text + cursor to the accessibility + /// text area, once a client has started reading it (no cost otherwise). + pub(crate) fn refresh_ax(&self) { + let Some(ax) = self.ax.as_ref().filter(|ax| ax.active()) else { + return; + }; + let Some(grid) = self + .focused_session_id() + .and_then(|id| self.renderer.grid(&id)) + else { + return; + }; + ax.set_text(grid.screen_text(), grid.cursor.row, grid.cursor.col); + } + /// A file dropped from the OS (Finder / file manager) onto this window: /// insert its shell-escaped path plus a trailing space at the focused /// pane's prompt, through the paste path (bracketed-paste aware) — diff --git a/crates/ember-platform/Cargo.toml b/crates/ember-platform/Cargo.toml index a168e42..ccbe070 100644 --- a/crates/ember-platform/Cargo.toml +++ b/crates/ember-platform/Cargo.toml @@ -14,7 +14,7 @@ arboard.workspace = true [target.'cfg(target_os = "macos")'.dependencies] objc2 = "0.6" objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSImage", "NSResponder", "NSWorkspace"] } -objc2-foundation = { version = "0.3", features = ["NSData", "NSString", "NSProcessInfo"] } +objc2-foundation = { version = "0.3", features = ["NSData", "NSString", "NSProcessInfo", "NSRange"] } # default features pull muda's Linux GTK backend (gtk3 → glib), which Ember # never uses — the menu is macOS-only (NSMenu via objc2); other platforms get # an inert stub. Dropping them keeps the vulnerable glib (RUSTSEC-2024-0429) diff --git a/crates/ember-platform/src/accessibility.rs b/crates/ember-platform/src/accessibility.rs new file mode 100644 index 0000000..4ffc6d8 --- /dev/null +++ b/crates/ember-platform/src/accessibility.rs @@ -0,0 +1,499 @@ +//! The terminal view as an accessibility text area (macOS) — an OS seam +//! (design §7). +//! +//! winit's `WinitView` exposes nothing to the macOS Accessibility API, so the +//! focused element an assistive client sees in an Ember window is the bare +//! `AXWindow`. Dictation tools (Speechify and friends) look for a focused text +//! element they can write to and, finding none, fall back to "copy the text +//! yourself". iTerm2's terminal view is a focused `AXTextArea` whose `AXValue` +//! and `AXSelectedTextRange` are settable; this module makes Ember's match. +//! +//! The NSAccessibility methods are added to winit's view class at runtime with +//! `class_addMethod`, and only ever ADDED: if `WinitView` already implements +//! any of them itself (a future winit release), nothing is installed and the +//! view keeps winit's behavior. They override NSView's inherited defaults, +//! which is what an accessibility-aware NSView subclass does anyway. +//! +//! The write path is deliberately narrow. Setting `AXValue` pastes the text the +//! client inserted (the new value minus what we last served it) into the +//! focused pane through the app's normal paste path, bracketed-paste included; +//! setting `AXSelectedTextRange` is accepted and ignored (a terminal has no +//! client-movable caret). Nothing else about the terminal's text changes. +//! +//! Main-thread only: AppKit calls these on the main thread, and the app owns +//! the per-window handles there too. Requests are queued and the event loop is +//! woken; the app drains them with [`drain`]. + +use std::sync::Arc; + +/// Something an accessibility client asked of a terminal view, for the app to +/// carry out on its next loop turn. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AxRequest { + /// The client wrote text into the view: paste it into the focused pane. + Insert { view: usize, text: String }, + /// A client started reading the view: start keeping its text current. + Refresh { view: usize }, +} + +/// The text a client inserted, given the value it last read (`old`) and the +/// value it wrote back (`new`): whatever lies between their common prefix and +/// common suffix in `new`. Characters the client removed are ignored (the +/// terminal's text is read-only), so a pure deletion inserts nothing. +pub fn inserted_text(old: &str, new: &str) -> String { + let prefix: usize = old + .chars() + .zip(new.chars()) + .take_while(|(a, b)| a == b) + .map(|(_, b)| b.len_utf8()) + .sum(); + let (old_rest, new_rest) = (&old[prefix..], &new[prefix..]); + let suffix: usize = old_rest + .chars() + .rev() + .zip(new_rest.chars().rev()) + .take_while(|(a, b)| a == b) + .map(|(_, b)| b.len_utf8()) + .sum(); + new_rest[..new_rest.len() - suffix].to_string() +} + +/// One window's registration as an accessibility text area. Dropping it +/// unregisters the view (it then answers as a plain, non-element NSView). +pub struct AxTextArea { + view: usize, +} + +impl AxTextArea { + /// Make `window`'s view an accessibility text area. `None` off macOS, or + /// when the methods could not be installed (see the module docs). + pub fn install( + window: &winit::window::Window, + wake: Arc, + ) -> Option { + imp::install(window, wake).map(|view| Self { view }) + } + + /// The registered view's key, as carried by [`AxRequest`]. + pub fn view(&self) -> usize { + self.view + } + + /// Whether a client has read this view, i.e. whether the app should keep + /// its text current with [`set_text`](Self::set_text). + pub fn active(&self) -> bool { + imp::active(self.view) + } + + /// Publish the view's current text: the focused pane's screen, rows joined + /// by `\n`, with the cursor at (`row`, `col`) (`col` in chars). + pub fn set_text(&self, text: String, row: u16, col: u16) { + imp::set_text(self.view, text, row, col); + } +} + +impl Drop for AxTextArea { + fn drop(&mut self) { + imp::unregister(self.view); + } +} + +/// Take every queued request (call from the event loop on the main thread). +pub fn drain() -> Vec { + imp::drain() +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::AxRequest; + use std::sync::Arc; + + pub fn install(_: &winit::window::Window, _: Arc) -> Option { + None + } + pub fn active(_: usize) -> bool { + false + } + pub fn set_text(_: usize, _: String, _: u16, _: u16) {} + pub fn unregister(_: usize) {} + pub fn drain() -> Vec { + Vec::new() + } +} + +#[cfg(target_os = "macos")] +#[allow(unsafe_code)] +mod imp { + use super::{AxRequest, inserted_text}; + use objc2::encode::{EncodeArguments, EncodeReturn, Encoding}; + use objc2::rc::Retained; + use objc2::runtime::{AnyClass, AnyObject, Bool, Imp, Sel}; + use objc2::{ClassType, msg_send, sel}; + use objc2_foundation::{NSInteger, NSRange, NSString}; + use std::cell::{OnceCell, RefCell}; + use std::collections::HashMap; + use std::ffi::CString; + use std::sync::Arc; + use winit::raw_window_handle::{HasWindowHandle, RawWindowHandle}; + + #[derive(Default)] + struct ViewState { + /// The latest published screen text. + text: String, + /// What we last answered `accessibilityValue` with: the base a + /// client's `setAccessibilityValue:` is diffed against. + served: Option, + cursor_row: u16, + cursor_col: u16, + /// A client has read this view; the app keeps `text` current. + active: bool, + } + + thread_local! { + static VIEWS: RefCell> = RefCell::new(HashMap::new()); + static QUEUE: RefCell> = const { RefCell::new(Vec::new()) }; + static WAKE: OnceCell> = const { OnceCell::new() }; + /// Per view class: whether our methods are installed on it. + static INSTALLED: RefCell> = RefCell::new(HashMap::new()); + } + + fn with_view(view: usize, f: impl FnOnce(&mut ViewState) -> R) -> Option { + VIEWS.with(|v| v.borrow_mut().get_mut(&view).map(f)) + } + + fn push(req: AxRequest) { + QUEUE.with(|q| q.borrow_mut().push(req)); + WAKE.with(|w| { + if let Some(w) = w.get() { + w(); + } + }); + } + + pub fn install( + window: &winit::window::Window, + wake: Arc, + ) -> Option { + let RawWindowHandle::AppKit(h) = window.window_handle().ok()?.as_raw() else { + return None; + }; + let view = h.ns_view.as_ptr() as usize; + // SAFETY: winit's AppKit handle is a live NSView for the window's life. + let obj: &AnyObject = unsafe { &*(view as *const AnyObject) }; + if !install_on(obj.class()) { + return None; + } + WAKE.with(|w| { + let _ = w.set(wake); + }); + VIEWS.with(|v| v.borrow_mut().insert(view, ViewState::default())); + Some(view) + } + + pub fn active(view: usize) -> bool { + with_view(view, |s| s.active).unwrap_or(false) + } + + pub fn set_text(view: usize, text: String, row: u16, col: u16) { + with_view(view, |s| { + s.text = text; + s.cursor_row = row; + s.cursor_col = col; + }); + } + + pub fn unregister(view: usize) { + VIEWS.with(|v| v.borrow_mut().remove(&view)); + } + + pub fn drain() -> Vec { + QUEUE.with(|q| std::mem::take(&mut *q.borrow_mut())) + } + + /// One method to add: selector, implementation, type encoding. + struct Method(Sel, Imp, CString); + + fn method(sel: Sel, imp: Imp) -> Method { + let mut types = format!( + "{}{}{}", + R::ENCODING_RETURN, + Encoding::Object, + Encoding::Sel + ); + for e in A::ENCODINGS { + types.push_str(&e.to_string()); + } + Method(sel, imp, CString::new(types).expect("encoding has no NUL")) + } + + /// Add the accessibility methods to `cls` (once per class). Returns whether + /// they are in place. Refuses, installing nothing, when the class itself + /// already implements any of them: we add, never replace. + fn install_on(cls: &AnyClass) -> bool { + let key = cls as *const AnyClass as usize; + if let Some(done) = INSTALLED.with(|i| i.borrow().get(&key).copied()) { + return done; + } + // SAFETY (all transmutes): each fn below is `extern "C-unwind"` with + // the receiver, `_cmd`, then the arguments its encoding declares; Imp + // is the erased form the runtime calls through. + #[allow(clippy::missing_transmute_annotations)] + let methods = unsafe { + use std::mem::transmute as t; + [ + method::( + sel!(isAccessibilityElement), + t(is_element as extern "C-unwind" fn(_, _) -> _), + ), + method::( + sel!(isAccessibilityFocused), + t(is_focused as extern "C-unwind" fn(_, _) -> _), + ), + method::<*mut NSString, ()>( + sel!(accessibilityRole), + t(role as extern "C-unwind" fn(_, _) -> _), + ), + method::<*mut AnyObject, ()>( + sel!(accessibilityValue), + t(value as extern "C-unwind" fn(_, _) -> _), + ), + method::<(), (*mut AnyObject,)>( + sel!(setAccessibilityValue:), + t(set_value as extern "C-unwind" fn(_, _, _)), + ), + method::<*mut NSString, ()>( + sel!(accessibilitySelectedText), + t(selected_text as extern "C-unwind" fn(_, _) -> _), + ), + method::( + sel!(accessibilitySelectedTextRange), + t(selected_range as extern "C-unwind" fn(_, _) -> _), + ), + method::<(), (NSRange,)>( + sel!(setAccessibilitySelectedTextRange:), + t(set_selected_range as extern "C-unwind" fn(_, _, _)), + ), + method::( + sel!(accessibilityVisibleCharacterRange), + t(visible_range as extern "C-unwind" fn(_, _) -> _), + ), + method::( + sel!(accessibilityNumberOfCharacters), + t(char_count as extern "C-unwind" fn(_, _) -> _), + ), + method::( + sel!(accessibilityInsertionPointLineNumber), + t(insertion_line as extern "C-unwind" fn(_, _) -> _), + ), + ] + }; + let own: Vec = cls.instance_methods().iter().map(|m| m.name()).collect(); + let clash: Vec<&str> = methods + .iter() + .filter(|m| own.contains(&m.0)) + .map(|m| m.0.name().to_str().unwrap_or("?")) + .collect(); + let ok = if clash.is_empty() { + methods.iter().all(|Method(sel, imp, types)| { + // SAFETY: `imp` matches `types` (built from the same signature). + unsafe { + objc2::ffi::class_addMethod( + cls as *const _ as *mut _, + *sel, + *imp, + types.as_ptr(), + ) + } + .as_bool() + }) + } else { + eprintln!( + "[ember] accessibility: {} already implements {clash:?}; leaving it as is", + cls.name().to_str().unwrap_or("?") + ); + false + }; + INSTALLED.with(|i| i.borrow_mut().insert(key, ok)); + ok + } + + fn key(this: &AnyObject) -> usize { + this as *const AnyObject as usize + } + + fn registered(this: &AnyObject) -> bool { + with_view(key(this), |_| ()).is_some() + } + + /// Mark the view read; the first read asks the app for current text. + fn touch(this: &AnyObject) { + let first = + with_view(key(this), |s| !std::mem::replace(&mut s.active, true)).unwrap_or(false); + if first { + push(AxRequest::Refresh { view: key(this) }); + } + } + + fn utf16_len(s: &str) -> usize { + s.encode_utf16().count() + } + + /// The cursor's offset into the text, in UTF-16 units (AppKit's index). + fn cursor_offset(s: &ViewState) -> usize { + let mut off = 0; + for (i, line) in s.text.split('\n').enumerate() { + if i == s.cursor_row as usize { + return off + + line + .chars() + .take(s.cursor_col as usize) + .map(char::len_utf16) + .sum::(); + } + off += utf16_len(line) + 1; + } + utf16_len(&s.text) + } + + fn ns_string(s: &str) -> *mut NSString { + Retained::autorelease_ptr(NSString::from_str(s)) + } + + extern "C-unwind" fn is_element(this: &AnyObject, _: Sel) -> Bool { + Bool::new(registered(this)) + } + + extern "C-unwind" fn is_focused(this: &AnyObject, _: Sel) -> Bool { + if !registered(this) { + return Bool::NO; + } + // SAFETY: `window` and `firstResponder` are plain getters on a live + // NSView / NSWindow (nil-safe). + let first: *mut AnyObject = unsafe { + let window: *mut AnyObject = msg_send![this, window]; + if window.is_null() { + return Bool::NO; + } + msg_send![window, firstResponder] + }; + Bool::new(std::ptr::eq(first, this)) + } + + extern "C-unwind" fn role(this: &AnyObject, _: Sel) -> *mut NSString { + ns_string(if registered(this) { + "AXTextArea" + } else { + "AXGroup" + }) + } + + extern "C-unwind" fn value(this: &AnyObject, _: Sel) -> *mut AnyObject { + touch(this); + let text = with_view(key(this), |s| { + s.served = Some(s.text.clone()); + s.text.clone() + }); + match text { + Some(t) => ns_string(&t).cast(), + None => std::ptr::null_mut(), + } + } + + extern "C-unwind" fn set_value(this: &AnyObject, _: Sel, new: *mut AnyObject) { + // SAFETY: a client passes an NSString (or an NSAttributedString, + // whose `string` is one); anything else is ignored. + let new = unsafe { + let Some(obj) = new.as_ref() else { return }; + let s: *mut NSString = if msg_send![obj, isKindOfClass: NSString::class()] { + new.cast() + } else if msg_send![obj, respondsToSelector: sel!(string)] { + msg_send![obj, string] + } else { + return; + }; + match s.as_ref() { + Some(s) => s.to_string(), + None => return, + } + }; + let Some(text) = with_view(key(this), |s| { + let base = s.served.clone().unwrap_or_else(|| s.text.clone()); + let text = inserted_text(&base, &new); + // The client's view of the value now includes its insertion. + s.served = Some(new); + text + }) else { + return; + }; + if !text.is_empty() { + push(AxRequest::Insert { + view: key(this), + text, + }); + } + } + + extern "C-unwind" fn selected_text(this: &AnyObject, _: Sel) -> *mut NSString { + touch(this); + ns_string("") + } + + extern "C-unwind" fn selected_range(this: &AnyObject, _: Sel) -> NSRange { + touch(this); + NSRange::new(with_view(key(this), |s| cursor_offset(s)).unwrap_or(0), 0) + } + + /// Accepted (so the attribute reads as settable, like iTerm2's) and + /// ignored: the terminal's caret belongs to the program running in it. + extern "C-unwind" fn set_selected_range(_: &AnyObject, _: Sel, _: NSRange) {} + + extern "C-unwind" fn visible_range(this: &AnyObject, _: Sel) -> NSRange { + NSRange::new(0, with_view(key(this), |s| utf16_len(&s.text)).unwrap_or(0)) + } + + extern "C-unwind" fn char_count(this: &AnyObject, _: Sel) -> NSInteger { + touch(this); + with_view(key(this), |s| utf16_len(&s.text)).unwrap_or(0) as NSInteger + } + + extern "C-unwind" fn insertion_line(this: &AnyObject, _: Sel) -> NSInteger { + touch(this); + with_view(key(this), |s| s.cursor_row).unwrap_or(0) as NSInteger + } +} + +#[cfg(test)] +mod tests { + use super::inserted_text; + + #[test] + fn append_at_end() { + assert_eq!(inserted_text("$ ls", "$ ls -la"), " -la"); + } + + #[test] + fn insert_in_middle() { + assert_eq!(inserted_text("a\n$ \nb", "a\n$ hello\nb"), "hello"); + } + + #[test] + fn from_empty_value() { + assert_eq!(inserted_text("", "dictated text"), "dictated text"); + } + + #[test] + fn replacement_keeps_only_the_new_text() { + assert_eq!(inserted_text("say foo now", "say barbaz now"), "barbaz"); + } + + #[test] + fn pure_deletion_inserts_nothing() { + assert_eq!(inserted_text("abc", "ac"), ""); + } + + #[test] + fn repeated_chars_and_multibyte() { + assert_eq!(inserted_text("aa", "aaa"), "a"); + assert_eq!(inserted_text("é$", "é café$"), " café"); + } +} diff --git a/crates/ember-platform/src/lib.rs b/crates/ember-platform/src/lib.rs index d0459d2..892116f 100644 --- a/crates/ember-platform/src/lib.rs +++ b/crates/ember-platform/src/lib.rs @@ -13,6 +13,7 @@ //! Wayland-specific clipboard quirk) without threading a new `#[cfg]` through //! every call site in `ember-app`. +pub mod accessibility; pub mod menu; pub use menu::{AppMenu, MenuAction, build_menu, menu_action};