From aa28e4a62df613f690793550bff1c6a5aef45141 Mon Sep 17 00:00:00 2001 From: Sajid Ali Date: Mon, 7 Sep 2026 21:06:32 +0500 Subject: [PATCH] [tvOS] Detect Siri Remote swipes from absolute pad position The fork's relative-space detection (UIKit indirect touches over 40 points) has several issues: it fires on drags across the ring buttons, generates phantom swipes from clicks, and counts slow drifts as swipes. Native tvOS only treats pad centre movement as a swipe, and never treats clicks as swipes. Switch to a per-container SiriRemoteTouchOracle that polls the GameController micro gamepad for absolute pad position (-1..1) while an indirect contact is active. A per-contact state machine fires a swipe once per contact when the dominant axis displacement reaches 0.30 within 250 ms, with the dominant axis at least twice the other. Contacts starting at radius 0.75 or more (ring zone) are ignored. Resting fingers (under 0.02 movement for 150 ms) reset the origin. A click or button press within 150 ms before or during the contact cancels the swipe. The dispatch holds for 80 ms to allow a late click to suppress it. Without a controller (simulator, first frames), fallback to the previous relative-space detection with the new duration, dead-zone and press gates. The COMPOSE_TVOS_SWIPE_DEBUG=1 environment variable prints all decisions. --- .../compose/ui/scene/ComposeContainer.tvos.kt | 8 + .../ui/scene/ComposeSceneMediator.tvos.kt | 536 ++++++++++++++++-- .../ui/scene/IosComposeSceneLayer.tvos.kt | 2 + .../ui/scene/SiriRemoteTouchOracle.tvos.kt | 270 +++++++++ .../ui/scene/TvPressDispatchLog.tvos.kt | 46 ++ 5 files changed, 799 insertions(+), 63 deletions(-) create mode 100644 compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt index 7360631056f37..50be50df2ab9a 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt @@ -102,6 +102,10 @@ internal class ComposeContainer( // responder chain delivers it back to the hosting view controller. private val pressDispatchLog = TvPressDispatchLog() + // Reports where the finger physically is on the Siri Remote clickpad, which UIKit indirect + // touches cannot tell. Shared by every mediator of this container, like the press log. + private val touchOracle = SiriRemoteTouchOracle() + @OptIn(InternalComposeUiApi::class) var rootForTestListener: PlatformContext.RootForTestListener? = null set(value) { @@ -243,6 +247,7 @@ internal class ComposeContainer( fun initializeComposeScene() { sceneJob = Job() + touchOracle.start() val frameChoreographer = frameChoreographer ?: error("No window scene found") val containerCoroutineContext = frameChoreographer.coroutineContext + motionDurationScale + sceneJob @@ -284,6 +289,7 @@ internal class ComposeContainer( coroutineContext = containerCoroutineContext, navigationEventInput = navigationEventInput, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, composeSceneFactory = { context -> PlatformLayersComposeScene( frameRecomposer = frameChoreographer.frameRecomposer, @@ -356,6 +362,7 @@ internal class ComposeContainer( mediator = null pressDispatchLog.clear() + touchOracle.stop() activeStateListener?.dispose() activeStateListener = null @@ -406,6 +413,7 @@ internal class ComposeContainer( consumePointerInputOutside = consumePointerInputOutside, parentCoroutineContext = containerCoroutineContext, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, ownerProvider = architectureComponentsOwner, interfaceOrientationState = interfaceOrientationState, invalidateLayout = { layersHolder.getLayersViewController().invalidateLayout() }, diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt index d925f08a0fc30..f6a89620c1409 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt @@ -96,6 +96,8 @@ import androidx.compose.ui.viewinterop.InteropSyncTransaction import androidx.compose.ui.window.FocusedViewsList import androidx.compose.ui.window.IosPrefetchScheduler import kotlin.coroutines.CoroutineContext +import kotlin.math.abs +import kotlin.math.hypot import kotlin.math.roundToInt import kotlinx.cinterop.CValue import kotlinx.cinterop.readValue @@ -111,6 +113,7 @@ import kotlinx.coroutines.launch import platform.CoreGraphics.CGPoint import platform.CoreGraphics.CGRectIsEmpty import platform.CoreGraphics.CGRectZero +import platform.Foundation.NSProcessInfo import platform.Foundation.NSTimeInterval import platform.QuartzCore.CACurrentMediaTime import platform.UIKit.UIEvent @@ -128,6 +131,97 @@ import platform.UIKit.UIView import platform.UIKit.endEditing import platform.UIKit.setAccessibilityElements +// Siri Remote swipe recognition tuning. Values measured on a second-generation Siri Remote +// (GameController micro gamepad, reportsAbsoluteDpadValues = true) with the display-link +// sampling of SiriRemoteTouchOracle, which catches the origin of a contact within a frame of the +// finger landing: centre swipes start at radius <= 0.33, ring drags at >= 0.94; genuine swipes +// cross 0.35 within 60 to 151 ms; a resting finger, a ring rest and the drift inside a ring click +// travel at most 0.25 and need 179 to 406 ms to get there. +// +// Radius, in the absolute clickpad space reported by SiriRemoteTouchOracle (|x| and |y| in +// [0, 1]), beyond which a contact started on the outer ring of arrow buttons rather than on the +// centre pad. Native tvOS only swipes for movement that starts on the centre pad. Ring contacts +// between 0.60 and 0.75 are not gated here but by the press latch and by the distance gate, +// since they travel at most 0.25. +private const val CENTER_PAD_RADIUS = 0.75f +// A contact that lasts longer than this is a rest or a drag, not a swipe: genuine swipes cross +// the distance gate within 151 ms, while the contact roll of a centre click that lands partly on +// the ring takes 300 ms or more to cover the same distance. +private const val SWIPE_MAX_DURATION_S = 0.25 +// Minimum travel along the dominant axis, in absolute clickpad units. +private const val SWIPE_DISTANCE_NORMALIZED = 0.30f +// A finger that stayed within this radius of its rest anchor for REST_RESET_DURATION_S is at +// rest: the origin and the timer move to where it rests, so a rest-then-move still swipes and a +// slow drift never accumulates into one. +private const val REST_ANCHOR_TOLERANCE = 0.02f +private const val REST_RESET_DURATION_S = 0.15 +// Minimum travel along the dominant axis, in dp, when no controller is reported and the relative +// UIKit location is the only signal. +private const val SWIPE_DISTANCE_FALLBACK_DP = 40f +// The dominant axis must travel at least this many times the other one, so a diagonal smear does +// not move focus in an arbitrary direction. +private const val SWIPE_AXIS_DOMINANCE = 2f +// A swipe that crossed the distance gate waits this long before it is dispatched, so a Select +// press that arrives right after the contact rolled far enough cancels it instead of firing both. +// A contact that ends while armed is dispatched at once: a lifted finger can no longer click. +private const val DISPATCH_HOLD_S = 0.08 +// A contact live for longer than this lost its terminal event: tvOS can absorb a touch's +// terminal event, e.g. when the keyboard overlay appears; without this the display link would +// poll forever. +private const val INDIRECT_CONTACT_MAX_AGE_S = 2.0 +// A clickpad press this long before a contact starts belongs to that contact: finger contact +// physically precedes the switch closing, and the two sensors stamp their events a few ms apart. +private const val PRESS_SUPPRESSION_WINDOW_S = 0.15 + +private val isSwipeDebugEnabled: Boolean by lazy { + NSProcessInfo.processInfo.environment["COMPOSE_TVOS_SWIPE_DEBUG"] == "1" +} + +private inline fun swipeDebug(message: () -> String) { + if (isSwipeDebugEnabled) { + println(message()) + } +} + +private fun directionName(key: Key): String = when (key) { + Key.DirectionRight -> "DirectionRight" + Key.DirectionLeft -> "DirectionLeft" + Key.DirectionDown -> "DirectionDown" + else -> "DirectionUp" +} + +/** + * The state of one Siri Remote indirect contact. + * + * A contact produces at most one directional key: it starts as a [CANDIDATE] (or as [IGNORED] + * when it started on the ring), becomes [ARMED] with a direction once it travelled far enough, + * and leaves that state for good once the swipe is [DISPATCHED] or a clickpad press or an + * overlong contact makes it [CANCELLED]. + * + * A contact whose BEGAN arrives before the oracle reported any position of that contact starts as + * [PENDING]: the pad reads exactly (0, 0) between contacts, so there is nothing to compare the + * origin against yet. The first later sample decides the origin and the verdict. + */ +private enum class IndirectTouchVerdict { + PENDING, CANDIDATE, ARMED, IGNORED, CANCELLED, DISPATCHED +} + +private class IndirectTouchState( + var origin: Offset, + var startTimestamp: Double, + val beginTimestamp: Double, + val usesOracle: Boolean, + var verdict: IndirectTouchVerdict, +) { + /** Position the rest detection measures against, and when it was last left. */ + var restAnchor: Offset = origin + var lastSignificantMoveTime: Double = startTimestamp + + /** The direction the contact armed, and when it armed it. */ + var armedKey: Key? = null + var armedAt: Double = 0.0 +} + /** * A reason for why touches are sent to Compose. * Mirrors the iOS [TouchesEventKind] which is not available in tvosMain. @@ -384,6 +478,7 @@ internal class ComposeSceneMediator( val coroutineContext: CoroutineContext, private val navigationEventInput: TvBackNavigationEventInput, private val pressDispatchLog: TvPressDispatchLog, + private val touchOracle: SiriRemoteTouchOracle, interfaceOrientationState: State, composeSceneFactory: (platformContext: PlatformContext) -> ComposeScene, private val schedulePendingInteropViewUpdates: () -> Unit = {}, @@ -406,26 +501,21 @@ internal class ComposeSceneMediator( // make the next Select press on another element disappear. private var swallowedSelectKeyId: Long? = null - // Tracks the start of each Siri Remote indirect touch for swipe-to-focus fallback. - private class IndirectTouchStart(val position: Offset, val timestamp: Double) - private val indirectTouchStarts = mutableMapOf() - // Timestamp (seconds since boot, same timebase as UITouch.timestamp) of the last clickpad - // press. The Siri Remote trackpad is itself a button (Select in the middle, arrow presses - // on the outer ring of 2nd-gen remotes): clicking it delivers a UIPress alongside an - // indirect UITouch whose ENDED position can drift several dp from where it began, so any - // touch whose lifetime overlaps a clickpad press is click contact, not a swipe. Both - // timestamps are hardware event times rather than delivery times, so comparing them is - // immune to UIKit delivering the press and touch callbacks of one click out of order. - private var lastClickpadPressTimestamp = Double.NEGATIVE_INFINITY + // Tracks each Siri Remote indirect contact for swipe-to-focus. + private val indirectTouches = mutableMapOf() + + // Guards the display-link evaluation against re-entering itself through a dispatched key. + private var isEvaluatingOracleSample = false + + // Kept as a value so the same instance can be removed from the shared oracle on dispose. + private val oracleSampleListener: (Offset?) -> Unit = ::onOracleSample + + // Whether this mediator holds a sampling session of the shared oracle, and the token of that + // session. + private var isSamplingOracle = false + private var oracleSamplingToken = 0 private val keyRepeatInitialDelayMs = 500L private val keyRepeatIntervalMs = 50L - // Minimum swipe distance (dp) on the Siri Remote trackpad to trigger a focus move. - private val INDIRECT_SWIPE_THRESHOLD_DP = 40f - // Finger contact physically precedes the click's switch closing, but the two sensors may - // stamp their events a few ms apart; this only needs to absorb that jitter. It must stay - // well below the time it takes to lift after a click and start a new touch, so a swipe - // immediately following a click is never falsely suppressed. - private val PRESS_TOUCH_TIMESTAMP_JITTER_SECONDS = 0.05 private val platformScreenReader = object : PlatformScreenReader { override var isActive by mutableStateOf(false) } @@ -644,6 +734,7 @@ internal class ComposeSceneMediator( init { coroutineContext.job.invokeOnCompletion { dispose() } frameChoreographer.addListener(frameChoreographerListener) + touchOracle.addSampleListener(oracleSampleListener) } private fun hitTestInteropView(point: CValue): UIView? = @@ -659,10 +750,352 @@ internal class ComposeSceneMediator( private fun onCancelAllTouches(touches: Set<*>) { activitiesHandler.onActivitiesEnded(touches.count()) - indirectTouchStarts.clear() + // Only the cancelled contacts are forgotten: another contact of the same gesture may + // still be live and must keep its verdict. + touches.forEach { indirectTouches.remove((it as UITouch).hashCode()) } + updateIndirectSampling() scene.cancelPointerInput() } + /** + * Recognizes Siri Remote swipes from indirect contacts and converts them to directional key + * events. Those contacts are never forwarded to the Compose pointer input pipeline. + * + * When [touchOracle] reports a controller the contact is tracked in the absolute clickpad + * space, so a movement that starts on the outer ring of arrow buttons is ignored the way + * native tvOS ignores it, and the swipe is dispatched as soon as it is long enough. Without a + * controller (the simulator, or the first frames before the remote connects) the relative + * UIKit location is the only signal and the contact is evaluated once, when it ends. + */ + private fun onIndirectTouchEvent(touch: UITouch, eventKind: TouchesEventKind) { + val key = touch.hashCode() + when (eventKind) { + TouchesEventKind.BEGAN -> { + val state = if (touchOracle.isAvailable) { + val oraclePosition = touchOracle.position() + if (oraclePosition == null) { + // No sample of this contact yet: stay pending until one arrives, so the + // ring gate still applies instead of dropping to the ungated fallback. + swipeDebug { "SWIPE began verdict=pending" } + IndirectTouchState( + origin = Offset.Zero, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = true, + verdict = IndirectTouchVerdict.PENDING, + ) + } else { + IndirectTouchState( + origin = oraclePosition, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = true, + verdict = IndirectTouchVerdict.PENDING, + ).also { + resolveIndirectTouchOrigin(it, oraclePosition, touch.timestamp) + } + } + } else { + val position = touch.offsetInView(_backgroundView, screenDensity.density) + swipeDebug { + "SWIPE began origin=(${position.x}, ${position.y}) r=0.0 verdict=fallback" + } + IndirectTouchState( + origin = position, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = false, + verdict = IndirectTouchVerdict.CANDIDATE, + ) + } + indirectTouches[key] = state + updateIndirectSampling() + } + TouchesEventKind.MOVED -> { + val state = indirectTouches[key] ?: return + if (!state.usesOracle) return + val position = touchOracle.position() + if (position == null) { + swipeDebug { "SWIPE moved sample=null" } + return + } + // The display link runs the same evaluation on every frame; this call is + // idempotent and only keeps the contact moving when UIKit is the earlier signal. + evaluateIndirectSample(state, position, touch.timestamp, logSample = true) + } + TouchesEventKind.ENDED -> { + val state = indirectTouches.remove(key) ?: return + updateIndirectSampling() + if (state.usesOracle) { + if (state.verdict == IndirectTouchVerdict.PENDING) { + // The only sample of this contact arrives at its end: an origin with no + // movement observed in oracle space, so nothing is dispatched. + touchOracle.position()?.let { + resolveIndirectTouchOrigin(state, it, touch.timestamp) + } + } + } else if (state.verdict == IndirectTouchVerdict.CANDIDATE) { + // Fallback only: the relative location is meaningful just once, at the end. + evaluateIndirectTouch( + state = state, + position = touch.offsetInView(_backgroundView, screenDensity.density), + timestamp = touch.timestamp, + ) + } + // A lifted finger can no longer be a click, so the hold is not waited out here: + // it only has to cover the press race while the finger is still down. + if (state.verdict == IndirectTouchVerdict.ARMED) { + dispatchArmedIndirectTouch(state, touch.timestamp) + } + } + } + } + + /** + * Decides the origin and the verdict of a pending oracle contact from [sample], its first + * known position. The contact keeps the timestamp of its BEGAN as start, so its duration is + * measured from the finger landing rather than from the first sample. + */ + private fun resolveIndirectTouchOrigin( + state: IndirectTouchState, + sample: Offset, + timestamp: Double, + ) { + state.origin = sample + state.restAnchor = sample + state.lastSignificantMoveTime = timestamp + val radius = hypot(sample.x, sample.y) + state.verdict = if (touchOracle.hasRing && radius >= CENTER_PAD_RADIUS) { + IndirectTouchVerdict.IGNORED + } else { + IndirectTouchVerdict.CANDIDATE + } + swipeDebug { + val verdict = if (state.verdict == IndirectTouchVerdict.IGNORED) { + "ignored-ring" + } else { + "candidate" + } + val sinceBegin = (timestamp - state.beginTimestamp) * 1000.0 + "SWIPE origin=(${sample.x}, ${sample.y}) r=$radius verdict=$verdict " + + "t=${sinceBegin}ms" + } + } + + /** + * Runs the origin resolution or the swipe evaluation of one oracle contact for [sample], + * whichever its verdict calls for. Called both from UIKit's sparse MOVED callbacks and from + * every display-link tick, so it must stay idempotent. + */ + private fun evaluateIndirectSample( + state: IndirectTouchState, + sample: Offset, + timestamp: Double, + logSample: Boolean, + ) { + if (!state.usesOracle) return + when (state.verdict) { + IndirectTouchVerdict.PENDING -> + // First sample of this contact: it is the origin, not a movement. + resolveIndirectTouchOrigin(state, sample, timestamp) + IndirectTouchVerdict.CANDIDATE -> + evaluateIndirectTouch(state, sample, timestamp, logSample) + IndirectTouchVerdict.ARMED -> + holdArmedIndirectTouch(state, timestamp) + else -> {} + } + } + + /** + * Feeds one display-link sample of the clickpad to every live oracle contact. UIKit reports + * indirect movement sparsely, so this is what catches the origin of a contact in time and + * what dispatches a swipe as soon as it is long enough. + */ + private fun onOracleSample(sample: Offset?) { + if (sample == null || isEvaluatingOracleSample) return + if (indirectTouches.isEmpty()) return + isEvaluatingOracleSample = true + try { + val timestamp = CACurrentMediaTime() + purgeStaleIndirectTouches(timestamp) + // A dispatch runs key event handlers, which may end contacts, so the states are + // snapshotted before they are evaluated. + for (state in indirectTouches.values.toList()) { + // A dispatch can dispose this mediator, which clears every contact. + if (indirectTouches.isEmpty()) break + evaluateIndirectSample(state, sample, timestamp, logSample = false) + } + } finally { + isEvaluatingOracleSample = false + } + } + + /** + * Drops the contacts of [timestamp] whose terminal event never arrived, so a lost ENDED does + * not keep the display link polling for the rest of the process. Contacts that already reached + * a terminal verdict are dropped as soon as they can no longer dispatch. + */ + private fun purgeStaleIndirectTouches(timestamp: Double) { + var removed = false + val iterator = indirectTouches.values.iterator() + while (iterator.hasNext()) { + val state = iterator.next() + val maxAge = when (state.verdict) { + IndirectTouchVerdict.IGNORED, + IndirectTouchVerdict.CANCELLED, + IndirectTouchVerdict.DISPATCHED -> SWIPE_MAX_DURATION_S + DISPATCH_HOLD_S + else -> INDIRECT_CONTACT_MAX_AGE_S + } + // The tick stamps with CACurrentMediaTime and the contacts with UITouch.timestamp, + // both of which are the system uptime. + if (timestamp - state.beginTimestamp > maxAge) { + iterator.remove() + removed = true + } + } + if (removed) { + updateIndirectSampling() + } + } + + /** Samples the pad per frame exactly while at least one indirect contact is live. */ + private fun updateIndirectSampling() { + val shouldSample = indirectTouches.isNotEmpty() + if (shouldSample == isSamplingOracle) return + isSamplingOracle = shouldSample + if (shouldSample) { + oracleSamplingToken = touchOracle.beginSampling() + } else { + touchOracle.endSampling(oracleSamplingToken) + } + } + + /** + * Moves [state] out of [IndirectTouchVerdict.CANDIDATE] if the contact reached [position] at + * [timestamp] is a swipe, or if it can no longer become one. + */ + private fun evaluateIndirectTouch( + state: IndirectTouchState, + position: Offset, + timestamp: Double, + logSample: Boolean = true, + ) { + if (updateIndirectRest(state, position, timestamp)) return + val dx = position.x - state.origin.x + val dy = position.y - state.origin.y + val elapsed = timestamp - state.startTimestamp + if (logSample) { + swipeDebug { "SWIPE moved d=($dx, $dy) t=${elapsed * 1000.0}ms" } + } + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + if (elapsed > SWIPE_MAX_DURATION_S) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=duration" } + return + } + val distance = if (state.usesOracle) { + SWIPE_DISTANCE_NORMALIZED + } else { + with(screenDensity) { SWIPE_DISTANCE_FALLBACK_DP.dp.toPx() } + } + val absDx = abs(dx) + val absDy = abs(dy) + val dominant = maxOf(absDx, absDy) + val other = minOf(absDx, absDy) + if (dominant < distance || dominant < SWIPE_AXIS_DOMINANCE * other) { + return + } + // Sign convention for dy differs by source: the oracle's dy is the GameController dpad's + // absolute clickpad y, which is positive toward the top of the remote, so a positive dy + // is an upward swipe. The fallback dy is a UIKit relative view location, positive + // downward as usual for screen coordinates, so a positive dy there is a downward swipe. + val key = if (absDx >= absDy) { + if (dx > 0) Key.DirectionRight else Key.DirectionLeft + } else if (state.usesOracle) { + if (dy > 0) Key.DirectionUp else Key.DirectionDown + } else { + if (dy > 0) Key.DirectionDown else Key.DirectionUp + } + state.verdict = IndirectTouchVerdict.ARMED + state.armedKey = key + state.armedAt = timestamp + swipeDebug { "SWIPE armed ${directionName(key)}" } + } + + /** + * Holds an armed swipe until [DISPATCH_HOLD_S] elapsed, so a Select press that lands right + * after the contact travelled far enough cancels the swipe instead of firing both. + */ + private fun holdArmedIndirectTouch(state: IndirectTouchState, timestamp: Double) { + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + if (timestamp - state.armedAt < DISPATCH_HOLD_S) return + dispatchIndirectSwipe(state) + } + + /** Dispatches an armed swipe unless a clickpad press claimed the contact meanwhile. */ + private fun dispatchArmedIndirectTouch(state: IndirectTouchState, timestamp: Double) { + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + dispatchIndirectSwipe(state) + } + + private fun dispatchIndirectSwipe(state: IndirectTouchState) { + val key = state.armedKey ?: return + swipeDebug { "SWIPE dispatch ${directionName(key)}" } + state.verdict = IndirectTouchVerdict.DISPATCHED + onKeyboardEvent(KeyEvent(key, KeyEventType.KeyDown)) + onKeyboardEvent(KeyEvent(key, KeyEventType.KeyUp)) + } + + /** + * Tracks whether the finger of [state] is resting: as long as it stays within + * [REST_ANCHOR_TOLERANCE] of its anchor for longer than [REST_RESET_DURATION_S], the origin + * and the swipe timer move to where it rests, so a rest followed by a real movement still + * swipes and a slow drift never accumulates into one. + * + * Returns `true` when the origin was just moved, i.e. when there is no displacement left to + * evaluate for this sample. + */ + private fun updateIndirectRest( + state: IndirectTouchState, + position: Offset, + timestamp: Double, + ): Boolean { + val moved = hypot(position.x - state.restAnchor.x, position.y - state.restAnchor.y) + if (moved >= REST_ANCHOR_TOLERANCE) { + state.restAnchor = position + state.lastSignificantMoveTime = timestamp + return false + } + if (timestamp - state.lastSignificantMoveTime <= REST_RESET_DURATION_S) return false + state.origin = position + state.startTimestamp = timestamp + state.lastSignificantMoveTime = timestamp + return true + } + + /** + * `true` if a clickpad press is held, or if one began during the contact that started at + * [startTimestamp] or shortly before it. Press and touch timestamps are hardware event times + * rather than delivery times, so comparing them is immune to UIKit delivering the press and + * the touch callbacks of one click out of order. + */ + private fun isClickpadPressOverlapping(startTimestamp: Double, timestamp: Double): Boolean = + touchOracle.anyButtonPressed() || + pressDispatchLog.isClickpadPressHeld(timestamp) || + pressDispatchLog.clickpadPressTimestamp >= startTimestamp - PRESS_SUPPRESSION_WINDOW_S + /** * Converts [UITouch] objects from [touches] to [ComposeScenePointer] and dispatches them to the appropriate handlers. * @param touches a [Set] of [UITouch] objects. Erasure happens due to K/N not supporting Obj-C lightweight generics. @@ -686,36 +1119,7 @@ internal class ComposeSceneMediator( for (anyTouch in touches) { val touch = anyTouch as UITouch if (touch.type == UITouchTypeIndirect) { - val position = touch.offsetInView(_backgroundView, screenDensity.density) - when (eventKind) { - TouchesEventKind.BEGAN -> { - indirectTouchStarts[touch.hashCode()] = - IndirectTouchStart(position, touch.timestamp) - } - TouchesEventKind.ENDED -> { - val start = indirectTouchStarts.remove(touch.hashCode()) ?: continue - // A clickpad press during this touch's lifetime means the drift is - // from a click, not a swipe. - if (lastClickpadPressTimestamp >= - start.timestamp - PRESS_TOUCH_TIMESTAMP_JITTER_SECONDS - ) { - continue - } - val dx = position.x - start.position.x - val dy = position.y - start.position.y - val threshold = with(screenDensity) { INDIRECT_SWIPE_THRESHOLD_DP.dp.toPx() } - if (dx * dx + dy * dy >= threshold * threshold) { - val key = if (kotlin.math.abs(dx) >= kotlin.math.abs(dy)) { - if (dx > 0) Key.DirectionRight else Key.DirectionLeft - } else { - if (dy > 0) Key.DirectionDown else Key.DirectionUp - } - onKeyboardEvent(KeyEvent(key, KeyEventType.KeyDown)) - onKeyboardEvent(KeyEvent(key, KeyEventType.KeyUp)) - } - } - TouchesEventKind.MOVED -> {} - } + onIndirectTouchEvent(touch, eventKind) } else { pointerTouches.add(touch) } @@ -852,6 +1256,12 @@ internal class ComposeSceneMediator( ) private fun dispose() { + touchOracle.removeSampleListener(oracleSampleListener) + if (isSamplingOracle) { + isSamplingOracle = false + touchOracle.endSampling(oracleSamplingToken) + } + indirectTouches.clear() repeatingKeys.values.forEach { it.cancel() } repeatingKeys.clear() consumedKeyIds.clear() @@ -991,26 +1401,26 @@ internal class ComposeSceneMediator( val keyId = press.key?.keyCode?.toLong() ?: -(press.type.toLong() + 1L) val phase = press.phase - if (!pressDispatchLog.shouldEvaluate(pressesEvent, keyId, phase)) { - // Already evaluated for this UIPressesEvent, by this mediator or by another - // mediator of the same container: this is the responder chain echoing back a - // press an overlay view forwarded to `super`. - reportUnconsumed(anyPress) - return@forEach - } - - // The Siri Remote trackpad doubles as its buttons (Select in the middle, arrows - // on the outer ring): clicking it produces a press alongside an indirect touch - // whose end position can drift enough to look like a swipe. Record the press time - // so the touch's ENDED branch doesn't also dispatch a phantom directional key. - if (phase == UIPressPhase.UIPressPhaseBegan && + // The Siri Remote clickpad doubles as its buttons (Select in the middle, arrows + // on the outer ring): clicking it produces a press alongside an indirect touch that + // drifts enough to look like a swipe. The press is recorded before the echo check, + // so the mediator that sees the echo rather than the original still latches it. + if ( when (event.key) { Key.DirectionCenter, Key.DirectionUp, Key.DirectionDown, Key.DirectionLeft, Key.DirectionRight -> true else -> false } ) { - lastClickpadPressTimestamp = press.timestamp + pressDispatchLog.recordClickpadPress(keyId, phase, press.timestamp) + } + + if (!pressDispatchLog.shouldEvaluate(pressesEvent, keyId, phase)) { + // Already evaluated for this UIPressesEvent, by this mediator or by another + // mediator of the same container: this is the responder chain echoing back a + // press an overlay view forwarded to `super`. + reportUnconsumed(anyPress) + return@forEach } when (phase) { diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt index d3cda1fc37278..99564cb8ec977 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt @@ -61,6 +61,7 @@ internal class IosComposeSceneLayer( consumePointerInputOutside: Boolean = focusedViewsList != null, parentCoroutineContext: CoroutineContext, private val pressDispatchLog: TvPressDispatchLog, + private val touchOracle: SiriRemoteTouchOracle, private val ownerProvider: PlatformArchitectureComponentsOwner, private val interfaceOrientationState: State, private var invalidateLayout: () -> Unit, @@ -110,6 +111,7 @@ internal class IosComposeSceneLayer( coroutineContext = layerCoroutineContext, navigationEventInput = navigationEventInput, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, composeSceneFactory = ::createComposeScene, interfaceOrientationState = interfaceOrientationState, schedulePendingInteropViewUpdates = layersViewController::invalidateDraw, diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt new file mode 100644 index 0000000000000..3bc1774fd4ffc --- /dev/null +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt @@ -0,0 +1,270 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.compose.ui.scene + +import androidx.compose.ui.geometry.Offset +import kotlinx.cinterop.BetaInteropApi +import kotlinx.cinterop.ObjCAction +import platform.Foundation.NSNotificationCenter +import platform.Foundation.NSOperationQueue +import platform.Foundation.NSRunLoop +import platform.Foundation.NSRunLoopCommonModes +import platform.Foundation.NSSelectorFromString +import platform.GameController.GCController +import platform.GameController.GCControllerDidConnectNotification +import platform.GameController.GCControllerDidDisconnectNotification +import platform.GameController.GCMicroGamepad +import platform.GameController.GCProductCategorySiriRemote1stGen +import platform.QuartzCore.CADisplayLink +import platform.darwin.NSObject +import platform.darwin.NSObjectProtocol + +/** + * Reports where the finger physically is on the Siri Remote clickpad. + * + * UIKit indirect touches ([platform.UIKit.UITouchTypeIndirect]) carry a relative location in a + * space that is re-centred on every new contact, so they cannot tell a movement that started on + * the centre pad from one that started on the outer ring of the second generation remote. The + * GameController micro gamepad does: with `reportsAbsoluteDpadValues` its dpad axes are the + * absolute finger position in [-1, 1] on both axes. + * + * This is an observer only: no `GCEventViewController` is installed and + * `controllerUserInteractionEnabled` is never touched, so UIKit keeps delivering presses and + * touches through the responder chain exactly as before. + * + * The gamepad is polled at the touch event that needs it rather than through + * `valueChangedHandler`. GameController keeps a single handler per element, so the last writer + * wins: an app that installs its own handler on the same remote would silently replace the + * oracle's and leave it without samples. Polling reads the live element values, so the oracle + * coexists with app-level GameController usage and never overwrites an app's handler. + * + * UIKit reports indirect movement sparsely: polling only inside `touchesBegan`/`touchesMoved` + * samples the origin of a contact up to 0.2 normalised units after the finger landed, which makes + * genuine centre swipes look like they started on the ring. [beginSampling] therefore drives a + * display link that polls the pad every frame while a contact is active and feeds the samples to + * the listeners installed by [addSampleListener]. + */ +internal class SiriRemoteTouchOracle { + private var connectObserver: NSObjectProtocol? = null + private var disconnectObserver: NSObjectProtocol? = null + + private var controller: GCController? = null + private var productCategory: String? = null + // The gamepad's own rotation mode, restored when the oracle lets go of it. + private var previousAllowsRotation: Boolean? = null + + private val sampleListeners = mutableListOf<(Offset?) -> Unit>() + private var samplingCount = 0 + // Bumped by every stop, so a session token of a previous run cannot close a current session. + private var samplingGeneration = 0 + private var displayLink: CADisplayLink? = null + private val displayLinkTarget = SampleDisplayLinkTarget { + val sample = position() + // A listener may dispose its mediator, and with it remove listeners, while being + // notified, so the list is snapshotted before it is walked. + sampleListeners.toList().forEach { it(sample) } + } + + /** `true` while a micro gamepad, i.e. a Siri Remote, is connected and reporting. */ + val isAvailable: Boolean get() = controller?.microGamepad != null + + /** + * `true` for remotes whose clickpad has an outer ring of arrow buttons. + * + * tvOS coalesces every paired remote into a single controller reporting the + * "Coalesced Remote" product category, so a physical remote's generation cannot be + * distinguished once connected. The ring gate is therefore the default, applying to the + * coalesced controller and any other unrecognised micro gamepad category. First-generation + * remotes have no ring and let a swipe start anywhere on the pad, so they are exempted here; + * so are genuine game controllers, which report an extended gamepad rather than a bare micro + * gamepad. + */ + val hasRing: Boolean + get() { + val category = productCategory ?: return false + if (category == GCProductCategorySiriRemote1stGen) return false + if (controller?.extendedGamepad != null) return false + return true + } + + fun start() { + if (connectObserver != null) return + val center = NSNotificationCenter.defaultCenter + val queue = NSOperationQueue.mainQueue + connectObserver = center.addObserverForName( + name = GCControllerDidConnectNotification, + `object` = null, + queue = queue, + ) { refresh() } + disconnectObserver = center.addObserverForName( + name = GCControllerDidDisconnectNotification, + `object` = null, + queue = queue, + ) { refresh() } + refresh() + } + + /** + * Adds a listener invoked with [position] on every display-link tick while any [beginSampling] + * session is open. Every mediator, the root one and each dialog layer, installs its own, so a + * contact live in one of them is sampled for all of them. + */ + fun addSampleListener(listener: (Offset?) -> Unit) { + sampleListeners.add(listener) + } + + /** Removes a listener installed by [addSampleListener]. */ + fun removeSampleListener(listener: (Offset?) -> Unit) { + sampleListeners.remove(listener) + } + + /** + * Opens a sampling session, starting the per-frame polling of the pad if it is the first one. + * Every call must be matched by an [endSampling] with the returned token. + */ + fun beginSampling(): Int { + samplingCount++ + if (displayLink != null) return samplingGeneration + val link = CADisplayLink.displayLinkWithTarget( + target = displayLinkTarget, + selector = NSSelectorFromString("tick:") + ) + link.preferredFramesPerSecond = 60L + link.addToRunLoop(NSRunLoop.mainRunLoop, NSRunLoopCommonModes) + displayLink = link + return samplingGeneration + } + + /** + * Closes the sampling session opened with [token], stopping the polling once the last one is + * closed. A token of an older generation is ignored: [stop] already dropped every session of + * it, so honouring it would underflow the count of the sessions opened since. + */ + fun endSampling(token: Int) { + if (token != samplingGeneration) return + if (samplingCount == 0) return + samplingCount-- + if (samplingCount > 0) return + stopSampling() + } + + private fun stopSampling() { + // Invalidating is what releases the display link's retain of the target, so it must run + // for every begun sampling session. + displayLink?.invalidate() + displayLink = null + } + + fun stop() { + samplingCount = 0 + samplingGeneration++ + stopSampling() + sampleListeners.clear() + val center = NSNotificationCenter.defaultCenter + connectObserver?.let { center.removeObserver(it) } + disconnectObserver?.let { center.removeObserver(it) } + connectObserver = null + disconnectObserver = null + detach() + } + + /** + * Current absolute finger position in [-1, 1] on both axes, `null` if no remote is connected + * or if the pad is at rest. The pad reports exactly (0, 0) whenever no finger is touching it, + * and no touching sample is ever exactly (0, 0), so an exact-zero sample is treated as "no + * finger": otherwise a BEGAN sampled before the pad reports a real position would be + * classified as a centre-pad contact at radius 0 using a stale origin instead of falling to + * fallback mode. + */ + fun position(): Offset? { + val microGamepad = controller?.microGamepad ?: return null + // An app is free to reset this flag on the shared gamepad; setting it back only takes + // effect from the next sample, so the sample read right after such a reset may still be + // relative. + if (!microGamepad.reportsAbsoluteDpadValues) { + microGamepad.reportsAbsoluteDpadValues = true + } + val pad = microGamepad.dpad + val x = pad.xAxis.value + val y = pad.yAxis.value + if (x == 0f && y == 0f) return null + return Offset(x, y) + } + + /** `true` while the clickpad or one of the ring buttons is physically held down. */ + fun anyButtonPressed(): Boolean { + val microGamepad = controller?.microGamepad ?: return false + return isAnyButtonPressed(microGamepad) + } + + private fun refresh() { + // The remote reports a bare micro gamepad; a game controller reports an extended gamepad + // and its micro gamepad projection, whose dpad is the thumbstick rather than a clickpad. + // Reevaluated on every connect and disconnect, so plugging a game controller in does not + // steal the oracle from the remote. + val controllers = GCController.controllers().filterIsInstance() + val connected = controllers.firstOrNull { + it.microGamepad != null && it.extendedGamepad == null + } ?: controllers.firstOrNull { it.microGamepad != null } + if (connected == null) { + detach() + return + } + if (connected == controller) return + detach() + attach(connected) + } + + private fun attach(connected: GCController) { + val microGamepad = connected.microGamepad ?: return + controller = connected + productCategory = connected.productCategory + previousAllowsRotation = microGamepad.allowsRotation + microGamepad.reportsAbsoluteDpadValues = true + microGamepad.allowsRotation = false + } + + private fun detach() { + // No handler is installed, so nothing is cleared here: an app's own + // `valueChangedHandler` on the same gamepad is never touched. The rotation mode is + // restored, since an app that enabled it did so for its own reading of the gamepad. + previousAllowsRotation?.let { controller?.microGamepad?.allowsRotation = it } + previousAllowsRotation = null + controller = null + productCategory = null + } + + // The dpad's up/down/left/right are virtual direction buttons synthesised from the axis + // sign, pressed whenever the finger is off-centre in that direction; they are NOT the + // physical ring click buttons. Treating them as presses would make every swipe candidate + // that leaves the centre pad look like a clickpad press. Only the physical action buttons + // (buttonA, buttonX) and the menu button count as a press here. + private fun isAnyButtonPressed(microGamepad: GCMicroGamepad): Boolean = + microGamepad.buttonA.pressed || + microGamepad.buttonX.pressed || + microGamepad.buttonMenu.pressed +} + +private class SampleDisplayLinkTarget( + private val onTick: () -> Unit +) : NSObject() { + @OptIn(BetaInteropApi::class) + @ObjCAction + fun tick(link: CADisplayLink) { + onTick() + } +} diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt index 623a77c079d51..95e5edf75fe69 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt @@ -36,6 +36,11 @@ import platform.UIKit.UIPressesEvent * Consumption itself stays per mediator ([ComposeSceneMediator.consumedKeyIds]): the mediator * that consumed the Began phase is the one that must dispatch the matching KeyUp. */ +// A held clickpad press older than this relative to the event being evaluated is stale: its +// Ended phase was absorbed before reaching the mediator (the tvOS keyboard overlay does that +// while it is up), and without ageing it out every later contact would be suppressed forever. +private const val HELD_PRESS_MAX_AGE_S = 1.0 + internal class TvPressDispatchLog { // The current event is identified by value rather than by reference: UIKit recycles // UIPressesEvent instances, so object identity alone would make a reused instance carrying a @@ -66,9 +71,50 @@ internal class TvPressDispatchLog { return true } + // Timestamp (seconds since boot, the same timebase as UITouch.timestamp) of the last + // clickpad press Began, and the ids of the clickpad presses currently held. The Siri Remote + // clickpad is itself a button (Select in the middle, arrows on the outer ring of 2nd + // generation remotes): clicking it delivers a UIPress alongside an indirect UITouch that + // drifts as the finger settles, so a contact whose lifetime overlaps a clickpad press is + // click contact, not a swipe. Shared like the log itself, because the press and the touch of + // one click can be delivered to different mediators of the same container. + private var lastClickpadPressTimestamp: Double = Double.NEGATIVE_INFINITY + private val heldClickpadKeyIds = mutableMapOf() + + /** + * `true` while at least one clickpad press is held, evaluated at [timestamp]: presses whose + * Began is older than [HELD_PRESS_MAX_AGE_S] are dropped as never-ended leftovers. + */ + fun isClickpadPressHeld(timestamp: Double): Boolean { + if (heldClickpadKeyIds.isEmpty()) return false + heldClickpadKeyIds.entries.removeAll { timestamp - it.value > HELD_PRESS_MAX_AGE_S } + return heldClickpadKeyIds.isNotEmpty() + } + + /** Timestamp of the last clickpad press Began, [Double.NEGATIVE_INFINITY] if there was none. */ + val clickpadPressTimestamp: Double get() = lastClickpadPressTimestamp + + /** Records the lifetime of a clickpad press identified by [keyId]. */ + fun recordClickpadPress(keyId: Long, phase: UIPressPhase, timestamp: Double) { + when (phase) { + UIPressPhase.UIPressPhaseBegan -> { + // A second Began for a key that is still held means the previous press's Ended + // never arrived, so that press ends here rather than staying held for good. + heldClickpadKeyIds.remove(keyId) + lastClickpadPressTimestamp = timestamp + heldClickpadKeyIds[keyId] = timestamp + } + UIPressPhase.UIPressPhaseEnded, UIPressPhase.UIPressPhaseCancelled -> + heldClickpadKeyIds.remove(keyId) + else -> {} + } + } + fun clear() { lastEventPtr = 0L lastEventTimestamp = Double.NaN evaluatedPhases.clear() + lastClickpadPressTimestamp = Double.NEGATIVE_INFINITY + heldClickpadKeyIds.clear() } }