diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt index 7360631056f37..50be50df2ab9a 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeContainer.tvos.kt @@ -102,6 +102,10 @@ internal class ComposeContainer( // responder chain delivers it back to the hosting view controller. private val pressDispatchLog = TvPressDispatchLog() + // Reports where the finger physically is on the Siri Remote clickpad, which UIKit indirect + // touches cannot tell. Shared by every mediator of this container, like the press log. + private val touchOracle = SiriRemoteTouchOracle() + @OptIn(InternalComposeUiApi::class) var rootForTestListener: PlatformContext.RootForTestListener? = null set(value) { @@ -243,6 +247,7 @@ internal class ComposeContainer( fun initializeComposeScene() { sceneJob = Job() + touchOracle.start() val frameChoreographer = frameChoreographer ?: error("No window scene found") val containerCoroutineContext = frameChoreographer.coroutineContext + motionDurationScale + sceneJob @@ -284,6 +289,7 @@ internal class ComposeContainer( coroutineContext = containerCoroutineContext, navigationEventInput = navigationEventInput, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, composeSceneFactory = { context -> PlatformLayersComposeScene( frameRecomposer = frameChoreographer.frameRecomposer, @@ -356,6 +362,7 @@ internal class ComposeContainer( mediator = null pressDispatchLog.clear() + touchOracle.stop() activeStateListener?.dispose() activeStateListener = null @@ -406,6 +413,7 @@ internal class ComposeContainer( consumePointerInputOutside = consumePointerInputOutside, parentCoroutineContext = containerCoroutineContext, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, ownerProvider = architectureComponentsOwner, interfaceOrientationState = interfaceOrientationState, invalidateLayout = { layersHolder.getLayersViewController().invalidateLayout() }, diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt index d925f08a0fc30..f6a89620c1409 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/ComposeSceneMediator.tvos.kt @@ -96,6 +96,8 @@ import androidx.compose.ui.viewinterop.InteropSyncTransaction import androidx.compose.ui.window.FocusedViewsList import androidx.compose.ui.window.IosPrefetchScheduler import kotlin.coroutines.CoroutineContext +import kotlin.math.abs +import kotlin.math.hypot import kotlin.math.roundToInt import kotlinx.cinterop.CValue import kotlinx.cinterop.readValue @@ -111,6 +113,7 @@ import kotlinx.coroutines.launch import platform.CoreGraphics.CGPoint import platform.CoreGraphics.CGRectIsEmpty import platform.CoreGraphics.CGRectZero +import platform.Foundation.NSProcessInfo import platform.Foundation.NSTimeInterval import platform.QuartzCore.CACurrentMediaTime import platform.UIKit.UIEvent @@ -128,6 +131,97 @@ import platform.UIKit.UIView import platform.UIKit.endEditing import platform.UIKit.setAccessibilityElements +// Siri Remote swipe recognition tuning. Values measured on a second-generation Siri Remote +// (GameController micro gamepad, reportsAbsoluteDpadValues = true) with the display-link +// sampling of SiriRemoteTouchOracle, which catches the origin of a contact within a frame of the +// finger landing: centre swipes start at radius <= 0.33, ring drags at >= 0.94; genuine swipes +// cross 0.35 within 60 to 151 ms; a resting finger, a ring rest and the drift inside a ring click +// travel at most 0.25 and need 179 to 406 ms to get there. +// +// Radius, in the absolute clickpad space reported by SiriRemoteTouchOracle (|x| and |y| in +// [0, 1]), beyond which a contact started on the outer ring of arrow buttons rather than on the +// centre pad. Native tvOS only swipes for movement that starts on the centre pad. Ring contacts +// between 0.60 and 0.75 are not gated here but by the press latch and by the distance gate, +// since they travel at most 0.25. +private const val CENTER_PAD_RADIUS = 0.75f +// A contact that lasts longer than this is a rest or a drag, not a swipe: genuine swipes cross +// the distance gate within 151 ms, while the contact roll of a centre click that lands partly on +// the ring takes 300 ms or more to cover the same distance. +private const val SWIPE_MAX_DURATION_S = 0.25 +// Minimum travel along the dominant axis, in absolute clickpad units. +private const val SWIPE_DISTANCE_NORMALIZED = 0.30f +// A finger that stayed within this radius of its rest anchor for REST_RESET_DURATION_S is at +// rest: the origin and the timer move to where it rests, so a rest-then-move still swipes and a +// slow drift never accumulates into one. +private const val REST_ANCHOR_TOLERANCE = 0.02f +private const val REST_RESET_DURATION_S = 0.15 +// Minimum travel along the dominant axis, in dp, when no controller is reported and the relative +// UIKit location is the only signal. +private const val SWIPE_DISTANCE_FALLBACK_DP = 40f +// The dominant axis must travel at least this many times the other one, so a diagonal smear does +// not move focus in an arbitrary direction. +private const val SWIPE_AXIS_DOMINANCE = 2f +// A swipe that crossed the distance gate waits this long before it is dispatched, so a Select +// press that arrives right after the contact rolled far enough cancels it instead of firing both. +// A contact that ends while armed is dispatched at once: a lifted finger can no longer click. +private const val DISPATCH_HOLD_S = 0.08 +// A contact live for longer than this lost its terminal event: tvOS can absorb a touch's +// terminal event, e.g. when the keyboard overlay appears; without this the display link would +// poll forever. +private const val INDIRECT_CONTACT_MAX_AGE_S = 2.0 +// A clickpad press this long before a contact starts belongs to that contact: finger contact +// physically precedes the switch closing, and the two sensors stamp their events a few ms apart. +private const val PRESS_SUPPRESSION_WINDOW_S = 0.15 + +private val isSwipeDebugEnabled: Boolean by lazy { + NSProcessInfo.processInfo.environment["COMPOSE_TVOS_SWIPE_DEBUG"] == "1" +} + +private inline fun swipeDebug(message: () -> String) { + if (isSwipeDebugEnabled) { + println(message()) + } +} + +private fun directionName(key: Key): String = when (key) { + Key.DirectionRight -> "DirectionRight" + Key.DirectionLeft -> "DirectionLeft" + Key.DirectionDown -> "DirectionDown" + else -> "DirectionUp" +} + +/** + * The state of one Siri Remote indirect contact. + * + * A contact produces at most one directional key: it starts as a [CANDIDATE] (or as [IGNORED] + * when it started on the ring), becomes [ARMED] with a direction once it travelled far enough, + * and leaves that state for good once the swipe is [DISPATCHED] or a clickpad press or an + * overlong contact makes it [CANCELLED]. + * + * A contact whose BEGAN arrives before the oracle reported any position of that contact starts as + * [PENDING]: the pad reads exactly (0, 0) between contacts, so there is nothing to compare the + * origin against yet. The first later sample decides the origin and the verdict. + */ +private enum class IndirectTouchVerdict { + PENDING, CANDIDATE, ARMED, IGNORED, CANCELLED, DISPATCHED +} + +private class IndirectTouchState( + var origin: Offset, + var startTimestamp: Double, + val beginTimestamp: Double, + val usesOracle: Boolean, + var verdict: IndirectTouchVerdict, +) { + /** Position the rest detection measures against, and when it was last left. */ + var restAnchor: Offset = origin + var lastSignificantMoveTime: Double = startTimestamp + + /** The direction the contact armed, and when it armed it. */ + var armedKey: Key? = null + var armedAt: Double = 0.0 +} + /** * A reason for why touches are sent to Compose. * Mirrors the iOS [TouchesEventKind] which is not available in tvosMain. @@ -384,6 +478,7 @@ internal class ComposeSceneMediator( val coroutineContext: CoroutineContext, private val navigationEventInput: TvBackNavigationEventInput, private val pressDispatchLog: TvPressDispatchLog, + private val touchOracle: SiriRemoteTouchOracle, interfaceOrientationState: State, composeSceneFactory: (platformContext: PlatformContext) -> ComposeScene, private val schedulePendingInteropViewUpdates: () -> Unit = {}, @@ -406,26 +501,21 @@ internal class ComposeSceneMediator( // make the next Select press on another element disappear. private var swallowedSelectKeyId: Long? = null - // Tracks the start of each Siri Remote indirect touch for swipe-to-focus fallback. - private class IndirectTouchStart(val position: Offset, val timestamp: Double) - private val indirectTouchStarts = mutableMapOf() - // Timestamp (seconds since boot, same timebase as UITouch.timestamp) of the last clickpad - // press. The Siri Remote trackpad is itself a button (Select in the middle, arrow presses - // on the outer ring of 2nd-gen remotes): clicking it delivers a UIPress alongside an - // indirect UITouch whose ENDED position can drift several dp from where it began, so any - // touch whose lifetime overlaps a clickpad press is click contact, not a swipe. Both - // timestamps are hardware event times rather than delivery times, so comparing them is - // immune to UIKit delivering the press and touch callbacks of one click out of order. - private var lastClickpadPressTimestamp = Double.NEGATIVE_INFINITY + // Tracks each Siri Remote indirect contact for swipe-to-focus. + private val indirectTouches = mutableMapOf() + + // Guards the display-link evaluation against re-entering itself through a dispatched key. + private var isEvaluatingOracleSample = false + + // Kept as a value so the same instance can be removed from the shared oracle on dispose. + private val oracleSampleListener: (Offset?) -> Unit = ::onOracleSample + + // Whether this mediator holds a sampling session of the shared oracle, and the token of that + // session. + private var isSamplingOracle = false + private var oracleSamplingToken = 0 private val keyRepeatInitialDelayMs = 500L private val keyRepeatIntervalMs = 50L - // Minimum swipe distance (dp) on the Siri Remote trackpad to trigger a focus move. - private val INDIRECT_SWIPE_THRESHOLD_DP = 40f - // Finger contact physically precedes the click's switch closing, but the two sensors may - // stamp their events a few ms apart; this only needs to absorb that jitter. It must stay - // well below the time it takes to lift after a click and start a new touch, so a swipe - // immediately following a click is never falsely suppressed. - private val PRESS_TOUCH_TIMESTAMP_JITTER_SECONDS = 0.05 private val platformScreenReader = object : PlatformScreenReader { override var isActive by mutableStateOf(false) } @@ -644,6 +734,7 @@ internal class ComposeSceneMediator( init { coroutineContext.job.invokeOnCompletion { dispose() } frameChoreographer.addListener(frameChoreographerListener) + touchOracle.addSampleListener(oracleSampleListener) } private fun hitTestInteropView(point: CValue): UIView? = @@ -659,10 +750,352 @@ internal class ComposeSceneMediator( private fun onCancelAllTouches(touches: Set<*>) { activitiesHandler.onActivitiesEnded(touches.count()) - indirectTouchStarts.clear() + // Only the cancelled contacts are forgotten: another contact of the same gesture may + // still be live and must keep its verdict. + touches.forEach { indirectTouches.remove((it as UITouch).hashCode()) } + updateIndirectSampling() scene.cancelPointerInput() } + /** + * Recognizes Siri Remote swipes from indirect contacts and converts them to directional key + * events. Those contacts are never forwarded to the Compose pointer input pipeline. + * + * When [touchOracle] reports a controller the contact is tracked in the absolute clickpad + * space, so a movement that starts on the outer ring of arrow buttons is ignored the way + * native tvOS ignores it, and the swipe is dispatched as soon as it is long enough. Without a + * controller (the simulator, or the first frames before the remote connects) the relative + * UIKit location is the only signal and the contact is evaluated once, when it ends. + */ + private fun onIndirectTouchEvent(touch: UITouch, eventKind: TouchesEventKind) { + val key = touch.hashCode() + when (eventKind) { + TouchesEventKind.BEGAN -> { + val state = if (touchOracle.isAvailable) { + val oraclePosition = touchOracle.position() + if (oraclePosition == null) { + // No sample of this contact yet: stay pending until one arrives, so the + // ring gate still applies instead of dropping to the ungated fallback. + swipeDebug { "SWIPE began verdict=pending" } + IndirectTouchState( + origin = Offset.Zero, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = true, + verdict = IndirectTouchVerdict.PENDING, + ) + } else { + IndirectTouchState( + origin = oraclePosition, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = true, + verdict = IndirectTouchVerdict.PENDING, + ).also { + resolveIndirectTouchOrigin(it, oraclePosition, touch.timestamp) + } + } + } else { + val position = touch.offsetInView(_backgroundView, screenDensity.density) + swipeDebug { + "SWIPE began origin=(${position.x}, ${position.y}) r=0.0 verdict=fallback" + } + IndirectTouchState( + origin = position, + startTimestamp = touch.timestamp, + beginTimestamp = touch.timestamp, + usesOracle = false, + verdict = IndirectTouchVerdict.CANDIDATE, + ) + } + indirectTouches[key] = state + updateIndirectSampling() + } + TouchesEventKind.MOVED -> { + val state = indirectTouches[key] ?: return + if (!state.usesOracle) return + val position = touchOracle.position() + if (position == null) { + swipeDebug { "SWIPE moved sample=null" } + return + } + // The display link runs the same evaluation on every frame; this call is + // idempotent and only keeps the contact moving when UIKit is the earlier signal. + evaluateIndirectSample(state, position, touch.timestamp, logSample = true) + } + TouchesEventKind.ENDED -> { + val state = indirectTouches.remove(key) ?: return + updateIndirectSampling() + if (state.usesOracle) { + if (state.verdict == IndirectTouchVerdict.PENDING) { + // The only sample of this contact arrives at its end: an origin with no + // movement observed in oracle space, so nothing is dispatched. + touchOracle.position()?.let { + resolveIndirectTouchOrigin(state, it, touch.timestamp) + } + } + } else if (state.verdict == IndirectTouchVerdict.CANDIDATE) { + // Fallback only: the relative location is meaningful just once, at the end. + evaluateIndirectTouch( + state = state, + position = touch.offsetInView(_backgroundView, screenDensity.density), + timestamp = touch.timestamp, + ) + } + // A lifted finger can no longer be a click, so the hold is not waited out here: + // it only has to cover the press race while the finger is still down. + if (state.verdict == IndirectTouchVerdict.ARMED) { + dispatchArmedIndirectTouch(state, touch.timestamp) + } + } + } + } + + /** + * Decides the origin and the verdict of a pending oracle contact from [sample], its first + * known position. The contact keeps the timestamp of its BEGAN as start, so its duration is + * measured from the finger landing rather than from the first sample. + */ + private fun resolveIndirectTouchOrigin( + state: IndirectTouchState, + sample: Offset, + timestamp: Double, + ) { + state.origin = sample + state.restAnchor = sample + state.lastSignificantMoveTime = timestamp + val radius = hypot(sample.x, sample.y) + state.verdict = if (touchOracle.hasRing && radius >= CENTER_PAD_RADIUS) { + IndirectTouchVerdict.IGNORED + } else { + IndirectTouchVerdict.CANDIDATE + } + swipeDebug { + val verdict = if (state.verdict == IndirectTouchVerdict.IGNORED) { + "ignored-ring" + } else { + "candidate" + } + val sinceBegin = (timestamp - state.beginTimestamp) * 1000.0 + "SWIPE origin=(${sample.x}, ${sample.y}) r=$radius verdict=$verdict " + + "t=${sinceBegin}ms" + } + } + + /** + * Runs the origin resolution or the swipe evaluation of one oracle contact for [sample], + * whichever its verdict calls for. Called both from UIKit's sparse MOVED callbacks and from + * every display-link tick, so it must stay idempotent. + */ + private fun evaluateIndirectSample( + state: IndirectTouchState, + sample: Offset, + timestamp: Double, + logSample: Boolean, + ) { + if (!state.usesOracle) return + when (state.verdict) { + IndirectTouchVerdict.PENDING -> + // First sample of this contact: it is the origin, not a movement. + resolveIndirectTouchOrigin(state, sample, timestamp) + IndirectTouchVerdict.CANDIDATE -> + evaluateIndirectTouch(state, sample, timestamp, logSample) + IndirectTouchVerdict.ARMED -> + holdArmedIndirectTouch(state, timestamp) + else -> {} + } + } + + /** + * Feeds one display-link sample of the clickpad to every live oracle contact. UIKit reports + * indirect movement sparsely, so this is what catches the origin of a contact in time and + * what dispatches a swipe as soon as it is long enough. + */ + private fun onOracleSample(sample: Offset?) { + if (sample == null || isEvaluatingOracleSample) return + if (indirectTouches.isEmpty()) return + isEvaluatingOracleSample = true + try { + val timestamp = CACurrentMediaTime() + purgeStaleIndirectTouches(timestamp) + // A dispatch runs key event handlers, which may end contacts, so the states are + // snapshotted before they are evaluated. + for (state in indirectTouches.values.toList()) { + // A dispatch can dispose this mediator, which clears every contact. + if (indirectTouches.isEmpty()) break + evaluateIndirectSample(state, sample, timestamp, logSample = false) + } + } finally { + isEvaluatingOracleSample = false + } + } + + /** + * Drops the contacts of [timestamp] whose terminal event never arrived, so a lost ENDED does + * not keep the display link polling for the rest of the process. Contacts that already reached + * a terminal verdict are dropped as soon as they can no longer dispatch. + */ + private fun purgeStaleIndirectTouches(timestamp: Double) { + var removed = false + val iterator = indirectTouches.values.iterator() + while (iterator.hasNext()) { + val state = iterator.next() + val maxAge = when (state.verdict) { + IndirectTouchVerdict.IGNORED, + IndirectTouchVerdict.CANCELLED, + IndirectTouchVerdict.DISPATCHED -> SWIPE_MAX_DURATION_S + DISPATCH_HOLD_S + else -> INDIRECT_CONTACT_MAX_AGE_S + } + // The tick stamps with CACurrentMediaTime and the contacts with UITouch.timestamp, + // both of which are the system uptime. + if (timestamp - state.beginTimestamp > maxAge) { + iterator.remove() + removed = true + } + } + if (removed) { + updateIndirectSampling() + } + } + + /** Samples the pad per frame exactly while at least one indirect contact is live. */ + private fun updateIndirectSampling() { + val shouldSample = indirectTouches.isNotEmpty() + if (shouldSample == isSamplingOracle) return + isSamplingOracle = shouldSample + if (shouldSample) { + oracleSamplingToken = touchOracle.beginSampling() + } else { + touchOracle.endSampling(oracleSamplingToken) + } + } + + /** + * Moves [state] out of [IndirectTouchVerdict.CANDIDATE] if the contact reached [position] at + * [timestamp] is a swipe, or if it can no longer become one. + */ + private fun evaluateIndirectTouch( + state: IndirectTouchState, + position: Offset, + timestamp: Double, + logSample: Boolean = true, + ) { + if (updateIndirectRest(state, position, timestamp)) return + val dx = position.x - state.origin.x + val dy = position.y - state.origin.y + val elapsed = timestamp - state.startTimestamp + if (logSample) { + swipeDebug { "SWIPE moved d=($dx, $dy) t=${elapsed * 1000.0}ms" } + } + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + if (elapsed > SWIPE_MAX_DURATION_S) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=duration" } + return + } + val distance = if (state.usesOracle) { + SWIPE_DISTANCE_NORMALIZED + } else { + with(screenDensity) { SWIPE_DISTANCE_FALLBACK_DP.dp.toPx() } + } + val absDx = abs(dx) + val absDy = abs(dy) + val dominant = maxOf(absDx, absDy) + val other = minOf(absDx, absDy) + if (dominant < distance || dominant < SWIPE_AXIS_DOMINANCE * other) { + return + } + // Sign convention for dy differs by source: the oracle's dy is the GameController dpad's + // absolute clickpad y, which is positive toward the top of the remote, so a positive dy + // is an upward swipe. The fallback dy is a UIKit relative view location, positive + // downward as usual for screen coordinates, so a positive dy there is a downward swipe. + val key = if (absDx >= absDy) { + if (dx > 0) Key.DirectionRight else Key.DirectionLeft + } else if (state.usesOracle) { + if (dy > 0) Key.DirectionUp else Key.DirectionDown + } else { + if (dy > 0) Key.DirectionDown else Key.DirectionUp + } + state.verdict = IndirectTouchVerdict.ARMED + state.armedKey = key + state.armedAt = timestamp + swipeDebug { "SWIPE armed ${directionName(key)}" } + } + + /** + * Holds an armed swipe until [DISPATCH_HOLD_S] elapsed, so a Select press that lands right + * after the contact travelled far enough cancels the swipe instead of firing both. + */ + private fun holdArmedIndirectTouch(state: IndirectTouchState, timestamp: Double) { + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + if (timestamp - state.armedAt < DISPATCH_HOLD_S) return + dispatchIndirectSwipe(state) + } + + /** Dispatches an armed swipe unless a clickpad press claimed the contact meanwhile. */ + private fun dispatchArmedIndirectTouch(state: IndirectTouchState, timestamp: Double) { + if (isClickpadPressOverlapping(state.beginTimestamp, timestamp)) { + state.verdict = IndirectTouchVerdict.CANCELLED + swipeDebug { "SWIPE cancelled reason=press" } + return + } + dispatchIndirectSwipe(state) + } + + private fun dispatchIndirectSwipe(state: IndirectTouchState) { + val key = state.armedKey ?: return + swipeDebug { "SWIPE dispatch ${directionName(key)}" } + state.verdict = IndirectTouchVerdict.DISPATCHED + onKeyboardEvent(KeyEvent(key, KeyEventType.KeyDown)) + onKeyboardEvent(KeyEvent(key, KeyEventType.KeyUp)) + } + + /** + * Tracks whether the finger of [state] is resting: as long as it stays within + * [REST_ANCHOR_TOLERANCE] of its anchor for longer than [REST_RESET_DURATION_S], the origin + * and the swipe timer move to where it rests, so a rest followed by a real movement still + * swipes and a slow drift never accumulates into one. + * + * Returns `true` when the origin was just moved, i.e. when there is no displacement left to + * evaluate for this sample. + */ + private fun updateIndirectRest( + state: IndirectTouchState, + position: Offset, + timestamp: Double, + ): Boolean { + val moved = hypot(position.x - state.restAnchor.x, position.y - state.restAnchor.y) + if (moved >= REST_ANCHOR_TOLERANCE) { + state.restAnchor = position + state.lastSignificantMoveTime = timestamp + return false + } + if (timestamp - state.lastSignificantMoveTime <= REST_RESET_DURATION_S) return false + state.origin = position + state.startTimestamp = timestamp + state.lastSignificantMoveTime = timestamp + return true + } + + /** + * `true` if a clickpad press is held, or if one began during the contact that started at + * [startTimestamp] or shortly before it. Press and touch timestamps are hardware event times + * rather than delivery times, so comparing them is immune to UIKit delivering the press and + * the touch callbacks of one click out of order. + */ + private fun isClickpadPressOverlapping(startTimestamp: Double, timestamp: Double): Boolean = + touchOracle.anyButtonPressed() || + pressDispatchLog.isClickpadPressHeld(timestamp) || + pressDispatchLog.clickpadPressTimestamp >= startTimestamp - PRESS_SUPPRESSION_WINDOW_S + /** * Converts [UITouch] objects from [touches] to [ComposeScenePointer] and dispatches them to the appropriate handlers. * @param touches a [Set] of [UITouch] objects. Erasure happens due to K/N not supporting Obj-C lightweight generics. @@ -686,36 +1119,7 @@ internal class ComposeSceneMediator( for (anyTouch in touches) { val touch = anyTouch as UITouch if (touch.type == UITouchTypeIndirect) { - val position = touch.offsetInView(_backgroundView, screenDensity.density) - when (eventKind) { - TouchesEventKind.BEGAN -> { - indirectTouchStarts[touch.hashCode()] = - IndirectTouchStart(position, touch.timestamp) - } - TouchesEventKind.ENDED -> { - val start = indirectTouchStarts.remove(touch.hashCode()) ?: continue - // A clickpad press during this touch's lifetime means the drift is - // from a click, not a swipe. - if (lastClickpadPressTimestamp >= - start.timestamp - PRESS_TOUCH_TIMESTAMP_JITTER_SECONDS - ) { - continue - } - val dx = position.x - start.position.x - val dy = position.y - start.position.y - val threshold = with(screenDensity) { INDIRECT_SWIPE_THRESHOLD_DP.dp.toPx() } - if (dx * dx + dy * dy >= threshold * threshold) { - val key = if (kotlin.math.abs(dx) >= kotlin.math.abs(dy)) { - if (dx > 0) Key.DirectionRight else Key.DirectionLeft - } else { - if (dy > 0) Key.DirectionDown else Key.DirectionUp - } - onKeyboardEvent(KeyEvent(key, KeyEventType.KeyDown)) - onKeyboardEvent(KeyEvent(key, KeyEventType.KeyUp)) - } - } - TouchesEventKind.MOVED -> {} - } + onIndirectTouchEvent(touch, eventKind) } else { pointerTouches.add(touch) } @@ -852,6 +1256,12 @@ internal class ComposeSceneMediator( ) private fun dispose() { + touchOracle.removeSampleListener(oracleSampleListener) + if (isSamplingOracle) { + isSamplingOracle = false + touchOracle.endSampling(oracleSamplingToken) + } + indirectTouches.clear() repeatingKeys.values.forEach { it.cancel() } repeatingKeys.clear() consumedKeyIds.clear() @@ -991,26 +1401,26 @@ internal class ComposeSceneMediator( val keyId = press.key?.keyCode?.toLong() ?: -(press.type.toLong() + 1L) val phase = press.phase - if (!pressDispatchLog.shouldEvaluate(pressesEvent, keyId, phase)) { - // Already evaluated for this UIPressesEvent, by this mediator or by another - // mediator of the same container: this is the responder chain echoing back a - // press an overlay view forwarded to `super`. - reportUnconsumed(anyPress) - return@forEach - } - - // The Siri Remote trackpad doubles as its buttons (Select in the middle, arrows - // on the outer ring): clicking it produces a press alongside an indirect touch - // whose end position can drift enough to look like a swipe. Record the press time - // so the touch's ENDED branch doesn't also dispatch a phantom directional key. - if (phase == UIPressPhase.UIPressPhaseBegan && + // The Siri Remote clickpad doubles as its buttons (Select in the middle, arrows + // on the outer ring): clicking it produces a press alongside an indirect touch that + // drifts enough to look like a swipe. The press is recorded before the echo check, + // so the mediator that sees the echo rather than the original still latches it. + if ( when (event.key) { Key.DirectionCenter, Key.DirectionUp, Key.DirectionDown, Key.DirectionLeft, Key.DirectionRight -> true else -> false } ) { - lastClickpadPressTimestamp = press.timestamp + pressDispatchLog.recordClickpadPress(keyId, phase, press.timestamp) + } + + if (!pressDispatchLog.shouldEvaluate(pressesEvent, keyId, phase)) { + // Already evaluated for this UIPressesEvent, by this mediator or by another + // mediator of the same container: this is the responder chain echoing back a + // press an overlay view forwarded to `super`. + reportUnconsumed(anyPress) + return@forEach } when (phase) { diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt index d3cda1fc37278..99564cb8ec977 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/IosComposeSceneLayer.tvos.kt @@ -61,6 +61,7 @@ internal class IosComposeSceneLayer( consumePointerInputOutside: Boolean = focusedViewsList != null, parentCoroutineContext: CoroutineContext, private val pressDispatchLog: TvPressDispatchLog, + private val touchOracle: SiriRemoteTouchOracle, private val ownerProvider: PlatformArchitectureComponentsOwner, private val interfaceOrientationState: State, private var invalidateLayout: () -> Unit, @@ -110,6 +111,7 @@ internal class IosComposeSceneLayer( coroutineContext = layerCoroutineContext, navigationEventInput = navigationEventInput, pressDispatchLog = pressDispatchLog, + touchOracle = touchOracle, composeSceneFactory = ::createComposeScene, interfaceOrientationState = interfaceOrientationState, schedulePendingInteropViewUpdates = layersViewController::invalidateDraw, diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt new file mode 100644 index 0000000000000..3bc1774fd4ffc --- /dev/null +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/SiriRemoteTouchOracle.tvos.kt @@ -0,0 +1,270 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.compose.ui.scene + +import androidx.compose.ui.geometry.Offset +import kotlinx.cinterop.BetaInteropApi +import kotlinx.cinterop.ObjCAction +import platform.Foundation.NSNotificationCenter +import platform.Foundation.NSOperationQueue +import platform.Foundation.NSRunLoop +import platform.Foundation.NSRunLoopCommonModes +import platform.Foundation.NSSelectorFromString +import platform.GameController.GCController +import platform.GameController.GCControllerDidConnectNotification +import platform.GameController.GCControllerDidDisconnectNotification +import platform.GameController.GCMicroGamepad +import platform.GameController.GCProductCategorySiriRemote1stGen +import platform.QuartzCore.CADisplayLink +import platform.darwin.NSObject +import platform.darwin.NSObjectProtocol + +/** + * Reports where the finger physically is on the Siri Remote clickpad. + * + * UIKit indirect touches ([platform.UIKit.UITouchTypeIndirect]) carry a relative location in a + * space that is re-centred on every new contact, so they cannot tell a movement that started on + * the centre pad from one that started on the outer ring of the second generation remote. The + * GameController micro gamepad does: with `reportsAbsoluteDpadValues` its dpad axes are the + * absolute finger position in [-1, 1] on both axes. + * + * This is an observer only: no `GCEventViewController` is installed and + * `controllerUserInteractionEnabled` is never touched, so UIKit keeps delivering presses and + * touches through the responder chain exactly as before. + * + * The gamepad is polled at the touch event that needs it rather than through + * `valueChangedHandler`. GameController keeps a single handler per element, so the last writer + * wins: an app that installs its own handler on the same remote would silently replace the + * oracle's and leave it without samples. Polling reads the live element values, so the oracle + * coexists with app-level GameController usage and never overwrites an app's handler. + * + * UIKit reports indirect movement sparsely: polling only inside `touchesBegan`/`touchesMoved` + * samples the origin of a contact up to 0.2 normalised units after the finger landed, which makes + * genuine centre swipes look like they started on the ring. [beginSampling] therefore drives a + * display link that polls the pad every frame while a contact is active and feeds the samples to + * the listeners installed by [addSampleListener]. + */ +internal class SiriRemoteTouchOracle { + private var connectObserver: NSObjectProtocol? = null + private var disconnectObserver: NSObjectProtocol? = null + + private var controller: GCController? = null + private var productCategory: String? = null + // The gamepad's own rotation mode, restored when the oracle lets go of it. + private var previousAllowsRotation: Boolean? = null + + private val sampleListeners = mutableListOf<(Offset?) -> Unit>() + private var samplingCount = 0 + // Bumped by every stop, so a session token of a previous run cannot close a current session. + private var samplingGeneration = 0 + private var displayLink: CADisplayLink? = null + private val displayLinkTarget = SampleDisplayLinkTarget { + val sample = position() + // A listener may dispose its mediator, and with it remove listeners, while being + // notified, so the list is snapshotted before it is walked. + sampleListeners.toList().forEach { it(sample) } + } + + /** `true` while a micro gamepad, i.e. a Siri Remote, is connected and reporting. */ + val isAvailable: Boolean get() = controller?.microGamepad != null + + /** + * `true` for remotes whose clickpad has an outer ring of arrow buttons. + * + * tvOS coalesces every paired remote into a single controller reporting the + * "Coalesced Remote" product category, so a physical remote's generation cannot be + * distinguished once connected. The ring gate is therefore the default, applying to the + * coalesced controller and any other unrecognised micro gamepad category. First-generation + * remotes have no ring and let a swipe start anywhere on the pad, so they are exempted here; + * so are genuine game controllers, which report an extended gamepad rather than a bare micro + * gamepad. + */ + val hasRing: Boolean + get() { + val category = productCategory ?: return false + if (category == GCProductCategorySiriRemote1stGen) return false + if (controller?.extendedGamepad != null) return false + return true + } + + fun start() { + if (connectObserver != null) return + val center = NSNotificationCenter.defaultCenter + val queue = NSOperationQueue.mainQueue + connectObserver = center.addObserverForName( + name = GCControllerDidConnectNotification, + `object` = null, + queue = queue, + ) { refresh() } + disconnectObserver = center.addObserverForName( + name = GCControllerDidDisconnectNotification, + `object` = null, + queue = queue, + ) { refresh() } + refresh() + } + + /** + * Adds a listener invoked with [position] on every display-link tick while any [beginSampling] + * session is open. Every mediator, the root one and each dialog layer, installs its own, so a + * contact live in one of them is sampled for all of them. + */ + fun addSampleListener(listener: (Offset?) -> Unit) { + sampleListeners.add(listener) + } + + /** Removes a listener installed by [addSampleListener]. */ + fun removeSampleListener(listener: (Offset?) -> Unit) { + sampleListeners.remove(listener) + } + + /** + * Opens a sampling session, starting the per-frame polling of the pad if it is the first one. + * Every call must be matched by an [endSampling] with the returned token. + */ + fun beginSampling(): Int { + samplingCount++ + if (displayLink != null) return samplingGeneration + val link = CADisplayLink.displayLinkWithTarget( + target = displayLinkTarget, + selector = NSSelectorFromString("tick:") + ) + link.preferredFramesPerSecond = 60L + link.addToRunLoop(NSRunLoop.mainRunLoop, NSRunLoopCommonModes) + displayLink = link + return samplingGeneration + } + + /** + * Closes the sampling session opened with [token], stopping the polling once the last one is + * closed. A token of an older generation is ignored: [stop] already dropped every session of + * it, so honouring it would underflow the count of the sessions opened since. + */ + fun endSampling(token: Int) { + if (token != samplingGeneration) return + if (samplingCount == 0) return + samplingCount-- + if (samplingCount > 0) return + stopSampling() + } + + private fun stopSampling() { + // Invalidating is what releases the display link's retain of the target, so it must run + // for every begun sampling session. + displayLink?.invalidate() + displayLink = null + } + + fun stop() { + samplingCount = 0 + samplingGeneration++ + stopSampling() + sampleListeners.clear() + val center = NSNotificationCenter.defaultCenter + connectObserver?.let { center.removeObserver(it) } + disconnectObserver?.let { center.removeObserver(it) } + connectObserver = null + disconnectObserver = null + detach() + } + + /** + * Current absolute finger position in [-1, 1] on both axes, `null` if no remote is connected + * or if the pad is at rest. The pad reports exactly (0, 0) whenever no finger is touching it, + * and no touching sample is ever exactly (0, 0), so an exact-zero sample is treated as "no + * finger": otherwise a BEGAN sampled before the pad reports a real position would be + * classified as a centre-pad contact at radius 0 using a stale origin instead of falling to + * fallback mode. + */ + fun position(): Offset? { + val microGamepad = controller?.microGamepad ?: return null + // An app is free to reset this flag on the shared gamepad; setting it back only takes + // effect from the next sample, so the sample read right after such a reset may still be + // relative. + if (!microGamepad.reportsAbsoluteDpadValues) { + microGamepad.reportsAbsoluteDpadValues = true + } + val pad = microGamepad.dpad + val x = pad.xAxis.value + val y = pad.yAxis.value + if (x == 0f && y == 0f) return null + return Offset(x, y) + } + + /** `true` while the clickpad or one of the ring buttons is physically held down. */ + fun anyButtonPressed(): Boolean { + val microGamepad = controller?.microGamepad ?: return false + return isAnyButtonPressed(microGamepad) + } + + private fun refresh() { + // The remote reports a bare micro gamepad; a game controller reports an extended gamepad + // and its micro gamepad projection, whose dpad is the thumbstick rather than a clickpad. + // Reevaluated on every connect and disconnect, so plugging a game controller in does not + // steal the oracle from the remote. + val controllers = GCController.controllers().filterIsInstance() + val connected = controllers.firstOrNull { + it.microGamepad != null && it.extendedGamepad == null + } ?: controllers.firstOrNull { it.microGamepad != null } + if (connected == null) { + detach() + return + } + if (connected == controller) return + detach() + attach(connected) + } + + private fun attach(connected: GCController) { + val microGamepad = connected.microGamepad ?: return + controller = connected + productCategory = connected.productCategory + previousAllowsRotation = microGamepad.allowsRotation + microGamepad.reportsAbsoluteDpadValues = true + microGamepad.allowsRotation = false + } + + private fun detach() { + // No handler is installed, so nothing is cleared here: an app's own + // `valueChangedHandler` on the same gamepad is never touched. The rotation mode is + // restored, since an app that enabled it did so for its own reading of the gamepad. + previousAllowsRotation?.let { controller?.microGamepad?.allowsRotation = it } + previousAllowsRotation = null + controller = null + productCategory = null + } + + // The dpad's up/down/left/right are virtual direction buttons synthesised from the axis + // sign, pressed whenever the finger is off-centre in that direction; they are NOT the + // physical ring click buttons. Treating them as presses would make every swipe candidate + // that leaves the centre pad look like a clickpad press. Only the physical action buttons + // (buttonA, buttonX) and the menu button count as a press here. + private fun isAnyButtonPressed(microGamepad: GCMicroGamepad): Boolean = + microGamepad.buttonA.pressed || + microGamepad.buttonX.pressed || + microGamepad.buttonMenu.pressed +} + +private class SampleDisplayLinkTarget( + private val onTick: () -> Unit +) : NSObject() { + @OptIn(BetaInteropApi::class) + @ObjCAction + fun tick(link: CADisplayLink) { + onTick() + } +} diff --git a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt index 623a77c079d51..95e5edf75fe69 100644 --- a/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt +++ b/compose/ui/ui/src/tvosMain/kotlin/androidx/compose/ui/scene/TvPressDispatchLog.tvos.kt @@ -36,6 +36,11 @@ import platform.UIKit.UIPressesEvent * Consumption itself stays per mediator ([ComposeSceneMediator.consumedKeyIds]): the mediator * that consumed the Began phase is the one that must dispatch the matching KeyUp. */ +// A held clickpad press older than this relative to the event being evaluated is stale: its +// Ended phase was absorbed before reaching the mediator (the tvOS keyboard overlay does that +// while it is up), and without ageing it out every later contact would be suppressed forever. +private const val HELD_PRESS_MAX_AGE_S = 1.0 + internal class TvPressDispatchLog { // The current event is identified by value rather than by reference: UIKit recycles // UIPressesEvent instances, so object identity alone would make a reused instance carrying a @@ -66,9 +71,50 @@ internal class TvPressDispatchLog { return true } + // Timestamp (seconds since boot, the same timebase as UITouch.timestamp) of the last + // clickpad press Began, and the ids of the clickpad presses currently held. The Siri Remote + // clickpad is itself a button (Select in the middle, arrows on the outer ring of 2nd + // generation remotes): clicking it delivers a UIPress alongside an indirect UITouch that + // drifts as the finger settles, so a contact whose lifetime overlaps a clickpad press is + // click contact, not a swipe. Shared like the log itself, because the press and the touch of + // one click can be delivered to different mediators of the same container. + private var lastClickpadPressTimestamp: Double = Double.NEGATIVE_INFINITY + private val heldClickpadKeyIds = mutableMapOf() + + /** + * `true` while at least one clickpad press is held, evaluated at [timestamp]: presses whose + * Began is older than [HELD_PRESS_MAX_AGE_S] are dropped as never-ended leftovers. + */ + fun isClickpadPressHeld(timestamp: Double): Boolean { + if (heldClickpadKeyIds.isEmpty()) return false + heldClickpadKeyIds.entries.removeAll { timestamp - it.value > HELD_PRESS_MAX_AGE_S } + return heldClickpadKeyIds.isNotEmpty() + } + + /** Timestamp of the last clickpad press Began, [Double.NEGATIVE_INFINITY] if there was none. */ + val clickpadPressTimestamp: Double get() = lastClickpadPressTimestamp + + /** Records the lifetime of a clickpad press identified by [keyId]. */ + fun recordClickpadPress(keyId: Long, phase: UIPressPhase, timestamp: Double) { + when (phase) { + UIPressPhase.UIPressPhaseBegan -> { + // A second Began for a key that is still held means the previous press's Ended + // never arrived, so that press ends here rather than staying held for good. + heldClickpadKeyIds.remove(keyId) + lastClickpadPressTimestamp = timestamp + heldClickpadKeyIds[keyId] = timestamp + } + UIPressPhase.UIPressPhaseEnded, UIPressPhase.UIPressPhaseCancelled -> + heldClickpadKeyIds.remove(keyId) + else -> {} + } + } + fun clear() { lastEventPtr = 0L lastEventTimestamp = Double.NaN evaluatedPhases.clear() + lastClickpadPressTimestamp = Double.NEGATIVE_INFINITY + heldClickpadKeyIds.clear() } }