android: make playback and prefetch recover instead of stall

android/README.md

@@ -61,8 +61,15 @@ the one core for the process through `CoreHost`:
   own headers) plus the gapless follow-up as a second playlist item; `SetNext` replaces everything after the current item; the transition is
   detected from `onMediaItemTransition(AUTO)`, reported as `Ended` (played item) then
   `TransitionedToNext`, and the played item removed. The player holds `C.WAKE_MODE_NETWORK` (wake +
-  Wi-Fi lock) so streams keep going with the screen off. Network errors (connection failed/timeout)
-  are retried with `prepare()` on a ~1 min backoff and reported non-fatal; only then fatal. `PreBuffer` prepares a
+  Wi-Fi lock) so streams keep going with the screen off. Errors a later `prepare()` can fix (the
+  connection, a timeout, a 5xx from the server, a stuck player, a core stream handle that went away
+  under the player) are retried with `prepare()` on a ~1 min backoff and reported non-fatal; while
+  the device is offline the retry waits instead of spending attempts, and the network coming back
+  (`NetworkMonitor` -> `ExoBackend.onConnectivityChanged`) retries at once; only when the attempts run
+  out is the error fatal and the core's own retry/skip takes over. `Paused` is reported only when the
+  player no longer means to play (`playWhenReady` false); a stall with it still set is `Buffering`,
+  and the service never stops itself under a player that is buffering or recovering
+  (`ExoBackend.isBusy`). `PreBuffer` prepares a
   second silent ExoPlayer at the requested position (`PreBufferReady` when READY); `DiscardPreBuffer`
   releases it. `gain_db` is applied as `10^(gain/20) * masterVolume` clamped to 1.0 — Media3 has no
   gain stage, so positive gain is an approximation (documented in `ExoBackend`).
@@ -74,7 +81,9 @@ the one core for the process through `CoreHost`:
   loader threads): a seek is a new open at the position; the core fetches from the server, caches a
   whole read and serves later plays and seeks from disk. An unknown/expired token is
   `ERROR_CODE_IO_FILE_NOT_FOUND`, never retried (`CoreStreamLoadErrorPolicy`), so the backend reports
-  a fatal error and the core resolves a fresh token. The factory routes the `hocket-stream` scheme to
+  a fatal error and the core resolves a fresh token; a handle the core closed under the player (idle
+  while the buffer was full) is retried with a fresh open at the same position, which the cache
+  serves. The factory routes the `hocket-stream` scheme to
   it and everything else (`file:`, a direct server URL for the fake core) to `DefaultDataSource`; there
   is no `CacheDataSource` (the core caches). `NativeCore` implements the `CoreStreams` seam.
 - Reports back: `Ready`, `Playing`, `Paused`, `Buffering`, `Position` every 750 ms while playing and

android/playback/src/main/java/app/hocket/playback/ExoBackend.kt

@@ -8,11 +8,13 @@ import androidx.media3.common.C
 import androidx.media3.common.MediaItem
 import androidx.media3.common.PlaybackException
 import androidx.media3.common.Player
+import androidx.media3.datasource.HttpDataSource
 import androidx.media3.exoplayer.DefaultLoadControl
 import androidx.media3.exoplayer.ExoPlayer
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
 import androidx.media3.exoplayer.source.MediaSource
 import app.hocket.core.Commands
+import app.hocket.core.CoreStreamException
 import app.hocket.core.CoreStreams
 import app.hocket.core.api.BackendCommand
 import app.hocket.core.api.BackendReport
@@ -62,9 +64,17 @@ import kotlin.math.pow
  *   then resumes by itself when focus returns and reports `Playing`).
  * - The player holds a wake lock and a Wi-Fi lock while playing ([C.WAKE_MODE_NETWORK]): without
  *   them a stream stalls once the screen is off and the CPU or Wi-Fi radio sleeps.
- * - Network failures leave ExoPlayer idle with an error, so they are retried here with backoff
- *   ([recoveryDelayMs]; `prepare()` resumes at the same position) and reported as non-fatal; only when
- *   the retries run out is the error fatal and the core's own retry/skip takes over.
+ * - `Paused` is reported only when the player no longer means to play (`playWhenReady` false: a
+ *   pause, becoming noisy, a lasting focus loss). A stall with `playWhenReady` still set (an empty
+ *   buffer, a load being retried, an error being recovered from) is `Buffering`, never `Paused`: the
+ *   core would otherwise show the item paused, and the service would count the player as idle and
+ *   could stop itself under a playback that was only rebuffering.
+ * - Errors a later `prepare()` can fix ([isRecoverable]: the connection, a timeout, a 5xx from the
+ *   server, a stream handle the core closed, a stuck player) leave ExoPlayer idle with an error, so
+ *   they are retried here with backoff ([recoveryDelayMs]; `prepare()` resumes at the same position)
+ *   and reported as non-fatal. While the device is offline ([onConnectivityChanged]) no attempt is
+ *   spent: the retry waits, and the network coming back retries at once with a fresh schedule. Only
+ *   when the attempts run out is the error fatal and the core's own retry/skip takes over.
  */
 class ExoBackend(
     private val context: Context,
@@ -73,6 +83,9 @@ class ExoBackend(
     /** The running core's stream reader, read on each open (a restarted service has a new core). */
     private val streams: () -> CoreStreams? = { CoreHost.current as? CoreStreams },
 ) {
+    /** What a player that stopped playing while it still has an item is reporting. */
+    internal enum class Stall { Paused, Buffering, Nothing }
+
     internal companion object {
         private const val TAG = "ExoBackend"
         private const val POSITION_INTERVAL_MS = 750L
@@ -81,10 +94,49 @@ class ExoBackend(
         /** Backoff before recovery attempt [attempt] (0-based) of a network error, or null when out of attempts. */
         internal fun recoveryDelayMs(attempt: Int): Long? = RECOVERY_DELAYS_MS.getOrNull(attempt)
 
-        /** Errors a later `prepare()` can fix: the connection, not the media. */
+        /** Errors a later `prepare()` can fix by their code alone: the connection, not the media. */
         internal fun isRecoverable(errorCode: Int): Boolean =
             errorCode == PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED ||
-                errorCode == PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT
+                errorCode == PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT ||
+                errorCode == PlaybackException.ERROR_CODE_TIMEOUT
+
+        /**
+         * Whether a later `prepare()` can fix [error]: a network failure or timeout, a stuck player
+         * (`ERROR_CODE_TIMEOUT`), a 5xx from the server (the core's stream status or a direct HTTP
+         * source), or a core stream handle that went away under the player (closed as idle, too many
+         * open, unknown): a re-open serves it again. A 4xx, an unknown token (the core re-resolves), a
+         * core that shut down, and anything about the media itself are not.
+         */
+        internal fun isRecoverable(error: PlaybackException): Boolean {
+            if (isRecoverable(error.errorCode)) return true
+            var t: Throwable? = error
+            while (t != null) {
+                when (t) {
+                    is CoreStreamException -> return when (t.kind) {
+                        CoreStreamException.Kind.Network, CoreStreamException.Kind.Closed, CoreStreamException.Kind.UnknownHandle,
+                        CoreStreamException.Kind.TooManyHandles, CoreStreamException.Kind.NoServer -> true
+                        CoreStreamException.Kind.Status -> (t.httpStatus ?: 0) >= 500
+                        else -> false
+                    }
+                    is HttpDataSource.InvalidResponseCodeException -> return t.responseCode >= 500
+                    is HttpDataSource.CleartextNotPermittedException -> return false
+                    is HttpDataSource.HttpDataSourceException -> return true
+                }
+                t = t.cause
+            }
+            return false
+        }
+
+        /**
+         * [Player.Listener.onIsPlayingChanged] `false`: [Stall.Paused] when the player no longer
+         * means to play, [Stall.Buffering] while it does but has nothing to play yet; idle (an error,
+         * a stop) and ended are reported by their own callbacks.
+         */
+        internal fun stall(playWhenReady: Boolean, playbackState: Int): Stall = when {
+            !playWhenReady -> if (playbackState == Player.STATE_IDLE || playbackState == Player.STATE_ENDED) Stall.Nothing else Stall.Paused
+            playbackState == Player.STATE_BUFFERING -> Stall.Buffering
+            else -> Stall.Nothing
+        }
     }
 
     val player: ExoPlayer = ExoPlayer.Builder(context)
@@ -103,6 +155,11 @@ class ExoBackend(
     private var recoveryJob: Job? = null
     /** Recovery attempts for the current error streak; reset once the player is ready again. */
     private var recoveryAttempts = 0
+    /** The key a pending recovery is for. */
+    private var recoveryKey: String? = null
+    /** Connectivity as the service last reported it; a retry waits while this is false. */
+    @Volatile
+    private var online = true
     /** The key of the last `Load`, so an error raised after the playlist emptied is still attributable. */
     private var lastLoadedKey: String? = null
     /** Keys by media id, so reports name the queue key the core gave us. */
@@ -117,7 +174,7 @@ class ExoBackend(
                 val key = currentKey() ?: return
                 when (playbackState) {
                     Player.STATE_READY -> {
-                        recoveryAttempts = 0
+                        cancelRecovery()
                         val duration = player.duration.takeIf { it != C.TIME_UNSET }?.toUInt()
                         report(BackendReport.Ready(BackendReportReadyInner(key, duration)))
                         report(BackendReport.Buffering(BackendReportBufferingInner(key, false)))
@@ -140,17 +197,22 @@ class ExoBackend(
                     startPositionLoop()
                 } else {
                     stopPositionLoop()
-                    if (player.playbackState != Player.STATE_ENDED && !player.isLoading) {
-                        report(BackendReport.Paused(BackendReportPausedInner(key, position())))
+                    when (stall(player.playWhenReady, player.playbackState)) {
+                        Stall.Paused -> report(BackendReport.Paused(BackendReportPausedInner(key, position())))
+                        Stall.Buffering -> report(BackendReport.Buffering(BackendReportBufferingInner(key, true)))
+                        Stall.Nothing -> Unit
                     }
                 }
             }
 
             override fun onPlayWhenReadyChanged(playWhenReady: Boolean, reason: Int) {
-                if (!playWhenReady && reason == Player.PLAY_WHEN_READY_CHANGE_REASON_AUDIO_FOCUS_LOSS) {
+                if (playWhenReady) return
+                if (reason == Player.PLAY_WHEN_READY_CHANGE_REASON_AUDIO_FOCUS_LOSS) {
                     report(BackendReport.AudioFocusLost(BackendReportAudioFocusLostInner(transient = false)))
                 }
-                if (!playWhenReady && reason == Player.PLAY_WHEN_READY_CHANGE_REASON_AUDIO_BECOMING_NOISY) {
+                // The player stopped meaning to play, whoever asked (a pause while buffering never
+                // reaches onIsPlayingChanged, so this is the only place that sees it).
+                if (player.playbackState == Player.STATE_BUFFERING || player.playbackState == Player.STATE_READY) {
                     currentKey()?.let { report(BackendReport.Paused(BackendReportPausedInner(it, position()))) }
                 }
             }
@@ -176,27 +238,72 @@ class ExoBackend(
                 val key = errorKey() ?: return
                 val message = error.errorCodeName + ": " + (error.message ?: "")
                 // Any player error leaves ExoPlayer idle: "non-fatal" only holds if we bring it back.
-                val delayMs = if (isRecoverable(error.errorCode)) recoveryDelayMs(recoveryAttempts) else null
+                val delayMs = if (isRecoverable(error)) recoveryDelayMs(recoveryAttempts) else null
                 if (delayMs == null) {
-                    recoveryAttempts = 0
+                    cancelRecovery()
                     report(BackendReport.Error(BackendReportErrorInner(key, message, true)))
                     return
                 }
                 recoveryAttempts++
+                Log.w(TAG, "playback error on $key: ${error.errorCodeName}; retrying in ${delayMs}ms (attempt $recoveryAttempts)")
                 report(BackendReport.Error(BackendReportErrorInner(key, message, false)))
                 report(BackendReport.Buffering(BackendReportBufferingInner(key, true)))
-                recoveryJob?.cancel()
-                recoveryJob = scope.launch {
-                    delay(delayMs)
-                    if (player.playerError != null && currentKey() == key) {
-                        Log.i(TAG, "retrying $key after ${error.errorCodeName} (attempt $recoveryAttempts)")
-                        player.prepare()
-                    }
-                }
+                scheduleRecovery(key, delayMs)
             }
         })
     }
 
+    private fun scheduleRecovery(key: String, delayMs: Long) {
+        recoveryJob?.cancel()
+        recoveryKey = key
+        recoveryJob = scope.launch {
+            delay(delayMs)
+            attemptRecovery(key)
+        }
+    }
+
+    /**
+     * Re-prepare after a recoverable error, if the error still stands and the item is still the one
+     * that failed. Offline, the attempt is not spent: it waits for [onConnectivityChanged], with the
+     * longest delay as a safety net should that never come.
+     */
+    private fun attemptRecovery(key: String) {
+        if (player.playerError == null || currentKey() != key) {
+            recoveryKey = null
+            return
+        }
+        if (!online) {
+            Log.i(TAG, "offline; waiting for connectivity before retrying $key")
+            scheduleRecovery(key, RECOVERY_DELAYS_MS.last())
+            return
+        }
+        Log.i(TAG, "retrying $key (attempt $recoveryAttempts)")
+        player.prepare()
+    }
+
+    /**
+     * Connectivity as the service sees it. Coming back online while a recovery is pending retries at
+     * once, with the attempts reset (the failures so far were the old network's).
+     */
+    fun onConnectivityChanged(online: Boolean) {
+        this.online = online
+        if (!online) return
+        val key = recoveryKey ?: return
+        if (recoveryJob?.isActive != true) return
+        recoveryJob?.cancel()
+        recoveryAttempts = 0
+        attemptRecovery(key)
+    }
+
+    /**
+     * True while the player still means to play something: playing, buffering, or recovering from
+     * an error. The service must not stop itself (releasing the player) on such a player, whatever
+     * the media session says about it.
+     */
+    fun isBusy(): Boolean =
+        recoveryJob?.isActive == true ||
+            (player.playWhenReady && (player.playbackState == Player.STATE_BUFFERING || player.playbackState == Player.STATE_READY))
+
     fun handle(command: BackendCommand) {
         when (command) {
             is BackendCommand.Load -> {
@@ -281,6 +388,7 @@ class ExoBackend(
     private fun cancelRecovery() {
         recoveryJob?.cancel()
         recoveryJob = null
+        recoveryKey = null
         recoveryAttempts = 0
     }
 

android/playback/src/main/java/app/hocket/playback/HocketStreamDataSource.kt

@@ -200,8 +200,10 @@ class CoreStreamDataSourceFactory(
 /**
  * `DefaultLoadErrorHandlingPolicy`, except that core stream failures a retry cannot fix fail at once:
  * an unknown/expired token (the core must resolve a fresh one), an offset past the end, and a core
- * that has shut down or closed the handle. Media3's default retries everything but
- * `FileNotFoundException`-typed errors, which would re-open an expired token three times.
+ * that has shut down. Media3's default retries everything but `FileNotFoundException`-typed errors,
+ * which would re-open an expired token three times. A handle the core closed under the player
+ * (idle for ten minutes while the player's buffer was full, or reaped) is retried: the retry opens a
+ * fresh handle at the same position, which the cache serves.
  */
 class CoreStreamLoadErrorPolicy : DefaultLoadErrorHandlingPolicy() {
     override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long {
@@ -214,7 +216,6 @@ class CoreStreamLoadErrorPolicy : DefaultLoadErrorHandlingPolicy() {
             CoreStreamException.Kind.UnknownToken,
             CoreStreamException.Kind.RangeNotSatisfiable,
             CoreStreamException.Kind.ShutDown,
-            CoreStreamException.Kind.Closed,
         )
 
         fun coreStreamFailure(e: Throwable?): CoreStreamException? {

android/playback/src/main/java/app/hocket/playback/Monitors.kt

@@ -23,7 +23,12 @@ import java.security.MessageDigest
  * network id so transcoding profiles can vary per network. The id is a hash of the SSID when it is
  * readable (needs location permission on 8.1+), otherwise the transport type.
  */
-class NetworkMonitor(private val context: Context, private val dispatch: (Command) -> Unit) {
+class NetworkMonitor(
+    private val context: Context,
+    private val dispatch: (Command) -> Unit,
+    /** Called with whether there is any connectivity, on every change of that (and once at start). */
+    private val onConnectivity: (online: Boolean) -> Unit = {},
+) {
     private val cm = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager
     private var last: NetworkState? = null
 
@@ -78,9 +83,12 @@ class NetworkMonitor(private val context: Context, private val dispatch: (Comman
 
     private fun publish() {
         val state = current()
-        if (state != last) {
+        val was = last
+        if (state != was) {
             last = state
             dispatch(Commands.setNetworkState(state))
+            val online = state.kind != NetworkKind.Offline
+            if (was == null || (was.kind != NetworkKind.Offline) != online) onConnectivity(online)
         }
     }
 }