feat: delegate narration playback through injected media manager

packages/vreo/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@realsee/vreo",
-  "version": "2.6.4-alpha.1",
+  "version": "2.6.4-alpha.5",
   "type": "module",
   "description": "Vreo (VR Video 缩写) 是基于如视三维渲染引擎 Five 和 用户界面构建库 React 实现的如视 3D 空间剧本播放器。",
   "keywords": [
@@ -75,10 +75,11 @@
     "preview": "vite preview",
     "docs": "npx typedoc",
     "packages": "node ./dev-tools/build-packages.js",
-    "postpublish": "cnpm sync @realsee/vreo"
+    "postpublish": "cnpm sync @realsee/vreo",
+    "test:audio-focus": "node --test test/audio-focus.test.mjs"
   },
   "peerDependencies": {
-    "@realsee/dnalogel": "3.67.0",
+    "@realsee/dnalogel": "3.81.10-alpha.4",
     "@realsee/five": "^6.4.0-alpha.1",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
@@ -91,7 +92,7 @@
     "@babel/preset-env": "^7.16.4",
     "@babel/preset-react": "^7.16.0",
     "@babel/preset-typescript": "^7.16.0",
-    "@realsee/dnalogel": "^3.67.2",
+    "@realsee/dnalogel": "3.81.10-alpha.4",
     "@realsee/five": "^6.4.0-alpha.38",
     "@tweenjs/tween.js": "18.6.4",
     "@types/node": "^18.11.17",

packages/vreo/resources/Player/App.tsx

@@ -34,9 +34,9 @@ const AppView = observer(({ controller }: { controller: Controller }) => {
               return
             }
             if (controller.playing) {
-              controller.setPlaying(false)
+              controller.playback.cancel()
             } else {
-              controller.setPlaying(true)
+              if (controller.playback.begin(true)) controller.setPlaying(true)
             }
           }}
           options={controller.configs?.videoAgentMeshOptions || {}}

packages/vreo/resources/Player/Controller.ts

@@ -1,3 +1,4 @@
+import { PlaybackLifecycle } from './PlaybackLifecycle'
 import { Five, Subscribe } from '@realsee/five'
 import { action, computed, makeObservable, observable, reaction } from 'mobx'
 import * as React from 'react'
@@ -127,7 +128,9 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
      * @param playing - 是否正在播放
      */
     setPlaying(playing: boolean) {
+        if (playing && !this.playback.active) return
         this.playing = playing
+        if (playing) this.resumeMedia()
     }
 
     /**
@@ -177,9 +180,21 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
      * @param params.container - DOM 容器元素
      * @param params.configs - 播放器配置
      */
+    private resumeGeneration = 0
+    private resuming: number | undefined
+    private disposers: (() => void)[] = []
+    readonly playback: PlaybackLifecycle
+
     constructor({five, container, configs}: { five: Five, container: Element, configs: PlayerConfigs }) {
         super()
 
+        this.playback = new PlaybackLifecycle(configs.mediaManager, () => {
+            this.resuming = undefined
+            ++this.resumeGeneration
+            this.setPlaying(false)
+            this.videoAgentScene?.videoAgentMesh.stop()
+            this.setWaitingForBgMusic(false)
+        })
         this.configs = configs
         this.container = container
         this.five = five
@@ -212,7 +227,7 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
             setWaitingForBgMusic: action,
         })
 
-        reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>(
+        this.disposers.push(reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>(
             () => [this.appearance.waveStyle, this.loading], 
             ([waveStyle, loading]) => {
                 if (loading === null) {
@@ -240,10 +255,10 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
                 }
             }, 
             { fireImmediately: true }
-        )
+        ))
 
         if (!this.appSize) {
-            reaction(
+            this.disposers.push(reaction(
                 () => this.containerSize,
                 (containerSize) => {
                     if (!containerSize?.width) return
@@ -263,29 +278,29 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
                     setElementDataset(this.container, { orientation })
                 },
                 { fireImmediately: true }
-            )
+            ))
         } else {
             setElementDataset(this.container, { size: this.appSize })
         }
 
         // 监听播放情况：抛出触发时机
-        reaction(
+        this.disposers.push(reaction(
             () => this.ended,
             (ended) => {
                 if (ended) {
                     this.emit('paused', true)
                 }
             }
-        )
+        ))
 
-        reaction(
+        this.disposers.push(reaction(
             () => this.playing,
             (playing) => {
                 if (!this.ended) {
                     this.emit(playing ? 'playing' : 'paused')
                 }
             }
-        )
+        ))
 
     }
 
@@ -344,43 +359,57 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
         // 如果正在等待背景音乐加载，暂停整个播放流程
         if (this.waitingForBgMusic) {
             if (!this.mediaInstance?.paused) {
-                this.mediaInstance?.pause()
+                this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause()
             }
             return
         }
 
         if (this.mediaInstance?.ended && this.mediaInstance.currentTime !== 0) {
             if (this.ended) return
             this.vreoUnit?.keyframes.forEach((keyframe) => (keyframe.parsed = false))
+            const media = this.videoAgentScene?.videoAgentMesh.mediaOperations
+            media?.pause()
+            if (media) media.currentTime = 0
+            this.playback.finish()
             this.setEnded(true)
             this.setPlaying(false)
-            this.mediaInstance.pause()
-            this.mediaInstance.currentTime = 0
             return
         }
 
         if (!this.playing) {
             if (!this.mediaInstance?.paused) {
-                this.mediaInstance?.pause()
+                this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause()
             }
             return
         }
 
-        if (this.mediaInstance?.paused && this.playing) {
-            this.mediaInstance.play()
-        }
+        this.resumeMedia()
 
         const currentKeyframes = this.currentKeyframes
+        const run = this.playback.capture()
         currentKeyframes.forEach((keyframe) => {
-            if (keyframe.parsed) return
+            if (!run?.valid() || keyframe.parsed) return
             keyframe.parsed = true
             this.emit(keyframe.type, keyframe, this.currentTime)
-            if (callback) {
+            if (callback && run.valid()) {
                 callback(keyframe.type, keyframe, this.currentTime)
             }
         })
     }
 
+    private resumeMedia() {
+        if (this.waitingForBgMusic) return
+        if (this.mediaInstance?.paused && this.playing && !this.resuming) {
+            const attempt = ++this.resumeGeneration
+            this.resuming = attempt
+            const run = this.playback.capture()
+            const valid = () => !!run?.valid()
+            void this.videoAgentScene?.videoAgentMesh.play().catch(error => {
+                if (valid()) { this.playback.cancel(); console.error(error) }
+            }).finally(() => { if (this.resuming === attempt) this.resuming = undefined })
+        }
+    }
+
     /**
      * 开始运行播放器逻辑循环
      * 
@@ -407,8 +436,8 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
          */
         this.vreoUnit = undefined
         if (this.mediaInstance) {
-            this.mediaInstance.pause()
-            this.mediaInstance.currentTime = 0
+            this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause()
+            if (this.videoAgentScene?.videoAgentMesh.mediaOperations) this.videoAgentScene.videoAgentMesh.mediaOperations.currentTime = 0
         }
 
         this.stopInterval?.()
@@ -421,6 +450,8 @@ export class Controller extends Subscribe<VreoKeyframeEvent> {
      * 清理所有状态和资源，释放内存
      */
     dispose() {
+        this.disposers.splice(0).forEach(dispose => dispose())
+        this.playback.dispose()
         this.clear()
     }
 }

packages/vreo/resources/Player/PlaybackLifecycle.ts

@@ -0,0 +1,59 @@
+import type { MediaOperations, PlaybackController, PlaybackManager, PlaybackSession } from './playback-types'
+
+export interface PlaybackRun {
+  readonly session?: PlaybackSession
+  readonly mediaManager?: PlaybackManager
+  valid(): boolean
+  bind(element: HTMLMediaElement): MediaOperations
+}
+
+/** Local timeline lifetime only. Playback permission is owned by the injected manager. */
+export class PlaybackLifecycle {
+  private controller?: PlaybackController
+  private run?: PlaybackRun
+  private generation = 0
+  private disposed = false
+  private members = new Set<() => void>()
+  constructor(manager: PlaybackManager | undefined, private onStop: () => void) {
+    this.controller = manager?.createController({ label: 'vreo', onCancel: () => this.stop() })
+  }
+  get active() { return !!this.run?.valid() }
+  capture() { return this.run }
+  begin(userAction: boolean) {
+    if (this.disposed) return false
+    if (this.active) return true
+    const session = this.controller?.begin({ userAction, audible: true }) ?? undefined
+    if (this.controller && !session) return false
+    const generation = ++this.generation
+    const mediaManager = session?.mediaManager
+    this.run = {
+      session, mediaManager,
+      valid: () => generation === this.generation && !this.disposed && !session?.signal.aborted,
+      bind: element => session ? session.bind(element) : element,
+    }
+    return true
+  }
+  add(stop: () => void) { this.members.add(stop); return () => { this.members.delete(stop) } }
+  private stop() {
+    ++this.generation
+    this.run = undefined
+    const errors: unknown[] = []
+    try { this.onStop() } catch (error) { errors.push(error) }
+    for (const stop of [...this.members]) { try { stop() } catch (error) { errors.push(error) } }
+    if (errors.length) throw errors[0]
+  }
+  cancel() {
+    const generation = this.generation
+    this.controller?.cancel()
+    if (generation === this.generation) this.stop()
+  }
+  finish() { const session = this.run?.session; this.stop(); session?.finish() }
+  dispose() {
+    if (this.disposed) return
+    this.disposed = true
+    const generation = this.generation
+    this.controller?.dispose()
+    if (generation === this.generation) this.stop()
+    this.members.clear()
+  }
+}