From 35c7fd978445c62b160c90be1556fcbf71b9ee68 Mon Sep 17 00:00:00 2001 From: lichengjie003 Date: Wed, 9 Sep 2026 14:00:06 +0800 Subject: [PATCH 1/4] feat: coordinate narration audio focus across media tracks --- packages/vreo/package.json | 9 +- packages/vreo/resources/Player/App.tsx | 4 +- packages/vreo/resources/Player/AudioFocus.ts | 67 +++++++ packages/vreo/resources/Player/Controller.ts | 50 +++-- packages/vreo/resources/Player/index.tsx | 56 ++++-- .../modules/VideoAgent/VideoAgentMesh.ts | 178 +++++++----------- .../modules/VideoAgent/VideoAgentScene.ts | 6 +- .../Player/modules/VideoAgent/index.tsx | 3 +- .../modules/keyframes/BgMusic/index.tsx | 99 ++++------ .../modules/keyframes/InfoPanel/index.tsx | 48 ++--- .../modules/keyframes/ModelVideo/index.tsx | 19 +- .../modules/keyframes/VideoEffect/index.tsx | 63 +++---- packages/vreo/resources/Player/typings.ts | 3 + .../fivePlugins/ModelTVVideoPlugin/index.ts | 61 ++++-- packages/vreo/resources/index.ts | 3 +- packages/vreo/resources/shared-utils/Audio.ts | 1 + .../vreo/resources/shared-utils/AudioLike.ts | 1 + packages/vreo/test/audio-focus.test.mjs | 152 +++++++++++++++ pnpm-lock.yaml | 8 +- 19 files changed, 546 insertions(+), 285 deletions(-) create mode 100644 packages/vreo/resources/Player/AudioFocus.ts create mode 100644 packages/vreo/test/audio-focus.test.mjs diff --git a/packages/vreo/package.json b/packages/vreo/package.json index 0b2a3109..ab0ad09d 100644 --- a/packages/vreo/package.json +++ b/packages/vreo/package.json @@ -1,6 +1,6 @@ { "name": "@realsee/vreo", - "version": "2.6.4-alpha.1", + "version": "2.6.4-alpha.2", "type": "module", "description": "Vreo (VR Video 缩写) 是基于如视三维渲染引擎 Five 和 用户界面构建库 React 实现的如视 3D 空间剧本播放器。", "keywords": [ @@ -75,10 +75,11 @@ "preview": "vite preview", "docs": "npx typedoc", "packages": "node ./dev-tools/build-packages.js", - "postpublish": "cnpm sync @realsee/vreo" + "postpublish": "cnpm sync @realsee/vreo", + "test:audio-focus": "node --test test/audio-focus.test.mjs" }, "peerDependencies": { - "@realsee/dnalogel": "3.67.0", + "@realsee/dnalogel": "3.81.10-alpha.0", "@realsee/five": "^6.4.0-alpha.1", "react": "^18.0.0", "react-dom": "^18.0.0", @@ -91,7 +92,7 @@ "@babel/preset-env": "^7.16.4", "@babel/preset-react": "^7.16.0", "@babel/preset-typescript": "^7.16.0", - "@realsee/dnalogel": "^3.67.2", + "@realsee/dnalogel": "3.81.10-alpha.0", "@realsee/five": "^6.4.0-alpha.38", "@tweenjs/tween.js": "18.6.4", "@types/node": "^18.11.17", diff --git a/packages/vreo/resources/Player/App.tsx b/packages/vreo/resources/Player/App.tsx index 48f6f0b1..25847f8c 100644 --- a/packages/vreo/resources/Player/App.tsx +++ b/packages/vreo/resources/Player/App.tsx @@ -34,9 +34,9 @@ const AppView = observer(({ controller }: { controller: Controller }) => { return } if (controller.playing) { - controller.setPlaying(false) + controller.audioFocus.cancel('paused') } else { - controller.setPlaying(true) + if (controller.audioFocus.acquire('user')) controller.setPlaying(true) } }} options={controller.configs?.videoAgentMeshOptions || {}} diff --git a/packages/vreo/resources/Player/AudioFocus.ts b/packages/vreo/resources/Player/AudioFocus.ts new file mode 100644 index 00000000..1b3a72f3 --- /dev/null +++ b/packages/vreo/resources/Player/AudioFocus.ts @@ -0,0 +1,67 @@ +export type AudioIntent = 'user' | 'auto' +export type AudioStopReason = 'preempted' | 'paused' | 'closed' | 'replaced' | 'disposed' +export interface AudioLease { readonly signal: AbortSignal; isCurrent(): boolean; release(): void } +export interface AudioSource { acquire(intent: AudioIntent): AudioLease | null; cancel(reason: AudioStopReason): void; dispose(): void } +export interface AudioFocusHost { register(options: { label: string; stop(reason: AudioStopReason): void }): AudioSource } + +/** One narration owns the outer lease; member tracks never compete with it. */ +export class NarrationAudioFocus { + private disposed = false + private source?: AudioSource + private lease?: AudioLease + private task?: AbortController + private members = new Set<(reason: AudioStopReason) => void>() + constructor(host: AudioFocusHost | undefined, private onStop: () => void) { + this.source = host?.register({ label: 'vreo', stop: reason => this.stop(reason) }) + } + get active() { return !!this.task && !this.task.signal.aborted && (!this.source || !!this.lease?.isCurrent()) } + capture() { const task = this.task; return () => !!task && task === this.task && this.active } + acquire(intent: AudioIntent) { + if (this.disposed) return false + if (this.active) return true + const lease = this.source?.acquire(intent) + if (this.source && !lease) return false + this.lease = lease ?? undefined + this.task = new AbortController() + return true + } + add(stop: (reason: AudioStopReason) => void) { this.members.add(stop); return () => { this.members.delete(stop) } } + stop(reason: AudioStopReason) { + this.task?.abort() + this.task = undefined + const errors: unknown[] = [] + try { this.onStop() } catch (error) { errors.push(error) } + for (const stop of [...this.members]) { + try { stop(reason) } catch (error) { errors.push(error) } + } + if (errors.length) throw errors[0] + } + cancel(reason: AudioStopReason) { this.stop(reason); this.source?.cancel(reason); this.lease = undefined } + dispose() { + if (this.disposed) return + this.disposed = true + this.cancel('disposed') + this.source?.dispose() + this.members.clear() + } + readonly host: AudioFocusHost = { + register: ({ stop }) => { + let disposed = false + let child: AbortController | undefined + const cancel = (reason: AudioStopReason) => { child?.abort(); child = undefined; stop(reason) } + const remove = this.add(cancel) + return { + acquire: () => { + if (disposed || !this.active) return null + child?.abort() + const current = child = new AbortController() + const valid = this.capture() + return { signal: current.signal, isCurrent: () => valid() && child === current && !current.signal.aborted, + release: () => { if (child === current) { current.abort(); child = undefined } } } + }, + cancel, + dispose: () => { if (disposed) return; disposed = true; cancel('disposed'); remove() }, + } + }, + } +} diff --git a/packages/vreo/resources/Player/Controller.ts b/packages/vreo/resources/Player/Controller.ts index 622bc259..0d3b3be1 100644 --- a/packages/vreo/resources/Player/Controller.ts +++ b/packages/vreo/resources/Player/Controller.ts @@ -1,3 +1,4 @@ +import { NarrationAudioFocus } from './AudioFocus' import { Five, Subscribe } from '@realsee/five' import { action, computed, makeObservable, observable, reaction } from 'mobx' import * as React from 'react' @@ -127,7 +128,9 @@ export class Controller extends Subscribe { * @param playing - 是否正在播放 */ setPlaying(playing: boolean) { + if (playing && !this.audioFocus.active) return this.playing = playing + if (playing) this.resumeMedia() } /** @@ -177,9 +180,21 @@ export class Controller extends Subscribe { * @param params.container - DOM 容器元素 * @param params.configs - 播放器配置 */ + private resumeGeneration = 0 + private resuming: number | undefined + private disposers: (() => void)[] = [] + readonly audioFocus: NarrationAudioFocus + constructor({five, container, configs}: { five: Five, container: Element, configs: PlayerConfigs }) { super() + this.audioFocus = new NarrationAudioFocus(configs.audioFocus, () => { + this.resuming = undefined + ++this.resumeGeneration + this.setPlaying(false) + this.videoAgentScene?.videoAgentMesh.stop() + this.setWaitingForBgMusic(false) + }) this.configs = configs this.container = container this.five = five @@ -212,7 +227,7 @@ export class Controller extends Subscribe { setWaitingForBgMusic: action, }) - reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>( + this.disposers.push(reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>( () => [this.appearance.waveStyle, this.loading], ([waveStyle, loading]) => { if (loading === null) { @@ -240,10 +255,10 @@ export class Controller extends Subscribe { } }, { fireImmediately: true } - ) + )) if (!this.appSize) { - reaction( + this.disposers.push(reaction( () => this.containerSize, (containerSize) => { if (!containerSize?.width) return @@ -263,29 +278,29 @@ export class Controller extends Subscribe { setElementDataset(this.container, { orientation }) }, { fireImmediately: true } - ) + )) } else { setElementDataset(this.container, { size: this.appSize }) } // 监听播放情况：抛出触发时机 - reaction( + this.disposers.push(reaction( () => this.ended, (ended) => { if (ended) { this.emit('paused', true) } } - ) + )) - reaction( + this.disposers.push(reaction( () => this.playing, (playing) => { if (!this.ended) { this.emit(playing ? 'playing' : 'paused') } } - ) + )) } @@ -352,6 +367,7 @@ export class Controller extends Subscribe { if (this.mediaInstance?.ended && this.mediaInstance.currentTime !== 0) { if (this.ended) return this.vreoUnit?.keyframes.forEach((keyframe) => (keyframe.parsed = false)) + this.audioFocus.cancel('paused') this.setEnded(true) this.setPlaying(false) this.mediaInstance.pause() @@ -366,9 +382,7 @@ export class Controller extends Subscribe { return } - if (this.mediaInstance?.paused && this.playing) { - this.mediaInstance.play() - } + this.resumeMedia() const currentKeyframes = this.currentKeyframes currentKeyframes.forEach((keyframe) => { @@ -381,6 +395,18 @@ export class Controller extends Subscribe { }) } + private resumeMedia() { + if (this.waitingForBgMusic) return + if (this.mediaInstance?.paused && this.playing && !this.resuming) { + const attempt = ++this.resumeGeneration + this.resuming = attempt + const valid = this.audioFocus.capture() + void this.videoAgentScene?.videoAgentMesh.play().catch(error => { + if (valid()) { this.audioFocus.cancel('paused'); console.error(error) } + }).finally(() => { if (this.resuming === attempt) this.resuming = undefined }) + } + } + /** * 开始运行播放器逻辑循环 * @@ -421,6 +447,8 @@ export class Controller extends Subscribe { * 清理所有状态和资源，释放内存 */ dispose() { + this.disposers.splice(0).forEach(dispose => dispose()) + this.audioFocus.dispose() this.clear() } } diff --git a/packages/vreo/resources/Player/index.tsx b/packages/vreo/resources/Player/index.tsx index a6facc02..71c3699b 100644 --- a/packages/vreo/resources/Player/index.tsx +++ b/packages/vreo/resources/Player/index.tsx @@ -1,6 +1,7 @@ // 下面这一行不能删 import * as React from 'react' -import * as ReactDOM from 'react-dom' +import type { Root } from 'react-dom/client' +import type { AudioIntent } from './AudioFocus' import { createRoot } from 'react-dom/client' import { Five, Subscribe } from '@realsee/five' @@ -49,6 +50,11 @@ export class Player extends Subscribe { $five: Five /** 内部控制器 */ private controller: Controller + private root: Root + private disposers: (() => void)[] = [] + private loadGeneration = 0 + private disposed = false + get audioFocus() { return this.controller.audioFocus.host } /** 播放器配置（只读） */ configs: Readonly @@ -90,8 +96,8 @@ export class Player extends Subscribe { this.controller = new Controller({five, container:configs.container, configs: this.configs}) - const root = createRoot(configs.container) - root.render( + this.root = createRoot(configs.container) + this.root.render( @@ -106,29 +112,30 @@ export class Player extends Subscribe { off: (name, callback) => this.off(name as any, callback as any), }} five={five} + audioFocus={this.audioFocus} /> ))} ) // 监听播放情况：抛出触发时机 - reaction( + this.disposers.push(reaction( () => this.controller.ended, (ended) => { if (ended) { this.emit('paused', true) } } - ) + )) - reaction( + this.disposers.push(reaction( () => this.controller.playing, (playing) => { if (!this.controller.ended) { this.emit(playing ? 'playing' : 'paused') } } - ) + )) } /** @@ -154,7 +161,12 @@ export class Player extends Subscribe { * await player.load(vreoUnit, 0, false, true) * ``` */ - async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false) { + async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false, intent: AudioIntent = 'auto') { + const generation = ++this.loadGeneration + this.controller.audioFocus.cancel('replaced') + if (this.disposed || !this.controller.audioFocus.acquire(intent)) return false + const valid = this.controller.audioFocus.capture() + try { this.controller.clear() this.controller.setLoading(true) if (force) { @@ -214,6 +226,7 @@ export class Player extends Subscribe { const panoIndexes = Object.keys(panoIndexMap) for (let i = 0; i < panoIndexes.length; i++) { await this.$five.preloadPano(Number(panoIndexes[i])) + if (!valid() || generation !== this.loadGeneration) return false } } @@ -229,6 +242,7 @@ export class Player extends Subscribe { await waitForBlankAudioGenerated() + if (!valid() || generation !== this.loadGeneration) return false await this.controller.videoAgentScene?.videoAgentMesh.play( vreoUnit.video.url, currentTime / 1000, @@ -236,12 +250,20 @@ export class Player extends Subscribe { ) + if (!valid() || generation !== this.loadGeneration) return false this.controller.setEnded(false) - this.play() + this.play(undefined, intent) this.controller.run((type, keyframe) => this.emit(type, keyframe, this.controller.currentTime)) this.controller.setLoading(false) return true + } catch (error) { + if (!valid()) return false + this.controller.audioFocus.cancel('paused') + throw error + } finally { + if (generation === this.loadGeneration) this.controller.setLoading(false) + } } /** @@ -267,7 +289,8 @@ export class Player extends Subscribe { * player.play(10000) // 从10秒处开始 * ``` */ - play(currentTime?: number) { + play(currentTime?: number, intent: AudioIntent = 'auto') { + if (this.disposed || !this.controller.audioFocus.acquire(intent)) return false if (this.controller.playing) return true if (currentTime && this.controller.mediaInstance) { this.controller.mediaInstance.currentTime = currentTime / 1000 @@ -302,7 +325,9 @@ export class Player extends Subscribe { * 暂停播放 */ pause() { - this.controller.setPlaying(false) + ++this.loadGeneration + this.controller.audioFocus.cancel('paused') + this.controller.setLoading(false) } /** @@ -333,11 +358,12 @@ export class Player extends Subscribe { * 清理所有资源、事件监听器和DOM元素 */ dispose() { + if (this.disposed) return + this.disposed = true + ++this.loadGeneration + this.disposers.forEach(dispose => dispose()) this.controller.dispose() - - if (this.configs.container) { - ReactDOM.unmountComponentAtNode(this.configs.container as Element) - } + this.root.unmount() } } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts index d0d26b2f..3b0969b4 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts +++ b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts @@ -59,15 +59,11 @@ void main(void) { } ` -const cacheInstance: { - videoInstance?: HTMLVideoElement - audioInstance?: HTMLAudioElement -} = {} - /** * 视频经纪人贴片的配置选项 */ export interface VideoAgentMeshOptions { + canPlay?: () => boolean /** * 自定义视频实例。 */ @@ -123,6 +119,11 @@ export class VideoAgentMesh extends THREE.Mesh { /** 是否暂停状态 */ paused: boolean /** 音频实例 */ + private ownsVideo = false + private ownsAudio = false + private generation = 0 + private removeStart?: () => void + private objectURL?: string audioInstance: HTMLAudioElement /** AudioLike 实例 */ audioLikeInstance: AudioLike @@ -180,20 +181,14 @@ export class VideoAgentMesh extends THREE.Mesh { preload: true }, ) { + const ownsVideo = !options.videoInstance if (!options.videoInstance) { - if (cacheInstance.videoInstance) { - options.videoInstance = cacheInstance.videoInstance - } else { - const videoInstance = document.createElement('video') - videoInstance.style.opacity = '0' - videoInstance.style.pointerEvents = 'none' - videoInstance.style.display = 'none' - document.body.append(videoInstance) - options.videoInstance = videoInstance - cacheInstance.videoInstance = videoInstance - videoInstance.playsInline = true - videoInstance.controls = false - } + const videoInstance = document.createElement('video') + videoInstance.style.display = 'none' + videoInstance.playsInline = true + videoInstance.muted = true + document.body.append(videoInstance) + options.videoInstance = videoInstance } const geometry = new THREE.PlaneGeometry(width, height, widthSegments, heightSegments) @@ -209,24 +204,17 @@ export class VideoAgentMesh extends THREE.Mesh { }) super(geometry, material) + this.ownsVideo = ownsVideo + this.ownsAudio = !options.audioInstance this.options = options this.freeze = false this.paused = true if (!options.audioInstance) { - if (cacheInstance.audioInstance) { - this.audioInstance = cacheInstance.audioInstance - } else { - const audioInstance = document.createElement('audio') - audioInstance.crossOrigin = '' - // videoInstance.muted = true - audioInstance.muted = false - audioInstance.autoplay = false - audioInstance.style.display = 'none' - document.body.appendChild(audioInstance) - this.audioInstance = audioInstance - cacheInstance.audioInstance = audioInstance - } + this.audioInstance = document.createElement('audio') + this.audioInstance.crossOrigin = '' + this.audioInstance.muted = true + this.audioInstance.autoplay = false } else { this.audioInstance = options.audioInstance } @@ -263,41 +251,29 @@ export class VideoAgentMesh extends THREE.Mesh { * @param videoUrl - 媒体文件URL * @private */ - private async update(videoUrl: string) { - - if (this.videoUrl === videoUrl) { - return - } - - this.videoUrl = videoUrl - - // // 兼容非视频场景 - // if (this.mediaInstance instanceof HTMLAudioElement) { - // this.mediaInstance.style.display = 'none' - // } else if (this.mediaInstance instanceof HTMLVideoElement) { - // this.mediaInstance.style.display = 'block' - // } - + private async update(videoUrl: string, valid: () => boolean) { + if (this.videoUrl === videoUrl) return + this.stop() this.freeze = true - await this.mediaInstance.pause() - - const uniforms = (this.material as THREE.ShaderMaterial).uniforms - this.mediaInstance.muted = true - - this.mediaInstance.src = (this.options.preload || this.options.preload === undefined || getMediaType(this.videoUrl) === 'video') ? - await URL.createObjectURL((await Preloader.blob(this.videoUrl)) as unknown as Blob) : this.videoUrl - this.mediaInstance.setAttribute('data-src', this.videoUrl) - + const media = getMediaType(videoUrl) === 'audio' ? this.audioInstance : this.options.videoInstance! + media.muted = true + const src = (this.options.preload !== false || getMediaType(videoUrl) === 'video') + ? URL.createObjectURL(await Preloader.blob(videoUrl) as unknown as Blob) : videoUrl + if (!valid()) { if (src !== videoUrl) URL.revokeObjectURL(src); return } + this.videoUrl = videoUrl + if (this.objectURL) URL.revokeObjectURL(this.objectURL) + this.objectURL = src !== videoUrl ? src : undefined + media.src = src + media.setAttribute('data-src', videoUrl) const onStart = () => { - if (this.mediaInstance.currentTime === 0) return + if (!valid() || media.currentTime === 0) return this.freeze = false - this.mediaInstance.muted = false - uniforms.enable.value = getMediaType(this.videoUrl) ? 1 : 0 - this.mediaInstance.removeEventListener('timeupdate', onStart, false) - + media.muted = false + ;(this.material as THREE.ShaderMaterial).uniforms.enable.value = getMediaType(videoUrl) ? 1 : 0 + this.removeStart?.() } - - this.mediaInstance.addEventListener('timeupdate', onStart, false) + this.removeStart = () => media.removeEventListener('timeupdate', onStart) + media.addEventListener('timeupdate', onStart) } /** @@ -326,41 +302,36 @@ export class VideoAgentMesh extends THREE.Mesh { * ``` */ async play(videoUrl = '', currentTime = 0, duration?: number) { - videoUrl = videoUrl || '' - - if (duration && !videoUrl) { - if (this.currentTime) { - this.mediaInstance.currentTime = currentTime - } - (this.mediaInstance as AudioLike).duration = duration - this.videoUrl = '' - this.mediaInstance.play() - return true - } - - if (!videoUrl) { - if (this.videoUrl) await this.mediaInstance.play() - else console.warn('警告：视频资源未初始化。') - return true - } - - if (videoUrl === this.videoUrl) { - this.mediaInstance.currentTime = currentTime - await this.mediaInstance.play() - return true + if (this.options.canPlay && !this.options.canPlay()) return false + // stop() invalidates all earlier loads before the new generation is captured. + if (videoUrl && videoUrl !== this.videoUrl) this.stop() + let generation = this.generation + const valid = () => generation === this.generation && (!this.options.canPlay || this.options.canPlay()) + if (videoUrl && videoUrl !== this.videoUrl) { + // update's synchronous stop is accounted for before awaiting its load. + generation++ + await this.update(videoUrl, valid) } + if (!valid()) return false + if (duration && !videoUrl) { this.videoUrl = ''; this.audioLikeInstance.duration = duration } + const media = this.mediaInstance + if (videoUrl || duration) media.currentTime = currentTime + media.muted = true + await media.play() + if (!valid()) return false + media.muted = false + return true + } - await this.update(videoUrl) - - this.mediaInstance.pause() - - return await new Promise((resolve) => - setTimeout(async () => { - this.mediaInstance.currentTime = currentTime - await this.mediaInstance.play() - resolve(true) - }, 20), - ) + stop() { + ++this.generation + this.removeStart?.() + this.removeStart = undefined + this.audioInstance.muted = true + this.audioInstance.pause() + this.options.videoInstance!.muted = true + this.options.videoInstance!.pause() + this.audioLikeInstance.pause() } /** @@ -385,16 +356,13 @@ export class VideoAgentMesh extends THREE.Mesh { * ``` */ dispose() { - // 销毁事件监听 + this.stop() this.$removeEventListener() - if (cacheInstance.audioInstance) { - document.body.removeChild(cacheInstance.audioInstance) - cacheInstance.audioInstance = undefined - } - - if (cacheInstance.videoInstance) { - document.body.removeChild(cacheInstance.videoInstance) - cacheInstance.videoInstance = undefined - } + if (this.ownsVideo) this.options.videoInstance?.remove() + if (this.ownsAudio) this.audioInstance.remove() + if (this.objectURL) URL.revokeObjectURL(this.objectURL) + this.geometry.dispose() + ;(this.material as THREE.ShaderMaterial).uniforms.map.value.dispose() + ;(this.material as THREE.ShaderMaterial).dispose() } } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts index 2d907612..9dbae9fe 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts +++ b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts @@ -9,6 +9,7 @@ export class VideoAgentScene { scene = new THREE.Scene() camera = new THREE.OrthographicCamera(-240, 240, 135, -135) renderer = new THREE.WebGLRenderer({ alpha: true }) + private frame = 0 disposers: (() => void)[] = [] constructor(container?: HTMLElement, options?: VideoAgentMeshOptions) { @@ -46,10 +47,13 @@ export class VideoAgentScene { this.renderer.render(this.scene, this.camera) } - requestAnimationFrame(this.run) + this.frame = requestAnimationFrame(this.run) } dispose = () => { + cancelAnimationFrame(this.frame) + this.renderer.dispose() + this.renderer.domElement.remove() this.videoAgentMesh?.dispose() this.disposers.forEach(disposer => disposer?.()) } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/index.tsx b/packages/vreo/resources/Player/modules/VideoAgent/index.tsx index 4b945f98..717d76ab 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/index.tsx +++ b/packages/vreo/resources/Player/modules/VideoAgent/index.tsx @@ -23,12 +23,13 @@ export function VideoAgent(props: { onClick?: () => void; options?: VideoAgentMe console.warn('VideoAgentScene" 重复初始化，已被过滤') return } - const videoAgentScene = new VideoAgentScene(ref.current, props.options) + const videoAgentScene = new VideoAgentScene(ref.current, { ...props.options, canPlay: () => controller.audioFocus.active }) controller.videoAgentScene = videoAgentScene return () => { controller.dispose() controller.videoAgentScene?.dispose() + controller.videoAgentScene = undefined } }, []) diff --git a/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx b/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx index 1d3e5a76..27e882fd 100644 --- a/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx @@ -5,80 +5,45 @@ import { useController } from '../../../hooks' export function BgMusic() { const controller = useController() - React.useEffect(() => { - const callback = async (keyframe: VreoKeyframe, currentTime: number) => { - const { start, end } = keyframe - - const _currentTime = (currentTime - start) / 1000 - - if (_currentTime < 0 || _currentTime >= keyframe.end - keyframe.start) { - return - } - - const audio = getAudio(keyframe.data.url) - audio.currentTime = Math.max(0, _currentTime) - - const waitForLoaded = controller.configs?.waitForBgMusicLoaded ?? false - - const cleanAudio = () => { - audio.removeEventListener('canplay', playOnCanPlay) - audio.removeEventListener('pause', play) + const tracks = new Set<() => void>() + const callback = (keyframe: VreoKeyframe, currentTime: number) => { + const valid = controller.audioFocus.capture() + if (!valid()) return + const audio = getAudio() + let cleaned = false + const clean = () => { + if (cleaned) return + cleaned = true + audio.removeEventListener('canplay', play) + audio.removeEventListener('ended', clean) + audio.muted = true audio.pause() audio.src = '' + remove() + tracks.delete(clean) + controller.setWaitingForBgMusic(false) } - + const remove = controller.audioFocus.add(clean) + tracks.add(clean) const play = () => { - if (audio.realSrc === keyframe.data.url) { - // 有可能会被其他音轨打断 - audio.play() - } - } - // canplay 事件比 canplaythrough 更早触发，表示可以开始播放了 - const playOnCanPlay = () => { - audio.removeEventListener('canplay', playOnCanPlay) - audio.play() - // 解除等待状态，恢复主时间线播放 + audio.removeEventListener('canplay', play) + if (!valid() || cleaned) return + audio.muted = false + void audio.play().catch(error => { if (valid() && !cleaned) { clean(); controller.audioFocus.cancel('paused'); console.error(error) } }) controller.setWaitingForBgMusic(false) } - - if (waitForLoaded) { - // 等待音频加载完成后再播放 - // readyState: 0=HAVE_NOTHING, 1=HAVE_METADATA, 2=HAVE_CURRENT_DATA, 3=HAVE_FUTURE_DATA, 4=HAVE_ENOUGH_DATA - if (audio.readyState >= 3) { - // readyState >= 3 表示有足够数据开始播放 - audio.play() - } else { - // 设置等待状态，阻塞整个播放流程 - controller.setWaitingForBgMusic(true) - // 监听 canplay 事件（可以开始播放） - audio.addEventListener('canplay', playOnCanPlay) - } - } else { - // 默认行为：立即播放，边加载边播放 - audio.play() - } - - audio.addEventListener('ended', () => { - cleanAudio() - }) - - audio.addEventListener('pause', play) - - controller.once('paused', () => { - // 如果播放器暂停，也要解除等待状态 - controller.setWaitingForBgMusic(false) - cleanAudio() - }) - + audio.muted = true + audio.src = keyframe.data.url + audio.currentTime = Math.max(0, (currentTime - keyframe.start) / 1000) + audio.addEventListener('ended', clean) + if (controller.configs.waitForBgMusicLoaded && audio.readyState < 3) { + controller.setWaitingForBgMusic(true) + audio.addEventListener('canplay', play) + } else play() } - controller.on(VreoKeyframeEnum.BgMusic, callback) - - return () => { - controller.off(VreoKeyframeEnum.BgMusic, callback) - } - }) - + return () => { controller.off(VreoKeyframeEnum.BgMusic, callback); for (const clean of [...tracks]) clean() } + }, [controller]) return <> -} \ No newline at end of file +} diff --git a/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx b/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx index ab809f48..d9a1f39d 100644 --- a/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx @@ -10,22 +10,6 @@ import { import { useController } from '../../../hooks' -const isWX = navigator.userAgent.toLowerCase().indexOf('micromessenger') !== -1 -const isIOS = navigator.userAgent.toLowerCase().indexOf('iphone') !== -1 - -const isIOSorWX = isIOS || isWX - - -const _videoElement = document.createElement('video') -_videoElement.setAttribute('playsinline', 'true') -_videoElement.setAttribute('webkit-playsinline', 'true') - -if (isIOSorWX) { - if (_videoElement.paused) { - _videoElement.addEventListener('click', () => _videoElement.play(), {once:true}) - } -} - function InfoPanelImg({ url, children }: { url: string; children?: ReactNode }) { return (
@@ -43,8 +27,14 @@ function InfoPanelVideo({ url, children }: { url: string; children?: ReactNode } React.useEffect(() => { // if (!isIOSorWX) return - const video = controller.configs?.videos?.videoPanel || _videoElement + const video = controller.configs?.videos?.videoPanel || document.createElement('video') + const valid = controller.audioFocus.capture() + let disposed = false + video.playsInline = true + video.muted = true + const stop = () => { disposed = true; video.muted = true; video.pause(); video.removeEventListener('canplaythrough', canplaythrough) } if (!videoWrapperRef.current) return + const remove = controller.audioFocus.add(stop) video.src = url if (!videoWrapperRef.current.contains(video)) { videoWrapperRef.current.appendChild(video) @@ -52,21 +42,22 @@ function InfoPanelVideo({ url, children }: { url: string; children?: ReactNode } const canplaythrough = () => { video.removeEventListener('canplaythrough', canplaythrough) - try { - video.play() - } catch (error) {} + if (disposed || !valid()) return + video.muted = false + void video.play().catch(error => { if (valid() && !disposed) { stop(); controller.audioFocus.cancel('paused'); console.error(error) } }) } video.addEventListener('canplaythrough', canplaythrough) video.load() // video.play() return () => { - video.pause() + stop() + remove() if (videoWrapperRef.current?.contains(video)) { videoWrapperRef.current.removeChild(video) } } - }, [videoWrapperRef.current]) + }, [controller, url]) return (
@@ -161,15 +152,16 @@ export function InfoPanel() { } controller.on(VreoKeyframeEnum.InfoPanel, callback) - controller.on('ended', () => { + const close = () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current) + controller.openDrawer(false) controller.openPopUp(false) - }) - - controller.on('paused', () => { - controller.openPopUp(false) - }) + } + const remove = controller.audioFocus.add(close) return () => { + remove() + close() controller.off(VreoKeyframeEnum.InfoPanel, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) diff --git a/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx b/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx index 8a393a4f..1678f0be 100644 --- a/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx @@ -12,9 +12,16 @@ export function ModelVideo() { const timeoutRef = React.useRef() React.useEffect(() => { + let generation = 0 + const stop = () => { ++generation; if (timeoutRef.current) clearTimeout(timeoutRef.current); ref.current?.disable() } + const remove = controller.audioFocus.add(stop) const callback = async (keyframe: VreoKeyframe) => { + stop() + const current = generation + const valid = controller.audioFocus.capture() + if (!valid()) return if (!ref.current) { - ref.current = ModelTVVideoPlugin(five, {}) + ref.current = ModelTVVideoPlugin(five, { canPlay: () => controller.audioFocus.active }) } const { start, end } = keyframe const { videoSrc, videoPosterSrc, vertexs, matrixWorld } = keyframe.data as ModelVideoData @@ -51,6 +58,7 @@ export function ModelVideo() { })() ref.current.disable() + try { await ref.current.load( { video_src: videoSrc, @@ -60,12 +68,21 @@ export function ModelVideo() { controller.configs?.videos?.modelTVVideo ) + } catch (error) { + if (current === generation && valid()) { controller.audioFocus.cancel('paused'); console.error(error) } + return + } + if (current !== generation || !valid()) return ref.current.enable() timeoutRef.current = setTimeout(() => ref.current?.disable(), end - start) } controller.on(VreoKeyframeEnum.ModelVideo, callback) return () => { + stop() + remove() + ref.current?.dispose() + ref.current = undefined controller.off(VreoKeyframeEnum.ModelVideo, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) diff --git a/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx b/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx index 19f4283c..ce8bb78a 100644 --- a/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx @@ -4,18 +4,6 @@ import * as React from 'react' import { VideoEffectData, VreoKeyframe, VreoKeyframeEnum } from '../../../../typings/VreoUnit' import { useController, useFiveInstance } from '../../../hooks' -const inlinePlay = (videoInstance?: HTMLVideoElement | null) => { - if (!videoInstance) return - const canplaythrough = () => { - videoInstance.removeEventListener('canplaythrough', canplaythrough) - try { - videoInstance.play() - } catch (error) {} - } - videoInstance.addEventListener('canplaythrough', canplaythrough) - videoInstance.load() -} - // const emptyVideo = '//vr-static.realsee-cdn.cn/release/web/leisure.69fd3522.mov' const PI = Math.PI const PI_2 = PI * 2 @@ -35,7 +23,10 @@ export function VideoEffect() { } React.useEffect(() => { + let generation = 0 + let removeCanPlay: (() => void) | undefined const callback = async (keyframe: VreoKeyframe) => { + const current = ++generation const { start, end } = keyframe const { videoSrc, fov, direction, panoIndex, vector } = keyframe.data as VideoEffectData const [longitude, latitude] = (() => { @@ -52,34 +43,49 @@ export function VideoEffect() { return [longitude, latitude] })() + if (timeoutRef.current) clearTimeout(timeoutRef.current) + if (videoRef.current) { videoRef.current.muted = true; videoRef.current.pause() } five.setState({ fov, panoIndex, longitude, latitude }, true) setBlobSrc(videoSrc) - inlinePlay(videoRef.current) + removeCanPlay?.() + const active = controller.audioFocus.capture() + const valid = () => current === generation && active() + const video = videoRef.current + if (!video || !valid()) return + const play = () => { + removeCanPlay?.() + if (!valid()) return + video.muted = false + void video.play().catch(error => { if (valid()) { controller.audioFocus.cancel('paused'); console.error(error) } }) + } + removeCanPlay = () => video.removeEventListener('canplaythrough', play) + video.addEventListener('canplaythrough', play) + video.load() setVisible(true) - timeoutRef.current = setTimeout(() => { - videoRef.current?.pause() - setVisible(false) - setBlobSrc('') - }, end - start) + timeoutRef.current = setTimeout(destroy, Math.max(0, end - start)) } controller.on(VreoKeyframeEnum.VideoEffect, callback) const destroy = () => { + ++generation if (timeoutRef.current) { clearTimeout(timeoutRef.current) } + removeCanPlay?.() + if (videoRef.current) videoRef.current.muted = true videoRef.current?.pause() setVisible(false) setBlobSrc('') } - controller.on('paused', () => destroy()) - controller.on('ended', () => destroy()) + const remove = controller.audioFocus.add(destroy) return () => { + destroy() + remove() controller.off(VreoKeyframeEnum.VideoEffect, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) @@ -93,26 +99,15 @@ export function VideoEffect() { //