diff --git a/packages/vreo/package.json b/packages/vreo/package.json index 0b2a3109..65d9c008 100644 --- a/packages/vreo/package.json +++ b/packages/vreo/package.json @@ -1,6 +1,6 @@ { "name": "@realsee/vreo", - "version": "2.6.4-alpha.1", + "version": "2.6.4-alpha.5", "type": "module", "description": "Vreo (VR Video 缩写) 是基于如视三维渲染引擎 Five 和 用户界面构建库 React 实现的如视 3D 空间剧本播放器。", "keywords": [ @@ -75,10 +75,11 @@ "preview": "vite preview", "docs": "npx typedoc", "packages": "node ./dev-tools/build-packages.js", - "postpublish": "cnpm sync @realsee/vreo" + "postpublish": "cnpm sync @realsee/vreo", + "test:audio-focus": "node --test test/audio-focus.test.mjs" }, "peerDependencies": { - "@realsee/dnalogel": "3.67.0", + "@realsee/dnalogel": "3.81.10-alpha.4", "@realsee/five": "^6.4.0-alpha.1", "react": "^18.0.0", "react-dom": "^18.0.0", @@ -91,7 +92,7 @@ "@babel/preset-env": "^7.16.4", "@babel/preset-react": "^7.16.0", "@babel/preset-typescript": "^7.16.0", - "@realsee/dnalogel": "^3.67.2", + "@realsee/dnalogel": "3.81.10-alpha.4", "@realsee/five": "^6.4.0-alpha.38", "@tweenjs/tween.js": "18.6.4", "@types/node": "^18.11.17", diff --git a/packages/vreo/resources/Player/App.tsx b/packages/vreo/resources/Player/App.tsx index 48f6f0b1..fa38abb6 100644 --- a/packages/vreo/resources/Player/App.tsx +++ b/packages/vreo/resources/Player/App.tsx @@ -34,9 +34,9 @@ const AppView = observer(({ controller }: { controller: Controller }) => { return } if (controller.playing) { - controller.setPlaying(false) + controller.playback.cancel() } else { - controller.setPlaying(true) + if (controller.playback.begin(true)) controller.setPlaying(true) } }} options={controller.configs?.videoAgentMeshOptions || {}} diff --git a/packages/vreo/resources/Player/Controller.ts b/packages/vreo/resources/Player/Controller.ts index 622bc259..af28b2a6 100644 --- a/packages/vreo/resources/Player/Controller.ts +++ b/packages/vreo/resources/Player/Controller.ts @@ -1,3 +1,4 @@ +import { PlaybackLifecycle } from './PlaybackLifecycle' import { Five, Subscribe } from '@realsee/five' import { action, computed, makeObservable, observable, reaction } from 'mobx' import * as React from 'react' @@ -127,7 +128,9 @@ export class Controller extends Subscribe { * @param playing - 是否正在播放 */ setPlaying(playing: boolean) { + if (playing && !this.playback.active) return this.playing = playing + if (playing) this.resumeMedia() } /** @@ -177,9 +180,21 @@ export class Controller extends Subscribe { * @param params.container - DOM 容器元素 * @param params.configs - 播放器配置 */ + private resumeGeneration = 0 + private resuming: number | undefined + private disposers: (() => void)[] = [] + readonly playback: PlaybackLifecycle + constructor({five, container, configs}: { five: Five, container: Element, configs: PlayerConfigs }) { super() + this.playback = new PlaybackLifecycle(configs.mediaManager, () => { + this.resuming = undefined + ++this.resumeGeneration + this.setPlaying(false) + this.videoAgentScene?.videoAgentMesh.stop() + this.setWaitingForBgMusic(false) + }) this.configs = configs this.container = container this.five = five @@ -212,7 +227,7 @@ export class Controller extends Subscribe { setWaitingForBgMusic: action, }) - reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>( + this.disposers.push(reaction<[typeof this.appearance.waveStyle, boolean | null], boolean>( () => [this.appearance.waveStyle, this.loading], ([waveStyle, loading]) => { if (loading === null) { @@ -240,10 +255,10 @@ export class Controller extends Subscribe { } }, { fireImmediately: true } - ) + )) if (!this.appSize) { - reaction( + this.disposers.push(reaction( () => this.containerSize, (containerSize) => { if (!containerSize?.width) return @@ -263,29 +278,29 @@ export class Controller extends Subscribe { setElementDataset(this.container, { orientation }) }, { fireImmediately: true } - ) + )) } else { setElementDataset(this.container, { size: this.appSize }) } // 监听播放情况：抛出触发时机 - reaction( + this.disposers.push(reaction( () => this.ended, (ended) => { if (ended) { this.emit('paused', true) } } - ) + )) - reaction( + this.disposers.push(reaction( () => this.playing, (playing) => { if (!this.ended) { this.emit(playing ? 'playing' : 'paused') } } - ) + )) } @@ -344,7 +359,7 @@ export class Controller extends Subscribe { // 如果正在等待背景音乐加载，暂停整个播放流程 if (this.waitingForBgMusic) { if (!this.mediaInstance?.paused) { - this.mediaInstance?.pause() + this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause() } return } @@ -352,35 +367,49 @@ export class Controller extends Subscribe { if (this.mediaInstance?.ended && this.mediaInstance.currentTime !== 0) { if (this.ended) return this.vreoUnit?.keyframes.forEach((keyframe) => (keyframe.parsed = false)) + const media = this.videoAgentScene?.videoAgentMesh.mediaOperations + media?.pause() + if (media) media.currentTime = 0 + this.playback.finish() this.setEnded(true) this.setPlaying(false) - this.mediaInstance.pause() - this.mediaInstance.currentTime = 0 return } if (!this.playing) { if (!this.mediaInstance?.paused) { - this.mediaInstance?.pause() + this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause() } return } - if (this.mediaInstance?.paused && this.playing) { - this.mediaInstance.play() - } + this.resumeMedia() const currentKeyframes = this.currentKeyframes + const run = this.playback.capture() currentKeyframes.forEach((keyframe) => { - if (keyframe.parsed) return + if (!run?.valid() || keyframe.parsed) return keyframe.parsed = true this.emit(keyframe.type, keyframe, this.currentTime) - if (callback) { + if (callback && run.valid()) { callback(keyframe.type, keyframe, this.currentTime) } }) } + private resumeMedia() { + if (this.waitingForBgMusic) return + if (this.mediaInstance?.paused && this.playing && !this.resuming) { + const attempt = ++this.resumeGeneration + this.resuming = attempt + const run = this.playback.capture() + const valid = () => !!run?.valid() + void this.videoAgentScene?.videoAgentMesh.play().catch(error => { + if (valid()) { this.playback.cancel(); console.error(error) } + }).finally(() => { if (this.resuming === attempt) this.resuming = undefined }) + } + } + /** * 开始运行播放器逻辑循环 * @@ -407,8 +436,8 @@ export class Controller extends Subscribe { */ this.vreoUnit = undefined if (this.mediaInstance) { - this.mediaInstance.pause() - this.mediaInstance.currentTime = 0 + this.videoAgentScene?.videoAgentMesh.mediaOperations?.pause() + if (this.videoAgentScene?.videoAgentMesh.mediaOperations) this.videoAgentScene.videoAgentMesh.mediaOperations.currentTime = 0 } this.stopInterval?.() @@ -421,6 +450,8 @@ export class Controller extends Subscribe { * 清理所有状态和资源，释放内存 */ dispose() { + this.disposers.splice(0).forEach(dispose => dispose()) + this.playback.dispose() this.clear() } } diff --git a/packages/vreo/resources/Player/PlaybackLifecycle.ts b/packages/vreo/resources/Player/PlaybackLifecycle.ts new file mode 100644 index 00000000..4609afa8 --- /dev/null +++ b/packages/vreo/resources/Player/PlaybackLifecycle.ts @@ -0,0 +1,59 @@ +import type { MediaOperations, PlaybackController, PlaybackManager, PlaybackSession } from './playback-types' + +export interface PlaybackRun { + readonly session?: PlaybackSession + readonly mediaManager?: PlaybackManager + valid(): boolean + bind(element: HTMLMediaElement): MediaOperations +} + +/** Local timeline lifetime only. Playback permission is owned by the injected manager. */ +export class PlaybackLifecycle { + private controller?: PlaybackController + private run?: PlaybackRun + private generation = 0 + private disposed = false + private members = new Set<() => void>() + constructor(manager: PlaybackManager | undefined, private onStop: () => void) { + this.controller = manager?.createController({ label: 'vreo', onCancel: () => this.stop() }) + } + get active() { return !!this.run?.valid() } + capture() { return this.run } + begin(userAction: boolean) { + if (this.disposed) return false + if (this.active) return true + const session = this.controller?.begin({ userAction, audible: true }) ?? undefined + if (this.controller && !session) return false + const generation = ++this.generation + const mediaManager = session?.mediaManager + this.run = { + session, mediaManager, + valid: () => generation === this.generation && !this.disposed && !session?.signal.aborted, + bind: element => session ? session.bind(element) : element, + } + return true + } + add(stop: () => void) { this.members.add(stop); return () => { this.members.delete(stop) } } + private stop() { + ++this.generation + this.run = undefined + const errors: unknown[] = [] + try { this.onStop() } catch (error) { errors.push(error) } + for (const stop of [...this.members]) { try { stop() } catch (error) { errors.push(error) } } + if (errors.length) throw errors[0] + } + cancel() { + const generation = this.generation + this.controller?.cancel() + if (generation === this.generation) this.stop() + } + finish() { const session = this.run?.session; this.stop(); session?.finish() } + dispose() { + if (this.disposed) return + this.disposed = true + const generation = this.generation + this.controller?.dispose() + if (generation === this.generation) this.stop() + this.members.clear() + } +} diff --git a/packages/vreo/resources/Player/index.tsx b/packages/vreo/resources/Player/index.tsx index a6facc02..bd4509ce 100644 --- a/packages/vreo/resources/Player/index.tsx +++ b/packages/vreo/resources/Player/index.tsx @@ -1,6 +1,7 @@ // 下面这一行不能删 import * as React from 'react' -import * as ReactDOM from 'react-dom' +import type { Root } from 'react-dom/client' + import { createRoot } from 'react-dom/client' import { Five, Subscribe } from '@realsee/five' @@ -49,6 +50,11 @@ export class Player extends Subscribe { $five: Five /** 内部控制器 */ private controller: Controller + private root: Root + private disposers: (() => void)[] = [] + private loadGeneration = 0 + private disposed = false + getMediaManager() { return this.controller.playback.capture()?.mediaManager } /** 播放器配置（只读） */ configs: Readonly @@ -90,8 +96,8 @@ export class Player extends Subscribe { this.controller = new Controller({five, container:configs.container, configs: this.configs}) - const root = createRoot(configs.container) - root.render( + this.root = createRoot(configs.container) + this.root.render( @@ -106,29 +112,30 @@ export class Player extends Subscribe { off: (name, callback) => this.off(name as any, callback as any), }} five={five} + getMediaManager={() => this.getMediaManager()} /> ))} ) // 监听播放情况：抛出触发时机 - reaction( + this.disposers.push(reaction( () => this.controller.ended, (ended) => { if (ended) { this.emit('paused', true) } } - ) + )) - reaction( + this.disposers.push(reaction( () => this.controller.playing, (playing) => { if (!this.controller.ended) { this.emit(playing ? 'playing' : 'paused') } } - ) + )) } /** @@ -154,7 +161,13 @@ export class Player extends Subscribe { * await player.load(vreoUnit, 0, false, true) * ``` */ - async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false) { + async load(vreoUnit: VreoUnit, currentTime = 0, preload = false, force = false, userAction = false) { + const generation = ++this.loadGeneration + this.controller.playback.cancel() + if (this.disposed || !this.controller.playback.begin(userAction)) return false + const run = this.controller.playback.capture()! + const valid = () => run.valid() + try { this.controller.clear() this.controller.setLoading(true) if (force) { @@ -176,7 +189,7 @@ export class Player extends Subscribe { this.controller.vreoUnit = vreoUnit - this.controller.mediaInstance?.pause() + this.controller.videoAgentScene?.videoAgentMesh.mediaOperations?.pause() // 预载逻辑 @@ -214,21 +227,24 @@ export class Player extends Subscribe { const panoIndexes = Object.keys(panoIndexMap) for (let i = 0; i < panoIndexes.length; i++) { await this.$five.preloadPano(Number(panoIndexes[i])) + if (!valid() || generation !== this.loadGeneration) return false } } // 新数据载入就绪 this.emit('loaded', vreoUnit) this.controller.emit('loaded', vreoUnit) + if (!valid() || generation !== this.loadGeneration) return false - if (this.controller.videoAgentScene?.videoAgentMesh.mediaInstance) { - this.controller.videoAgentScene.videoAgentMesh.mediaInstance.currentTime = currentTime / 1000 + if (this.controller.videoAgentScene?.videoAgentMesh.mediaOperations) { + this.controller.videoAgentScene.videoAgentMesh.mediaOperations.currentTime = currentTime / 1000 } this.controller.setAvatar(vreoUnit.video.avatar) await waitForBlankAudioGenerated() + if (!valid() || generation !== this.loadGeneration) return false await this.controller.videoAgentScene?.videoAgentMesh.play( vreoUnit.video.url, currentTime / 1000, @@ -236,12 +252,20 @@ export class Player extends Subscribe { ) + if (!valid() || generation !== this.loadGeneration) return false this.controller.setEnded(false) - this.play() + this.play(undefined, userAction) this.controller.run((type, keyframe) => this.emit(type, keyframe, this.controller.currentTime)) this.controller.setLoading(false) return true + } catch (error) { + if (!valid()) return false + this.controller.playback.cancel() + throw error + } finally { + if (generation === this.loadGeneration) this.controller.setLoading(false) + } } /** @@ -267,10 +291,11 @@ export class Player extends Subscribe { * player.play(10000) // 从10秒处开始 * ``` */ - play(currentTime?: number) { + play(currentTime?: number, userAction = false) { + if (this.disposed || !this.controller.playback.begin(userAction)) return false if (this.controller.playing) return true - if (currentTime && this.controller.mediaInstance) { - this.controller.mediaInstance.currentTime = currentTime / 1000 + if (currentTime && this.controller.videoAgentScene?.videoAgentMesh.mediaOperations) { + this.controller.videoAgentScene.videoAgentMesh.mediaOperations.currentTime = currentTime / 1000 } Object.assign(window, { $vreoController: this.controller }) this.controller.setEnded(false) @@ -302,7 +327,9 @@ export class Player extends Subscribe { * 暂停播放 */ pause() { - this.controller.setPlaying(false) + ++this.loadGeneration + this.controller.playback.cancel() + this.controller.setLoading(false) } /** @@ -333,11 +360,12 @@ export class Player extends Subscribe { * 清理所有资源、事件监听器和DOM元素 */ dispose() { + if (this.disposed) return + this.disposed = true + ++this.loadGeneration + this.disposers.forEach(dispose => dispose()) this.controller.dispose() - - if (this.configs.container) { - ReactDOM.unmountComponentAtNode(this.configs.container as Element) - } + this.root.unmount() } } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts index d0d26b2f..a92abf74 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts +++ b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentMesh.ts @@ -1,3 +1,5 @@ +import type { PlaybackRun } from '../../PlaybackLifecycle' +import type { MediaOperations } from '../../playback-types' import * as THREE from 'three' import { Preloader } from '../../../shared-utils/Preloader' import { makeObservable, observable, runInAction } from 'mobx' @@ -59,15 +61,11 @@ void main(void) { } ` -const cacheInstance: { - videoInstance?: HTMLVideoElement - audioInstance?: HTMLAudioElement -} = {} - /** * 视频经纪人贴片的配置选项 */ export interface VideoAgentMeshOptions { + getPlayback?: () => PlaybackRun | undefined /** * 自定义视频实例。 */ @@ -123,6 +121,17 @@ export class VideoAgentMesh extends THREE.Mesh { /** 是否暂停状态 */ paused: boolean /** 音频实例 */ + private outputs = new Map() + private operations(run: PlaybackRun | undefined, element: HTMLMediaElement) { + const operations = run ? run.bind(element) : element + this.outputs.set(element, operations) + return operations + } + private ownsVideo = false + private ownsAudio = false + private generation = 0 + private removeStart?: () => void + private objectURL?: string audioInstance: HTMLAudioElement /** AudioLike 实例 */ audioLikeInstance: AudioLike @@ -180,20 +189,14 @@ export class VideoAgentMesh extends THREE.Mesh { preload: true }, ) { + const ownsVideo = !options.videoInstance if (!options.videoInstance) { - if (cacheInstance.videoInstance) { - options.videoInstance = cacheInstance.videoInstance - } else { - const videoInstance = document.createElement('video') - videoInstance.style.opacity = '0' - videoInstance.style.pointerEvents = 'none' - videoInstance.style.display = 'none' - document.body.append(videoInstance) - options.videoInstance = videoInstance - cacheInstance.videoInstance = videoInstance - videoInstance.playsInline = true - videoInstance.controls = false - } + const videoInstance = document.createElement('video') + videoInstance.style.display = 'none' + videoInstance.playsInline = true + videoInstance.muted = true + document.body.append(videoInstance) + options.videoInstance = videoInstance } const geometry = new THREE.PlaneGeometry(width, height, widthSegments, heightSegments) @@ -209,24 +212,17 @@ export class VideoAgentMesh extends THREE.Mesh { }) super(geometry, material) + this.ownsVideo = ownsVideo + this.ownsAudio = !options.audioInstance this.options = options this.freeze = false this.paused = true if (!options.audioInstance) { - if (cacheInstance.audioInstance) { - this.audioInstance = cacheInstance.audioInstance - } else { - const audioInstance = document.createElement('audio') - audioInstance.crossOrigin = '' - // videoInstance.muted = true - audioInstance.muted = false - audioInstance.autoplay = false - audioInstance.style.display = 'none' - document.body.appendChild(audioInstance) - this.audioInstance = audioInstance - cacheInstance.audioInstance = audioInstance - } + this.audioInstance = document.createElement('audio') + this.audioInstance.crossOrigin = '' + this.audioInstance.muted = true + this.audioInstance.autoplay = false } else { this.audioInstance = options.audioInstance } @@ -263,41 +259,30 @@ export class VideoAgentMesh extends THREE.Mesh { * @param videoUrl - 媒体文件URL * @private */ - private async update(videoUrl: string) { - - if (this.videoUrl === videoUrl) { - return - } - - this.videoUrl = videoUrl - - // // 兼容非视频场景 - // if (this.mediaInstance instanceof HTMLAudioElement) { - // this.mediaInstance.style.display = 'none' - // } else if (this.mediaInstance instanceof HTMLVideoElement) { - // this.mediaInstance.style.display = 'block' - // } - + private async update(videoUrl: string, valid: () => boolean, run?: PlaybackRun) { + if (this.videoUrl === videoUrl) return + this.stop() this.freeze = true - await this.mediaInstance.pause() - - const uniforms = (this.material as THREE.ShaderMaterial).uniforms - this.mediaInstance.muted = true - - this.mediaInstance.src = (this.options.preload || this.options.preload === undefined || getMediaType(this.videoUrl) === 'video') ? - await URL.createObjectURL((await Preloader.blob(this.videoUrl)) as unknown as Blob) : this.videoUrl - this.mediaInstance.setAttribute('data-src', this.videoUrl) - + const element = getMediaType(videoUrl) === 'audio' ? this.audioInstance : this.options.videoInstance! + const media = this.operations(run, element) + media.muted = true + const src = (this.options.preload !== false || getMediaType(videoUrl) === 'video') + ? URL.createObjectURL(await Preloader.blob(videoUrl) as unknown as Blob) : videoUrl + if (!valid()) { if (src !== videoUrl) URL.revokeObjectURL(src); return } + this.videoUrl = videoUrl + if (this.objectURL) URL.revokeObjectURL(this.objectURL) + this.objectURL = src !== videoUrl ? src : undefined + media.src = src + element.setAttribute('data-src', videoUrl) const onStart = () => { - if (this.mediaInstance.currentTime === 0) return + if (!valid() || media.currentTime === 0) return this.freeze = false - this.mediaInstance.muted = false - uniforms.enable.value = getMediaType(this.videoUrl) ? 1 : 0 - this.mediaInstance.removeEventListener('timeupdate', onStart, false) - + media.muted = false + ;(this.material as THREE.ShaderMaterial).uniforms.enable.value = getMediaType(videoUrl) ? 1 : 0 + this.removeStart?.() } - - this.mediaInstance.addEventListener('timeupdate', onStart, false) + this.removeStart = () => element.removeEventListener('timeupdate', onStart) + element.addEventListener('timeupdate', onStart) } /** @@ -326,41 +311,41 @@ export class VideoAgentMesh extends THREE.Mesh { * ``` */ async play(videoUrl = '', currentTime = 0, duration?: number) { - videoUrl = videoUrl || '' - - if (duration && !videoUrl) { - if (this.currentTime) { - this.mediaInstance.currentTime = currentTime - } - (this.mediaInstance as AudioLike).duration = duration - this.videoUrl = '' - this.mediaInstance.play() - return true - } - - if (!videoUrl) { - if (this.videoUrl) await this.mediaInstance.play() - else console.warn('警告：视频资源未初始化。') - return true - } - - if (videoUrl === this.videoUrl) { - this.mediaInstance.currentTime = currentTime - await this.mediaInstance.play() - return true + const run = this.options.getPlayback?.() + if (this.options.getPlayback && !run?.valid()) return false + // stop() invalidates all earlier loads before the new generation is captured. + if (videoUrl && videoUrl !== this.videoUrl) this.stop() + let generation = this.generation + const valid = () => generation === this.generation && (!this.options.getPlayback || !!run?.valid()) + if (videoUrl && videoUrl !== this.videoUrl) { + // update's synchronous stop is accounted for before awaiting its load. + generation++ + await this.update(videoUrl, valid, run) } + if (!valid()) return false + if (duration && !videoUrl) { this.videoUrl = ''; this.audioLikeInstance.duration = duration } + const media = this.videoUrl ? this.operations(run, this.mediaInstance as HTMLMediaElement) : this.audioLikeInstance + if (videoUrl || duration) media.currentTime = currentTime + media.muted = true + await media.play() + if (!valid()) return false + media.muted = false + return true + } - await this.update(videoUrl) - - this.mediaInstance.pause() + get mediaOperations(): MediaOperations | AudioLike | undefined { + const run = this.options.getPlayback?.() + if (this.options.getPlayback && !run?.valid()) return undefined + return this.videoUrl ? this.operations(run, this.mediaInstance as HTMLMediaElement) : this.audioLikeInstance + } - return await new Promise((resolve) => - setTimeout(async () => { - this.mediaInstance.currentTime = currentTime - await this.mediaInstance.play() - resolve(true) - }, 20), - ) + stop() { + ++this.generation + this.removeStart?.() + this.removeStart = undefined + for (const output of this.outputs.values()) { output.muted = true; output.pause() } + this.outputs.clear() + this.audioLikeInstance.pause() } /** @@ -385,16 +370,13 @@ export class VideoAgentMesh extends THREE.Mesh { * ``` */ dispose() { - // 销毁事件监听 + this.stop() this.$removeEventListener() - if (cacheInstance.audioInstance) { - document.body.removeChild(cacheInstance.audioInstance) - cacheInstance.audioInstance = undefined - } - - if (cacheInstance.videoInstance) { - document.body.removeChild(cacheInstance.videoInstance) - cacheInstance.videoInstance = undefined - } + if (this.ownsVideo) this.options.videoInstance?.remove() + if (this.ownsAudio) this.audioInstance.remove() + if (this.objectURL) URL.revokeObjectURL(this.objectURL) + this.geometry.dispose() + ;(this.material as THREE.ShaderMaterial).uniforms.map.value.dispose() + ;(this.material as THREE.ShaderMaterial).dispose() } } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts index 2d907612..9dbae9fe 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts +++ b/packages/vreo/resources/Player/modules/VideoAgent/VideoAgentScene.ts @@ -9,6 +9,7 @@ export class VideoAgentScene { scene = new THREE.Scene() camera = new THREE.OrthographicCamera(-240, 240, 135, -135) renderer = new THREE.WebGLRenderer({ alpha: true }) + private frame = 0 disposers: (() => void)[] = [] constructor(container?: HTMLElement, options?: VideoAgentMeshOptions) { @@ -46,10 +47,13 @@ export class VideoAgentScene { this.renderer.render(this.scene, this.camera) } - requestAnimationFrame(this.run) + this.frame = requestAnimationFrame(this.run) } dispose = () => { + cancelAnimationFrame(this.frame) + this.renderer.dispose() + this.renderer.domElement.remove() this.videoAgentMesh?.dispose() this.disposers.forEach(disposer => disposer?.()) } diff --git a/packages/vreo/resources/Player/modules/VideoAgent/index.tsx b/packages/vreo/resources/Player/modules/VideoAgent/index.tsx index 4b945f98..861c734d 100644 --- a/packages/vreo/resources/Player/modules/VideoAgent/index.tsx +++ b/packages/vreo/resources/Player/modules/VideoAgent/index.tsx @@ -23,12 +23,13 @@ export function VideoAgent(props: { onClick?: () => void; options?: VideoAgentMe console.warn('VideoAgentScene" 重复初始化，已被过滤') return } - const videoAgentScene = new VideoAgentScene(ref.current, props.options) + const videoAgentScene = new VideoAgentScene(ref.current, { ...props.options, getPlayback: () => controller.playback.capture() }) controller.videoAgentScene = videoAgentScene return () => { controller.dispose() controller.videoAgentScene?.dispose() + controller.videoAgentScene = undefined } }, []) diff --git a/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx b/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx index 1d3e5a76..547063ea 100644 --- a/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/BgMusic/index.tsx @@ -5,80 +5,47 @@ import { useController } from '../../../hooks' export function BgMusic() { const controller = useController() - React.useEffect(() => { - const callback = async (keyframe: VreoKeyframe, currentTime: number) => { - const { start, end } = keyframe - - const _currentTime = (currentTime - start) / 1000 - - if (_currentTime < 0 || _currentTime >= keyframe.end - keyframe.start) { - return - } - - const audio = getAudio(keyframe.data.url) - audio.currentTime = Math.max(0, _currentTime) - - const waitForLoaded = controller.configs?.waitForBgMusicLoaded ?? false - - const cleanAudio = () => { - audio.removeEventListener('canplay', playOnCanPlay) - audio.removeEventListener('pause', play) + const tracks = new Set<() => void>() + const callback = (keyframe: VreoKeyframe, currentTime: number) => { + const run = controller.playback.capture() + const valid = () => !!run?.valid() + if (!valid()) return + const element = getAudio() + const audio = run!.bind(element) + let cleaned = false + const clean = () => { + if (cleaned) return + cleaned = true + element.removeEventListener('canplay', play) + element.removeEventListener('ended', clean) + audio.muted = true audio.pause() audio.src = '' + remove() + tracks.delete(clean) + controller.setWaitingForBgMusic(false) } - + const remove = controller.playback.add(clean) + tracks.add(clean) const play = () => { - if (audio.realSrc === keyframe.data.url) { - // 有可能会被其他音轨打断 - audio.play() - } - } - // canplay 事件比 canplaythrough 更早触发，表示可以开始播放了 - const playOnCanPlay = () => { - audio.removeEventListener('canplay', playOnCanPlay) - audio.play() - // 解除等待状态，恢复主时间线播放 + element.removeEventListener('canplay', play) + if (!valid() || cleaned) return + audio.muted = false + void audio.play().catch(error => { if (valid() && !cleaned) { clean(); controller.playback.cancel(); console.error(error) } }) controller.setWaitingForBgMusic(false) } - - if (waitForLoaded) { - // 等待音频加载完成后再播放 - // readyState: 0=HAVE_NOTHING, 1=HAVE_METADATA, 2=HAVE_CURRENT_DATA, 3=HAVE_FUTURE_DATA, 4=HAVE_ENOUGH_DATA - if (audio.readyState >= 3) { - // readyState >= 3 表示有足够数据开始播放 - audio.play() - } else { - // 设置等待状态，阻塞整个播放流程 - controller.setWaitingForBgMusic(true) - // 监听 canplay 事件（可以开始播放） - audio.addEventListener('canplay', playOnCanPlay) - } - } else { - // 默认行为：立即播放，边加载边播放 - audio.play() - } - - audio.addEventListener('ended', () => { - cleanAudio() - }) - - audio.addEventListener('pause', play) - - controller.once('paused', () => { - // 如果播放器暂停，也要解除等待状态 - controller.setWaitingForBgMusic(false) - cleanAudio() - }) - + audio.muted = true + audio.src = keyframe.data.url + audio.currentTime = Math.max(0, (currentTime - keyframe.start) / 1000) + element.addEventListener('ended', clean) + if (controller.configs.waitForBgMusicLoaded && element.readyState < 3) { + controller.setWaitingForBgMusic(true) + element.addEventListener('canplay', play) + } else play() } - controller.on(VreoKeyframeEnum.BgMusic, callback) - - return () => { - controller.off(VreoKeyframeEnum.BgMusic, callback) - } - }) - + return () => { controller.off(VreoKeyframeEnum.BgMusic, callback); for (const clean of [...tracks]) clean() } + }, [controller]) return <> -} \ No newline at end of file +} diff --git a/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx b/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx index ab809f48..4430ea2c 100644 --- a/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/InfoPanel/index.tsx @@ -10,22 +10,6 @@ import { import { useController } from '../../../hooks' -const isWX = navigator.userAgent.toLowerCase().indexOf('micromessenger') !== -1 -const isIOS = navigator.userAgent.toLowerCase().indexOf('iphone') !== -1 - -const isIOSorWX = isIOS || isWX - - -const _videoElement = document.createElement('video') -_videoElement.setAttribute('playsinline', 'true') -_videoElement.setAttribute('webkit-playsinline', 'true') - -if (isIOSorWX) { - if (_videoElement.paused) { - _videoElement.addEventListener('click', () => _videoElement.play(), {once:true}) - } -} - function InfoPanelImg({ url, children }: { url: string; children?: ReactNode }) { return (
@@ -43,30 +27,40 @@ function InfoPanelVideo({ url, children }: { url: string; children?: ReactNode } React.useEffect(() => { // if (!isIOSorWX) return - const video = controller.configs?.videos?.videoPanel || _videoElement + const video = controller.configs?.videos?.videoPanel || document.createElement('video') + const run = controller.playback.capture() + const valid = () => !!run?.valid() + if (!run?.valid()) return + const output = run.bind(video) + let disposed = false + video.playsInline = true + output.muted = true + const stop = () => { disposed = true; output.muted = true; output.pause(); video.removeEventListener('canplaythrough', canplaythrough) } if (!videoWrapperRef.current) return - video.src = url + const remove = controller.playback.add(stop) + output.src = url if (!videoWrapperRef.current.contains(video)) { videoWrapperRef.current.appendChild(video) } const canplaythrough = () => { video.removeEventListener('canplaythrough', canplaythrough) - try { - video.play() - } catch (error) {} + if (disposed || !valid()) return + output.muted = false + void output.play().catch(error => { if (valid() && !disposed) { stop(); controller.playback.cancel(); console.error(error) } }) } video.addEventListener('canplaythrough', canplaythrough) - video.load() - // video.play() + output.load() + // output.play() return () => { - video.pause() + stop() + remove() if (videoWrapperRef.current?.contains(video)) { videoWrapperRef.current.removeChild(video) } } - }, [videoWrapperRef.current]) + }, [controller, url]) return (
@@ -161,15 +155,16 @@ export function InfoPanel() { } controller.on(VreoKeyframeEnum.InfoPanel, callback) - controller.on('ended', () => { + const close = () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current) + controller.openDrawer(false) controller.openPopUp(false) - }) - - controller.on('paused', () => { - controller.openPopUp(false) - }) + } + const remove = controller.playback.add(close) return () => { + remove() + close() controller.off(VreoKeyframeEnum.InfoPanel, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) diff --git a/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx b/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx index 8a393a4f..1eb5ae54 100644 --- a/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/ModelVideo/index.tsx @@ -12,10 +12,16 @@ export function ModelVideo() { const timeoutRef = React.useRef() React.useEffect(() => { + let generation = 0 + const stop = () => { ++generation; if (timeoutRef.current) clearTimeout(timeoutRef.current); ref.current?.dispose(); ref.current = undefined } + const remove = controller.playback.add(stop) const callback = async (keyframe: VreoKeyframe) => { - if (!ref.current) { - ref.current = ModelTVVideoPlugin(five, {}) - } + stop() + const current = generation + const run = controller.playback.capture() + const valid = () => !!run?.valid() + if (!valid()) return + const plugin = ref.current = ModelTVVideoPlugin(five, { mediaManager: run?.mediaManager }) const { start, end } = keyframe const { videoSrc, videoPosterSrc, vertexs, matrixWorld } = keyframe.data as ModelVideoData @@ -50,8 +56,9 @@ export function ModelVideo() { return [position] })() - ref.current.disable() - await ref.current.load( + plugin.disable() + try { + await plugin.load( { video_src: videoSrc, video_poster_src: videoPosterSrc, @@ -60,12 +67,21 @@ export function ModelVideo() { controller.configs?.videos?.modelTVVideo ) - ref.current.enable() - timeoutRef.current = setTimeout(() => ref.current?.disable(), end - start) + } catch (error) { + if (current === generation && valid()) { controller.playback.cancel(); console.error(error) } + return + } + if (current !== generation || !valid()) return + plugin.enable() + timeoutRef.current = setTimeout(stop, end - start) } controller.on(VreoKeyframeEnum.ModelVideo, callback) return () => { + stop() + remove() + ref.current?.dispose() + ref.current = undefined controller.off(VreoKeyframeEnum.ModelVideo, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) diff --git a/packages/vreo/resources/Player/modules/keyframes/PanoTag/index.tsx b/packages/vreo/resources/Player/modules/keyframes/PanoTag/index.tsx index 9118212f..1cdb3fe6 100644 --- a/packages/vreo/resources/Player/modules/keyframes/PanoTag/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/PanoTag/index.tsx @@ -27,7 +27,8 @@ export function PanoTag() { return } const callback = (keyframe: VreoKeyframe) => { - if (!panoTagPlugin.current) return + const run = controller.playback.capture() + if (!panoTagPlugin.current || !run?.valid()) return const { start, end, data } = keyframe @@ -50,6 +51,7 @@ export function PanoTag() { const tagInstance = panoTagPlugin.current.getTagById(id) if (tagInstance) { + tagInstance.setMediaManager(run.mediaManager) tagInstance.state.unfolded = true ;(panoTagPlugin.current as any).updateRenderAllTags() } diff --git a/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx b/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx index 19f4283c..c38af59c 100644 --- a/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx +++ b/packages/vreo/resources/Player/modules/keyframes/VideoEffect/index.tsx @@ -1,21 +1,10 @@ +import type { MediaOperations } from '../../../playback-types' import classNames from 'classnames' import * as React from 'react' // import { Preloader } from '../../../../shared-utils/Preloader' import { VideoEffectData, VreoKeyframe, VreoKeyframeEnum } from '../../../../typings/VreoUnit' import { useController, useFiveInstance } from '../../../hooks' -const inlinePlay = (videoInstance?: HTMLVideoElement | null) => { - if (!videoInstance) return - const canplaythrough = () => { - videoInstance.removeEventListener('canplaythrough', canplaythrough) - try { - videoInstance.play() - } catch (error) {} - } - videoInstance.addEventListener('canplaythrough', canplaythrough) - videoInstance.load() -} - // const emptyVideo = '//vr-static.realsee-cdn.cn/release/web/leisure.69fd3522.mov' const PI = Math.PI const PI_2 = PI * 2 @@ -28,13 +17,10 @@ export function VideoEffect() { const [visible, setVisible] = React.useState(false) const five = useFiveInstance() - const setBlobSrc = (blob: string) => { - if (!videoRef.current) return - - videoRef.current.src = blob - } - React.useEffect(() => { + let output: MediaOperations | undefined + let generation = 0 + let removeCanPlay: (() => void) | undefined const callback = async (keyframe: VreoKeyframe) => { const { start, end } = keyframe const { videoSrc, fov, direction, panoIndex, vector } = keyframe.data as VideoEffectData @@ -52,34 +38,53 @@ export function VideoEffect() { return [longitude, latitude] })() + destroy() + const currentGeneration = generation five.setState({ fov, panoIndex, longitude, latitude }, true) - setBlobSrc(videoSrc) - inlinePlay(videoRef.current) + + removeCanPlay?.() + const run = controller.playback.capture() + const active = () => !!run?.valid() + const valid = () => currentGeneration === generation && active() + const video = videoRef.current + if (!video || !valid()) return + const media = output = run!.bind(video) + media.muted = true + media.src = videoSrc + const play = () => { + removeCanPlay?.() + if (!valid()) return + media.muted = false + void media.play().catch(error => { if (valid()) { controller.playback.cancel(); console.error(error) } }) + } + removeCanPlay = () => video.removeEventListener('canplaythrough', play) + video.addEventListener('canplaythrough', play) + media.load() setVisible(true) - timeoutRef.current = setTimeout(() => { - videoRef.current?.pause() - setVisible(false) - setBlobSrc('') - }, end - start) + timeoutRef.current = setTimeout(destroy, Math.max(0, end - start)) } controller.on(VreoKeyframeEnum.VideoEffect, callback) const destroy = () => { + ++generation if (timeoutRef.current) { clearTimeout(timeoutRef.current) } - videoRef.current?.pause() + removeCanPlay?.() + if (output) { output.muted = true; output.pause(); output.src = '' } + output = undefined setVisible(false) - setBlobSrc('') + } - controller.on('paused', () => destroy()) - controller.on('ended', () => destroy()) + const remove = controller.playback.add(destroy) return () => { + destroy() + remove() controller.off(VreoKeyframeEnum.VideoEffect, callback) if (timeoutRef.current) { clearTimeout(timeoutRef.current) @@ -93,26 +98,15 @@ export function VideoEffect() { //