diff --git a/project/include/media/VideoDecoder.h b/project/include/media/VideoDecoder.h index efdd688f88..27a667bd9b 100644 --- a/project/include/media/VideoDecoder.h +++ b/project/include/media/VideoDecoder.h @@ -244,6 +244,11 @@ namespace lime { // Deletes the backend's GL objects, with the GL context current. virtual void ReleaseTextures () {} + // Deletes the GL objects that backends of this kind left when they closed + // away from the GL thread, where their context is current. Called on the + // GL thread before each texture frame is read. + virtual void ReleaseOrphans () {} + static VideoBackend* Create (); static bool IsSupported (); @@ -308,7 +313,7 @@ namespace lime { static int GetFrameLength (VideoFrameFormat format, int width, int height); static int WriteFrame (const VideoPlanes* planes, VideoFrameFormat format, unsigned char* data, int length); - double audioTime; + std::atomic audioTime; std::atomic dropLateFrames; std::atomic droppedFrames; VideoFrameFormat format; diff --git a/project/src/media/video/AVFVideoBackend.mm b/project/src/media/video/AVFVideoBackend.mm index 3e5a9f70d8..c71ea0569a 100644 --- a/project/src/media/video/AVFVideoBackend.mm +++ b/project/src/media/video/AVFVideoBackend.mm @@ -342,6 +342,29 @@ static bool IsFramebufferComplete (bool ext, GLenum target) { #endif + #if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS) + + // GL objects of a backend closed away from the GL thread, deleted the next + // time a decoder reads a texture frame where their context is current + struct AVFOrphan { + + void* context; + GLuint copyFramebuffer; + GLuint copyTexture; + bool framebufferEXT; + GLuint sourceFramebuffer; + GLuint sourceTexture; + + }; + + // Never destroyed, as exiting runs static destructors while threads can + // still be closing videos + static std::mutex* orphanMutex = new std::mutex (); + static std::vector* orphans = new std::vector (); + + #endif + + // Only iOS uses a CoreVideo texture cache #if defined (LIME_VIDEO_AVF_TEXTURES) && !defined (HX_MACOS) typedef CVOpenGLESTextureCacheRef AVFTextureCacheRef; @@ -425,6 +448,7 @@ static bool IsFramebufferComplete (bool ext, GLenum target) { virtual unsigned int LockTexture (VideoTextureFrame* frame); virtual bool Open (const char* path, bool hardwareDecoding, VideoStreamInfo* info); virtual void ReleaseTextureFrame (VideoTextureFrame* frame); + virtual void ReleaseOrphans (); virtual void ReleaseTextures (); virtual bool Seek (double time); virtual void SetTextureOutput (bool enabled); @@ -529,6 +553,23 @@ static bool IsFramebufferComplete (bool ext, GLenum target) { void AVFVideoBackend::Close () { + #if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS) + // GL objects still held were left by closing away from the GL thread, as + // closing there releases them first + if (copyTexture || sourceTexture || copyFramebuffer || sourceFramebuffer) { + + AVFOrphan orphan = { textureContext, copyFramebuffer, copyTexture, framebufferEXT, sourceFramebuffer, sourceTexture }; + + { + std::lock_guard lock (*orphanMutex); + orphans->push_back (orphan); + } + + ForgetTextures (); + + } + #endif + @autoreleasepool { StopAudio (); @@ -1119,6 +1160,36 @@ static bool IsFramebufferComplete (bool ext, GLenum target) { } + void AVFVideoBackend::ReleaseOrphans () { + + #if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS) + std::vector list; + + { + std::lock_guard lock (*orphanMutex); + if (orphans->empty ()) return; + list.swap (*orphans); + } + + void* context = (void*)CGLGetCurrentContext (); + + for (size_t i = 0; i < list.size (); i++) { + + // Objects from a context that is no longer current are left with it + AVFOrphan& orphan = list[i]; + if (orphan.context != context) continue; + + if (orphan.copyTexture) glDeleteTextures (1, &orphan.copyTexture); + if (orphan.sourceTexture) glDeleteTextures (1, &orphan.sourceTexture); + if (orphan.copyFramebuffer) DeleteFramebuffer (orphan.framebufferEXT, orphan.copyFramebuffer); + if (orphan.sourceFramebuffer) DeleteFramebuffer (orphan.framebufferEXT, orphan.sourceFramebuffer); + + } + #endif + + } + + void AVFVideoBackend::ReleaseTextures () { #ifdef LIME_VIDEO_AVF_TEXTURES diff --git a/project/src/media/video/MFVideoBackend.cpp b/project/src/media/video/MFVideoBackend.cpp index f769d104d7..6c33222395 100644 --- a/project/src/media/video/MFVideoBackend.cpp +++ b/project/src/media/video/MFVideoBackend.cpp @@ -345,6 +345,8 @@ namespace lime { unsigned int glTexture; HANDLE interop; + // Locked for OpenGL, as the frame being shown is + bool locked; ID3D11Texture2D* texture; ID3D11VideoProcessorOutputView* view; @@ -658,6 +660,7 @@ namespace lime { virtual VideoDecodeResult DecodeVideoTexture (VideoTextureFrame** frame, double* time, double* duration); virtual unsigned int LockTexture (VideoTextureFrame* frame); virtual void ReleaseTextureFrame (VideoTextureFrame* frame); + virtual void ReleaseOrphans (); virtual void ReleaseTextures (); virtual bool SupportsTextures (); virtual void UnlockTexture (VideoTextureFrame* frame); @@ -799,6 +802,7 @@ namespace lime { outputs[i].glTexture = 0; outputs[i].interop = NULL; + outputs[i].locked = false; outputs[i].texture = NULL; outputs[i].view = NULL; outputFrames[i].index = i; @@ -811,6 +815,68 @@ namespace lime { } + #ifdef LIME_VIDEO_WGL_INTEROP + // Textures shared with OpenGL by a backend closed away from the GL thread. + // The device is kept so the interop objects can still be released. + struct MFOrphan { + + void* context; + ID3D11Device* device; + HANDLE interopDevice; + std::vector interops; + std::vector glTextures; + std::vector locked; + std::vector textures; + + }; + + // Never destroyed, like the other video code's, as threads can still be + // closing videos while static destructors run + static std::mutex* orphanMutex = new std::mutex (); + static std::vector* orphans = new std::vector (); + #endif + + + void MFVideoBackend::ReleaseOrphans () { + + #ifdef LIME_VIDEO_WGL_INTEROP + std::vector list; + + { + std::lock_guard lock (*orphanMutex); + if (orphans->empty ()) return; + list.swap (*orphans); + } + + void* context = (void*)wglGetCurrentContext (); + + for (size_t i = 0; i < list.size (); i++) { + + MFOrphan& orphan = list[i]; + + // Objects from a context that is no longer current are left with it + bool current = orphan.interopDevice && orphan.context == context; + + for (size_t j = 0; j < orphan.interops.size (); j++) { + + // The frame shown when the backend closed is still locked, and is + // unlocked first, as closing on the GL thread does + if (current && orphan.interops[j] && orphan.locked[j]) _wglDXUnlockObjectsNV (orphan.interopDevice, 1, &orphan.interops[j]); + if (current && orphan.interops[j]) _wglDXUnregisterObjectNV (orphan.interopDevice, orphan.interops[j]); + if (current && orphan.glTextures[j]) glDeleteTextures (1, &orphan.glTextures[j]); + if (orphan.textures[j]) orphan.textures[j]->Release (); + + } + + if (current) _wglDXCloseDeviceNV (orphan.interopDevice); + if (orphan.device) orphan.device->Release (); + + } + #endif + + } + + MFVideoBackend::~MFVideoBackend () { Close (); @@ -822,20 +888,51 @@ namespace lime { void MFVideoBackend::Close () { - // Closing away from the GL thread leaves the textures it shares with - // OpenGL behind, rather than using them with the next device + // Closing away from the GL thread queues the textures it shares with + // OpenGL, rather than using them with the next device + #ifdef LIME_VIDEO_WGL_INTEROP + MFOrphan orphan; + orphan.context = interopContext; + orphan.device = NULL; + orphan.interopDevice = interopDevice; + #endif + for (int i = 0; i < OUTPUT_TEXTURES; i++) { if (outputs[i].interop) { + #ifdef LIME_VIDEO_WGL_INTEROP + orphan.interops.push_back (outputs[i].interop); + orphan.glTextures.push_back (outputs[i].glTexture); + orphan.locked.push_back (outputs[i].locked); + orphan.textures.push_back (outputs[i].texture); + #endif + outputs[i].glTexture = 0; outputs[i].interop = NULL; + outputs[i].locked = false; outputs[i].texture = NULL; } } + #ifdef LIME_VIDEO_WGL_INTEROP + if (orphan.interopDevice || !orphan.interops.empty ()) { + + if (device) { + + device->AddRef (); + orphan.device = device; + + } + + std::lock_guard lock (*orphanMutex); + orphans->push_back (orphan); + + } + #endif + interopContext = NULL; interopDevice = NULL; interopFailed = false; @@ -2475,6 +2572,7 @@ namespace lime { outputs[i].glTexture = 0; outputs[i].interop = NULL; + outputs[i].locked = false; } @@ -2525,6 +2623,7 @@ namespace lime { } if (!_wglDXLockObjectsNV (interopDevice, 1, &output.interop)) return 0; + output.locked = true; // Without mipmaps, the default minification filter would leave the // texture incomplete @@ -2541,7 +2640,9 @@ namespace lime { } - return _wglDXLockObjectsNV (interopDevice, 1, &output.interop) ? output.glTexture : 0; + if (!_wglDXLockObjectsNV (interopDevice, 1, &output.interop)) return 0; + output.locked = true; + return output.glTexture; #else return 0; #endif @@ -2607,8 +2708,10 @@ namespace lime { if (outputs[i].interop) { + if (current && outputs[i].locked) _wglDXUnlockObjectsNV (interopDevice, 1, &outputs[i].interop); if (current) _wglDXUnregisterObjectNV (interopDevice, outputs[i].interop); outputs[i].interop = NULL; + outputs[i].locked = false; } @@ -2720,7 +2823,8 @@ namespace lime { #ifdef LIME_VIDEO_WGL_INTEROP MFOutputTexture& output = outputs[((MFTextureFrame*)frame)->index]; - if (interopDevice && output.interop) _wglDXUnlockObjectsNV (interopDevice, 1, &output.interop); + if (interopDevice && output.interop && output.locked) _wglDXUnlockObjectsNV (interopDevice, 1, &output.interop); + output.locked = false; #endif } diff --git a/project/src/media/video/MediaCodecVideoBackend.cpp b/project/src/media/video/MediaCodecVideoBackend.cpp index 60d3c468d9..c34ca68bb6 100644 --- a/project/src/media/video/MediaCodecVideoBackend.cpp +++ b/project/src/media/video/MediaCodecVideoBackend.cpp @@ -718,6 +718,26 @@ namespace lime { } + // GL objects of a backend closed away from the GL thread, deleted the next + // time a decoder reads a texture frame where their context is current + struct MediaCodecOrphan { + + EGLContext context; + GLuint copyBuffer; + GLuint copyFramebuffer; + GLuint copyProgram; + GLuint copyTexture; + GLuint copyVertexArray; + GLuint imageTexture; + + }; + + // Never destroyed, as exiting runs static destructors while threads can + // still be closing videos + static std::mutex* orphanMutex = new std::mutex (); + static std::vector* orphans = new std::vector (); + + static void RestoreGLState (const SavedGLState* state, bool gles3) { glBindFramebuffer (GL_FRAMEBUFFER, state->framebuffer); @@ -1301,6 +1321,7 @@ namespace lime { virtual bool Open (const char* path, bool hardwareDecoding, VideoStreamInfo* info); virtual void PrepareSeek (); virtual void ReleaseTextureFrame (VideoTextureFrame* frame); + virtual void ReleaseOrphans (); virtual void ReleaseTextures (); virtual bool Seek (double time); virtual void SetTextureOutput (bool enabled); @@ -1530,6 +1551,21 @@ namespace lime { void MediaCodecVideoBackend::Close () { + // GL objects still held were left by closing away from the GL thread, as + // closing there releases them first + if (textureContext != EGL_NO_CONTEXT && (copyBuffer || copyFramebuffer || copyProgram || copyTexture || copyVertexArray || imageTexture)) { + + MediaCodecOrphan orphan = { textureContext, copyBuffer, copyFramebuffer, copyProgram, copyTexture, copyVertexArray, imageTexture }; + + { + std::lock_guard lock (*orphanMutex); + orphans->push_back (orphan); + } + + ForgetTextures (); + + } + video.Close (); audio.Close (); @@ -2687,6 +2723,36 @@ namespace lime { } + void MediaCodecVideoBackend::ReleaseOrphans () { + + std::vector list; + + { + std::lock_guard lock (*orphanMutex); + if (orphans->empty ()) return; + list.swap (*orphans); + } + + EGLContext context = eglGetCurrentContext (); + + for (size_t i = 0; i < list.size (); i++) { + + // Objects from a context that is no longer current are left with it + MediaCodecOrphan& orphan = list[i]; + if (orphan.context != context) continue; + + if (orphan.copyBuffer) glDeleteBuffers (1, &orphan.copyBuffer); + if (orphan.copyFramebuffer) glDeleteFramebuffers (1, &orphan.copyFramebuffer); + if (orphan.copyProgram) glDeleteProgram (orphan.copyProgram); + if (orphan.copyTexture) glDeleteTextures (1, &orphan.copyTexture); + if (orphan.copyVertexArray && _glDeleteVertexArrays) _glDeleteVertexArrays (1, &orphan.copyVertexArray); + if (orphan.imageTexture) glDeleteTextures (1, &orphan.imageTexture); + + } + + } + + void MediaCodecVideoBackend::ReleaseTextures () { // GL names only mean these objects in the context that made them diff --git a/project/src/media/video/VideoDecoder.cpp b/project/src/media/video/VideoDecoder.cpp index 6304b9f954..19efcf0777 100644 --- a/project/src/media/video/VideoDecoder.cpp +++ b/project/src/media/video/VideoDecoder.cpp @@ -33,6 +33,46 @@ namespace lime { + // Textures of decoders closed away from the GL thread, deleted the next + // time any decoder reads a texture frame where their context is current + struct OrphanTexture { + + unsigned int name; + void* context; + + }; + + // Never destroyed, as exiting on Android and Linux runs static destructors + // while threads can still be closing videos + static std::mutex* orphanMutex = new std::mutex (); + static std::vector* orphanTextures = new std::vector (); + + + static void ReleaseOrphans (VideoBackend* backend) { + + #ifdef LIME_VIDEO_TEXTURES + std::vector textures; + + { + std::lock_guard lock (*orphanMutex); + textures.swap (*orphanTextures); + } + + void* context = VideoBackend::GetCurrentGLContext (); + + for (size_t i = 0; i < textures.size (); i++) { + + // A texture made in a context that has since been replaced went with it + if (textures[i].context == context) glDeleteTextures (1, &textures[i].name); + + } + #endif + + backend->ReleaseOrphans (); + + } + + // Decoding stops once this many frames or seconds of audio are queued static const size_t MAX_QUEUED_FRAMES = 4; static const double MAX_QUEUED_AUDIO = 0.75; @@ -366,8 +406,31 @@ namespace lime { } // GL objects can only be deleted where their context is current, so - // closing on another thread leaves them - if (IsTextureThread ()) ReleaseTextures (); + // closing on another thread queues them for the next texture read there + if (IsTextureThread ()) { + + ReleaseTextures (); + + } else { + + #ifdef LIME_VIDEO_TEXTURES + if (texture) { + + std::lock_guard lock (*orphanMutex); + OrphanTexture orphan = { texture, textureContext }; + orphanTextures->push_back (orphan); + + } + #endif + + texture = 0; + textureContext = 0; + frameTexture = 0; + textureHeight = 0; + textureThreadKnown = false; + textureWidth = 0; + + } backend->Close (); @@ -651,6 +714,7 @@ namespace lime { textureThread = std::this_thread::get_id (); textureThreadKnown = true; + ReleaseOrphans (backend); unsigned int name = frame->texture ? backend->LockTexture (frame->texture) : UploadTexture (frame); diff --git a/tests/video-smoke/Source/Main.hx b/tests/video-smoke/Source/Main.hx index e90376c3dc..c11293b16a 100644 --- a/tests/video-smoke/Source/Main.hx +++ b/tests/video-smoke/Source/Main.hx @@ -4,6 +4,7 @@ import haxe.Timer; import lime.app.Application; import lime.graphics.opengl.GLBuffer; import lime.graphics.opengl.GLProgram; +import lime.graphics.opengl.GLTexture; import lime.graphics.opengl.GLUniformLocation; import lime.graphics.RenderContext; import lime.graphics.WebGLRenderContext; @@ -12,6 +13,9 @@ import lime.media.VideoFrame; import lime.system.System; import lime.utils.Float32Array; import lime.utils.UInt8Array; +#if target.threaded +import sys.thread.Thread; +#end /** Plays assets/video.mp4 as texture frames and checks every frame on the @@ -26,13 +30,22 @@ import lime.utils.UInt8Array; them show up too. After the last frame, the test seeks several times in a row and checks the frames from the final position. Platforms without a decoder for the clip skip the test. + + Then it closes the video on another thread, while its last frame is still + shown, and checks that its GL objects are released the next time another + video reads a frame here. With hxcpp and HashLink, it also drops a video + without closing it and collects it on another thread, and checks the same. **/ class Main extends Application { private static inline var FRAME_RATE = 30; private static inline var SEEK_TIME = 1.0; - private static inline var TIMEOUT = 20.0; + private static inline var TIMEOUT = 30.0; + private var abandoned:VideoDecoder; + private var abandonedFrames:Int; + private var abandonedTextures:Array; + private var awayFrames:Int; private var blackFrames:Array; private var decoder:VideoDecoder; private var finished:Bool; @@ -44,24 +57,37 @@ class Main extends Application private var glProgram:GLProgram; private var glRegionUniform:GLUniformLocation; private var mismatches:Array; + private var other:VideoDecoder; + private var otherFrames:Int; + private var otherTextures:Array; private var pixel:UInt8Array; + private var playbackSummary:String; + private var playedTextures:Array; private var redChecked:Bool; + private var releaseStep:Int; private var seekFrames:Int; private var seekStartTime:Float; private var seeked:Bool; private var start:Float; private var values:Map; + private var waiting:Bool; public function new() { super(); + abandonedFrames = 0; + abandonedTextures = []; blackFrames = []; firstTime = -1; frame = new VideoFrame(); frames = 0; mismatches = []; + otherFrames = 0; + otherTextures = []; pixel = new UInt8Array(4); + playedTextures = []; + releaseStep = 0; seekFrames = 0; seekStartTime = -1; values = new Map(); @@ -90,7 +116,7 @@ class Main extends Application override public function render(context:RenderContext):Void { - if (finished || decoder == null) return; + if (finished || (decoder == null && releaseStep == 0)) return; switch (context.type) { @@ -109,6 +135,12 @@ class Main extends Application return; } + if (releaseStep > 0) + { + checkRelease(gl); + return; + } + // Read every frame in order, checking each one as it arrives while (decoder.readFrame(frame)) { @@ -153,6 +185,8 @@ class Main extends Application return; } + addTexture(playedTextures, frame.texture); + if (frame.width != 640 || frame.height != 360) { fail("frame size " + frame.width + "x" + frame.height); @@ -229,12 +263,172 @@ class Main extends Application return; } - trace("Video smoke passed: " + frames + " frames, " + distinct + " distinct, " + seekFrames + " after seeking"); + playbackSummary = frames + " frames, " + distinct + " distinct, " + seekFrames + " after seeking"; + releaseStep = 1; + } + + private function addTexture(textures:Array, texture:GLTexture):Void + { + if (textures.indexOf(texture) < 0) textures.push(texture); + } + + private function checkRelease(gl:WebGLRenderContext):Void + { + if (waiting) return; + + switch (releaseStep) + { + case 1: + // A second video plays until it has made all its GL objects, so the + // names it is given are not those of the video being released + other = VideoDecoder.fromFile("assets/video.mp4", TEXTURE); + + if (other == null) + { + fail("a second video did not open"); + return; + } + + releaseStep++; + + case 2: + otherFrames += readFrames(other, otherTextures); + if (otherFrames >= 12) releaseStep++; + + case 3: + // Closed on another thread, while its last frame is still shown + var played = decoder; + decoder = null; + runAway(function() played.close()); + releaseStep++; + + case 4: + if (readFrames(other, otherTextures) == 0) return; + + var left = countAlive(gl, playedTextures); + + if (left > 0) + { + fail(left + " of the " + playedTextures.length + " textures of a video closed on another thread were not released"); + return; + } + + #if (cpp || hl) + abandoned = VideoDecoder.fromFile("assets/video.mp4", TEXTURE); + releaseStep++; + #else + finishRelease(); + #end + + case 5: + abandonedFrames += readFrames(abandoned, abandonedTextures); + if (abandonedFrames < 12) return; + + // Dropped without closing, and collected on another thread once + // nothing on the stack still refers to it + abandoned = null; + awayFrames = 0; + releaseStep++; + + case 6: + if (++awayFrames < 10) return; + runAway(collectGarbageHere); + releaseStep++; + + case 7: + if (readFrames(other, otherTextures) == 0) return; + + var left = countAlive(gl, abandonedTextures); + + if (left > 0) + { + fail(left + " of the " + abandonedTextures.length + " textures of a video collected on another thread were not released"); + return; + } + + finishRelease(); + + default: + } + } + + private static function collectGarbageHere():Void + { + #if cpp + cpp.vm.Gc.run(true); + cpp.vm.Gc.run(true); + #elseif hl + hl.Gc.major(); + hl.Gc.major(); + #end + } + + private function countAlive(gl:WebGLRenderContext, textures:Array):Int + { + // Names the second video holds are its own, even where they were used before + var count = 0; + + for (texture in textures) + { + if (!gl.isTexture(texture)) continue; + + var reused = false; + + for (otherTexture in otherTextures) + { + if (idOf(otherTexture) == idOf(texture)) reused = true; + } + + if (!reused) count++; + } + + return count; + } + + private function finishRelease():Void + { + trace("Video smoke passed: " + playbackSummary + ", and textures were released after closing on another thread"); finished = true; - decoder.close(); + other.close(); System.exit(0); } + private static function idOf(texture:GLTexture):Int + { + var object:lime.graphics.opengl.GL.GLObject = texture; + return @:privateAccess object.id; + } + + private function readFrames(video:VideoDecoder, textures:Array):Int + { + var count = 0; + + while (video.readFrame(frame)) + { + if (frame.texture == null) + { + fail("a frame came without a texture"); + return 0; + } + + addTexture(textures, frame.texture); + count++; + } + + return count; + } + + private function runAway(work:Void->Void):Void + { + waiting = true; + + Thread.create(function() + { + work(); + waiting = false; + }); + } + private function initGL(gl:WebGLRenderContext):Void { var precision = #if !desktop "precision mediump float;" #else "" #end;