fix video texture leaks

project/include/media/VideoDecoder.h

@@ -244,6 +244,11 @@ namespace lime {
 			// Deletes the backend's GL objects, with the GL context current.
 			virtual void ReleaseTextures () {}
 
+			// Deletes the GL objects that backends of this kind left when they closed
+			// away from the GL thread, where their context is current. Called on the
+			// GL thread before each texture frame is read.
+			virtual void ReleaseOrphans () {}
+
 			static VideoBackend* Create ();
 			static bool IsSupported ();
 
@@ -308,7 +313,7 @@ namespace lime {
 			static int GetFrameLength (VideoFrameFormat format, int width, int height);
 			static int WriteFrame (const VideoPlanes* planes, VideoFrameFormat format, unsigned char* data, int length);
 
-			double audioTime;
+			std::atomic<double> audioTime;
 			std::atomic<bool> dropLateFrames;
 			std::atomic<int> droppedFrames;
 			VideoFrameFormat format;

project/src/media/video/AVFVideoBackend.mm

@@ -342,6 +342,29 @@ static bool IsFramebufferComplete (bool ext, GLenum target) {
 	#endif
 
 
+	#if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS)
+
+	// GL objects of a backend closed away from the GL thread, deleted the next
+	// time a decoder reads a texture frame where their context is current
+	struct AVFOrphan {
+
+		void* context;
+		GLuint copyFramebuffer;
+		GLuint copyTexture;
+		bool framebufferEXT;
+		GLuint sourceFramebuffer;
+		GLuint sourceTexture;
+
+	};
+
+	// Never destroyed, as exiting runs static destructors while threads can
+	// still be closing videos
+	static std::mutex* orphanMutex = new std::mutex ();
+	static std::vector<AVFOrphan>* orphans = new std::vector<AVFOrphan> ();
+
+	#endif
+
+
 	// Only iOS uses a CoreVideo texture cache
 	#if defined (LIME_VIDEO_AVF_TEXTURES) && !defined (HX_MACOS)
 	typedef CVOpenGLESTextureCacheRef AVFTextureCacheRef;
@@ -425,6 +448,7 @@ static bool IsFramebufferComplete (bool ext, GLenum target) {
 			virtual unsigned int LockTexture (VideoTextureFrame* frame);
 			virtual bool Open (const char* path, bool hardwareDecoding, VideoStreamInfo* info);
 			virtual void ReleaseTextureFrame (VideoTextureFrame* frame);
+			virtual void ReleaseOrphans ();
 			virtual void ReleaseTextures ();
 			virtual bool Seek (double time);
 			virtual void SetTextureOutput (bool enabled);
@@ -529,6 +553,23 @@ static bool IsFramebufferComplete (bool ext, GLenum target) {
 
 	void AVFVideoBackend::Close () {
 
+		#if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS)
+		// GL objects still held were left by closing away from the GL thread, as
+		// closing there releases them first
+		if (copyTexture || sourceTexture || copyFramebuffer || sourceFramebuffer) {
+
+			AVFOrphan orphan = { textureContext, copyFramebuffer, copyTexture, framebufferEXT, sourceFramebuffer, sourceTexture };
+
+			{
+				std::lock_guard<std::mutex> lock (*orphanMutex);
+				orphans->push_back (orphan);
+			}
+
+			ForgetTextures ();
+
+		}
+		#endif
+
 		@autoreleasepool {
 
 			StopAudio ();
@@ -1119,6 +1160,36 @@ static bool IsFramebufferComplete (bool ext, GLenum target) {
 	}
 
 
+	void AVFVideoBackend::ReleaseOrphans () {
+
+		#if defined (LIME_VIDEO_AVF_TEXTURES) && defined (HX_MACOS)
+		std::vector<AVFOrphan> list;
+
+		{
+			std::lock_guard<std::mutex> lock (*orphanMutex);
+			if (orphans->empty ()) return;
+			list.swap (*orphans);
+		}
+
+		void* context = (void*)CGLGetCurrentContext ();
+
+		for (size_t i = 0; i < list.size (); i++) {
+
+			// Objects from a context that is no longer current are left with it
+			AVFOrphan& orphan = list[i];
+			if (orphan.context != context) continue;
+
+			if (orphan.copyTexture) glDeleteTextures (1, &orphan.copyTexture);
+			if (orphan.sourceTexture) glDeleteTextures (1, &orphan.sourceTexture);
+			if (orphan.copyFramebuffer) DeleteFramebuffer (orphan.framebufferEXT, orphan.copyFramebuffer);
+			if (orphan.sourceFramebuffer) DeleteFramebuffer (orphan.framebufferEXT, orphan.sourceFramebuffer);
+
+		}
+		#endif
+
+	}
+
+
 	void AVFVideoBackend::ReleaseTextures () {
 
 		#ifdef LIME_VIDEO_AVF_TEXTURES

project/src/media/video/MFVideoBackend.cpp

@@ -345,6 +345,8 @@ namespace lime {
 
 		unsigned int glTexture;
 		HANDLE interop;
+		// Locked for OpenGL, as the frame being shown is
+		bool locked;
 		ID3D11Texture2D* texture;
 		ID3D11VideoProcessorOutputView* view;
 
@@ -658,6 +660,7 @@ namespace lime {
 			virtual VideoDecodeResult DecodeVideoTexture (VideoTextureFrame** frame, double* time, double* duration);
 			virtual unsigned int LockTexture (VideoTextureFrame* frame);
 			virtual void ReleaseTextureFrame (VideoTextureFrame* frame);
+			virtual void ReleaseOrphans ();
 			virtual void ReleaseTextures ();
 			virtual bool SupportsTextures ();
 			virtual void UnlockTexture (VideoTextureFrame* frame);
@@ -799,6 +802,7 @@ namespace lime {
 
 			outputs[i].glTexture = 0;
 			outputs[i].interop = NULL;
+			outputs[i].locked = false;
 			outputs[i].texture = NULL;
 			outputs[i].view = NULL;
 			outputFrames[i].index = i;
@@ -811,6 +815,68 @@ namespace lime {
 	}
 
 
+	#ifdef LIME_VIDEO_WGL_INTEROP
+	// Textures shared with OpenGL by a backend closed away from the GL thread.
+	// The device is kept so the interop objects can still be released.
+	struct MFOrphan {
+
+		void* context;
+		ID3D11Device* device;
+		HANDLE interopDevice;
+		std::vector<HANDLE> interops;
+		std::vector<unsigned int> glTextures;
+		std::vector<bool> locked;
+		std::vector<ID3D11Texture2D*> textures;
+
+	};
+
+	// Never destroyed, like the other video code's, as threads can still be
+	// closing videos while static destructors run
+	static std::mutex* orphanMutex = new std::mutex ();
+	static std::vector<MFOrphan>* orphans = new std::vector<MFOrphan> ();
+	#endif
+
+
+	void MFVideoBackend::ReleaseOrphans () {
+
+		#ifdef LIME_VIDEO_WGL_INTEROP
+		std::vector<MFOrphan> list;
+
+		{
+			std::lock_guard<std::mutex> lock (*orphanMutex);
+			if (orphans->empty ()) return;
+			list.swap (*orphans);
+		}
+
+		void* context = (void*)wglGetCurrentContext ();
+
+		for (size_t i = 0; i < list.size (); i++) {
+
+			MFOrphan& orphan = list[i];
+
+			// Objects from a context that is no longer current are left with it
+			bool current = orphan.interopDevice && orphan.context == context;
+
+			for (size_t j = 0; j < orphan.interops.size (); j++) {
+
+				// The frame shown when the backend closed is still locked, and is
+				// unlocked first, as closing on the GL thread does
+				if (current && orphan.interops[j] && orphan.locked[j]) _wglDXUnlockObjectsNV (orphan.interopDevice, 1, &orphan.interops[j]);
+				if (current && orphan.interops[j]) _wglDXUnregisterObjectNV (orphan.interopDevice, orphan.interops[j]);
+				if (current && orphan.glTextures[j]) glDeleteTextures (1, &orphan.glTextures[j]);
+				if (orphan.textures[j]) orphan.textures[j]->Release ();
+
+			}
+
+			if (current) _wglDXCloseDeviceNV (orphan.interopDevice);
+			if (orphan.device) orphan.device->Release ();
+
+		}
+		#endif
+
+	}
+
+
 	MFVideoBackend::~MFVideoBackend () {
 
 		Close ();
@@ -822,20 +888,51 @@ namespace lime {
 
 	void MFVideoBackend::Close () {
 
-		// Closing away from the GL thread leaves the textures it shares with
-		// OpenGL behind, rather than using them with the next device
+		// Closing away from the GL thread queues the textures it shares with
+		// OpenGL, rather than using them with the next device
+		#ifdef LIME_VIDEO_WGL_INTEROP
+		MFOrphan orphan;
+		orphan.context = interopContext;
+		orphan.device = NULL;
+		orphan.interopDevice = interopDevice;
+		#endif
+
 		for (int i = 0; i < OUTPUT_TEXTURES; i++) {
 
 			if (outputs[i].interop) {
 
+				#ifdef LIME_VIDEO_WGL_INTEROP
+				orphan.interops.push_back (outputs[i].interop);
+				orphan.glTextures.push_back (outputs[i].glTexture);
+				orphan.locked.push_back (outputs[i].locked);
+				orphan.textures.push_back (outputs[i].texture);
+				#endif
+
 				outputs[i].glTexture = 0;
 				outputs[i].interop = NULL;
+				outputs[i].locked = false;
 				outputs[i].texture = NULL;
 
 			}
 
 		}
 
+		#ifdef LIME_VIDEO_WGL_INTEROP
+		if (orphan.interopDevice || !orphan.interops.empty ()) {
+
+			if (device) {
+
+				device->AddRef ();
+				orphan.device = device;
+
+			}
+
+			std::lock_guard<std::mutex> lock (*orphanMutex);
+			orphans->push_back (orphan);
+
+		}
+		#endif
+
 		interopContext = NULL;
 		interopDevice = NULL;
 		interopFailed = false;
@@ -2475,6 +2572,7 @@ namespace lime {
 
 				outputs[i].glTexture = 0;
 				outputs[i].interop = NULL;
+				outputs[i].locked = false;
 
 			}
 
@@ -2525,6 +2623,7 @@ namespace lime {
 			}
 
 			if (!_wglDXLockObjectsNV (interopDevice, 1, &output.interop)) return 0;
+			output.locked = true;
 
 			// Without mipmaps, the default minification filter would leave the
 			// texture incomplete
@@ -2541,7 +2640,9 @@ namespace lime {
 
 		}
 
-		return _wglDXLockObjectsNV (interopDevice, 1, &output.interop) ? output.glTexture : 0;
+		if (!_wglDXLockObjectsNV (interopDevice, 1, &output.interop)) return 0;
+		output.locked = true;
+		return output.glTexture;
 		#else
 		return 0;
 		#endif
@@ -2607,8 +2708,10 @@ namespace lime {
 
 			if (outputs[i].interop) {
 
+				if (current && outputs[i].locked) _wglDXUnlockObjectsNV (interopDevice, 1, &outputs[i].interop);
 				if (current) _wglDXUnregisterObjectNV (interopDevice, outputs[i].interop);
 				outputs[i].interop = NULL;
+				outputs[i].locked = false;
 
 			}
 
@@ -2720,7 +2823,8 @@ namespace lime {
 
 		#ifdef LIME_VIDEO_WGL_INTEROP
 		MFOutputTexture& output = outputs[((MFTextureFrame*)frame)->index];
-		if (interopDevice && output.interop) _wglDXUnlockObjectsNV (interopDevice, 1, &output.interop);
+		if (interopDevice && output.interop && output.locked) _wglDXUnlockObjectsNV (interopDevice, 1, &output.interop);
+		output.locked = false;
 		#endif
 
 	}