From f3123925e25c852bf76f7093e16bd3e51db19fda Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Wed, 30 Sep 2026 15:23:01 -0400 Subject: [PATCH 1/2] fix(webgl): deleteProgram deleted a framebuffer with the program's name Both deleteProgram bindings called canvas_native_webgl_delete_framebuffer, so programs were never deleted, and a framebuffer sharing the program's name was deleted in its place. On iOS that is the canvas's own framebuffer when it and the program both got name 1: three.js disposing its PMREM programs left the canvas drawing into nothing. The spec deletes a fresh context's first program and checks it is gone and the canvas still takes a clear. --- .../ios/src/cpp/webgl/WebGLRenderingContext.cpp | 2 +- .../ios/src/cpp/webgl/WebGLRenderingContext.h | 2 +- tools/demo/canvas/spec/webgl.ts | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.cpp b/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.cpp index 498302d53..93d4e28b9 100644 --- a/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.cpp +++ b/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.cpp @@ -1685,7 +1685,7 @@ void WebGLRenderingContext::DeleteProgram(const v8::FunctionCallbackInfo()); if (program != nullptr) { - canvas_native_webgl_delete_framebuffer( + canvas_native_webgl_delete_program( program->GetProgram(), ptr->GetState() ); diff --git a/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.h b/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.h index 8cc1b67f3..ab482a962 100644 --- a/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.h +++ b/packages/canvas/platforms/ios/src/cpp/webgl/WebGLRenderingContext.h @@ -1218,7 +1218,7 @@ class WebGLRenderingContext : public WebGLRenderingContextBase { if (type == NativeType::WebGLProgram) { auto program = WebGLProgram::GetPointer(value); if (program != nullptr) { - canvas_native_webgl_delete_framebuffer( + canvas_native_webgl_delete_program( program->GetProgram(), ptr->GetState() ); diff --git a/tools/demo/canvas/spec/webgl.ts b/tools/demo/canvas/spec/webgl.ts index 66ea6e21d..4f0b1c4e8 100644 --- a/tools/demo/canvas/spec/webgl.ts +++ b/tools/demo/canvas/spec/webgl.ts @@ -303,6 +303,21 @@ function registerCommon(version: 1 | 2) { equal(gl.getUniformLocation(program, 'nope'), null, 'an unknown uniform must be null'); }); + test('deleteProgram deletes the program and nothing else', () => { + const { gl } = makeGL(version); + // A fresh context's first program shares its name with the first framebuffer, which is + // the canvas's own on iOS. + const program = gl.createProgram(); + gl.deleteProgram(program); + equal(gl.isProgram(program), false, 'the program should be gone'); + gl.clearColor(0, 1, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT); + const pixel = new Uint8Array(4); + gl.readPixels(32, 32, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixel); + arrayEqual(pixel, [0, 255, 0, 255], 'the canvas should still take the clear'); + equal(gl.getError(), gl.NO_ERROR, 'getError'); + }); + test('getActiveAttrib / getActiveUniform describe the program', () => { const { gl } = makeGL(version); const program = buildProgram(gl); From 88984e6de415747285a1419d4dfe7334a085f727 Mon Sep 17 00:00:00 2001 From: Osei Fortune Date: Wed, 30 Sep 2026 15:23:02 -0400 Subject: [PATCH 2/2] feat(webgl): run iOS WebGL contexts on the WebGL thread iOS WebGL drew through a GLKView, whose display (the per-frame present) only runs on the main thread, which is the JS thread: a heavy scene held the UI to the GPU's pace. The GLKView is replaced by a view backed by a CAEAGLLayer. The context owns its drawing buffer: a framebuffer whose color buffer is stored in the layer (renderbufferStorage:fromDrawable:) and presented with presentRenderbuffer:, both of which run on any thread, plus a packed depth/stencil buffer from the context attributes. An offscreen context gets plain renderbuffers instead of an unshown GLKView. That lets an iOS context be threaded as on Android (the default; getContext(..., { threaded: false }) opts out): it is created, drawn, presented and dropped on the shared WebGL thread, and requestAnimationFrame is held back while a canvas is behind. - Resizing reallocates the drawing buffer, queued behind earlier calls; a GL-backed 2D context reallocates its own as it resizes. - Snapshots read the drawing buffer on the context's thread. - The texImage helpers and video frame uploads (NSCRender) run on the context's thread; NSCRender makes its GL objects on first use there. - Depth and stencil are discarded after each present unless preserveDrawingBuffer is set, as GLKView did. --- crates/canvas-c/src/c2d/context.rs | 7 + crates/canvas-core/src/gpu/gl/ios.rs | 568 ++++++++++-------- crates/canvas-ios/src/lib.rs | 246 +++++--- crates/canvas-webgl/src/prelude.rs | 6 + .../typings/objc!CanvasNative.d.ts | 4 +- packages/canvas-media/video/index.ios.ts | 11 +- packages/canvas/Canvas/index.ios.ts | 18 + .../CanvasNative/Source/CanvasHelpers.swift | 58 +- .../CanvasNative/Source/GLRenderer.swift | 80 +-- .../CanvasNative/Source/NSCCanvas.swift | 116 ++-- .../CanvasNative/include/canvas_ios.h | 26 +- .../src-native/canvas-ios/NSCRender.swift | 77 ++- 12 files changed, 678 insertions(+), 539 deletions(-) diff --git a/crates/canvas-c/src/c2d/context.rs b/crates/canvas-c/src/c2d/context.rs index e4b5ca9b1..597c33807 100644 --- a/crates/canvas-c/src/c2d/context.rs +++ b/crates/canvas-c/src/c2d/context.rs @@ -124,6 +124,13 @@ pub fn resize_gl(context: &mut CanvasRenderingContext2D, width: f32, height: f32 context.clear_rect(0., 0., width, height); context.flush_and_render_to_surface(); + // The view's layer changed size; its drawable follows, and is what the surface wraps. + #[cfg(any(target_os = "ios", target_os = "visionos", target_os = "tvos"))] + if let Some(gl) = context.gl_context.as_mut() { + gl.resize_drawable(width as i32, height as i32); + gl.bind_drawable(); + } + unsafe { gl_bindings::Viewport(0, 0, width as i32, height as i32); match context.window_framebuffer() { diff --git a/crates/canvas-core/src/gpu/gl/ios.rs b/crates/canvas-core/src/gpu/gl/ios.rs index 4c9b8793a..89783e9e3 100644 --- a/crates/canvas-core/src/gpu/gl/ios.rs +++ b/crates/canvas-core/src/gpu/gl/ios.rs @@ -7,8 +7,7 @@ use std::sync::OnceLock; use objc2::ffi::BOOL; use objc2::{class, msg_send, msg_send_id, rc::Id, Encode, Encoding}; -use objc2_foundation::{NSData, NSInteger, NSObject, NSUInteger}; -use objc2_foundation::{NSPoint, NSRect, NSSize}; +use objc2_foundation::{NSObject, NSUInteger}; use crate::context_attributes::ContextAttributes; @@ -57,18 +56,17 @@ fn forget_current_binding() { set_current_binding(UNBOUND); } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub(crate) struct GLContextInner { context: Option, sharegroup: EAGLSharegroup, - view: Option, + drawable: Drawable, } #[derive(Debug, Clone)] pub struct GLContextRaw { context: Option, sharegroup: EAGLSharegroup, - view: Option, } impl GLContextRaw { @@ -90,6 +88,19 @@ unsafe impl Sync for GLContextInner {} unsafe impl Send for GLContextInner {} +impl Drop for GLContextInner { + fn drop(&mut self) { + let Some(context) = self.context.as_ref() else { + return; + }; + if EAGLContext::set_current_context(Some(context)) { + self.drawable.delete(); + // EAGL keeps the current context alive until another replaces it. + EAGLContext::set_current_context(None); + } + } +} + #[derive(Debug, Default)] pub struct GLContext(GLContextInner); @@ -234,221 +245,255 @@ impl EAGLContext { let result: BOOL = unsafe { msg_send![&self.0, presentRenderbuffer: 0x8d41 as NSUInteger] }; result.into() } -} -#[derive(Debug)] -#[repr(i32)] -pub enum GLKViewDrawableColorFormat { - RGBA8888 = 0, - RGB565 = 1, - SRGBA8888 = 2, -} - -unsafe impl Encode for GLKViewDrawableColorFormat { - const ENCODING: Encoding = Encoding::ULong; -} - -impl TryFrom for GLKViewDrawableColorFormat { - type Error = &'static str; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(GLKViewDrawableColorFormat::RGBA8888), - 1 => Ok(GLKViewDrawableColorFormat::RGB565), - 2 => Ok(GLKViewDrawableColorFormat::SRGBA8888), - _ => Err("Invalid GLKViewDrawableColorFormat"), - } + /// Stores the bound renderbuffer in `layer` (a `CAEAGLLayer`), at the layer's size in pixels. + pub fn renderbuffer_storage_from_drawable(&self, layer: &NSObject) -> bool { + let result: BOOL = unsafe { + msg_send![&self.0, renderbufferStorage: 0x8d41 as NSUInteger, fromDrawable: layer] + }; + result.into() } } -#[derive(Debug)] -#[repr(i32)] -pub enum GLKViewDrawableDepthFormat { - DepthFormatNone = 0, - DepthFormat16 = 1, - DepthFormat24 = 2, -} +type DiscardFramebuffer = unsafe extern "C" fn(target: u32, count: i32, attachments: *const u32); -unsafe impl Encode for GLKViewDrawableDepthFormat { - const ENCODING: Encoding = Encoding::ULong; +/// `glDiscardFramebufferEXT`, for GLES2 contexts, which lack `glInvalidateFramebuffer`. +fn discard_framebuffer_ext() -> Option { + static DISCARD: OnceLock> = OnceLock::new(); + *DISCARD.get_or_init(|| { + let function = super::get_proc_address("glDiscardFramebufferEXT"); + (!function.is_null()) + .then(|| unsafe { std::mem::transmute::<*const c_void, DiscardFramebuffer>(function) }) + }) } -impl TryFrom for GLKViewDrawableDepthFormat { - type Error = &'static str; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(GLKViewDrawableDepthFormat::DepthFormatNone), - 1 => Ok(GLKViewDrawableDepthFormat::DepthFormat16), - 2 => Ok(GLKViewDrawableDepthFormat::DepthFormat24), - _ => Err("Invalid GLKViewDrawableDepthFormat"), - } - } +fn bound(binding: u32) -> u32 { + let mut name = 0; + unsafe { gl_bindings::GetIntegerv(binding, &mut name) }; + name as u32 } -#[derive(Debug)] -#[repr(i32)] -pub enum GLKViewDrawableStencilFormat { - StencilFormatNone = 0, - StencilFormat8 = 1, -} - -unsafe impl Encode for GLKViewDrawableStencilFormat { - const ENCODING: Encoding = Encoding::ULong; +/// What a context draws to by default (WebGL's null framebuffer, a 2D surface): a framebuffer +/// whose color buffer is stored in a view's `CAEAGLLayer` and presented to it, or a plain one +/// offscreen. Everything here runs with the context current, on whichever thread owns it: none of +/// it needs the main thread. +#[derive(Debug, Default)] +struct Drawable { + layer: Option>, + framebuffer: u32, + color: u32, + /// Packed depth and stencil, attached as either or both. + depth_stencil: u32, + depth: bool, + stencil: bool, + /// Depth and stencil outlive a present (preserveDrawingBuffer). + preserve: bool, + legacy: bool, + width: i32, + height: i32, } -impl TryFrom for GLKViewDrawableStencilFormat { - type Error = &'static str; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(GLKViewDrawableStencilFormat::StencilFormatNone), - 1 => Ok(GLKViewDrawableStencilFormat::StencilFormat8), - _ => Err("Invalid GLKViewDrawableStencilFormat"), +impl Drawable { + fn new(layer: Option>, attrs: &ContextAttributes) -> Self { + let mut drawable = Drawable { + layer, + depth: attrs.get_depth(), + // 2D clips through the stencil. + stencil: attrs.get_stencil() || attrs.get_is_canvas(), + preserve: attrs.get_preserve_drawing_buffer(), + legacy: attrs.get_gl_legacy(), + ..Default::default() + }; + unsafe { + gl_bindings::GenFramebuffers(1, &mut drawable.framebuffer); + gl_bindings::GenRenderbuffers(1, &mut drawable.color); + if drawable.depth || drawable.stencil { + gl_bindings::GenRenderbuffers(1, &mut drawable.depth_stencil); + } } + drawable } -} - -#[derive(Debug)] -#[repr(i32)] -pub enum GLKViewDrawableMultisample { - DrawableMultisampleNone = 0, - DrawableMultisample4X = 1, -} -unsafe impl Encode for GLKViewDrawableMultisample { - const ENCODING: Encoding = Encoding::ULong; -} - -impl TryFrom for GLKViewDrawableMultisample { - type Error = &'static str; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(GLKViewDrawableMultisample::DrawableMultisampleNone), - 1 => Ok(GLKViewDrawableMultisample::DrawableMultisample4X), - _ => Err("Invalid GLKViewDrawableMultisample"), + /// (Re)allocates the buffers, at the layer's size in pixels or `width` x `height` offscreen, and + /// leaves the framebuffer bound. A layer with no size yet leaves it incomplete. + fn store(&mut self, context: &EAGLContext, width: i32, height: i32) { + unsafe { + let renderbuffer = bound(gl_bindings::RENDERBUFFER_BINDING); + gl_bindings::BindRenderbuffer(gl_bindings::RENDERBUFFER, self.color); + match self.layer.as_ref() { + Some(layer) => { + context.renderbuffer_storage_from_drawable(layer); + } + None => gl_bindings::RenderbufferStorage( + gl_bindings::RENDERBUFFER, + gl_bindings::RGBA8, + width.max(1), + height.max(1), + ), + } + gl_bindings::GetRenderbufferParameteriv( + gl_bindings::RENDERBUFFER, + gl_bindings::RENDERBUFFER_WIDTH, + &mut self.width, + ); + gl_bindings::GetRenderbufferParameteriv( + gl_bindings::RENDERBUFFER, + gl_bindings::RENDERBUFFER_HEIGHT, + &mut self.height, + ); + if self.depth_stencil != 0 && self.width > 0 && self.height > 0 { + gl_bindings::BindRenderbuffer(gl_bindings::RENDERBUFFER, self.depth_stencil); + gl_bindings::RenderbufferStorage( + gl_bindings::RENDERBUFFER, + gl_bindings::DEPTH24_STENCIL8, + self.width, + self.height, + ); + } + gl_bindings::BindRenderbuffer(gl_bindings::RENDERBUFFER, renderbuffer); + + gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, self.framebuffer); + gl_bindings::FramebufferRenderbuffer( + gl_bindings::FRAMEBUFFER, + gl_bindings::COLOR_ATTACHMENT0, + gl_bindings::RENDERBUFFER, + self.color, + ); + if self.depth { + gl_bindings::FramebufferRenderbuffer( + gl_bindings::FRAMEBUFFER, + gl_bindings::DEPTH_ATTACHMENT, + gl_bindings::RENDERBUFFER, + self.depth_stencil, + ); + } + if self.stencil { + gl_bindings::FramebufferRenderbuffer( + gl_bindings::FRAMEBUFFER, + gl_bindings::STENCIL_ATTACHMENT, + gl_bindings::RENDERBUFFER, + self.depth_stencil, + ); + } } } -} -#[derive(Clone, Debug)] -pub(crate) struct GLKView(Id); - -impl GLKView { - pub fn new() -> Self { - let cls = class!(CanvasGLKView); - let instance = unsafe { msg_send_id![cls, alloc] }; - GLKView(unsafe { - msg_send_id![ - instance, - initWithFrame: NSRect::default() - ] - }) + fn bind(&self) { + unsafe { gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, self.framebuffer) }; } - pub fn new_with_frame(x: f32, y: f32, width: f32, height: f32) -> Self { + /// Tells the GPU depth and stencil needn't be written back to memory: the next frame starts + /// them over. + fn discard_depth_stencil(&self) { + let mut attachments = [0u32; 2]; + let mut count = 0; + if self.depth { + attachments[count] = gl_bindings::DEPTH_ATTACHMENT; + count += 1; + } + if self.stencil { + attachments[count] = gl_bindings::STENCIL_ATTACHMENT; + count += 1; + } + if count == 0 { + return; + } unsafe { - let cls = class!(CanvasGLKView); - let instance = msg_send_id![cls, alloc]; - let point = NSPoint::new(x as f64, y as f64); - let size = NSSize::new(width as f64, height as f64); - let frame = NSRect::new(point, size); - GLKView(msg_send_id![instance, initWithFrame: frame]) + let framebuffer = bound(gl_bindings::FRAMEBUFFER_BINDING); + self.bind(); + if self.legacy { + if let Some(discard) = discard_framebuffer_ext() { + discard(gl_bindings::FRAMEBUFFER, count as i32, attachments.as_ptr()); + } + } else { + gl_bindings::InvalidateFramebuffer( + gl_bindings::FRAMEBUFFER, + count as i32, + attachments.as_ptr(), + ); + } + gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, framebuffer); } } - pub fn snapshot(&self) -> Vec { - let width = self.drawable_width(); - let height = self.drawable_height(); - let size = width * height * 4; - let mut buf = vec![0u8; size as usize]; - let mut data = unsafe { - NSData::dataWithBytesNoCopy_length( - NonNull::new(buf.as_mut_ptr() as _).unwrap(), - size as NSUInteger, - ) - }; - let _: () = unsafe { msg_send![&self.0, snapshotWithData: &*data] }; - forget_current_binding(); // GLKView binds its own context - buf - } - - pub fn display(&self) { - let _: () = unsafe { msg_send![&self.0, display] }; - forget_current_binding(); // GLKView binds its own context - } - - pub fn drawable_width(&self) -> NSInteger { - return unsafe { msg_send![&self.0, drawableWidth] }; - } - - pub fn drawable_height(&self) -> NSInteger { - return unsafe { msg_send![&self.0, drawableHeight] }; - } - - pub fn bind_drawable(&self) { - let _: () = unsafe { msg_send![&self.0, bindDrawable] }; - forget_current_binding(); // GLKView binds its own context - } - - pub fn delete_drawable(&self) { - let _: () = unsafe { msg_send![&self.0, deleteDrawable] }; - } - - pub fn set_alpha(&self, alpha: bool) { - let layer: Id = unsafe { msg_send_id![&self.0, layer] }; - let _: () = unsafe { msg_send![&layer, setOpaque: !alpha] }; - } - - pub fn get_alpha(&self) -> bool { - let layer: Id = unsafe { msg_send_id![&self.0, layer] }; - let ret: bool = unsafe { msg_send![&layer, isOpaque] }; - !ret - } - - pub fn set_drawable_depth_format(&self, format: GLKViewDrawableDepthFormat) { - let _: () = unsafe { msg_send![&self.0, drawableDepthFormat: format] }; - } - - pub fn get_drawable_depth_format(&self) -> GLKViewDrawableDepthFormat { - let depth: i32 = unsafe { msg_send![&self.0, drawableDepthFormat] }; - GLKViewDrawableDepthFormat::try_from(depth).unwrap() - } - - pub fn set_drawable_stencil_format(&self, format: GLKViewDrawableStencilFormat) { - let _: () = unsafe { msg_send![&self.0, drawableStencilFormat: format] }; - } - - pub fn get_drawable_stencil_format(&self) -> GLKViewDrawableStencilFormat { - let stencil: i32 = unsafe { msg_send![&self.0, drawableStencilFormat] }; - GLKViewDrawableStencilFormat::try_from(stencil).unwrap() - } - - pub fn set_drawable_multisample(&self, sample: GLKViewDrawableMultisample) { - let _: () = unsafe { msg_send![&self.0, drawableMultisample: sample] }; - } - - pub fn get_drawable_multisample(&self) -> GLKViewDrawableMultisample { - let sample: i32 = unsafe { msg_send![&self.0, drawableMultisample] }; - GLKViewDrawableMultisample::try_from(sample).unwrap() + /// Hands the color buffer to the layer. Callable from any thread: the layer takes it with the + /// next Core Animation commit, without waiting on the main thread. + fn present(&self, context: &EAGLContext) -> bool { + if self.layer.is_none() || self.width == 0 || self.height == 0 { + return false; + } + if !self.preserve { + self.discard_depth_stencil(); + } + unsafe { + let renderbuffer = bound(gl_bindings::RENDERBUFFER_BINDING); + gl_bindings::BindRenderbuffer(gl_bindings::RENDERBUFFER, self.color); + let presented = context.present_renderbuffer(); + gl_bindings::BindRenderbuffer(gl_bindings::RENDERBUFFER, renderbuffer); + presented + } } - pub fn set_context(&mut self, context: Option<&EAGLContext>) { - match context { - Some(context) => { - let _: () = unsafe { msg_send![&self.0, setContext: &*context.0] }; + /// The color buffer as top-down RGBA. + fn read(&self) -> Vec { + let (width, height) = (self.width.max(0) as usize, self.height.max(0) as usize); + let row = width * 4; + let mut pixels = vec![0u8; row * height]; + if pixels.is_empty() { + return pixels; + } + unsafe { + let framebuffer = bound(gl_bindings::FRAMEBUFFER_BINDING); + let pack_buffer = if self.legacy { + 0 + } else { + bound(gl_bindings::PIXEL_PACK_BUFFER_BINDING) + }; + let mut alignment = 4; + gl_bindings::GetIntegerv(gl_bindings::PACK_ALIGNMENT, &mut alignment); + if pack_buffer != 0 { + gl_bindings::BindBuffer(gl_bindings::PIXEL_PACK_BUFFER, 0); } - None => { - let nil: *mut NSObject = std::ptr::null_mut(); - let _: () = unsafe { msg_send![&self.0, setContext: nil] }; + self.bind(); + gl_bindings::PixelStorei(gl_bindings::PACK_ALIGNMENT, 4); + gl_bindings::ReadPixels( + 0, + 0, + width as i32, + height as i32, + gl_bindings::RGBA, + gl_bindings::UNSIGNED_BYTE, + pixels.as_mut_ptr() as *mut c_void, + ); + gl_bindings::PixelStorei(gl_bindings::PACK_ALIGNMENT, alignment); + gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, framebuffer); + if pack_buffer != 0 { + gl_bindings::BindBuffer(gl_bindings::PIXEL_PACK_BUFFER, pack_buffer); } } + // GL reads bottom-up. + for y in 0..height / 2 { + let (upper, lower) = pixels.split_at_mut((height - 1 - y) * row); + upper[y * row..(y + 1) * row].swap_with_slice(&mut lower[..row]); + } + pixels } - pub fn get_context(&self) -> Option { - let context: Option> = unsafe { msg_send_id![&self.0, context] }; - context.map(EAGLContext) + fn delete(&mut self) { + unsafe { + if self.framebuffer != 0 { + gl_bindings::DeleteFramebuffers(1, &self.framebuffer); + } + if self.color != 0 { + gl_bindings::DeleteRenderbuffers(1, &self.color); + } + if self.depth_stencil != 0 { + gl_bindings::DeleteRenderbuffers(1, &self.depth_stencil); + } + } + self.framebuffer = 0; + self.color = 0; + self.depth_stencil = 0; } } @@ -457,57 +502,68 @@ impl GLContext { GLContextRaw { context: self.0.context.clone(), sharegroup: self.0.sharegroup.clone(), - view: self.0.view.clone(), } } - pub fn set_surface(&mut self, view: NonNull) -> bool { - let glview = unsafe { Id::::from_raw(view.as_ptr() as _) }; - match glview { - None => false, - Some(glview) => { - let id = glview.clone(); - let glview = GLKView(id); - self.0.view = Some(glview); - true - } + /// Moves the drawable to another `CAEAGLLayer`. + pub fn set_surface(&mut self, layer: NonNull) -> bool { + let Some(layer) = (unsafe { Id::::retain(layer.as_ptr() as _) }) else { + return false; + }; + let Some(context) = self.0.context.clone() else { + return false; + }; + if !EAGLContext::set_current_context(Some(&context)) { + return false; } + let framebuffer = bound(gl_bindings::FRAMEBUFFER_BINDING); + self.0.drawable.layer = Some(layer); + self.0.drawable.store(&context, 0, 0); + unsafe { gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, framebuffer) }; + true } + + /// Reallocates the drawable after its layer changed size (`width` and `height` size an + /// offscreen one). Its contents are lost; the bindings are kept. + pub fn resize_drawable(&mut self, width: i32, height: i32) { + let Some(context) = self.0.context.clone() else { + return; + }; + if !EAGLContext::set_current_context(Some(&context)) { + return; + } + let framebuffer = bound(gl_bindings::FRAMEBUFFER_BINDING); + self.0.drawable.store(&context, width, height); + unsafe { gl_bindings::BindFramebuffer(gl_bindings::FRAMEBUFFER, framebuffer) }; + } + + /// `layer` is the view's `CAEAGLLayer`. pub fn create_shared_window_context( context_attrs: &mut ContextAttributes, - view: NonNull, + layer: NonNull, context: &GLContext, ) -> Option { - let glview = unsafe { Id::::retain(view.as_ptr() as _) }; - match glview { - None => None, - Some(glview) => { - let glview = GLKView(glview); - GLContext::create_window_context_with_gl_view(context_attrs, glview, Some(context)) - } - } + let layer = unsafe { Id::::retain(layer.as_ptr() as _) }?; + GLContext::create(context_attrs, Some(layer), 0, 0, Some(context)) } + /// `layer` is the view's `CAEAGLLayer`. pub fn create_window_context( context_attrs: &mut ContextAttributes, - view: NonNull, + layer: NonNull, ) -> Option { - let glview = unsafe { Id::::from_raw(view.as_ptr() as _) }; - match glview { - None => None, - Some(glview) => { - let glview = GLKView(glview); - GLContext::create_window_context_with_gl_view(context_attrs, glview, None) - } - } + let layer = unsafe { Id::::retain(layer.as_ptr() as _) }?; + GLContext::create(context_attrs, Some(layer), 0, 0, None) } - pub(crate) fn create_window_context_with_gl_view( + /// Makes the context current on the calling thread, with its drawable bound. + fn create( context_attrs: &mut ContextAttributes, - mut view: GLKView, + layer: Option>, + width: i32, + height: i32, shared_context: Option<&GLContext>, ) -> Option { - let gl_view = view.clone(); IS_GL_SYMBOLS_LOADED.get_or_init(|| { gl_bindings::load_with(|symbol| super::get_proc_address(symbol).cast()); true @@ -531,24 +587,21 @@ impl GLContext { } }; - let context = EAGLContext::new_with_api_sharegroup(api, &share_group); + let context = EAGLContext::new_with_api_sharegroup(api, &share_group)?; - if context.is_none() { + if !EAGLContext::set_current_context(Some(&context)) { return None; } - view.set_context(context.as_ref()); - - EAGLContext::set_current_context(context.as_ref()); - view.display(); + let mut drawable = Drawable::new(layer, context_attrs); + drawable.store(&context, width, height); + unsafe { gl_bindings::Viewport(0, 0, drawable.width, drawable.height) }; - let inner = GLContextInner { - context, + Some(GLContext(GLContextInner { + context: Some(context), sharegroup: share_group, - view: Some(view), - }; - - Some(GLContext(inner)) + drawable, + })) } pub fn create_offscreen_context( @@ -556,9 +609,7 @@ impl GLContext { width: i32, height: i32, ) -> Option { - let view = GLKView::new_with_frame(0., 0., width as f32, height as f32); - - GLContext::create_window_context_with_gl_view(context_attrs, view, None) + GLContext::create(context_attrs, None, width, height, None) } pub fn create_shared_offscreen_context( @@ -567,8 +618,7 @@ impl GLContext { height: i32, shared_context: &GLContext, ) -> Option { - let view = GLKView::new_with_frame(0., 0., width as f32, height as f32); - GLContext::create_window_context_with_gl_view(context_attrs, view, Some(shared_context)) + GLContext::create(context_attrs, None, width, height, Some(shared_context)) } fn has_extension(extensions: &str, name: &str) -> bool { @@ -579,8 +629,12 @@ impl GLContext { true } + /// The drawable as top-down RGBA. pub fn snapshot(&self) -> Option> { - self.0.view.as_ref().map(|view| view.snapshot()) + if !self.make_current() { + return None; + } + Some(self.0.drawable.read()) } pub fn set_vsync(&self, _sync: bool) -> bool { @@ -603,45 +657,25 @@ impl GLContext { } pub fn bind_drawable(&self) { - if let Some(view) = self.0.view.as_ref() { - view.bind_drawable(); - } + self.0.drawable.bind(); } pub fn swap_buffers(&self) -> bool { - if let Some(view) = self.0.view.as_ref() { - view.display(); - return true; + match self.0.context.as_ref() { + Some(context) => self.0.drawable.present(context), + None => false, } - false } pub fn get_surface_width(&self) -> i32 { - self.0 - .view - .as_ref() - .map(|v| v.drawable_width().try_into().unwrap_or_default()) - .unwrap() + self.0.drawable.width } pub fn get_surface_height(&self) -> i32 { - self.0 - .view - .as_ref() - .map(|v| v.drawable_height().try_into().unwrap_or_default()) - .unwrap() + self.0.drawable.height } pub fn get_surface_dimensions(&self) -> (i32, i32) { - self.0 - .view - .as_ref() - .map(|v| { - ( - v.drawable_width().try_into().unwrap_or_default(), - v.drawable_height().try_into().unwrap_or_default(), - ) - }) - .unwrap() + (self.0.drawable.width, self.0.drawable.height) } } diff --git a/crates/canvas-ios/src/lib.rs b/crates/canvas-ios/src/lib.rs index 3db5fe815..ac2711c98 100644 --- a/crates/canvas-ios/src/lib.rs +++ b/crates/canvas-ios/src/lib.rs @@ -16,9 +16,11 @@ use std::ffi::{c_int, c_longlong, c_void, CStr}; use std::os::raw::c_char; use std::ptr::NonNull; +/// A WebGL context drawing to `layer`, the view's `CAEAGLLayer`. A threaded one lives on the WebGL +/// thread, so presents never wait on the GPU here. #[no_mangle] pub extern "C" fn canvas_native_ios_create_webgl_context( - view: *mut c_void, + layer: *mut c_void, alpha: bool, antialias: bool, depth: bool, @@ -30,17 +32,21 @@ pub extern "C" fn canvas_native_ios_create_webgl_context( desynchronized: bool, xr_compatible: bool, version: u32, + threaded: bool, ) -> c_longlong { if version == 2 && !GLContext::has_gl2support() { return 0; } - - // let _ = env_logger::try_init(); - - if let Some(power_preference) = PowerPreference::try_from(power_preference).ok() { - return Box::into_raw(Box::new(WebGLState::new_with_view( - view, - WebGLVersion::try_from(version as i32).unwrap(), + let (Ok(version), Ok(power_preference)) = ( + WebGLVersion::try_from(version as i32), + PowerPreference::try_from(power_preference), + ) else { + return 0; + }; + let create = || { + WebGLState::new_with_view( + layer, + version, alpha, antialias, depth, @@ -52,10 +58,14 @@ pub extern "C" fn canvas_native_ios_create_webgl_context( desynchronized, xr_compatible, false, - ))) as i64; - } - - 0 + ) + }; + let state = if threaded { + WebGLState::create_threaded(create) + } else { + create().map_or(std::ptr::null_mut(), |state| Box::into_raw(Box::new(state))) + }; + state as i64 } #[no_mangle] @@ -63,12 +73,60 @@ pub extern "C" fn canvas_native_ios_flush_webgl(context: i64) -> bool { if context == 0 { return false; } + unsafe { &*(context as *const WebGLState) }.present() +} - let context = context as *mut WebGLState; - let context = unsafe { &mut *context }; +/// Reallocates the drawing buffer after the view's layer changed size. Queued: whatever reads the +/// size next runs after it. +#[no_mangle] +pub extern "C" fn canvas_native_ios_resize_webgl(context: i64, width: i32, height: i32) { + if context == 0 { + return; + } + unsafe { &*(context as *const WebGLState) } + .post(move |state| state.resize_drawable(width, height)); +} - context.get_inner().make_current(); - context.get_inner().swap_buffers() +/// The drawing buffer as top-down RGBA, `width` x `height`. +#[no_mangle] +pub extern "C" fn canvas_native_ios_webgl_read_pixels( + context: i64, + width: *mut i32, + height: *mut i32, +) -> *mut U8Buffer { + if context == 0 || width.is_null() || height.is_null() { + return std::ptr::null_mut(); + } + let state = unsafe { &*(context as *const WebGLState) }; + let (w, h, pixels) = state.sync(|state| { + let (w, h) = state.get_dimensions(); + (w, h, state.snapshot().unwrap_or_default()) + }); + unsafe { + *width = w; + *height = h; + } + Box::into_raw(Box::new(U8Buffer::from(pixels))) +} + +/// Runs `callback(data)` with the context current, on the thread that owns it, and waits for it: +/// native code that draws with a threaded context (video frames) has to run there. +#[no_mangle] +pub extern "C" fn canvas_native_ios_webgl_run( + context: i64, + callback: Option, + data: *mut c_void, +) { + let Some(callback) = callback else { + return; + }; + if context == 0 { + return; + } + unsafe { &*(context as *const WebGLState) }.sync(|state| { + state.make_current(); + callback(data); + }); } #[no_mangle] @@ -189,12 +247,8 @@ pub extern "C" fn canvas_native_ios_update_webgl_surface( return; } - if let Some(ios_view) = NonNull::new(view as *mut c_void) { - let context = context as *mut WebGLState; - let context = unsafe { &mut *context }; - - let context = context.get_inner_mut(); - context.set_surface(ios_view); + if let Some(layer) = NonNull::new(view as *mut c_void) { + unsafe { &*(context as *const WebGLState) }.sync(|state| state.set_surface(layer)); } } @@ -221,7 +275,7 @@ pub extern "C" fn canvas_native_ios_gl_make_current(context: i64) { } let gl_context = context as *mut WebGLState; let gl_context = unsafe { &*gl_context }; - gl_context.get_inner().make_current(); + gl_context.sync(|state| state.make_current()); } #[no_mangle] @@ -629,43 +683,44 @@ pub extern "C" fn canvas_native_ios_webgl_tex_image_2d( let gl_context = context as *mut WebGLState; let gl_context = unsafe { &*gl_context }; - gl_context.get_inner().make_current(); - - unsafe { - if flip_y { - let mut buffer = bytes.to_vec(); - canvas_webgl::utils::gl::flip_in_place( - buffer.as_mut_ptr(), - buffer.len(), - canvas_webgl::utils::gl::bytes_per_pixel(type_ as _, format as _) as _, - height as usize, - ); - - gl_bindings::TexImage2D( - target as u32, - level, - internalformat, - width as i32, - height as i32, - 0, - format as u32, - type_ as u32, - buffer.as_ptr() as *const std::os::raw::c_void, - ); - } else { - gl_bindings::TexImage2D( - target as u32, - level, - internalformat, - width as i32, - height as i32, - 0, - format as u32, - type_ as u32, - bytes.as_ptr() as *const std::os::raw::c_void, - ); + gl_context.sync(|state| { + state.make_current(); + unsafe { + if flip_y { + let mut buffer = bytes.to_vec(); + canvas_webgl::utils::gl::flip_in_place( + buffer.as_mut_ptr(), + buffer.len(), + canvas_webgl::utils::gl::bytes_per_pixel(type_ as _, format as _) as _, + height as usize, + ); + + gl_bindings::TexImage2D( + target as u32, + level, + internalformat, + width as i32, + height as i32, + 0, + format as u32, + type_ as u32, + buffer.as_ptr() as *const std::os::raw::c_void, + ); + } else { + gl_bindings::TexImage2D( + target as u32, + level, + internalformat, + width as i32, + height as i32, + 0, + format as u32, + type_ as u32, + bytes.as_ptr() as *const std::os::raw::c_void, + ); + } } - } + }); } #[no_mangle] @@ -690,41 +745,42 @@ pub extern "C" fn canvas_native_ios_webgl_tex_sub_image_2d( let gl_context = context as *mut WebGLState; let gl_context = unsafe { &*gl_context }; - gl_context.get_inner().make_current(); - - unsafe { - if flip_y { - let mut buffer = bytes.to_vec(); - canvas_webgl::utils::gl::flip_in_place( - buffer.as_mut_ptr(), - buffer.len(), - canvas_webgl::utils::gl::bytes_per_pixel(type_ as _, format as _) as _, - height as usize, - ); - - gl_bindings::TexSubImage2D( - target as u32, - level, - xoffset, - yoffset, - width as i32, - height as i32, - format as u32, - type_ as u32, - buffer.as_ptr() as *const std::os::raw::c_void, - ); - } else { - gl_bindings::TexSubImage2D( - target as u32, - level, - xoffset, - yoffset, - width as i32, - height as i32, - format as u32, - type_ as u32, - bytes.as_ptr() as *const std::os::raw::c_void, - ); + gl_context.sync(|state| { + state.make_current(); + unsafe { + if flip_y { + let mut buffer = bytes.to_vec(); + canvas_webgl::utils::gl::flip_in_place( + buffer.as_mut_ptr(), + buffer.len(), + canvas_webgl::utils::gl::bytes_per_pixel(type_ as _, format as _) as _, + height as usize, + ); + + gl_bindings::TexSubImage2D( + target as u32, + level, + xoffset, + yoffset, + width as i32, + height as i32, + format as u32, + type_ as u32, + buffer.as_ptr() as *const std::os::raw::c_void, + ); + } else { + gl_bindings::TexSubImage2D( + target as u32, + level, + xoffset, + yoffset, + width as i32, + height as i32, + format as u32, + type_ as u32, + bytes.as_ptr() as *const std::os::raw::c_void, + ); + } } - } + }); } diff --git a/crates/canvas-webgl/src/prelude.rs b/crates/canvas-webgl/src/prelude.rs index 6f35861cd..ad19a448b 100644 --- a/crates/canvas-webgl/src/prelude.rs +++ b/crates/canvas-webgl/src/prelude.rs @@ -264,6 +264,12 @@ impl WebGLState { self.context.set_surface(view) } + /// After the view's layer changed size (`width` and `height` size an offscreen drawing buffer). + #[cfg(any(target_os = "ios", target_os = "visionos", target_os = "tvos"))] + pub fn resize_drawable(&mut self, width: i32, height: i32) { + self.context.resize_drawable(width, height) + } + #[cfg(target_os = "android")] pub fn set_surface( &mut self, diff --git a/packages/canvas-media/typings/objc!CanvasNative.d.ts b/packages/canvas-media/typings/objc!CanvasNative.d.ts index 6183d3554..a02c7433d 100644 --- a/packages/canvas-media/typings/objc!CanvasNative.d.ts +++ b/packages/canvas-media/typings/objc!CanvasNative.d.ts @@ -1203,7 +1203,7 @@ declare class NSCCanvas extends UIView { forceLayout(width: number, height: number): void; - getGlViewPtr(): interop.Pointer | interop.Reference; + getGlLayerPtr(): interop.Pointer | interop.Reference; getMtlLayerPtr(): interop.Pointer | interop.Reference; @@ -1499,6 +1499,8 @@ declare class NSCRender extends NSObject { drawFrame(player: AVPlayer, output: AVPlayerItemVideoOutput, videoSize: CGSize, internalFormat: number, format: number, flipYWebGL: boolean): void; + drawFrame(player: AVPlayer, output: AVPlayerItemVideoOutput, videoSize: CGSize, internalFormat: number, format: number, flipYWebGL: boolean, context: number): void; + drawFrameWithBufferWidthHeightInternalFormatFormatFlipYWebGL(buffer: any, width: number, height: number, internalFormat: number, format: number, flipYWebGL: boolean): void; initWithDevice(device: MTLDevice): this; diff --git a/packages/canvas-media/video/index.ios.ts b/packages/canvas-media/video/index.ios.ts index 580f6f966..f4888cd37 100644 --- a/packages/canvas-media/video/index.ios.ts +++ b/packages/canvas-media/video/index.ios.ts @@ -4,6 +4,11 @@ import { controlsProperty, playsinlineProperty, mutedProperty, currentTimeProper import { booleanConverter, knownFolders, path } from '@nativescript/core'; declare const NSCCanvasUtils, NSCVideoHelper, NSCRender, NSCVideoFrameBridge; +/** The native context behind a WebGL context, whose thread video frames are uploaded on. */ +function webglContextPointer(context: any): number { + return Number(context?._contextPtr ?? 0); +} + interface NSCVideoHelperListener {} //@ts-ignore @@ -126,7 +131,7 @@ export class Video extends VideoBase { if (!this._renderer) { this._renderer = NSCRender.alloc().init(); } - this._renderer.drawFrame(this.helper.player, this.helper.assetOutput, this.helper.videoSize, arguments[4], arguments[5], flipY); + this._renderer.drawFrame(this.helper.player, this.helper.assetOutput, this.helper.videoSize, arguments[4], arguments[5], flipY, webglContextPointer(arguments[1])); // NSCCanvasUtils.drawFrame(this.helper.player, this.helper.assetOutput, this.helper.videoSize, arguments[4], arguments[5], flipY); } catch (e) { console.error('getCurrentFrame error:', e); @@ -146,7 +151,7 @@ export class Video extends VideoBase { } try { //@ts-ignore - this._renderer.drawFrameTexImage3D(this.helper.player, this.helper.assetOutput, this.helper.videoSize, target, level, internalformat, width, height, depth, border, format, type, flipY); + this._renderer.drawFrameTexImage3D(this.helper.player, this.helper.assetOutput, this.helper.videoSize, target, level, internalformat, width, height, depth, border, format, type, flipY, webglContextPointer(ctx)); return; } catch (e) { console.error('getFrameForTexImage3D error:', e); @@ -187,7 +192,7 @@ export class Video extends VideoBase { } try { //@ts-ignore - this._renderer.drawFrameTexSubImage3D(this.helper.player, this.helper.assetOutput, this.helper.videoSize, target, level, xoffset, yoffset, zoffset, width, height, depth, format, type, flipY); + this._renderer.drawFrameTexSubImage3D(this.helper.player, this.helper.assetOutput, this.helper.videoSize, target, level, xoffset, yoffset, zoffset, width, height, depth, format, type, flipY, webglContextPointer(ctx)); return; } catch (e) { console.error('getFrameForTexSubImage3D error:', e); diff --git a/packages/canvas/Canvas/index.ios.ts b/packages/canvas/Canvas/index.ios.ts index 2967b7c54..2f870b33b 100644 --- a/packages/canvas/Canvas/index.ios.ts +++ b/packages/canvas/Canvas/index.ios.ts @@ -60,6 +60,12 @@ export class Canvas extends CanvasBase { /** Default for `getContext('2d', { threaded })`: rasterize on a shared render thread. */ static threaded2D = true; + + /** + * Default for `getContext('webgl' | 'webgl2', { threaded })`: run the context on the WebGL + * thread, so GL work and presenting never block the UI thread. + */ + static threadedWebGL = true; surfaceOnTop = false; _renderer; @@ -521,6 +527,16 @@ export class Canvas extends CanvasBase { return context; } + private __setThreadedWebGL(options?: any) { + const threaded = !!(options?.threaded ?? Canvas.threadedWebGL); + if ('threadedWebGL' in this._canvas) { + this._canvas.threadedWebGL = threaded; + } + if (threaded) { + holdBackAnimationFramesWhileBehind(); + } + } + _hasContext(): boolean { return this._contextType !== ContextType.None; } @@ -569,6 +585,7 @@ export class Canvas extends CanvasBase { if (!this._webglContext) { const opts = { version: 1, ...defaultOpts, ...handleContextOptions(type, options) }; + this.__setThreadedWebGL(options); this._canvas.initContext(type, opts.alpha, false, opts.depth, opts.failIfMajorPerformanceCaveat, opts.powerPreference, opts.premultipliedAlpha, opts.preserveDrawingBuffer, opts.stencil, opts.desynchronized, opts.xrCompatible, false, opts.colorSpace ?? 0); this._webglContext = new (WebGLRenderingContext as any)(this._canvas, opts); @@ -585,6 +602,7 @@ export class Canvas extends CanvasBase { if (!this._webgl2Context) { const opts = { version: 2, ...defaultOpts, ...handleContextOptions(type, options) }; + this.__setThreadedWebGL(options); this._canvas.initContext(type, opts.alpha, false, opts.depth, opts.failIfMajorPerformanceCaveat, opts.powerPreference, opts.premultipliedAlpha, opts.preserveDrawingBuffer, opts.stencil, opts.desynchronized, opts.xrCompatible, false, opts.colorSpace ?? 0); this._webgl2Context = new (WebGL2RenderingContext as any)(this._canvas, opts); diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/CanvasHelpers.swift b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/CanvasHelpers.swift index 0c5a9bb2d..0e731d46a 100644 --- a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/CanvasHelpers.swift +++ b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/CanvasHelpers.swift @@ -134,16 +134,10 @@ public class CanvasHelpers: NSObject { _ stencil:Bool, _ desynchronized:Bool, _ xr_compatible:Bool, - _ version: Int32) -> Int64{ - - let ret = canvas_native_webgl_create(view.getGlViewPtr(), version, alpha, antialias, depth, fail_if_major_performance_caveat, power_preference, premultiplied_alpha, preserve_drawing_buffer, stencil, desynchronized, xr_compatible) - - - if(ret == nil){ - return 0 - } - return unsafeBitCast(ret, to: Int64.self) + _ version: Int32, + _ threaded: Bool) -> Int64{ + return canvas_native_ios_create_webgl_context(view.getGlLayerPtr(), alpha, antialias, depth, fail_if_major_performance_caveat, power_preference, premultiplied_alpha, preserve_drawing_buffer, stencil, desynchronized, xr_compatible, UInt32(version), threaded) } @@ -212,6 +206,50 @@ public class CanvasHelpers: NSObject { public static func flushWebGL(_ context: Int64)-> Bool { return canvas_native_ios_flush_webgl(context) } + + /// After the GL view's layer changed size. + public static func resizeWebGL(_ context: Int64, _ width: Int, _ height: Int) { + canvas_native_ios_resize_webgl(context, Int32(width), Int32(height)) + } + + /// A WebGL context's drawing buffer as an image, read on the thread that owns the context. + public static func snapshotWebGL(_ context: Int64) -> UIImage? { + var width: Int32 = 0 + var height: Int32 = 0 + guard context != 0, let buffer = canvas_native_ios_webgl_read_pixels(context, &width, &height) else { + return nil + } + defer { canvas_native_u8_buffer_release(buffer) } + let (w, h) = (Int(width), Int(height)) + let length = Int(canvas_native_u8_buffer_get_length(buffer)) + guard w > 0, h > 0, length >= w * h * 4, let bytes = canvas_native_u8_buffer_get_bytes(buffer), + let provider = CGDataProvider(data: Data(bytes: bytes, count: length) as CFData), + let image = CGImage(width: w, height: h, bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: w * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, decode: nil, shouldInterpolate: false, + intent: .defaultIntent) else { + return nil + } + return UIImage(cgImage: image) + } + + /// Runs `block` with the WebGL context current, on the thread that owns it, and waits for it: a + /// threaded context is only ever current on the WebGL thread, so GL work against it runs there. + public static func runWithWebGL(_ context: Int64, _ block: () -> Void) { + if context == 0 { + block() + return + } + withoutActuallyEscaping(block) { block in + var block = block + withUnsafeMutablePointer(to: &block) { pointer in + canvas_native_ios_webgl_run(context, { data in + data?.assumingMemoryBound(to: (() -> Void).self).pointee() + }, pointer) + } + } + } public static func releaseWebGL(_ context: Int64) { @@ -242,7 +280,7 @@ public class CanvasHelpers: NSObject { cs = CanvasColorSpaceP3 } - return canvas_native_ios_create_2d_context(view.getGlViewPtr(), width, height, alpha, density, fontColor, ppi, direction, cs) + return canvas_native_ios_create_2d_context(view.getGlLayerPtr(), width, height, alpha, density, fontColor, ppi, direction, cs) } public static func create2DContextMetal( diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/GLRenderer.swift b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/GLRenderer.swift index 1d7d4095c..03d87e136 100644 --- a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/GLRenderer.swift +++ b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/GLRenderer.swift @@ -7,87 +7,33 @@ import Foundation #if !os(visionOS) -import GLKit +import OpenGLES #endif import UIKit #if !os(visionOS) +/// Shows a GL context's drawing buffer. The context stores its color buffer in this view's layer +/// and presents it there itself, from whichever thread it runs on. @objcMembers -@objc(CanvasGLKView) -public class CanvasGLKView: GLKView, GLKViewDelegate { - var isDirty: Bool = false +@objc(CanvasGLView) +public class CanvasGLView: UIView { internal(set) public weak var canvas: NSCCanvas? = nil - - private(set) var fbo: UInt32 = 0 - - public init() { - super.init(frame: .zero) - } - public override init(frame: CGRect) { - super.init(frame: frame) - delegate = self - } - - required init?(coder: NSCoder) { - super.init(coder: coder) - delegate = self - } - - public override func bindDrawable() { - super.bindDrawable() - var fbo: UInt32 = 0 - glGetIntegerv(GLenum(GL_FRAMEBUFFER_BINDING), &fbo) - self.fbo = fbo - } - - public override func deleteDrawable() { - super.deleteDrawable() - self.fbo = 0 - } - - public func glkView(_ view: GLKView, drawIn rect: CGRect) { -// guard let canvas = canvas else {return} -// if(canvas.is2D){ -// CanvasHelpers.flush2DContext(canvas.nativeContext) -// } - } - -} + public override class var layerClass: AnyClass { + return CAEAGLLayer.self + } -extension GLKView { - @objc public func snapshotWithData(_ data: Data){ - let pixels = self.snapshot - - var cgImage: CGImage? - - if let image = pixels.cgImage { - cgImage = image - } else if let image = pixels.ciImage { - let ctx = CIContext() - cgImage = ctx.createCGImage(image, from: image.extent) - } - - if let image = cgImage { - let width = Int(pixels.size.width) - let height = Int(pixels.size.height) - let row = width * 4 - var buffer = [UInt8](data) - let colorSpace = CGColorSpaceCreateDeviceRGB() - let imageCtx = CGContext(data: &buffer, width: width, height: height, bitsPerComponent: 8, bytesPerRow: row, space: colorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Big.rawValue) - imageCtx!.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) - } + public var eaglLayer: CAEAGLLayer { + return layer as! CAEAGLLayer } } #else -// visionOS has no OpenGL ES. CanvasGLKView is a no-op UIView stand-in so NSCCanvas's +// visionOS has no OpenGL ES. CanvasGLView is a no-op UIView stand-in so NSCCanvas's // layout/lifecycle code compiles; the GL engine is never selected on visionOS. @objcMembers -@objc(CanvasGLKView) -public class CanvasGLKView: UIView { - var isDirty: Bool = false +@objc(CanvasGLView) +public class CanvasGLView: UIView { internal(set) public weak var canvas: NSCCanvas? = nil - private(set) var fbo: UInt32 = 0 } #endif diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCCanvas.swift b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCCanvas.swift index be7959d84..fa8cc0cfc 100644 --- a/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCCanvas.swift +++ b/packages/canvas/src-native/canvas-ios/CanvasNative/Source/NSCCanvas.swift @@ -8,7 +8,7 @@ import Foundation import UIKit #if !os(visionOS) -import GLKit +import OpenGLES #endif #if canImport(WebKit) import WebKit @@ -100,7 +100,7 @@ public class NSCCanvas: UIView { didSet { CATransaction.begin() CATransaction.setDisableActions(true) - glkView.layer.transform = CATransform3DIdentity + glView.layer.transform = CATransform3DIdentity mtlView.layer.transform = CATransform3DIdentity cpuView.layer.transform = CATransform3DIdentity CATransaction.commit() @@ -122,9 +122,10 @@ public class NSCCanvas: UIView { var mtlPtr: UnsafeMutableRawPointer? = nil var mtlLayerPtr: UnsafeMutableRawPointer? = nil - public func getGlViewPtr() -> UnsafeMutableRawPointer { + /// The GL view's `CAEAGLLayer`, which a GL context stores its drawing buffer in. + public func getGlLayerPtr() -> UnsafeMutableRawPointer { if(glPtr == nil){ - glPtr = Unmanaged.passRetained(glkView).toOpaque() + glPtr = Unmanaged.passRetained(glView.layer).toOpaque() } return glPtr! } @@ -147,9 +148,9 @@ public class NSCCanvas: UIView { public var autoScale: Bool = true { didSet { if(!autoScale){ - glkView.contentScaleFactor = 1 + glView.contentScaleFactor = 1 }else { - glkView.contentScaleFactor = nscNativeScale() + glView.contentScaleFactor = nscNativeScale() } } } @@ -162,12 +163,15 @@ public class NSCCanvas: UIView { /// Read when the 2D context is created. @objc public var threaded2D = false + + /// Read when the WebGL context is created: it then lives on the WebGL thread, which presents it. + @objc public var threadedWebGL = false internal var engine = Engine.None internal var mtlView: NSCMTLView - internal var glkView: CanvasGLKView + internal var glView: CanvasGLView internal var cpuView: CanvasCPUView @@ -203,7 +207,7 @@ public class NSCCanvas: UIView { return cpuView.frame.size.width } - return glkView.frame.size.width + return glView.frame.size.width } var drawingBufferHeightRaw: CGFloat { @@ -215,7 +219,7 @@ public class NSCCanvas: UIView { return cpuView.frame.size.height } - return glkView.frame.size.height + return glView.frame.size.height } public var width: Float { @@ -406,56 +410,33 @@ public class NSCCanvas: UIView { if(alpha){ properties[kEAGLDrawablePropertyColorFormat] = kEAGLColorFormatRGBA8 isOpaque = false - glkView.isOpaque = false - (glkView.layer as! CAEAGLLayer).isOpaque = false + glView.isOpaque = false + glView.eaglLayer.isOpaque = false }else { properties[kEAGLDrawablePropertyColorFormat] = kEAGLColorFormatRGBA8 isOpaque = true - (glkView.layer as! CAEAGLLayer).isOpaque = true - glkView.isOpaque = true + glView.eaglLayer.isOpaque = true + glView.isOpaque = true } if(!properties.isEmpty){ - let eaglLayer = self.glkView.layer as! CAEAGLLayer - eaglLayer.drawableProperties = properties - } - - if(useWebGL && depth){ - glkView.drawableDepthFormat = .format24 - }else { - glkView.drawableDepthFormat = .formatNone - } - - if(useWebGL && stencil){ - glkView.drawableStencilFormat = .format8 - }else if(isCanvas) { - glkView.drawableStencilFormat = .format8 + glView.eaglLayer.drawableProperties = properties } - // antialias fails in 2D - if(useWebGL && antialias){ - glkView.drawableMultisample = .multisample4X - } - - - + // Depth and stencil come with the context's drawing buffer, from its attributes. if(is2D){ // Use surfaceWidth/Height directly — at this point engine is still .None so - // drawingBufferWidth would return glkView.frame * scale, which has float + // drawingBufferWidth would return glView.frame * scale, which has float // precision loss from the divide-then-multiply round-trip in forceLayout. nativeContext = CanvasHelpers.create2DContext(self, Int32(surfaceWidth), Int32(surfaceHeight), alpha, density, -16777216, density * 160, direction, colorSpace) }else { - nativeContext = CanvasHelpers.initWebGLWithView(self, alpha, antialias, depth, failIfMajorPerformanceCaveat, powerPreference, premultipliedAlpha, preserveDrawingBuffer, stencil, desynchronized, xrCompatible, version) + nativeContext = CanvasHelpers.initWebGLWithView(self, alpha, antialias, depth, failIfMajorPerformanceCaveat, powerPreference, premultipliedAlpha, preserveDrawingBuffer, stencil, desynchronized, xrCompatible, version, threadedWebGL) } engine = .GL - if(glkView.drawableWidth == 0 && glkView.drawableHeight == 0){ - glkView.bindDrawable() - } - - glkView.isHidden = false + glView.isHidden = false #endif }else if(is2D) { isOpaque = !alpha @@ -524,13 +505,11 @@ public class NSCCanvas: UIView { } var snapshot: UIImage? = nil if(engine == .GL){ - #if !os(visionOS) - if(nativeContext != 0){ - glkView.display() + if(is2D){ + snapshot = CanvasHelpers.snapshot2DContext(nativeContext, surfaceWidth, surfaceHeight) + }else { + snapshot = CanvasHelpers.snapshotWebGL(nativeContext) } - - snapshot = glkView.snapshot - #endif }else if(engine == .GPU){ if(is2D && nativeContext != 0){ snapshot = CanvasHelpers.snapshot2DContext(nativeContext, surfaceWidth, surfaceHeight) @@ -653,7 +632,7 @@ public class NSCCanvas: UIView { let frame = CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight) mtlView = NSCMTLView(frame: frame) - glkView = CanvasGLKView(frame: frame) + glView = CanvasGLView(frame: frame) cpuView = CanvasCPUView(frame: frame) mtlView.drawableSize = CGSize(width: 300, height: 150) super.init(coder: coder) @@ -669,7 +648,7 @@ public class NSCCanvas: UIView { let unscaledHeight = (150 / scale).rounded(.down) mtlView = NSCMTLView(frame: CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight)) - glkView = CanvasGLKView(frame: CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight)) + glView = CanvasGLView(frame: CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight)) cpuView = CanvasCPUView(frame: CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight)) mtlView.drawableSize = CGSize(width: 300, height: 150) super.init(frame: frame) @@ -681,21 +660,18 @@ public class NSCCanvas: UIView { private func initializeView(){ setup() let scale = nscNativeScale() - glkView.contentScaleFactor = scale + glView.contentScaleFactor = scale mtlView.contentScaleFactor = scale - glkView.canvas = self + glView.canvas = self mtlView.canvas = self cpuView.canvas = self handler = NSCTouchHandler(canvas: self) backgroundColor = .clear - #if !os(visionOS) - glkView.enableSetNeedsDisplay = false - #endif - glkView.isHidden = true + glView.isHidden = true mtlView.isHidden = true cpuView.isHidden = true - addSubview(glkView) + addSubview(glView) addSubview(mtlView) addSubview(cpuView) scaleSurface() @@ -774,16 +750,10 @@ public class NSCCanvas: UIView { scaleSurface() return } - #if !os(visionOS) - if(engine == .GL){ - EAGLContext.setCurrent(glkView.context) - } - - if(engine == .GL){ - glkView.deleteDrawable() - glkView.bindDrawable() + // A 2D context reallocates its GL drawable as it resizes. + if(engine == .GL && !is2D){ + CanvasHelpers.resizeWebGL(nativeContext, surfaceWidth, surfaceHeight) } - #endif if(is2D){ CanvasHelpers.resize2DContext(nativeContext, Float(surfaceWidth), Float(surfaceHeight)) } @@ -818,14 +788,14 @@ public class NSCCanvas: UIView { // causes UIKit to compute wrong layer.position, resulting in misplaced child views. CATransaction.begin() CATransaction.setDisableActions(true) - glkView.layer.transform = CATransform3DIdentity + glView.layer.transform = CATransform3DIdentity mtlView.layer.transform = CATransform3DIdentity cpuView.layer.transform = CATransform3DIdentity CATransaction.commit() lastScaledSurfaceFrame = .null lastScaledSurfaceTransform = CATransform3DIdentity - glkView.frame = CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight) + glView.frame = CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight) mtlView.frame = CGRect(x: 0, y: 0, width: unscaledWidth, height: unscaledHeight) mtlView.drawableSize = CGSize(width: width.rounded(.down), height: height.rounded(.down)) @@ -833,10 +803,10 @@ public class NSCCanvas: UIView { cpuView.data = NSMutableData(length: Int(surfaceWidth * surfaceHeight) * 4) - glkView.setNeedsLayout() + glView.setNeedsLayout() mtlView.setNeedsLayout() cpuView.setNeedsLayout() - glkView.layoutIfNeeded() + glView.layoutIfNeeded() mtlView.layoutIfNeeded() cpuView.layoutIfNeeded() } @@ -909,7 +879,7 @@ public class NSCCanvas: UIView { guard let transform = makeSurfaceTransform() else { CATransaction.begin() CATransaction.setDisableActions(true) - glkView.layer.transform = CATransform3DIdentity + glView.layer.transform = CATransform3DIdentity mtlView.layer.transform = CATransform3DIdentity cpuView.layer.transform = CATransform3DIdentity CATransaction.commit() @@ -952,17 +922,17 @@ public class NSCCanvas: UIView { CATransaction.begin() CATransaction.setDisableActions(true) if(frameChanged){ - glkView.layer.transform = CATransform3DIdentity + glView.layer.transform = CATransform3DIdentity mtlView.layer.transform = CATransform3DIdentity cpuView.layer.transform = CATransform3DIdentity - glkView.frame = surfaceFrame + glView.frame = surfaceFrame mtlView.frame = surfaceFrame cpuView.frame = surfaceFrame } if(transformChanged){ - glkView.layer.transform = transform + glView.layer.transform = transform mtlView.layer.transform = transform cpuView.layer.transform = transform } diff --git a/packages/canvas/src-native/canvas-ios/CanvasNative/include/canvas_ios.h b/packages/canvas/src-native/canvas-ios/CanvasNative/include/canvas_ios.h index e407e3468..27d87d316 100644 --- a/packages/canvas/src-native/canvas-ios/CanvasNative/include/canvas_ios.h +++ b/packages/canvas/src-native/canvas-ios/CanvasNative/include/canvas_ios.h @@ -8,7 +8,11 @@ #include #include -long long canvas_native_ios_create_webgl_context(void *view, +/** + * A WebGL context drawing to `layer`, the view's `CAEAGLLayer`. A threaded one lives on the WebGL + * thread, so presents never wait on the GPU here. + */ +long long canvas_native_ios_create_webgl_context(void *layer, bool alpha, bool antialias, bool depth, @@ -19,10 +23,28 @@ long long canvas_native_ios_create_webgl_context(void *view, bool stencil, bool desynchronized, bool xr_compatible, - uint32_t version); + uint32_t version, + bool threaded); bool canvas_native_ios_flush_webgl(int64_t context); +/** + * Reallocates the drawing buffer after the view's layer changed size. Queued: whatever reads the + * size next runs after it. + */ +void canvas_native_ios_resize_webgl(int64_t context, int32_t width, int32_t height); + +/** + * The drawing buffer as top-down RGBA, `width` x `height`. + */ +U8Buffer *canvas_native_ios_webgl_read_pixels(int64_t context, int32_t *width, int32_t *height); + +/** + * Runs `callback(data)` with the context current, on the thread that owns it, and waits for it: + * native code that draws with a threaded context (video frames) has to run there. + */ +void canvas_native_ios_webgl_run(int64_t context, void (*callback)(void*), void *data); + void canvas_native_ios_flush_2d_context(int64_t context); void canvas_native_ios_present_drawable(int64_t context); diff --git a/packages/canvas/src-native/canvas-ios/NSCRender.swift b/packages/canvas/src-native/canvas-ios/NSCRender.swift index ba34fdfc2..10f397be1 100644 --- a/packages/canvas/src-native/canvas-ios/NSCRender.swift +++ b/packages/canvas/src-native/canvas-ios/NSCRender.swift @@ -29,9 +29,19 @@ public class NSCRender: NSObject { private var context: EAGLContext? private var width: GLuint = 0 private var height: GLuint = 0 + private var glReady = false public override init() { super.init() + } + + /// Made on first use, with the context current then: a threaded WebGL context is only current + /// on its own thread, not where this was made. + private func setupGL() { + if(glReady){ + return + } + glReady = true context = EAGLContext.current() guard let context = context else {return} #if !targetEnvironment(simulator) @@ -95,22 +105,31 @@ public class NSCRender: NSObject { return (tex, c) } - public func drawFrame(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput,_ videoSize: CGSize, _ internalFormat: Int32,_ format: Int32,_ flipYWebGL: Bool){ + private func nextPixelBuffer(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput) -> CVPixelBuffer? { let currentTime = player.currentTime() - if(!output.hasNewPixelBuffer(forItemTime: currentTime)) {return} + if(!output.hasNewPixelBuffer(forItemTime: currentTime)) {return nil} var presentationTime = CMTime.zero - let buffer = output.copyPixelBuffer(forItemTime: currentTime, itemTimeForDisplay: &presentationTime) - - guard let pixel_buffer = buffer else {return} - + return output.copyPixelBuffer(forItemTime: currentTime, itemTimeForDisplay: &presentationTime) + } + + public func drawFrame(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput,_ videoSize: CGSize, _ internalFormat: Int32,_ format: Int32,_ flipYWebGL: Bool){ + drawFrame(player, output, videoSize, internalFormat, format, flipYWebGL, 0) + } + + /// Uploads the current frame into the texture bound in `context` (a WebGL context), on the thread + /// that owns it. + public func drawFrame(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput,_ videoSize: CGSize, _ internalFormat: Int32,_ format: Int32,_ flipYWebGL: Bool, _ context: Int64){ + guard let pixel_buffer = nextPixelBuffer(player, output) else {return} let width = CVPixelBufferGetWidth(pixel_buffer) let height = CVPixelBufferGetHeight(pixel_buffer) - drawFrame(buffer: pixel_buffer, width: Int(width), height: Int(height), internalFormat: internalFormat, format: format, flipYWebGL: flipYWebGL) + CanvasHelpers.runWithWebGL(context) { + drawFrame(buffer: pixel_buffer, width: Int(width), height: Int(height), internalFormat: internalFormat, format: format, flipYWebGL: flipYWebGL) + } } @@ -410,6 +429,7 @@ public class NSCRender: NSObject { public func drawFrame(buffer:CVPixelBuffer, width: Int, height: Int, internalFormat: Int32, format: Int32, flipYWebGL: Bool){ + setupGL() #if targetEnvironment(simulator) drawBuffer(buffer: buffer, width: width, height: height, internalFormat: internalFormat, format: format, flipYWebGL: flipYWebGL) #else @@ -561,13 +581,20 @@ public class NSCRender: NSObject { _ target: Int32, _ level: Int32, _ internalFormat: Int32, _ width: Int32, _ height: Int32, _ depth: Int32, _ border: Int32, _ format: Int32, _ type: Int32, _ flipYWebGL: Bool) { - let currentTime = player.currentTime() - guard output.hasNewPixelBuffer(forItemTime: currentTime) else { return } - var presentationTime = CMTime.zero - guard let buffer = output.copyPixelBuffer(forItemTime: currentTime, itemTimeForDisplay: &presentationTime) else { return } - uploadPixelBufferTexImage3D(buffer: buffer, target: target, level: level, - internalFormat: internalFormat, width: width, height: height, - depth: depth, border: border, flipYWebGL: flipYWebGL) + drawFrameTexImage3D(player, output, videoSize, target, level, internalFormat, width, height, depth, border, format, type, flipYWebGL, 0) + } + + /// `drawFrameTexImage3D`, on the thread that owns `context` (a WebGL context). + public func drawFrameTexImage3D(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput, _ videoSize: CGSize, + _ target: Int32, _ level: Int32, _ internalFormat: Int32, + _ width: Int32, _ height: Int32, _ depth: Int32, _ border: Int32, + _ format: Int32, _ type: Int32, _ flipYWebGL: Bool, _ context: Int64) { + guard let buffer = nextPixelBuffer(player, output) else { return } + CanvasHelpers.runWithWebGL(context) { + uploadPixelBufferTexImage3D(buffer: buffer, target: target, level: level, + internalFormat: internalFormat, width: width, height: height, + depth: depth, border: border, flipYWebGL: flipYWebGL) + } } public func drawFrameTexSubImage3D(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput, _ videoSize: CGSize, @@ -575,13 +602,21 @@ public class NSCRender: NSObject { _ xoffset: Int32, _ yoffset: Int32, _ zoffset: Int32, _ width: Int32, _ height: Int32, _ depth: Int32, _ format: Int32, _ type: Int32, _ flipYWebGL: Bool) { - let currentTime = player.currentTime() - guard output.hasNewPixelBuffer(forItemTime: currentTime) else { return } - var presentationTime = CMTime.zero - guard let buffer = output.copyPixelBuffer(forItemTime: currentTime, itemTimeForDisplay: &presentationTime) else { return } - uploadPixelBufferTexSubImage3D(buffer: buffer, target: target, level: level, - xoffset: xoffset, yoffset: yoffset, zoffset: zoffset, - width: width, height: height, depth: depth, flipYWebGL: flipYWebGL) + drawFrameTexSubImage3D(player, output, videoSize, target, level, xoffset, yoffset, zoffset, width, height, depth, format, type, flipYWebGL, 0) + } + + /// `drawFrameTexSubImage3D`, on the thread that owns `context` (a WebGL context). + public func drawFrameTexSubImage3D(_ player: AVPlayer, _ output: AVPlayerItemVideoOutput, _ videoSize: CGSize, + _ target: Int32, _ level: Int32, + _ xoffset: Int32, _ yoffset: Int32, _ zoffset: Int32, + _ width: Int32, _ height: Int32, _ depth: Int32, + _ format: Int32, _ type: Int32, _ flipYWebGL: Bool, _ context: Int64) { + guard let buffer = nextPixelBuffer(player, output) else { return } + CanvasHelpers.runWithWebGL(context) { + uploadPixelBufferTexSubImage3D(buffer: buffer, target: target, level: level, + xoffset: xoffset, yoffset: yoffset, zoffset: zoffset, + width: width, height: height, depth: depth, flipYWebGL: flipYWebGL) + } }