Use UIImage instead of CGImage for brush-texture interfaces

ink/rendering/metal/BUILD.bazel

@@ -67,15 +67,13 @@ cc_library(
 )
 
 objc_library(
-    name = "texture_bitmap_store",
-    hdrs = ["INKTextureBitmapStore.h"],
-    module_name = "INKTextureBitmapStore",
-    sdk_frameworks = [
-        "CoreGraphics",
-        "Foundation",
-    ],
+    name = "texture_image_source",
+    hdrs = ["INKTextureImageSource.h"],
+    module_name = "INKTextureImageSource",
+    sdk_frameworks = ["Foundation"],
     target_compatible_with = ["@platforms//os:ios"],
     visibility = ["//visibility:public"],
+    deps = ["//third_party/apple_frameworks:UIKit"],
 )
 
 objc_library(
@@ -92,9 +90,11 @@ objc_library(
     visibility = ["//visibility:private"],
     deps = [
         ":ink_metal_shaders_embedded",
-        ":texture_bitmap_store",
+        ":texture_image_source",
         "//ink/brush:brush_paint",
         "//ink/geometry:mesh",
+        "//third_party/apple_frameworks:CoreImage",
+        "//third_party/apple_frameworks:UIKit",
         "@abseil-cpp//absl/base:nullability",
         "@abseil-cpp//absl/status",
         "@abseil-cpp//absl/status:statusor",

ink/rendering/metal/INKTextureBitmapStore.h → ink/rendering/metal/INKTextureImageSource.h

@@ -12,22 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#import <CoreGraphics/CoreGraphics.h>
 #import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
 /**
- * Protocol for a callback to allow the caller to provide a particular CGImageRef corresponding to a
+ * Protocol for a callback to allow the caller to provide a particular UIImage corresponding to a
  * client-provided texture ID.
  */
-@protocol INKTextureBitmapStore <NSObject>
+@protocol INKTextureImageSource <NSObject>
 
 /**
- * Retrieve a CGImageRef for the given texture ID. This may be called synchronously during drawing,
- * so loading of texture files from disk and decoding them into CGImageRef objects should be done
- * on init. The result may be cached by consumers, so this should return a deterministic result for
- * a given input.
+ * Retrieve a UIImage for the given texture ID. This may be called synchronously during drawing,
+ * so preferably this should use pre-loaded UIImage objects (that is, initialized from CGImage
+ * instead of CIImage).
  *
  * Textures can be disabled by having load always return null. null should also be returned when a
  * texture can not be loaded. If null is returned, the texture layer in question should be
@@ -37,7 +36,7 @@ NS_ASSUME_NONNULL_BEGIN
  * @param textureID The client-provided texture ID.
  * @return The texture image, if any, associated with the given ID.
  */
-- (nullable CGImageRef)textureForID:(NSString *)textureID;
+- (nullable UIImage *)textureForID:(NSString *)textureID;
 
 @end
 

ink/rendering/metal/metal_renderer.cc

@@ -246,7 +246,7 @@ void MetalRenderer::DrawWithStencilAlreadySet(
 
   if (request_shader_aa && (forward_offset < 0 || side_offset < 0)) {
     ABSL_LOG(WARNING)
-        << "INKMetalRenderer shader-based anti-aliasing requested, but mesh "
+        << "MetalRenderer shader-based anti-aliasing requested, but mesh "
            "format does not contain side derivative and forward derivative "
            "attributes.";
     return;
@@ -385,7 +385,7 @@ void MetalRenderer::DrawWithStencilAlreadySet(
 
   if (request_shader_aa && (forward_offset < 0 || side_offset < 0)) {
     ABSL_LOG(WARNING)
-        << "INKMetalRenderer shader-based anti-aliasing requested, but mesh "
+        << "MetalRenderer shader-based anti-aliasing requested, but mesh "
            "format does not contain side derivative and forward derivative "
            "attributes.";
     return;

ink/rendering/metal/metal_renderer.h

@@ -178,7 +178,7 @@ class MetalRenderer {
   // `sample_count`: The number of samples used for multisampling. Set to
   //   1 to enable shader-based anti-aliasing instead.
   // `texture_bitmap_store`: Optional opaque pointer to
-  // `id<INKTextureBitmapStore>`.
+  // `id<INKTextureImageSource>`.
   static absl::StatusOr<MetalRenderer> Create(
       void* device, uint64_t color_pixel_format, uint64_t stencil_pixel_format,
       std::optional<int> sample_count = std::nullopt,

ink/rendering/metal/metal_renderer_native.h

@@ -17,7 +17,6 @@
 
 // C-compatible library header for Kotlin-native bindings.
 
-#include <simd/types.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
@@ -31,7 +30,7 @@ extern "C" {
 // samples per pixel for MSAA. If -1, shader-based antialiasing will be used
 // instead. `texture_for_id_callback` is a callback used to retrieve textures
 // for given texture ID strings, and returns a nullable raw pointer to a
-// `CGImage`.
+// `UIImage`.
 int64_t MetalRendererNative_create(
     void* device, uint64_t color_pixel_format, uint64_t stencil_pixel_format,
     int sample_count,

ink/rendering/metal/metal_renderer_objc_helper.h

@@ -40,15 +40,15 @@ namespace ink::rendering::metal_objc {
 // `sample_count`: The number of samples used for multisampling, or nullopt to
 //   use shader-based AA.
 // `texture_bitmap_store`: Optional opaque pointer to
-// `id<INKTextureBitmapStore>` used for
+// `id<INKTextureImageSource>` used for
 //   texture lookup.
 absl::StatusOr<std::unique_ptr<void, std::function<void(void*)>>>
 CreateINKMetalRendererState(
     void* device_ptr, uint64_t color_pixel_format_val,
     uint64_t stencil_pixel_format_val, std::optional<int> sample_count,
     void* absl_nullable texture_bitmap_store_ptr = nullptr);
 
-// Creates a wrapper implementing `INKTextureBitmapStore` that delegates to a
+// Creates a wrapper implementing `INKTextureImageSource` that delegates to a
 // Kotlin callback.
 // Returns an opaque pointer to the wrapper, which should be passed to
 // `CreateINKMetalRendererState`.

ink/rendering/metal/metal_renderer_objc_helper.mm

@@ -15,9 +15,11 @@
 #import "ink/rendering/metal/metal_renderer_objc_helper.h"
 
 #import <CoreFoundation/CFBase.h>
+#import <CoreImage/CIImage.h>
 #import <Foundation/Foundation.h>
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
+#import <UIKit/UIKit.h>
 
 #import <functional>
 
@@ -26,7 +28,7 @@
 #include "absl/strings/str_cat.h"
 #include "ink/brush/brush_paint.h"
 #include "ink/geometry/mesh.h"
-#import "ink/rendering/metal/INKTextureBitmapStore.h"
+#import "ink/rendering/metal/INKTextureImageSource.h"
 #include "ink/rendering/metal/ink_metal_shaders_embedded.h"
 
 __attribute__((objc_subclassing_restricted))
@@ -57,7 +59,7 @@ - (instancetype)initWithVertexBuffer:(id<MTLBuffer>)vertexBuffer
 @end
 
 __attribute__((objc_subclassing_restricted))
-@interface KotlinTextureStoreWrapper : NSObject<INKTextureBitmapStore>
+@interface KotlinTextureStoreWrapper : NSObject<INKTextureImageSource>
 @property(nonatomic) int64_t kotlinMetalRendererNativePtr;
 @property(nonatomic, readonly) void* (*callback)(int64_t, const char*);
 
@@ -86,10 +88,10 @@ - (instancetype)initWithCallback:(void* (*)(int64_t, const char*))callback {
   return self;
 }
 
-- (CGImageRef)textureForID:(NSString*)textureID {
+- (UIImage*)textureForID:(NSString*)textureID {
   if (self.callback == nullptr || self.kotlinMetalRendererNativePtr == 0) return nullptr;
   const char* c_texture_id = [textureID UTF8String];
-  return (CGImageRef)(*self.callback)(self.kotlinMetalRendererNativePtr, c_texture_id);
+  return (__bridge UIImage*)(*self.callback)(self.kotlinMetalRendererNativePtr, c_texture_id);
 }
 
 @end
@@ -103,12 +105,13 @@ @interface INKMetalRendererState : NSObject
 @property(nonatomic, readonly, nonnull) id<MTLDepthStencilState> discardSelfOverlapStencilState;
 @property(nonatomic, readonly, nonnull) id<MTLDepthStencilState> clearStencilBufferStencilState;
 @property(nonatomic, readonly, nonnull) id<MTLDepthStencilState> noOpStencilState;
-@property(nonatomic, readonly, nullable) id<INKTextureBitmapStore> textureBitmapStore;
+@property(nonatomic, readonly, nullable) id<INKTextureImageSource> textureBitmapStore;
 @property(nonatomic, readonly, nonnull) MTKTextureLoader* textureLoader;
 @property(nonatomic, readonly, nonnull)
     NSMutableDictionary<NSString*, id<MTLTexture>>* textureCache;
 @property(nonatomic, readonly, nonnull)
     NSMutableArray<NSMutableArray<id<MTLSamplerState>>*>* samplerCache;
+@property(nonatomic, readwrite, nullable) CIContext* ciContext;
 
 - (instancetype)init NS_UNAVAILABLE;
 
@@ -118,7 +121,7 @@ - (instancetype)init NS_UNAVAILABLE;
     discardSelfOverlapStencilState:(nullable id<MTLDepthStencilState>)discardSelfOverlapStencilState
     clearStencilBufferStencilState:(nullable id<MTLDepthStencilState>)clearStencilBufferStencilState
                   noOpStencilState:(nullable id<MTLDepthStencilState>)noOpStencilState
-                textureBitmapStore:(nullable id<INKTextureBitmapStore>)textureBitmapStore
+                textureBitmapStore:(nullable id<INKTextureImageSource>)textureBitmapStore
     NS_DESIGNATED_INITIALIZER;
 
 @end
@@ -131,7 +134,7 @@ @implementation INKMetalRendererState
     discardSelfOverlapStencilState:(nullable id<MTLDepthStencilState>)discardSelfOverlapStencilState
     clearStencilBufferStencilState:(nullable id<MTLDepthStencilState>)clearStencilBufferStencilState
                   noOpStencilState:(nullable id<MTLDepthStencilState>)noOpStencilState
-                textureBitmapStore:(nullable id<INKTextureBitmapStore>)textureBitmapStore {
+                textureBitmapStore:(nullable id<INKTextureImageSource>)textureBitmapStore {
   self = [super init];
   if (self) {
     _device = device;
@@ -177,7 +180,7 @@ MTLSamplerAddressMode TextureWrapToMTLSamplerAddressMode(BrushPaint::TextureWrap
 
 absl::StatusOr<std::unique_ptr<void, std::function<void(void*)>>> CreateINKMetalRendererState(
     id<MTLDevice> device, MTLPixelFormat color_pixel_format, MTLPixelFormat stencil_pixel_format,
-    std::optional<int> sample_count, id<INKTextureBitmapStore> texture_bitmap_store) {
+    std::optional<int> sample_count, id<INKTextureImageSource> texture_bitmap_store) {
   if (!device) {
     return absl::InvalidArgumentError("Device cannot be nil");
   }
@@ -343,7 +346,7 @@ void DrawIndexedTriangles(id<MTLRenderCommandEncoder> render_encoder, const void
 
 id<MTLTexture> GetOrLoadMTLTexture(INKMetalRendererState* state, const char* client_texture_id) {
   if (!client_texture_id) return nullptr;
-  id<INKTextureBitmapStore> texture_bitmap_store = state.textureBitmapStore;
+  id<INKTextureImageSource> texture_bitmap_store = state.textureBitmapStore;
   if (!texture_bitmap_store) return nullptr;
 
   NSMutableDictionary<NSString*, id<MTLTexture>>* texture_cache = state.textureCache;
@@ -352,7 +355,15 @@ void DrawIndexedTriangles(id<MTLRenderCommandEncoder> render_encoder, const void
   NSString* ns_texture_id = [NSString stringWithUTF8String:client_texture_id];
   id<MTLTexture> texture = texture_cache[ns_texture_id];
   if (!texture && texture_bitmap_store) {
-    CGImageRef cg_image = [texture_bitmap_store textureForID:ns_texture_id];
+    UIImage* ui_image = [texture_bitmap_store textureForID:ns_texture_id];
+    CGImageRef cg_image = ui_image.CGImage;
+    if (!cg_image && ui_image.CIImage) {
+      if (!state.ciContext) {
+        state.ciContext = [CIContext contextWithOptions:nil];
+      }
+      cg_image = [state.ciContext createCGImage:ui_image.CIImage
+                                       fromRect:[ui_image.CIImage extent]];
+    }
     if (cg_image) {
       NSError* error = nil;
       texture = [texture_loader newTextureWithCGImage:cg_image options:nil error:&error];
@@ -409,7 +420,7 @@ bool SupportsSelfOverlapDiscard(INKMetalRendererState* state) {
   return CreateINKMetalRendererState(
       (__bridge id<MTLDevice>)device_ptr, static_cast<MTLPixelFormat>(color_pixel_format_val),
       static_cast<MTLPixelFormat>(stencil_pixel_format_val), sample_count,
-      (__bridge id<INKTextureBitmapStore>)texture_bitmap_store_ptr);
+      (__bridge id<INKTextureImageSource>)texture_bitmap_store_ptr);
 }
 
 std::shared_ptr<void> CreateKotlinTextureStoreWrapper(