feat(swift)!: read pasted and dropped files lazily, off the main thread

swift/.changeset/lazy-data-transfer-files.md

@@ -0,0 +1,9 @@
+---
+'lexical-swift': minor
+---
+
+**Breaking:** pasted and dropped files are no longer read into memory on the main thread. As a web `File` is a lazy `Blob`, a `DataTransferFile` now has a `size` and reads its contents only when a handler awaits `bytes()`, off the main thread, so a paste or drop of large files no longer blocks the view, even when no handler wants the files. `DataTransferFile.data` is gone, and the type is no longer `Equatable`.
+
+To migrate, read a file's contents with `try await file.bytes()` where you read `file.data`, as upstream's `mediaFileReader` awaits each `File`. A `DRAG_DROP_PASTE` handler that needs the bytes returns `true` and reads them in a `Task`. `DataTransferFile(name:type:data:)` still makes a file in memory, and `DataTransferFile(name:type:size:load:)` makes a lazy one. Compare files by `name` and `type` where you compared them with `==`.
+
+On macOS, a dropped or pasted file is read from its URL; an image on the pasteboard with no file behind it is read from the pasteboard and throws once the pasteboard has changed. On iOS, UIKit lets items load only during the paste or drop, so each file starts loading into a temporary file then, `bytes()` waits for it, and `size` is `nil`. iOS drops and pastes now also carry files that aren't images, such as a PDF from Files, which were lost before.

swift/DESIGN.md

@@ -717,7 +717,13 @@ Reversible choices made porting `@lexical/extension` under Q18.
   `application/x-lexical-editor` as `dev.lexical.editor`, the drag marker as
   `dev.lexical.drag`, `text/html` as `public.html`, `text/plain` and
   `text/uri-list` as their public types, and file URLs and images as
-  `files`. `copy:` and `cut:` dispatch `COPY_COMMAND` / `CUT_COMMAND` with a
+  `files`. Files load lazily, as a web `File` is a `Blob` read through a
+  promise: a `DataTransferFile` has its name, type and `size` up front and
+  reads its contents in `async bytes()`, off the main thread, so a paste or
+  drop of large files reads nothing until a handler asks. On iOS, UIKit
+  lets items load only during the paste or drop, so each file starts
+  loading into a temporary file then and `bytes()` waits for it; iOS drops
+  and pastes carry files of any type, not only images. `copy:` and `cut:` dispatch `COPY_COMMAND` / `CUT_COMMAND` with a
   `ClipboardEvent` and write its data when a handler prevented the default;
   `paste:` dispatches `PASTE_COMMAND`, and paste as plain text passes only
   `text/plain`. RTF without HTML (from native apps) is converted to HTML,

swift/Sources/Lexical/Lexical.docc/JSParity.md

@@ -84,4 +84,6 @@ Listed with their category in the repository's `swift/parity/deviations.json`:
 - **Clipboard.** RTF without HTML is converted to HTML before
   `PASTE_COMMAND`. Lexical's own clipboard data travels as the pasteboard
   type `dev.lexical.editor`. A drag moved into another editor is removed
-  from the source by the source view.
+  from the source by the source view. A pasted or dropped file is read when
+  a handler awaits `bytes()`; on iOS its load starts at the paste or drop,
+  since UIKit allows it only then, and its `size` is `nil`.

swift/Sources/Lexical/LexicalEvents.swift

@@ -361,16 +361,41 @@ public func compileKeyboardShortcuts<S: KeyboardShortcutMatch>(
 
 // MARK: - Other DOM-shaped events
 
-/// One file of a `DataTransfer`.
-public struct DataTransferFile: Equatable, Sendable {
+/// One file of a `DataTransfer`, as a web `File`: its name, type and size
+/// are known up front, and its contents are read only when `bytes()` asks
+/// for them, as `File.bytes()` reads a lazy `Blob`.
+public struct DataTransferFile: Sendable {
+  /// `File.name`.
   public var name: String
+  /// `File.type`, a MIME type.
   public var type: String
-  public var data: [UInt8]
+  /// `File.size` in bytes, or `nil` when it isn't known without reading
+  /// the file (an item that is still loading, on iOS).
+  public var size: Int?
+  private let load: @Sendable () async throws -> [UInt8]
 
+  /// A file whose contents are `data`, in memory.
   public init(name: String, type: String, data: [UInt8] = []) {
+    self.init(name: name, type: type, size: data.count) { data }
+  }
+
+  /// A file that `load` reads when `bytes()` is called, each time it is
+  /// called.
+  public init(
+    name: String, type: String, size: Int?,
+    load: @escaping @Sendable () async throws -> [UInt8]
+  ) {
     self.name = name
     self.type = type
-    self.data = data
+    self.size = size
+    self.load = load
+  }
+
+  /// `File.bytes()`: the file's contents. Throws when the file can no
+  /// longer be read, such as one deleted since it was dropped, as a web
+  /// `File` rejects with `NotReadableError`.
+  public func bytes() async throws -> [UInt8] {
+    try await load()
   }
 }
 

swift/Sources/LexicalAppKit/NSPasteboard+DataTransfer.swift

@@ -42,24 +42,41 @@
           || (dataTransfer.types.isEmpty && availableType(from: [.png, .tiff]) != nil)
         return dataTransfer
       }
+      // The files are read only when a handler asks for their bytes, off
+      // the main thread, as a web `File` is a lazy blob.
       for url in urls {
-        guard let data = try? Data(contentsOf: url) else { continue }
+        let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .fileSizeKey])
+        guard values?.isRegularFile == true else { continue }
         let type =
           UTType(filenameExtension: url.pathExtension)?.preferredMIMEType
           ?? TextKitPasteboard.mimeType(forPathExtension: url.pathExtension)
         dataTransfer.files.append(
-          DataTransferFile(name: url.lastPathComponent, type: type, data: Array(data)))
+          DataTransferFile(name: url.lastPathComponent, type: type, size: values?.fileSize) {
+            try await readFile(url)
+          })
       }
       if dataTransfer.files.isEmpty, dataTransfer.types.isEmpty {
         // An image copied from an app, with no file behind it.
-        for (type, ext) in [(NSPasteboard.PasteboardType.png, "png"), (.tiff, "tiff")] {
-          if let data = data(forType: type) {
-            dataTransfer.files.append(
-              DataTransferFile(
-                name: "image.\(ext)", type: TextKitPasteboard.mimeType(forPathExtension: ext),
-                data: Array(data)))
-            break
-          }
+        for (type, ext) in [(NSPasteboard.PasteboardType.png, "png"), (.tiff, "tiff")]
+        where availableType(from: [type]) != nil {
+          nonisolated(unsafe) let pasteboard = self
+          let changeCount = changeCount
+          dataTransfer.files.append(
+            DataTransferFile(
+              name: "image.\(ext)", type: TextKitPasteboard.mimeType(forPathExtension: ext),
+              size: nil
+            ) {
+              // A pasteboard is read on the main thread, and only while it
+              // still holds what was pasted or dropped.
+              let data = try await MainActor.run {
+                guard pasteboard.changeCount == changeCount,
+                  let data = pasteboard.data(forType: type)
+                else { throw CocoaError(.fileReadNoSuchFile) }
+                return data
+              }
+              return Array(data)
+            })
+          break
         }
       }
       return dataTransfer
@@ -86,6 +103,12 @@
     }
   }
 
+  /// The contents of the file at `url`, read off the main thread.
+  @concurrent
+  private func readFile(_ url: URL) async throws -> [UInt8] {
+    Array(try Data(contentsOf: url))
+  }
+
   extension DataTransfer {
     /// A pasteboard item with this transfer's data, for the clipboard or a
     /// drag.

swift/Sources/LexicalRichText/RichText.swift

@@ -14,7 +14,23 @@ import LexicalUtils
 // paste, because a native control inside a decorator handles its own
 // clipboard; and the iOS text-file drop fall-through, a WebKit quirk.
 
-/// `DRAG_DROP_PASTE`: files dropped or pasted into the editor.
+/// `DRAG_DROP_PASTE`: files dropped or pasted into the editor. Nothing has
+/// read them yet; a handler that wants their contents awaits each file's
+/// `bytes()`, as upstream's `mediaFileReader` reads each `File`:
+///
+/// ```swift
+/// editor.registerCommand(DRAG_DROP_PASTE, { files, editor in
+///   let images = files.filter { isMimeType($0, ["image/"]) }
+///   if images.isEmpty { return false }
+///   Task { @MainActor in
+///     for image in images {
+///       let bytes = try await image.bytes()
+///       // insert the image
+///     }
+///   }
+///   return true
+/// }, COMMAND_PRIORITY_LOW)
+/// ```
 public let DRAG_DROP_PASTE = LexicalCommand<[DataTransferFile]>("DRAG_DROP_PASTE_FILE")
 
 /// `EscapeFormatTrigger`: the interactions that can escape a format.

swift/Sources/LexicalUIKit/LexicalUITextView.swift

@@ -1099,8 +1099,9 @@
     public func dropInteraction(
       _ interaction: UIDropInteraction, canHandle session: any UIDropSession
     ) -> Bool {
+      // Text, or any file (`DataTransfer.load`).
       session.hasItemsConforming(
-        toTypeIdentifiers: TextKitPasteboard.types.map(\.type) + ["public.image"])
+        toTypeIdentifiers: TextKitPasteboard.types.map(\.type) + ["public.data"])
     }
 
     public func dropInteraction(

swift/Sources/LexicalUIKit/UIPasteboard+DataTransfer.swift

@@ -33,36 +33,35 @@
   }
 
   extension UIPasteboard {
-    /// The first item's contents as a `DataTransfer`.
+    /// The first item's contents as a `DataTransfer`, reading only the
+    /// types it carries. When it has no text, the items' images and other
+    /// files are its `files`, which start loading off the main thread now
+    /// (UIKit lets a paste's items load only while it happens) and which
+    /// `bytes()` waits for.
     public func lexicalDataTransfer(plainTextOnly: Bool = false) -> DataTransfer {
       let dataTransfer = DataTransfer()
-      guard let item = items.first else { return dataTransfer }
+      guard numberOfItems > 0 else { return dataTransfer }
+      let first = IndexSet(integer: 0)
+      func value(_ type: String) -> Any? {
+        values(forPasteboardType: type, inItemSet: first)?.first
+      }
       for (mime, type) in TextKitPasteboard.types {
         if plainTextOnly && mime != "text/plain" { continue }
-        if let value = pasteboardString(item[type]) { dataTransfer.setData(mime, value) }
+        if let value = pasteboardString(value(type)) { dataTransfer.setData(mime, value) }
       }
       if dataTransfer.getData("text/plain").isEmpty,
-        let value = pasteboardString(item["public.plain-text"])
+        let value = pasteboardString(value("public.plain-text"))
       {
         dataTransfer.setData("text/plain", value)
       }
       if plainTextOnly { return dataTransfer }
-      if dataTransfer.getData("text/html").isEmpty, let rtf = item["public.rtf"] as? Data,
+      if dataTransfer.getData("text/html").isEmpty, let rtf = value("public.rtf") as? Data,
         let html = htmlFromRTF(rtf)
       {
         dataTransfer.setData("text/html", html)
       }
       if dataTransfer.types.isEmpty {
-        for (type, ext) in [("public.png", "png"), ("public.jpeg", "jpg")] {
-          if let data = (item[type] as? Data) ?? (item[type] as? UIImage).flatMap({ $0.pngData() })
-          {
-            dataTransfer.files.append(
-              DataTransferFile(
-                name: "image.\(ext)", type: TextKitPasteboard.mimeType(forPathExtension: ext),
-                data: Array(data)))
-            break
-          }
-        }
+        dataTransfer.files = itemProviders.compactMap(DataTransferFile.loading)
       }
       return dataTransfer
     }
@@ -99,15 +98,18 @@
       return provider
     }
 
-    /// Loads the data of drag items, calling `completion` on the main queue.
+    /// Loads the text of drag items, calling `completion` on the main queue.
+    /// The items' images and other files, such as a PDF from Files, are its
+    /// `files`: they start loading off the main thread now, since UIKit lets
+    /// a drop's items load only while it is performed, and `completion`
+    /// doesn't wait for them; `bytes()` does.
     @MainActor
     public static func load(
       _ providers: [NSItemProvider], completion: @escaping @MainActor (DataTransfer) -> Void
     ) {
       let dataTransfer = DataTransfer()
       let group = DispatchGroup()
       nonisolated(unsafe) var strings: [(String, String)] = []
-      nonisolated(unsafe) var files: [DataTransferFile] = []
       let lock = NSLock()
       for provider in providers {
         for (mime, type) in TextKitPasteboard.types
@@ -120,23 +122,10 @@
             group.leave()
           }
         }
-        if let type = provider.registeredTypeIdentifiers.first(where: {
-          UTType($0)?.conforms(to: .image) == true
-        }) {
-          group.enter()
-          provider.loadDataRepresentation(forTypeIdentifier: type) { data, _ in
-            if let data {
-              let utType = UTType(type)
-              let ext = utType?.preferredFilenameExtension ?? "png"
-              let file = DataTransferFile(
-                name: (provider.suggestedName ?? "image") + ".\(ext)",
-                type: utType?.preferredMIMEType ?? "image/png", data: Array(data))
-              lock.withLock { files.append(file) }
-            }
-            group.leave()
-          }
-        }
       }
+      // Started here: UIKit lets a drop's items load only while it is
+      // performed.
+      let files = providers.compactMap(DataTransferFile.loading)
       group.notify(queue: .main) {
         MainActor.assumeIsolated {
           for (mime, value) in strings where dataTransfer.getData(mime).isEmpty {
@@ -148,4 +137,81 @@
       }
     }
   }
+
+  extension DataTransferFile {
+    /// The file an item provider carries: an image, or an item with none
+    /// of `TextKitPasteboard`'s types, such as a PDF from Files; `nil` for
+    /// an item of text. UIKit lets a drop's or a paste's items load only
+    /// while it happens, so the file starts loading now, off the main
+    /// thread, into a temporary file that `bytes()` waits for and reads.
+    /// `size` is `nil`, since it is known only once the file has loaded.
+    static func loading(_ provider: NSItemProvider) -> DataTransferFile? {
+      let types = provider.registeredTypeIdentifiers
+      let isText = TextKitPasteboard.types.contains { types.contains($0.type) }
+      guard
+        let type = types.first(where: { UTType($0)?.conforms(to: .image) == true })
+          ?? (isText ? nil : types.first { UTType($0)?.conforms(to: .data) == true })
+      else { return nil }
+      let utType = UTType(type)
+      var name =
+        provider.suggestedName ?? (utType?.conforms(to: .image) == true ? "image" : "file")
+      if let ext = utType?.preferredFilenameExtension,
+        UTType(filenameExtension: (name as NSString).pathExtension) != utType
+      {
+        name += ".\(ext)"
+      }
+      let file = ItemProviderFile(provider, type)
+      return DataTransferFile(
+        name: name,
+        type: utType?.preferredMIMEType
+          ?? TextKitPasteboard.mimeType(forPathExtension: (name as NSString).pathExtension),
+        size: nil
+      ) { try await file.bytes() }
+    }
+  }
+
+  /// An item provider's file, moved into a temporary directory as it
+  /// loads, and removed once no `DataTransferFile` holds it.
+  private final class ItemProviderFile: Sendable {
+    private let location: Task<URL, any Error>
+
+    init(_ provider: NSItemProvider, _ type: String) {
+      let (loaded, continuation) = AsyncThrowingStream<URL, any Error>.makeStream()
+      // The provider removes its file when this returns, so it is moved.
+      _ = provider.loadFileRepresentation(forTypeIdentifier: type) { url, error in
+        do {
+          guard let url else { throw error ?? CocoaError(.fileReadUnknown) }
+          let directory = FileManager.default.temporaryDirectory
+            .appending(path: "lexical-\(UUID().uuidString)", directoryHint: .isDirectory)
+          try FileManager.default.createDirectory(
+            at: directory, withIntermediateDirectories: true)
+          let destination = directory.appending(path: url.lastPathComponent)
+          try FileManager.default.moveItem(at: url, to: destination)
+          continuation.yield(destination)
+          continuation.finish()
+        } catch {
+          continuation.finish(throwing: error)
+        }
+      }
+      location = Task.detached {
+        for try await url in loaded { return url }
+        throw CocoaError(.fileReadUnknown)
+      }
+    }
+
+    deinit {
+      let location = location
+      Task.detached {
+        if let url = try? await location.value {
+          try? FileManager.default.removeItem(at: url.deletingLastPathComponent())
+        }
+      }
+    }
+
+    /// The file's contents, once it has loaded, read off the main thread.
+    @concurrent
+    func bytes() async throws -> [UInt8] {
+      Array(try Data(contentsOf: try await location.value))
+    }
+  }
 #endif