Add support for Object Detector models

.claude-plugin/marketplace.json

@@ -12,7 +12,7 @@
     {
       "name": "prototypekit",
       "source": "./plugin",
-      "description": "Scaffold and wire up PrototypeKit SwiftUI views for rapid on-device ML prototyping (camera, image/text/barcode/hand-pose/sound recognition, natural-language analysis).",
+      "description": "Scaffold and wire up PrototypeKit SwiftUI views for rapid on-device ML prototyping (camera, image classification, object detection, text/barcode/hand-pose/sound recognition, natural-language analysis).",
       "version": "0.1.0",
       "author": {
         "name": "Friday Technologies",

CHANGELOG.md

@@ -9,6 +9,14 @@ once tagged releases begin.
 ## [Unreleased]
 
 ### Added
+- **Object detection.** A new `ObjectDetectorView` runs a Create ML / Core ML **Object Detector**
+  model on the live camera feed and publishes the objects found in each frame. Two initializers are
+  offered: one takes a `detectedObjects: Binding<[String]>` for the labels only, and one takes a
+  `detectedObjects: Binding<[DetectedObject]>` — a new public value type carrying each object's
+  `label`, `confidence`, and normalized `boundingBox` — so you can draw bounding boxes or reason about
+  position. Like the other model-backed views it degrades gracefully on a bad/missing model (camera
+  feed with no detections, logged, reported via the optional `onError` closure) and accepts a
+  `camera: CameraOptions?`.
 - **Error reporting.** A public `PrototypeKitError` type and an optional `onError` closure on the
   entry points that can fail at setup time: `ImageClassifierView`, `HandPoseClassifierView`, the
   `classifyActivity(...)` modifier (model-load failures), and the `recognizeSounds(...)` modifier

Examples/PrototypeKitExamples.swift

@@ -23,6 +23,8 @@ struct ExampleGallery: View {
                     NavigationLink("Live barcodes", destination: LiveBarcodeExample())
                     NavigationLink("Face detection", destination: FaceCountExample())
                     NavigationLink("Image classification", destination: ImageClassifierExample())
+                    NavigationLink("Object detection", destination: ObjectDetectorExample())
+                    NavigationLink("Object detection (boxes)", destination: ObjectDetectorBoxesExample())
                 }
                 Section("Sound") {
                     NavigationLink("Sound recognition", destination: SoundExample())
@@ -118,6 +120,78 @@ struct ImageClassifierExample: View {
     }
 }
 
+/// Detects objects in each camera frame with a Create ML object detector.
+///
+/// Replace `modelURL` with your own model, e.g. `MyObjectDetector.urlOfModelInThisBundle`.
+struct ObjectDetectorExample: View {
+    @State private var detectedObjects: [String] = []
+    @State private var errorMessage: String?
+
+    // Point this at your Create ML object detector's `urlOfModelInThisBundle`.
+    private let modelURL = URL(fileURLWithPath: "/path/to/YourObjectDetector.mlmodelc")
+
+    var body: some View {
+        ZStack(alignment: .bottom) {
+            ObjectDetectorView(modelURL: modelURL,
+                               detectedObjects: $detectedObjects) { error in
+                // React to a model-load failure instead of showing a silent, detection-less feed.
+                errorMessage = error.localizedDescription
+            }
+            Text(errorMessage ?? (detectedObjects.isEmpty ? "Detecting…" : detectedObjects.joined(separator: ", ")))
+                .padding()
+                .frame(maxWidth: .infinity)
+                .background(.thinMaterial)
+        }
+        .ignoresSafeArea()
+    }
+}
+
+/// Detects objects and draws a bounding box + label around each one.
+///
+/// Replace `modelURL` with your own model, e.g. `MyObjectDetector.urlOfModelInThisBundle`.
+struct ObjectDetectorBoxesExample: View {
+    @State private var detectedObjects: [DetectedObject] = []
+    @State private var errorMessage: String?
+
+    private let modelURL = URL(fileURLWithPath: "/path/to/YourObjectDetector.mlmodelc")
+
+    var body: some View {
+        ZStack {
+            ObjectDetectorView(modelURL: modelURL,
+                               detectedObjects: $detectedObjects) { error in
+                errorMessage = error.localizedDescription
+            }
+
+            GeometryReader { geometry in
+                ForEach(Array(detectedObjects.enumerated()), id: \.offset) { _, object in
+                    let box = object.boundingBox
+                    Rectangle()
+                        .stroke(.red, lineWidth: 2)
+                        // Vision's origin is bottom-left; SwiftUI's is top-left, so flip Y.
+                        .frame(width: box.width * geometry.size.width,
+                               height: box.height * geometry.size.height)
+                        .position(x: box.midX * geometry.size.width,
+                                  y: (1 - box.midY) * geometry.size.height)
+                        .overlay(
+                            Text(object.label)
+                                .font(.caption2)
+                                .padding(2)
+                                .background(.red)
+                                .foregroundStyle(.white)
+                                .position(x: box.midX * geometry.size.width,
+                                          y: (1 - box.maxY) * geometry.size.height)
+                        )
+                }
+            }
+
+            if let errorMessage = errorMessage {
+                Text(errorMessage).foregroundStyle(.red).padding().background(.thinMaterial)
+            }
+        }
+        .ignoresSafeArea()
+    }
+}
+
 // MARK: - Sound
 
 /// Recognizes sounds from the microphone using the built-in classifier.

README.md

@@ -154,6 +154,95 @@ struct ImageClassifierViewSample: View {
 </details>
 
 
+### Live Object Detection
+
+Detect and locate objects in the live camera feed using a Create ML / Core ML **Object Detector** model.
+
+1. **Required Step:** Drag in your Create ML / Core ML object detector model into Xcode.
+2. Change `MyObjectDetector` below to the name of your Model.
+3. `detectedObjects` holds the labels of the objects found in the latest frame; use it as you would any other state variable.
+
+Utilise `ObjectDetectorView`
+
+```swift
+ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+                   detectedObjects: $detectedObjects)
+```
+
+<details>
+<summary>Full Example</summary>
+<br>
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorViewSample: View {
+
+    @State var detectedObjects: [String] = []
+
+    var body: some View {
+        VStack {
+            ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+                               detectedObjects: $detectedObjects)
+
+            ScrollView {
+                ForEach(Array(detectedObjects.enumerated()), id: \.offset) { index, object in
+                    Text(object)
+                }
+            }
+        }
+    }
+}
+```
+</details>
+
+Need to know **where** each object is (for example, to draw bounding boxes)? Bind an array of
+`DetectedObject` instead of `[String]`. Each `DetectedObject` carries the `label`, a `confidence`
+(`0`–`1`), and a normalized `boundingBox` (`CGRect`, origin bottom-left as Vision reports it):
+
+```swift
+ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+                   detectedObjects: $detectedObjects) // $detectedObjects is [DetectedObject]
+```
+
+<details>
+<summary>Full Example (with bounding boxes)</summary>
+<br>
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorBoxesSample: View {
+
+    @State var detectedObjects: [DetectedObject] = []
+
+    var body: some View {
+        ZStack {
+            ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+                               detectedObjects: $detectedObjects)
+
+            GeometryReader { geometry in
+                ForEach(Array(detectedObjects.enumerated()), id: \.offset) { _, object in
+                    let box = object.boundingBox
+                    Rectangle()
+                        .stroke(.red, lineWidth: 2)
+                        // Vision's origin is bottom-left; SwiftUI's is top-left, so flip Y.
+                        .frame(width: box.width * geometry.size.width,
+                               height: box.height * geometry.size.height)
+                        .position(x: box.midX * geometry.size.width,
+                                  y: (1 - box.midY) * geometry.size.height)
+                        .overlay(Text(object.label))
+                }
+            }
+        }
+    }
+}
+```
+</details>
+
+
 ### Live Hand Pose Classification
 
 Classify hand poses in real-time using a Create ML / Core ML hand action classifier.

Sources/PrototypeKit/PrototypeKit.docc/PrototypeKit.md

@@ -59,6 +59,8 @@ directory for a runnable gallery covering every feature.
 ### Vision & Core ML
 
 - ``ImageClassifierView``
+- ``ObjectDetectorView``
+- ``DetectedObject``
 - ``HandPoseClassifierView``
 - ``LiveTextRecognizerView``
 - ``LiveBarcodeRecognizerView``