diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index 01c3cad..1f65a31 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -12,7 +12,7 @@
{
"name": "prototypekit",
"source": "./plugin",
- "description": "Scaffold and wire up PrototypeKit SwiftUI views for rapid on-device ML prototyping (camera, image/text/barcode/hand-pose/sound recognition, natural-language analysis).",
+ "description": "Scaffold and wire up PrototypeKit SwiftUI views for rapid on-device ML prototyping (camera, image classification, object detection, text/barcode/hand-pose/sound recognition, natural-language analysis).",
"version": "0.1.0",
"author": {
"name": "Friday Technologies",
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a91dd18..675b815 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,14 @@ once tagged releases begin.
## [Unreleased]
### Added
+- **Object detection.** A new `ObjectDetectorView` runs a Create ML / Core ML **Object Detector**
+ model on the live camera feed and publishes the objects found in each frame. Two initializers are
+ offered: one takes a `detectedObjects: Binding<[String]>` for the labels only, and one takes a
+ `detectedObjects: Binding<[DetectedObject]>` — a new public value type carrying each object's
+ `label`, `confidence`, and normalized `boundingBox` — so you can draw bounding boxes or reason about
+ position. Like the other model-backed views it degrades gracefully on a bad/missing model (camera
+ feed with no detections, logged, reported via the optional `onError` closure) and accepts a
+ `camera: CameraOptions?`.
- **Error reporting.** A public `PrototypeKitError` type and an optional `onError` closure on the
entry points that can fail at setup time: `ImageClassifierView`, `HandPoseClassifierView`, the
`classifyActivity(...)` modifier (model-load failures), and the `recognizeSounds(...)` modifier
diff --git a/Examples/PrototypeKitExamples.swift b/Examples/PrototypeKitExamples.swift
index f8bb3a4..86790c3 100644
--- a/Examples/PrototypeKitExamples.swift
+++ b/Examples/PrototypeKitExamples.swift
@@ -23,6 +23,8 @@ struct ExampleGallery: View {
NavigationLink("Live barcodes", destination: LiveBarcodeExample())
NavigationLink("Face detection", destination: FaceCountExample())
NavigationLink("Image classification", destination: ImageClassifierExample())
+ NavigationLink("Object detection", destination: ObjectDetectorExample())
+ NavigationLink("Object detection (boxes)", destination: ObjectDetectorBoxesExample())
}
Section("Sound") {
NavigationLink("Sound recognition", destination: SoundExample())
@@ -118,6 +120,78 @@ struct ImageClassifierExample: View {
}
}
+/// Detects objects in each camera frame with a Create ML object detector.
+///
+/// Replace `modelURL` with your own model, e.g. `MyObjectDetector.urlOfModelInThisBundle`.
+struct ObjectDetectorExample: View {
+ @State private var detectedObjects: [String] = []
+ @State private var errorMessage: String?
+
+ // Point this at your Create ML object detector's `urlOfModelInThisBundle`.
+ private let modelURL = URL(fileURLWithPath: "/path/to/YourObjectDetector.mlmodelc")
+
+ var body: some View {
+ ZStack(alignment: .bottom) {
+ ObjectDetectorView(modelURL: modelURL,
+ detectedObjects: $detectedObjects) { error in
+ // React to a model-load failure instead of showing a silent, detection-less feed.
+ errorMessage = error.localizedDescription
+ }
+ Text(errorMessage ?? (detectedObjects.isEmpty ? "Detecting…" : detectedObjects.joined(separator: ", ")))
+ .padding()
+ .frame(maxWidth: .infinity)
+ .background(.thinMaterial)
+ }
+ .ignoresSafeArea()
+ }
+}
+
+/// Detects objects and draws a bounding box + label around each one.
+///
+/// Replace `modelURL` with your own model, e.g. `MyObjectDetector.urlOfModelInThisBundle`.
+struct ObjectDetectorBoxesExample: View {
+ @State private var detectedObjects: [DetectedObject] = []
+ @State private var errorMessage: String?
+
+ private let modelURL = URL(fileURLWithPath: "/path/to/YourObjectDetector.mlmodelc")
+
+ var body: some View {
+ ZStack {
+ ObjectDetectorView(modelURL: modelURL,
+ detectedObjects: $detectedObjects) { error in
+ errorMessage = error.localizedDescription
+ }
+
+ GeometryReader { geometry in
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { _, object in
+ let box = object.boundingBox
+ Rectangle()
+ .stroke(.red, lineWidth: 2)
+ // Vision's origin is bottom-left; SwiftUI's is top-left, so flip Y.
+ .frame(width: box.width * geometry.size.width,
+ height: box.height * geometry.size.height)
+ .position(x: box.midX * geometry.size.width,
+ y: (1 - box.midY) * geometry.size.height)
+ .overlay(
+ Text(object.label)
+ .font(.caption2)
+ .padding(2)
+ .background(.red)
+ .foregroundStyle(.white)
+ .position(x: box.midX * geometry.size.width,
+ y: (1 - box.maxY) * geometry.size.height)
+ )
+ }
+ }
+
+ if let errorMessage = errorMessage {
+ Text(errorMessage).foregroundStyle(.red).padding().background(.thinMaterial)
+ }
+ }
+ .ignoresSafeArea()
+ }
+}
+
// MARK: - Sound
/// Recognizes sounds from the microphone using the built-in classifier.
diff --git a/README.md b/README.md
index c69a93a..809500e 100644
--- a/README.md
+++ b/README.md
@@ -154,6 +154,95 @@ struct ImageClassifierViewSample: View {
+### Live Object Detection
+
+Detect and locate objects in the live camera feed using a Create ML / Core ML **Object Detector** model.
+
+1. **Required Step:** Drag in your Create ML / Core ML object detector model into Xcode.
+2. Change `MyObjectDetector` below to the name of your Model.
+3. `detectedObjects` holds the labels of the objects found in the latest frame; use it as you would any other state variable.
+
+Utilise `ObjectDetectorView`
+
+```swift
+ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+```
+
+
+Full Example
+
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorViewSample: View {
+
+ @State var detectedObjects: [String] = []
+
+ var body: some View {
+ VStack {
+ ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+
+ ScrollView {
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { index, object in
+ Text(object)
+ }
+ }
+ }
+ }
+}
+```
+
+
+Need to know **where** each object is (for example, to draw bounding boxes)? Bind an array of
+`DetectedObject` instead of `[String]`. Each `DetectedObject` carries the `label`, a `confidence`
+(`0`–`1`), and a normalized `boundingBox` (`CGRect`, origin bottom-left as Vision reports it):
+
+```swift
+ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects) // $detectedObjects is [DetectedObject]
+```
+
+
+Full Example (with bounding boxes)
+
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorBoxesSample: View {
+
+ @State var detectedObjects: [DetectedObject] = []
+
+ var body: some View {
+ ZStack {
+ ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+
+ GeometryReader { geometry in
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { _, object in
+ let box = object.boundingBox
+ Rectangle()
+ .stroke(.red, lineWidth: 2)
+ // Vision's origin is bottom-left; SwiftUI's is top-left, so flip Y.
+ .frame(width: box.width * geometry.size.width,
+ height: box.height * geometry.size.height)
+ .position(x: box.midX * geometry.size.width,
+ y: (1 - box.midY) * geometry.size.height)
+ .overlay(Text(object.label))
+ }
+ }
+ }
+ }
+}
+```
+
+
+
### Live Hand Pose Classification
Classify hand poses in real-time using a Create ML / Core ML hand action classifier.
diff --git a/Sources/PrototypeKit/PrototypeKit.docc/PrototypeKit.md b/Sources/PrototypeKit/PrototypeKit.docc/PrototypeKit.md
index a4360a2..beba268 100644
--- a/Sources/PrototypeKit/PrototypeKit.docc/PrototypeKit.md
+++ b/Sources/PrototypeKit/PrototypeKit.docc/PrototypeKit.md
@@ -59,6 +59,8 @@ directory for a runnable gallery covering every feature.
### Vision & Core ML
- ``ImageClassifierView``
+- ``ObjectDetectorView``
+- ``DetectedObject``
- ``HandPoseClassifierView``
- ``LiveTextRecognizerView``
- ``LiveBarcodeRecognizerView``
diff --git a/Sources/PrototypeKit/Public/ObjectDetectorView.swift b/Sources/PrototypeKit/Public/ObjectDetectorView.swift
new file mode 100644
index 0000000..38913b6
--- /dev/null
+++ b/Sources/PrototypeKit/Public/ObjectDetectorView.swift
@@ -0,0 +1,215 @@
+//
+// ObjectDetectorView.swift
+//
+//
+// Created by James Dale.
+//
+
+import SwiftUI
+import CoreML
+import Vision
+
+/// A single object located by an ``ObjectDetectorView`` in a camera frame.
+///
+/// This is the richer counterpart to the label-only (`[String]`) API: alongside the most likely
+/// label it carries the detection confidence and the object's location in the frame, so you can draw
+/// bounding boxes or reason about where things are.
+public struct DetectedObject: Equatable {
+
+ /// The most likely label for the object (the top Vision classification).
+ public let label: String
+
+ /// The confidence of ``label``, from `0` to `1`.
+ public let confidence: Float
+
+ /// The object's location in the image in normalized coordinates (`0`–`1`), with the origin at the
+ /// bottom-left, exactly as Vision reports it (`VNRecognizedObjectObservation.boundingBox`).
+ ///
+ /// SwiftUI's coordinate space has its origin at the top-left, so flip the `y` axis
+ /// (`1 - boundingBox.maxY`) when positioning an overlay.
+ public let boundingBox: CGRect
+
+ /// Creates a detected object.
+ ///
+ /// - Parameters:
+ /// - label: The most likely label for the object.
+ /// - confidence: The confidence of `label`, from `0` to `1`.
+ /// - boundingBox: The object's normalized bounding box (origin bottom-left).
+ public init(label: String, confidence: Float, boundingBox: CGRect) {
+ self.label = label
+ self.confidence = confidence
+ self.boundingBox = boundingBox
+ }
+}
+
+final class ObjectDetectorReceiver: PKCameraViewReceiver, ObservableObject {
+
+ private let vnCoreMLModel: VNCoreMLModel?
+
+ @Published var detectedObjects: [DetectedObject] = []
+
+ private lazy var request: VNCoreMLRequest? = {
+ guard let model = self.vnCoreMLModel else { return nil }
+ return VNCoreMLRequest(model: model) { [weak self] request, error in
+ if let error = error {
+ PKLog.vision.error("Object detection request failed: \(error.localizedDescription)")
+ return
+ }
+ let results = request.results as? [VNRecognizedObjectObservation] ?? []
+ let objects = results.compactMap { observation -> DetectedObject? in
+ guard let top = observation.labels.first else { return nil }
+ return DetectedObject(label: top.identifier,
+ confidence: top.confidence,
+ boundingBox: observation.boundingBox)
+ }
+
+ DispatchQueue.main.async {
+ self?.detectedObjects = objects
+ }
+ }
+ }()
+
+ /// Creates a receiver for the given Vision model.
+ ///
+ /// - Parameter vnMLModel: The Vision Core ML object-detection model to run on frames, or `nil` when
+ /// the model failed to load. When `nil`, frames are ignored and no detections are published.
+ init(vnMLModel: VNCoreMLModel?) {
+ self.vnCoreMLModel = vnMLModel
+ }
+
+ func processImage(_ cgImage: CGImage) {
+ guard let request = request else { return }
+
+ let handler = VNImageRequestHandler(cgImage: cgImage,
+ orientation: .up,
+ options: [:])
+
+ // Object detectors localize objects across the whole frame, so scale (rather than crop) the
+ // image into the model's input size to avoid discarding objects near the edges.
+ request.imageCropAndScaleOption = .scaleFill
+ do {
+ try handler.perform([request])
+ } catch {
+ PKLog.vision.error("Failed to perform object detection: \(error.localizedDescription)")
+ }
+ }
+}
+
+/// A SwiftUI view that shows a live camera feed and detects objects in each frame using a Core ML
+/// object detector.
+///
+/// Provide the URL of a Create ML / Core ML **Object Detector** model and a binding to receive the
+/// objects found in the latest frame. The view drives a ``PKCameraView`` internally, so your app target
+/// must declare the `NSCameraUsageDescription` (Privacy - Camera Usage Description) key in its Info
+/// properties.
+///
+/// Two flavours of binding are available. Pass a `Binding<[String]>` when you only care about *what*
+/// was detected:
+///
+/// ```swift
+/// @State var detectedObjects: [String] = []
+///
+/// ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+/// detectedObjects: $detectedObjects)
+/// ```
+///
+/// Or pass a `Binding<[DetectedObject]>` when you also want *where* — each ``DetectedObject`` carries a
+/// label, a confidence, and a normalized bounding box, so you can draw overlays:
+///
+/// ```swift
+/// @State var detectedObjects: [DetectedObject] = []
+///
+/// ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+/// detectedObjects: $detectedObjects)
+/// ```
+public struct ObjectDetectorView: View {
+
+ @State var receiver: ObjectDetectorReceiver
+
+ /// Forwards the receiver's detections to the caller's binding. Set once at init to map the rich
+ /// ``DetectedObject`` values into whichever binding flavour the caller chose.
+ private let publish: ([DetectedObject]) -> Void
+
+ private let cameraOptions: CameraOptions?
+
+ private let onError: ((PrototypeKitError) -> Void)?
+
+ private let loadError: PrototypeKitError?
+
+ /// Loads the model and wires up the receiver. Shared by the public initializers, which differ only
+ /// in how they forward detections to the caller.
+ private init(modelURL: URL,
+ camera: CameraOptions?,
+ onError: ((PrototypeKitError) -> Void)?,
+ publish: @escaping ([DetectedObject]) -> Void) {
+ self.publish = publish
+ self.cameraOptions = camera
+ self.onError = onError
+ do {
+ let mlModel = try MLModel(contentsOf: modelURL)
+ let vnModel = try VNCoreMLModel(for: mlModel)
+ self.receiver = ObjectDetectorReceiver(vnMLModel: vnModel)
+ self.loadError = nil
+ } catch {
+ // Degrade gracefully: show the camera feed without detection rather than
+ // crashing the host app when the model can't be loaded.
+ PKLog.model.error("Failed to load object detection model at \(modelURL.path): \(error.localizedDescription)")
+ self.receiver = ObjectDetectorReceiver(vnMLModel: nil)
+ self.loadError = .modelLoadFailed(url: modelURL, underlying: error)
+ }
+ }
+
+ /// Creates an object detector view that reports the *labels* of detected objects.
+ ///
+ /// - Parameters:
+ /// - modelURL: The location of the compiled Core ML / Create ML object-detection model to load,
+ /// typically `YourModel.urlOfModelInThisBundle`.
+ /// - detectedObjects: A binding updated with the labels of the objects detected in the latest
+ /// frame. Defaults to a constant empty array when you only need the on-screen camera feed.
+ /// - camera: Optional ``CameraOptions`` selecting the camera position and device type. Pass `nil`
+ /// to use the default back wide-angle camera. (Ignored on macOS.)
+ /// - onError: An optional closure called (on the main thread) if the model fails to load. The view
+ /// still shows the camera feed without detection; use this to surface the failure in your UI.
+ public init(modelURL: URL,
+ detectedObjects: Binding<[String]> = .constant([]),
+ camera: CameraOptions? = nil,
+ onError: ((PrototypeKitError) -> Void)? = nil) {
+ self.init(modelURL: modelURL, camera: camera, onError: onError) { objects in
+ detectedObjects.wrappedValue = objects.map(\.label)
+ }
+ }
+
+ /// Creates an object detector view that reports each detected object's label, confidence, and
+ /// bounding box.
+ ///
+ /// Use this initializer when you need to know *where* objects are (for example, to draw bounding
+ /// boxes) rather than only *what* was detected.
+ ///
+ /// - Parameters:
+ /// - modelURL: The location of the compiled Core ML / Create ML object-detection model to load,
+ /// typically `YourModel.urlOfModelInThisBundle`.
+ /// - detectedObjects: A binding updated with the ``DetectedObject`` values found in the latest
+ /// frame, each carrying a label, a confidence, and a normalized bounding box.
+ /// - camera: Optional ``CameraOptions`` selecting the camera position and device type. Pass `nil`
+ /// to use the default back wide-angle camera. (Ignored on macOS.)
+ /// - onError: An optional closure called (on the main thread) if the model fails to load. The view
+ /// still shows the camera feed without detection; use this to surface the failure in your UI.
+ public init(modelURL: URL,
+ detectedObjects: Binding<[DetectedObject]>,
+ camera: CameraOptions? = nil,
+ onError: ((PrototypeKitError) -> Void)? = nil) {
+ self.init(modelURL: modelURL, camera: camera, onError: onError) { objects in
+ detectedObjects.wrappedValue = objects
+ }
+ }
+
+ public var body: some View {
+ PKCameraView(receiver: receiver, options: cameraOptions)
+ .onReceive(receiver.$detectedObjects, perform: { newObjects in
+ self.publish(newObjects)
+ })
+ .onAppear {
+ if let loadError = loadError { onError?(loadError) }
+ }
+ }
+}
diff --git a/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift b/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift
index 7ff1e38..f08cef8 100644
--- a/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift
+++ b/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift
@@ -34,6 +34,11 @@ final class GracefulFailureTests: XCTestCase {
XCTAssertNoThrow(view.body)
}
+ func testObjectDetectorViewInitWithBadURLDoesNotCrash() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL)
+ XCTAssertNoThrow(view.body)
+ }
+
// MARK: - The optional onError hook is accepted and does not crash construction
func testImageClassifierViewAcceptsOnError() throws {
@@ -46,6 +51,11 @@ final class GracefulFailureTests: XCTestCase {
XCTAssertNoThrow(view.body)
}
+ func testObjectDetectorViewAcceptsOnError() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL, onError: { _ in })
+ XCTAssertNoThrow(view.body)
+ }
+
// MARK: - PrototypeKitError provides human-readable descriptions
func testPrototypeKitErrorDescriptions() {
@@ -77,6 +87,15 @@ final class GracefulFailureTests: XCTestCase {
XCTAssertNil(receiver.latestPrediction)
}
+ func testObjectDetectorReceiverWithNilModelIgnoresFrames() throws {
+ let receiver = ObjectDetectorReceiver(vnMLModel: nil)
+ let frame = try makeSolidColorCGImage(width: 64, height: 64)
+
+ receiver.processImage(frame)
+
+ XCTAssertTrue(receiver.detectedObjects.isEmpty)
+ }
+
// MARK: - Activity classification (iOS only) must stay inert without a model
#if os(iOS)
diff --git a/Tests/PrototypeKitTests/OD/ObjectDetectorTests.swift b/Tests/PrototypeKitTests/OD/ObjectDetectorTests.swift
new file mode 100644
index 0000000..4392eeb
--- /dev/null
+++ b/Tests/PrototypeKitTests/OD/ObjectDetectorTests.swift
@@ -0,0 +1,72 @@
+//
+// ObjectDetectorTests.swift
+//
+//
+// Created by PrototypeKit.
+//
+
+import XCTest
+import Vision
+import CoreML
+import Combine
+
+@testable
+import PrototypeKit
+
+@available(iOS 16.0, *)
+final class ObjectDetectorTests: XCTestCase {
+
+ private var cancellables = [AnyCancellable]()
+
+ /// A URL that does not point at a real Core ML model.
+ private var bogusModelURL: URL {
+ URL(fileURLWithPath: "/nonexistent/PrototypeKit/DoesNotExist.mlmodelc")
+ }
+
+ // A bad model URL must degrade gracefully (camera feed, no detections) rather than crash.
+ func testObjectDetectorViewInitWithBadURLDoesNotCrash() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL)
+ XCTAssertNoThrow(view.body)
+ }
+
+ func testObjectDetectorViewAcceptsOnError() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL, onError: { _ in })
+ XCTAssertNoThrow(view.body)
+ }
+
+ // The DetectedObject-based initializer must also degrade gracefully on a bad model URL.
+ func testObjectDetectorViewWithDetectedObjectsBindingDoesNotCrash() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL,
+ detectedObjects: .constant([DetectedObject]()))
+ XCTAssertNoThrow(view.body)
+ }
+
+ func testObjectDetectorViewWithDetectedObjectsBindingAcceptsOnError() throws {
+ let view = ObjectDetectorView(modelURL: bogusModelURL,
+ detectedObjects: .constant([DetectedObject]()),
+ onError: { _ in })
+ XCTAssertNoThrow(view.body)
+ }
+
+ // With no model loaded, the receiver must ignore frames and publish no detections.
+ func testObjectDetectorReceiverWithNilModelIgnoresFrames() throws {
+ let receiver = ObjectDetectorReceiver(vnMLModel: nil)
+ let frame = try makeSolidColorCGImage(width: 64, height: 64)
+
+ receiver.processImage(frame)
+
+ XCTAssertTrue(receiver.detectedObjects.isEmpty)
+ }
+
+ // DetectedObject is a simple value type carrying label, confidence, and bounding box.
+ func testDetectedObjectStoresItsValues() {
+ let box = CGRect(x: 0.1, y: 0.2, width: 0.3, height: 0.4)
+ let object = DetectedObject(label: "dog", confidence: 0.9, boundingBox: box)
+
+ XCTAssertEqual(object.label, "dog")
+ XCTAssertEqual(object.confidence, 0.9, accuracy: 0.0001)
+ XCTAssertEqual(object.boundingBox, box)
+ XCTAssertEqual(object, DetectedObject(label: "dog", confidence: 0.9, boundingBox: box))
+ XCTAssertNotEqual(object, DetectedObject(label: "cat", confidence: 0.9, boundingBox: box))
+ }
+}
diff --git a/plugin/README.md b/plugin/README.md
index 9566a0f..937caba 100644
--- a/plugin/README.md
+++ b/plugin/README.md
@@ -31,6 +31,7 @@ claude --plugin-dir ./plugin
| `/prototypekit:setup` | Add the SwiftPM package and configure Info.plist privacy keys |
| `/prototypekit:camera` | Scaffold a live camera feed (`PKCameraView`) |
| `/prototypekit:image-classifier` | Scaffold live image classification (`ImageClassifierView`, Core ML) |
+| `/prototypekit:object-detector` | Scaffold live object detection (`ObjectDetectorView`, Core ML) |
| `/prototypekit:text-recognizer` | Scaffold live text recognition / OCR (`LiveTextRecognizerView`) |
| `/prototypekit:barcode-scanner` | Scaffold live barcode/QR scanning (`LiveBarcodeRecognizerView`) |
| `/prototypekit:animal-recognizer` | Scaffold live animal recognition — cats & dogs (`LiveAnimalRecognizerView`) |
diff --git a/plugin/commands/object-detector.md b/plugin/commands/object-detector.md
new file mode 100644
index 0000000..d411944
--- /dev/null
+++ b/plugin/commands/object-detector.md
@@ -0,0 +1,55 @@
+---
+description: Scaffold a PrototypeKit live object detector (ObjectDetectorView) backed by a Core ML model.
+argument-hint: "[optional: Core ML model class name, e.g. MyObjectDetector]"
+---
+
+Add live camera object detection using PrototypeKit's `ObjectDetectorView`.
+Follow the exact API in the `prototypekit` skill.
+
+Signatures to use (pick based on whether the user needs positions):
+`ObjectDetectorView(modelURL:, detectedObjects: Binding<[String]>, camera: CameraOptions? = nil, onError: ((PrototypeKitError) -> Void)? = nil)` — labels only.
+`ObjectDetectorView(modelURL:, detectedObjects: Binding<[DetectedObject]>, camera: CameraOptions? = nil, onError: ((PrototypeKitError) -> Void)? = nil)` — labels + confidence + bounding boxes.
+
+`DetectedObject` exposes `label: String`, `confidence: Float`, and `boundingBox: CGRect` (normalized,
+origin bottom-left as Vision reports it — flip Y for SwiftUI overlays).
+
+1. Generate a SwiftUI `View` that:
+ - declares `@State var detectedObjects: [String] = []` (or `[DetectedObject]` if the user wants
+ positions / bounding boxes),
+ - embeds `ObjectDetectorView(modelURL: .urlOfModelInThisBundle, detectedObjects: $detectedObjects)`,
+ - displays the detected objects (a `ScrollView` + `ForEach` of labels, or a bounding-box overlay
+ using each `DetectedObject`'s `boundingBox`).
+ Use the model class name from `$ARGUMENTS` if provided; otherwise use a clear
+ placeholder like `MyObjectDetector` and tell the user to replace it.
+2. Ensure `import SwiftUI` and `import PrototypeKit`.
+3. Remind the user to:
+ - drag their `.mlmodel` (a Create ML **Object Detector**) into Xcode (Xcode generates the class used above), and
+ - add `NSCameraUsageDescription` to Info.plist.
+ Note that a bad/missing model URL degrades gracefully (camera feed, no detections) and is
+ reported through the optional `onError` closure rather than crashing.
+
+Reference example:
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorViewSample: View {
+ @State var detectedObjects: [String] = []
+
+ var body: some View {
+ VStack {
+ ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+
+ ScrollView {
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { index, object in
+ Text(object)
+ }
+ }
+ }
+ }
+}
+```
+
+Prefer editing the user's actual files over just printing code when a target is clear.
diff --git a/plugin/skills/prototypekit/SKILL.md b/plugin/skills/prototypekit/SKILL.md
index d08b485..88400d8 100644
--- a/plugin/skills/prototypekit/SKILL.md
+++ b/plugin/skills/prototypekit/SKILL.md
@@ -4,8 +4,9 @@ description: >-
Reference and code-generation guide for PrototypeKit, a SwiftUI framework for
rapid on-device machine-learning prototyping on iOS/macOS. Use whenever the
user is building with PrototypeKit or asks for a live camera feed, on-device
- image classification, live text/OCR recognition, barcode scanning, animal
- recognition, face detection, body pose detection, rectangle detection, hand-pose
+ image classification, object detection, live text/OCR recognition, barcode
+ scanning, animal recognition, face detection, body pose detection, rectangle
+ detection, hand-pose
classification, sound recognition, or natural-language analysis (sentiment,
language identification, named-entity recognition) in a SwiftUI app. Provides exact
initializer signatures, @State/@Binding wiring, required Info.plist privacy
@@ -45,7 +46,7 @@ Then `import PrototypeKit` in any file that uses it.
## Required Info.plist permissions (most common cause of crashes)
-Any camera-based view (`PKCameraView`, `ImageClassifierView`,
+Any camera-based view (`PKCameraView`, `ImageClassifierView`, `ObjectDetectorView`,
`LiveTextRecognizerView`, `LiveBarcodeRecognizerView`, `LiveAnimalRecognizerView`,
`LiveFaceDetectorView`, `LiveBodyPoseDetectorView`, `LiveRectangleDetectorView`,
`HandPoseClassifierView`) requires a camera-usage description, or the app crashes on
@@ -64,8 +65,8 @@ To add: select the project ▸ your target ▸ **Info** tab ▸ right-click the
## Core ML models
-Views that classify (`ImageClassifierView`, `HandPoseClassifierView`) take a
-`modelURL: URL` pointing at a compiled Core ML model.
+Views that classify or detect (`ImageClassifierView`, `ObjectDetectorView`,
+`HandPoseClassifierView`) take a `modelURL: URL` pointing at a compiled Core ML model.
1. Drag your `.mlmodel` (from Create ML / Core ML) into the Xcode project.
2. Xcode generates a Swift class named after the model.
@@ -131,6 +132,95 @@ struct ImageClassifierViewSample: View {
}
```
+### `ObjectDetectorView` — live object detection (Core ML)
+
+Runs a Create ML / Core ML **Object Detector** model on each camera frame and publishes the
+objects found. There are two initializers — pick by whether you need only *what* was detected or
+also *where*:
+
+```swift
+// Labels only:
+public init(modelURL: URL,
+ detectedObjects: Binding<[String]> = .constant([]),
+ camera: CameraOptions? = nil,
+ onError: ((PrototypeKitError) -> Void)? = nil)
+
+// Labels + confidence + bounding boxes:
+public init(modelURL: URL,
+ detectedObjects: Binding<[DetectedObject]>,
+ camera: CameraOptions? = nil,
+ onError: ((PrototypeKitError) -> Void)? = nil)
+```
+
+`DetectedObject` is a public value type:
+
+```swift
+public struct DetectedObject: Equatable {
+ public let label: String // top Vision label
+ public let confidence: Float // 0...1
+ public let boundingBox: CGRect // normalized (0...1), origin bottom-left (Vision convention)
+}
+```
+
+The overload is chosen by the binding's element type: pass `Binding<[String]>` for labels only, or
+`Binding<[DetectedObject]>` for positions. Vision's bounding-box origin is bottom-left, so flip the
+`y` axis (`1 - boundingBox.midY`) when positioning a SwiftUI overlay.
+
+Full example (labels only):
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorViewSample: View {
+ @State var detectedObjects: [String] = []
+
+ var body: some View {
+ VStack {
+ ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+
+ ScrollView {
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { index, object in
+ Text(object)
+ }
+ }
+ }
+ }
+}
+```
+
+Full example (bounding boxes):
+
+```swift
+import SwiftUI
+import PrototypeKit
+
+struct ObjectDetectorBoxesSample: View {
+ @State var detectedObjects: [DetectedObject] = []
+
+ var body: some View {
+ ZStack {
+ ObjectDetectorView(modelURL: MyObjectDetector.urlOfModelInThisBundle,
+ detectedObjects: $detectedObjects)
+
+ GeometryReader { geometry in
+ ForEach(Array(detectedObjects.enumerated()), id: \.offset) { _, object in
+ let box = object.boundingBox
+ Rectangle()
+ .stroke(.red, lineWidth: 2)
+ .frame(width: box.width * geometry.size.width,
+ height: box.height * geometry.size.height)
+ .position(x: box.midX * geometry.size.width,
+ y: (1 - box.midY) * geometry.size.height)
+ .overlay(Text(object.label))
+ }
+ }
+ }
+ }
+}
+```
+
### `LiveTextRecognizerView` — live text recognition / OCR
Signature:
@@ -438,9 +528,11 @@ struct SentimentView: View {
- **Missing privacy key = crash.** Add `NSCameraUsageDescription` (camera views)
and `NSMicrophoneUsageDescription` (`recognizeSounds`) before running.
-- **Bad model URL.** `ImageClassifierView` / `HandPoseClassifierView` currently
- call `fatalError()` if the model fails to load. Verify the model is in the
- target's bundle and the generated class name matches.
+- **Bad model URL.** `ImageClassifierView` / `ObjectDetectorView` /
+ `HandPoseClassifierView` degrade gracefully if the model fails to load — the camera
+ feed still shows but no predictions/detections are produced, and the failure is logged
+ (and reported to `onError` where available). Verify the model is in the target's bundle
+ and the generated class name matches.
- **Simulator limits.** The camera isn't available in the iOS Simulator (limited
preview), and inference falls back to CPU-only in the simulator and under XCTest.
- **`recognizeSounds` is iOS-only** and requires iOS 15+. It is not available on