From 719ae314dd2ee95e96fc516e3318935da97a137c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 08:01:31 +0000 Subject: [PATCH] Add PrototypeKitError and optional onError handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets consuming apps react to setup failures instead of only reading the log (the library still degrades gracefully and logs as before). - New public `PrototypeKitError` (LocalizedError): `.modelLoadFailed(url:underlying:)` and `.soundClassificationFailed(underlying:)`. - Optional `onError:` closure (defaults to nil, so source-compatible) on the entry points that can fail at setup: - ImageClassifierView, HandPoseClassifierView, classifyActivity(...) — fires on model-load failure (on appear). - recognizeSounds(...) — fires on a failed start (bad custom model) and on session errors (denied mic access, audio interruptions), sourced from a new long-lived `errors` relay on SystemAudioClassifier. - Transient per-frame Vision errors stay log-only by design (they would spam a callback). - Tests for the error descriptions and that the onError-accepting inits don't crash. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014wwQvH5kuGmjQ9nW3gJ6zD --- CHANGELOG.md | 9 ++++- .../PrototypeKit/Audio/AudioClassifier.swift | 6 +++ .../Public/ClassifyActivityModifier.swift | 24 +++++++++-- .../Public/HandPoseClassifierView.swift | 18 ++++++++- .../Public/ImageClassifierView.swift | 17 +++++++- .../Public/PrototypeKitError.swift | 40 +++++++++++++++++++ .../Public/RecognizeSoundsModifier.swift | 35 ++++++++++++++-- .../ErrorHandling/GracefulFailureTests.swift | 23 +++++++++++ 8 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 Sources/PrototypeKit/Public/PrototypeKitError.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c43f0..c7f009a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,14 @@ once tagged releases begin. ## [Unreleased] -_Nothing yet._ +### Added +- **Error reporting.** A public `PrototypeKitError` type and an optional `onError` closure on the + entry points that can fail at setup time: `ImageClassifierView`, `HandPoseClassifierView`, the + `classifyActivity(...)` modifier (model-load failures), and the `recognizeSounds(...)` modifier + (failed starts, denied microphone access, audio-session interruptions). Apps can now *react* to + these failures — show a message, offer a retry — instead of only reading the log. The parameter + defaults to `nil`, so existing call sites are unaffected, and errors are still logged as before. + Transient per-frame Vision errors remain log-only by design. ## [0.1.0] - 2026-07-02 diff --git a/Sources/PrototypeKit/Audio/AudioClassifier.swift b/Sources/PrototypeKit/Audio/AudioClassifier.swift index 3965b4a..4847798 100644 --- a/Sources/PrototypeKit/Audio/AudioClassifier.swift +++ b/Sources/PrototypeKit/Audio/AudioClassifier.swift @@ -52,6 +52,11 @@ final class SystemAudioClassifier: NSObject { /// restart classification after an interruption and keep receiving results on the same relay. let results = PassthroughSubject() + /// A long-lived relay that publishes classification session errors (denied microphone access, + /// audio interruptions, failed starts). Like ``results`` it never completes, so callers can keep + /// observing across sessions. + let errors = PassthroughSubject() + /// The subject for the current classification session, if any. private var sessionSubject: PassthroughSubject? @@ -164,6 +169,7 @@ final class SystemAudioClassifier: NSObject { receiveCompletion: { [weak self] completion in if case .failure(let error) = completion { PKLog.audio.error("Sound classification session ended: \(error.localizedDescription)") + self?.errors.send(error) } self?.sessionCancellable = nil self?.sessionSubject = nil diff --git a/Sources/PrototypeKit/Public/ClassifyActivityModifier.swift b/Sources/PrototypeKit/Public/ClassifyActivityModifier.swift index bb792c7..ec5876f 100644 --- a/Sources/PrototypeKit/Public/ClassifyActivityModifier.swift +++ b/Sources/PrototypeKit/Public/ClassifyActivityModifier.swift @@ -285,14 +285,18 @@ extension View { /// model's feature names. Defaults match a standard Create ML Activity Classifier. /// - latestActivity: A binding updated with the most recently predicted activity label, or `nil` /// when nothing has been classified yet. + /// - onError: An optional closure called (on the main thread) if the model fails to load. The + /// modifier stays inert; use this to surface the failure in your UI. /// - Returns: A view that classifies activity while visible. /// - Important: Activity classification relies on `CoreMotion` and is available on iOS only. public func classifyActivity(modelURL: URL, configuration: ActivityClassifierConfiguration = .init(), - latestActivity: Binding) -> some View { + latestActivity: Binding, + onError: ((PrototypeKitError) -> Void)? = nil) -> some View { modifier(ClassifyActivityModifier(modelURL: modelURL, configuration: configuration, - latestActivity: latestActivity)) + latestActivity: latestActivity, + onError: onError)) } } @@ -302,11 +306,18 @@ struct ClassifyActivityModifier: ViewModifier { @Binding var latestActivity: String? + private let onError: ((PrototypeKitError) -> Void)? + + private let loadError: PrototypeKitError? + init(modelURL: URL, configuration: ActivityClassifierConfiguration, - latestActivity: Binding) { + latestActivity: Binding, + onError: ((PrototypeKitError) -> Void)? = nil) { self._latestActivity = latestActivity + self.onError = onError let loadedModel: MLModel? + var loadError: PrototypeKitError? do { loadedModel = try MLModel(contentsOf: modelURL) } catch { @@ -314,14 +325,19 @@ struct ClassifyActivityModifier: ViewModifier { // when the model can't be loaded. PKLog.model.error("Failed to load activity model at \(modelURL.path): \(error.localizedDescription)") loadedModel = nil + loadError = .modelLoadFailed(url: modelURL, underlying: error) } + self.loadError = loadError self._receiver = StateObject(wrappedValue: ActivityClassifierReceiver(mlModel: loadedModel, configuration: configuration)) } func body(content: Content) -> some View { content - .onAppear { receiver.start() } + .onAppear { + receiver.start() + if let loadError = loadError { onError?(loadError) } + } .onDisappear { receiver.stop() } .onReceive(receiver.$latestPrediction) { newPrediction in guard let newPrediction = newPrediction else { return } diff --git a/Sources/PrototypeKit/Public/HandPoseClassifierView.swift b/Sources/PrototypeKit/Public/HandPoseClassifierView.swift index 7b71bc7..308542b 100644 --- a/Sources/PrototypeKit/Public/HandPoseClassifierView.swift +++ b/Sources/PrototypeKit/Public/HandPoseClassifierView.swift @@ -89,6 +89,10 @@ public struct HandPoseClassifierView: View { @Binding var latestPrediction: String + private let onError: ((PrototypeKitError) -> Void)? + + private let loadError: PrototypeKitError? + /// Creates a hand pose classifier view backed by a Core ML model. /// /// - Parameters: @@ -96,24 +100,34 @@ public struct HandPoseClassifierView: View { /// `YourModel.urlOfModelInThisBundle`. /// - latestPrediction: A binding updated with the most recent predicted hand-pose label. Defaults to a /// constant empty string when you only need the on-screen camera feed. - public init(modelURL: URL, latestPrediction: Binding = .constant("")) { + /// - onError: An optional closure called (on the main thread) if the model fails to load. The view + /// still shows the camera feed without classification; use this to surface the failure in your UI. + public init(modelURL: URL, + latestPrediction: Binding = .constant(""), + onError: ((PrototypeKitError) -> Void)? = nil) { self._latestPrediction = latestPrediction + self.onError = onError do { let mlModel = try MLModel(contentsOf: modelURL) self.receiver = HandPoseClassifierReceiver(mlModel: mlModel) + self.loadError = nil } catch { // Degrade gracefully: show the camera feed without classification rather than // crashing the host app when the model can't be loaded. PKLog.model.error("Failed to load hand pose model at \(modelURL.path): \(error.localizedDescription)") self.receiver = HandPoseClassifierReceiver(mlModel: nil) + self.loadError = .modelLoadFailed(url: modelURL, underlying: error) } } - + public var body: some View { PKCameraView(receiver: receiver) .onReceive(receiver.$latestPrediction, perform: { newPrediction in guard let newPrediction = newPrediction else { return } self.latestPrediction = newPrediction }) + .onAppear { + if let loadError = loadError { onError?(loadError) } + } } } diff --git a/Sources/PrototypeKit/Public/ImageClassifierView.swift b/Sources/PrototypeKit/Public/ImageClassifierView.swift index 250ac1c..c08d1b5 100644 --- a/Sources/PrototypeKit/Public/ImageClassifierView.swift +++ b/Sources/PrototypeKit/Public/ImageClassifierView.swift @@ -77,6 +77,10 @@ public struct ImageClassifierView: View { private let cameraOptions: CameraOptions? + private let onError: ((PrototypeKitError) -> Void)? + + private let loadError: PrototypeKitError? + /// Creates an image classifier view backed by a Core ML model. /// /// - Parameters: @@ -86,28 +90,37 @@ public struct ImageClassifierView: View { /// constant empty string when you only need the on-screen camera feed. /// - camera: Optional ``CameraOptions`` selecting the camera position and device type. Pass `nil` /// to use the default back wide-angle camera. (Ignored on macOS.) + /// - onError: An optional closure called (on the main thread) if the model fails to load. The view + /// still shows the camera feed without classification; use this to surface the failure in your UI. public init(modelURL: URL, latestPrediction: Binding = .constant(""), - camera: CameraOptions? = nil) { + camera: CameraOptions? = nil, + onError: ((PrototypeKitError) -> Void)? = nil) { self._latestPrediction = latestPrediction self.cameraOptions = camera + self.onError = onError do { let mlModel = try MLModel(contentsOf: modelURL) let vnModel = try VNCoreMLModel(for: mlModel) self.receiver = ImageClassifierReceiver(vnMLModel: vnModel) + self.loadError = nil } catch { // Degrade gracefully: show the camera feed without classification rather than // crashing the host app when the model can't be loaded. PKLog.model.error("Failed to load image classification model at \(modelURL.path): \(error.localizedDescription)") self.receiver = ImageClassifierReceiver(vnMLModel: nil) + self.loadError = .modelLoadFailed(url: modelURL, underlying: error) } } - + public var body: some View { PKCameraView(receiver: receiver, options: cameraOptions) .onReceive(receiver.$latestPrediction, perform: { newPrediction in guard let newPrediction = newPrediction else { return } self.latestPrediction = newPrediction }) + .onAppear { + if let loadError = loadError { onError?(loadError) } + } } } diff --git a/Sources/PrototypeKit/Public/PrototypeKitError.swift b/Sources/PrototypeKit/Public/PrototypeKitError.swift new file mode 100644 index 0000000..1addf74 --- /dev/null +++ b/Sources/PrototypeKit/Public/PrototypeKitError.swift @@ -0,0 +1,40 @@ +// +// PrototypeKitError.swift +// +// +// A public error type that PrototypeKit surfaces to consuming apps. +// + +import Foundation + +/// An error PrototypeKit reports to your app through an `onError` handler. +/// +/// PrototypeKit degrades gracefully — a failure never crashes the host app — but sometimes an app +/// wants to *react* to a failure (show a message, offer a retry) rather than silently show a camera +/// feed with no predictions. The camera and sound views/modifiers that load a model or start an audio +/// session accept an optional `onError` closure that receives one of these values. +/// +/// Errors are always also written to the unified log (subsystem `com.prototypekit.PrototypeKit`). +public enum PrototypeKitError: Error { + + /// A Core ML / Create ML model could not be loaded from the given URL. + /// + /// The affected view still shows the camera feed but produces no predictions. + case modelLoadFailed(url: URL, underlying: Error) + + /// Live sound classification could not start or was interrupted (for example, microphone + /// access was denied or the audio session was interrupted). + case soundClassificationFailed(underlying: Error) +} + +extension PrototypeKitError: LocalizedError { + + public var errorDescription: String? { + switch self { + case .modelLoadFailed(let url, let underlying): + return "PrototypeKit couldn't load the model \"\(url.lastPathComponent)\": \(underlying.localizedDescription)" + case .soundClassificationFailed(let underlying): + return "PrototypeKit sound classification failed: \(underlying.localizedDescription)" + } + } +} diff --git a/Sources/PrototypeKit/Public/RecognizeSoundsModifier.swift b/Sources/PrototypeKit/Public/RecognizeSoundsModifier.swift index ac5aca0..a57bbcc 100644 --- a/Sources/PrototypeKit/Public/RecognizeSoundsModifier.swift +++ b/Sources/PrototypeKit/Public/RecognizeSoundsModifier.swift @@ -67,11 +67,16 @@ extension View { /// when nothing has been classified yet. /// - configuration: A ``SoundAnalysisConfiguration`` controlling the analysis window and optional /// custom Core ML model. Defaults to the built-in system sound classifier. + /// - onError: An optional closure called (on the main thread) if classification can't start or is + /// interrupted — for example, denied microphone access or an audio-session interruption. /// - Returns: A view that performs live sound recognition while visible. /// - Important: Available on iOS 15.0+ only; sound recognition is not supported on macOS. - public func recognizeSounds(recognizedSound: Binding, configuration: SoundAnalysisConfiguration = .init()) -> some View { + public func recognizeSounds(recognizedSound: Binding, + configuration: SoundAnalysisConfiguration = .init(), + onError: ((PrototypeKitError) -> Void)? = nil) -> some View { ModifiedContent(content: self, modifier: RecognizeSoundsModifier(recognizedSound: recognizedSound, - configuration: configuration)) + configuration: configuration, + onError: onError)) } } @@ -80,10 +85,18 @@ struct RecognizeSoundsModifier: ViewModifier { @Binding var recognizedSound: String? - init(recognizedSound: Binding, configuration: SoundAnalysisConfiguration = .init()) { + private let onError: ((PrototypeKitError) -> Void)? + + private let setupError: PrototypeKitError? + + init(recognizedSound: Binding, + configuration: SoundAnalysisConfiguration = .init(), + onError: ((PrototypeKitError) -> Void)? = nil) { self._recognizedSound = recognizedSound + self.onError = onError SystemAudioClassifier.singleton.stopSoundClassification() + var setupError: PrototypeKitError? if let customModel = configuration.mlModel { // Use custom Core ML model for sound classification do { @@ -96,6 +109,7 @@ struct RecognizeSoundsModifier: ViewModifier { // Degrade gracefully: no classification starts rather than crashing the host app // when the custom model can't back a sound request. PKLog.audio.error("Failed to create sound request from custom model: \(error.localizedDescription)") + setupError = .soundClassificationFailed(underlying: error) } } else { // Use system sound classifier @@ -103,6 +117,7 @@ struct RecognizeSoundsModifier: ViewModifier { inferenceWindowSize: configuration.inferenceWindowSize, overlapFactor: configuration.overlapFactor) } + self.setupError = setupError } func body(content: Content) -> some View { @@ -110,6 +125,12 @@ struct RecognizeSoundsModifier: ViewModifier { .onReceive(classificationPublisher, perform: { result in self.recognizedSound = result.classifications.first?.identifier }) + .onReceive(errorPublisher, perform: { error in + onError?(.soundClassificationFailed(underlying: error)) + }) + .onAppear { + if let setupError = setupError { onError?(setupError) } + } } /// A main-thread publisher of classification results. @@ -123,5 +144,13 @@ struct RecognizeSoundsModifier: ViewModifier { .receive(on: DispatchQueue.main) .eraseToAnyPublisher() } + + /// A main-thread publisher of classification session errors, sourced from the classifier's + /// long-lived `errors` relay (which never completes). + private var errorPublisher: AnyPublisher { + SystemAudioClassifier.singleton.errors + .receive(on: DispatchQueue.main) + .eraseToAnyPublisher() + } } #endif diff --git a/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift b/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift index ffb2020..7ff1e38 100644 --- a/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift +++ b/Tests/PrototypeKitTests/ErrorHandling/GracefulFailureTests.swift @@ -34,6 +34,29 @@ final class GracefulFailureTests: XCTestCase { XCTAssertNoThrow(view.body) } + // MARK: - The optional onError hook is accepted and does not crash construction + + func testImageClassifierViewAcceptsOnError() throws { + let view = ImageClassifierView(modelURL: bogusModelURL, onError: { _ in }) + XCTAssertNoThrow(view.body) + } + + func testHandPoseClassifierViewAcceptsOnError() throws { + let view = HandPoseClassifierView(modelURL: bogusModelURL, onError: { _ in }) + XCTAssertNoThrow(view.body) + } + + // MARK: - PrototypeKitError provides human-readable descriptions + + func testPrototypeKitErrorDescriptions() { + let modelError = PrototypeKitError.modelLoadFailed(url: bogusModelURL, underlying: CocoaError(.fileNoSuchFile)) + XCTAssertNotNil(modelError.errorDescription) + XCTAssertTrue(modelError.errorDescription?.contains(bogusModelURL.lastPathComponent) ?? false) + + let soundError = PrototypeKitError.soundClassificationFailed(underlying: CocoaError(.fileReadUnknown)) + XCTAssertNotNil(soundError.errorDescription) + } + // MARK: - Receivers with no model must ignore frames rather than crash func testImageClassifierReceiverWithNilModelIgnoresFrames() throws {