diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..8d171ee --- /dev/null +++ b/Package.resolved @@ -0,0 +1,24 @@ +{ + "originHash" : "99b1b74f562a107d6351d636f42866c3bd7401e8aad60aeb4b29a8afa9130ebd", + "pins" : [ + { + "identity" : "fluidaudio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/FluidInference/FluidAudio", + "state" : { + "revision" : "503b4bd1bbf7220882de39fe8ae6716aae4132da", + "version" : "0.17.7" + } + }, + { + "identity" : "sparkle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/sparkle-project/Sparkle", + "state" : { + "revision" : "ab7d71291238df6e0492143b120c9b696a35f603", + "version" : "2.10.1" + } + } + ], + "version" : 3 +} diff --git a/Package.swift b/Package.swift index 8b7da6e..d58253b 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.10 +// swift-tools-version: 6.2 import PackageDescription let package = Package( @@ -11,12 +11,14 @@ let package = Package( products: [ .library(name: "AIProcessingCore", targets: ["AIProcessingCore"]), .library(name: "ASRCore", targets: ["ASRCore"]), + .library(name: "FluidASRMac", targets: ["FluidASRMac"]), .library(name: "AudioCore", targets: ["AudioCore"]), .library(name: "SessionCore", targets: ["SessionCore"]), .library(name: "SnippetCore", targets: ["SnippetCore"]), .library(name: "TextTargetMac", targets: ["TextTargetMac"]), .library(name: "CapabilityCore", targets: ["CapabilityCore"]) ], + dependencies: [.package(url: "https://github.com/FluidInference/FluidAudio", exact: "0.17.7", traits: [])], targets: [ .target( name: "AIProcessingCore" @@ -24,6 +26,7 @@ let package = Package( .target( name: "ASRCore" ), + .target(name: "FluidASRMac", dependencies: ["ASRCore", .product(name: "FluidAudio", package: "FluidAudio", condition: .when(platforms: [.macOS]))]), .target( name: "AudioCore", dependencies: ["ASRCore"] @@ -46,6 +49,7 @@ let package = Package( dependencies: [ "AIProcessingCore", "ASRCore", + "FluidASRMac", "AudioCore", "CapabilityCore", "SessionCore", @@ -62,6 +66,10 @@ let package = Package( name: "AIProcessingCoreTests", dependencies: ["AIProcessingCore"] ), + .testTarget(name: "FluidASRMacTests", dependencies: [ + "FluidASRMac", "ASRCore", + .product(name: "FluidAudio", package: "FluidAudio", condition: .when(platforms: [.macOS])) + ]), .testTarget( name: "ASRCoreTests", dependencies: ["ASRCore"] @@ -90,5 +98,6 @@ let package = Package( name: "AppShellSupportTests", dependencies: ["AppShellSupport"] ) - ] + ], + swiftLanguageModes: [.v5] ) diff --git a/Sources/ASRCore/ASRTypes.swift b/Sources/ASRCore/ASRTypes.swift index 7ef6db1..fc84585 100644 --- a/Sources/ASRCore/ASRTypes.swift +++ b/Sources/ASRCore/ASRTypes.swift @@ -2,6 +2,7 @@ import Foundation public enum ASRBackend: String, Sendable, Codable { case whisperCpp + case fluidAudio case coreMLHybrid case nemoSpeech } diff --git a/Sources/ASRCore/NativeVoiceModelBundle.swift b/Sources/ASRCore/NativeVoiceModelBundle.swift new file mode 100644 index 0000000..1b5739e --- /dev/null +++ b/Sources/ASRCore/NativeVoiceModelBundle.swift @@ -0,0 +1,84 @@ +import CryptoKit +import Foundation + +public struct NativeVoiceModelFile: Codable, Sendable { + public let path: String + public let bytes: Int64 + public let sha256: String +} + +public struct NativeVoiceModelBundle: Codable, Sendable { + public let modelID: String + public let repository: String + public let revision: String + public let bundleName: String + public let files: [NativeVoiceModelFile] + + public var expectedBytes: Int64 { files.reduce(0) { $0 + $1.bytes } } + + public static func manifest(modelID: String) throws -> NativeVoiceModelBundle { + guard let result = all.first(where: { $0.modelID == modelID }) else { + throw VoiceModelInstallerError.modelNotDownloadable(modelID) + } + return result + } + + public func validate(at directory: URL) throws { + for file in files { + let url = directory.appendingPathComponent(file.path) + let values = try url.resourceValues(forKeys: [.isRegularFileKey, .fileSizeKey, .isSymbolicLinkKey]) + guard values.isRegularFile == true, values.isSymbolicLink != true, + values.fileSize.map(Int64.init) == file.bytes else { + throw VoiceModelInstallerError.runtimeInstallFailed("Invalid model file: \(file.path)") + } + let data = try Data(contentsOf: url, options: .mappedIfSafe) + let checksum = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + guard checksum == file.sha256 else { + throw VoiceModelInstallerError.runtimeInstallFailed("Model checksum mismatch: \(file.path)") + } + } + } + + public static let all: [NativeVoiceModelBundle] = [ + NativeVoiceModelBundle(modelID: "fermion.phonon2", repository: "FluidInference/phonon-2-coreml", revision: "1702fc22e76f203d5ca464cd5d3ef130a45ce2c0", bundleName: "phonon-2-coreml", files: [ + NativeVoiceModelFile(path: "Decoder.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "3d1a9cd4d7b7956d20c48d9a8041717f9f02c6db72dfeda565080e1cff1db3a3"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/coremldata.bin", bytes: 554, sha256: "2590e4588972f8d3d1ddb10ce603decffd2e9bb79a88c6096a58c5732f6369fd"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/model.mil", bytes: 13110, sha256: "956f600207f88396017ca5c96cfa3acd5bfee60835a5b44b762e033a8fb28955"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/weights/weight.bin", bytes: 23604992, sha256: "05181616ed1af70b2aad0463d08d1d73a245af4ccb6159df79994f28c135d9d6"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "3d674b78b1a6916586611089d7ac80b4d20d94f691c0497f92b2a58339c5a0e2"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/coremldata.bin", bytes: 500, sha256: "4b661445d3e93201e3d70d616eaa9f061117a4ee1e4a6191d18a23dc23068b84"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/model.mil", bytes: 2734921, sha256: "b8954fb3ef2fedcb9ce01c67f0490cfd905b79862a33912dea12100cd24a5170"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/weights/weight.bin", bytes: 338391488, sha256: "d6a51593fc07b3c81989d5f871f687e78c52998aae0b06d000faced47313edb0"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "7b53f663c4400205f8bc00243076de50f95d8a1a64978dee7232eb0ddc7d683c"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/coremldata.bin", bytes: 586, sha256: "8d2170f28b28466476f62484a0c8b07c2d31713ac254d1b7cf80eba8070e9554"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/model.mil", bytes: 11777, sha256: "791b3c3cf3eb2079c84623fc880f6bba008d1366e5f8b03e9a8ed8bd4d7194a0"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/weights/weight.bin", bytes: 12642764, sha256: "5f922e335bc57174645fd6ede581395f1ae846f7227898301b5a1a7585f4ab45"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "c9beeb989c8d66f8be11df59bc6df277ec76cee404f6865b46243835ef562f6d"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/coremldata.bin", bytes: 486, sha256: "dbde3f2300842c1fd51ef3ff948a0bcffe65ffd2dca10707f2509f32c1d65b1d"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/metadata.json", bytes: 2841, sha256: "2a98699e22d279dd37fa1d238aeb1c6db1df0d6fad687775324157689d8f3acf"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/model.mil", bytes: 28181, sha256: "4b8518a956450fec57f06c2a21bdffc26973f7f1fa6842fb38fe917f896b6b93"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/weights/weight.bin", bytes: 491072, sha256: "129b76e3aeafa8afa3ea76d995b964b145fe83700d579f6ff42c4c38fa0968ea"), + NativeVoiceModelFile(path: "parakeet_vocab.json", bytes: 151122, sha256: "7ec60e05f1b24480736ec0eed40900f4626bce1fa9a60fd700ec7e2a59198735"), + ]), + NativeVoiceModelBundle(modelID: "moondream.parakeet-redux", repository: "FluidInference/parakeet-redux-coreml", revision: "8c5ef97a29cd120dc76b354b3f22b7fec3b486f9", bundleName: "parakeet-redux-coreml", files: [ + NativeVoiceModelFile(path: "Decoder.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "7f24e7248d57024c6f1c5a2ea306f4a6a89db45a363218523bb6410c4b1aecf0"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/coremldata.bin", bytes: 560, sha256: "422da51d983e93a3e48cd6d46a17d6c32844452414c8e242fb0caaca2b8af5ed"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/model.mil", bytes: 13110, sha256: "956f600207f88396017ca5c96cfa3acd5bfee60835a5b44b762e033a8fb28955"), + NativeVoiceModelFile(path: "Decoder.mlmodelc/weights/weight.bin", bytes: 23604992, sha256: "6c6c88c23ee5492a6b229e8cc636ddadb2b01ae0dd559d9cb63123bebf507d77"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "7862e972e09315df099d4bfd320743bf106acbf7fe4343317f9de45b0635d7b7"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/coremldata.bin", bytes: 506, sha256: "ca0b212fe3c06c3ccd35e508fa61e4dfaad19dd0c55b82e4f5bf8711e7758fd9"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/model.mil", bytes: 965906, sha256: "165487e314ed93c4a44dfec99d57be4e436e7c3dba6eef49fd3f0daf9de6d0d4"), + NativeVoiceModelFile(path: "Encoder.mlmodelc/weights/weight.bin", bytes: 182389312, sha256: "adbb5550dc6488717d3a1407b53122fbdb1af5e99c330ee20f8d8cfd9ec49252"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "fc857a756b8c9c9f31aa9e1062766cc0a82ac035ef369d6e1cbe97184921479c"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/coremldata.bin", bytes: 592, sha256: "611ea653e2adede7cc1cfe70da36296844f97e95e2c7dd961d45230e5599389d"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/model.mil", bytes: 11777, sha256: "791b3c3cf3eb2079c84623fc880f6bba008d1366e5f8b03e9a8ed8bd4d7194a0"), + NativeVoiceModelFile(path: "JointDecisionv3.mlmodelc/weights/weight.bin", bytes: 12642764, sha256: "73b0288acd115fc4c36038f66b6cbf49233182488febf5d8c787bea4009535f7"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/analytics/coremldata.bin", bytes: 243, sha256: "c9beeb989c8d66f8be11df59bc6df277ec76cee404f6865b46243835ef562f6d"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/coremldata.bin", bytes: 486, sha256: "dbde3f2300842c1fd51ef3ff948a0bcffe65ffd2dca10707f2509f32c1d65b1d"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/metadata.json", bytes: 2841, sha256: "2a98699e22d279dd37fa1d238aeb1c6db1df0d6fad687775324157689d8f3acf"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/model.mil", bytes: 28181, sha256: "4b8518a956450fec57f06c2a21bdffc26973f7f1fa6842fb38fe917f896b6b93"), + NativeVoiceModelFile(path: "Preprocessor.mlmodelc/weights/weight.bin", bytes: 491072, sha256: "129b76e3aeafa8afa3ea76d995b964b145fe83700d579f6ff42c4c38fa0968ea"), + NativeVoiceModelFile(path: "parakeet_vocab.json", bytes: 151122, sha256: "7ec60e05f1b24480736ec0eed40900f4626bce1fa9a60fd700ec7e2a59198735"), + ]), + ] +} diff --git a/Sources/ASRCore/VoiceModelCatalog.swift b/Sources/ASRCore/VoiceModelCatalog.swift index c39fbd6..230f4bf 100644 --- a/Sources/ASRCore/VoiceModelCatalog.swift +++ b/Sources/ASRCore/VoiceModelCatalog.swift @@ -3,6 +3,8 @@ import Foundation public enum VoiceProviderID: String, Codable, CaseIterable, Sendable { case whisperCpp = "whisper.cpp" case nvidiaParakeet = "nvidia.parakeet" + case fermionPhonon = "fermion.phonon2" + case moondreamRedux = "moondream.parakeet-redux" } public struct VoiceProviderDescriptor: Identifiable, Codable, Equatable, Sendable { @@ -166,6 +168,11 @@ public enum LocalVoiceModelCatalog { ) ) + for (id, name) in [(VoiceProviderID.fermionPhonon, "Fermion Research"), (.moondreamRedux, "Moondream")] { + providers.append(VoiceProviderDescriptor(id: id.rawValue, displayName: name, + summary: "Lokale Core-ML-Spracherkennung über FluidAudio (Apple Silicon, macOS 15+).", + isAvailable: nativeModelsAvailable)) + } return providers } @@ -174,6 +181,7 @@ public enum LocalVoiceModelCatalog { if includeParakeet { models.append(contentsOf: parakeetModels) } + models.append(contentsOf: nativeModels) return models } @@ -181,6 +189,29 @@ public enum LocalVoiceModelCatalog { availableModels(includeParakeet: includeParakeet).first(where: { $0.id == id }) } + public static var nativeModelsAvailable: Bool { + #if os(macOS) && arch(arm64) + if #available(macOS 15, *) { return true } + #endif + return false + } + + private static let nativeModels: [VoiceModelDescriptor] = NativeVoiceModelBundle.all.map { bundle in + let english = bundle.modelID == "fermion.phonon2" + return VoiceModelDescriptor( + id: bundle.modelID, providerID: bundle.modelID, + displayName: english ? "Phonon-2 (English)" : "Parakeet Redux", + languageCode: english ? "en" : nil, languageScope: english ? .english : .multilingual, + supportsTranslationToEnglish: false, supportsLiveTranscription: true, + supportsAutomaticLanguageDetection: !english, supportsLanguageSelection: !english, + supportsQualityProfile: false, + supportedLanguageCodes: english ? ["en"] : ["bg", "hr", "cs", "da", "nl", "en", "et", "fi", "fr", "de", "el", "hu", "it", "lv", "lt", "mt", "pl", "pt", "ro", "sk", "sl", "es", "sv", "ru", "uk"], + speedScore: 5, accuracyScore: 5, + sizeLabel: formattedDownloadSize(bundle.expectedBytes), expectedDownloadBytes: bundle.expectedBytes, + installState: nativeModelsAvailable ? .downloadable : .unavailable, + localFileName: bundle.bundleName, downloadIdentifier: bundle.modelID, runtimeID: "fluidAudio") + } + private static let whisperModels: [VoiceModelDescriptor] = [ VoiceModelDescriptor( id: "whisper.ultra-v3-turbo", providerID: VoiceProviderID.whisperCpp.rawValue, diff --git a/Sources/ASRCore/VoiceModelInstaller.swift b/Sources/ASRCore/VoiceModelInstaller.swift index f8302b1..95b3085 100644 --- a/Sources/ASRCore/VoiceModelInstaller.swift +++ b/Sources/ASRCore/VoiceModelInstaller.swift @@ -606,3 +606,91 @@ enum NemoSpeechRuntimeManager { return String(data: outData, encoding: .utf8) ?? "" } } + +extension VoiceModelInstaller { + public static func nativeModelURL(modelID: String) throws -> URL { + let manifest = try NativeVoiceModelBundle.manifest(modelID: modelID) + let support = try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + return support.appendingPathComponent("Cortexa/NativeVoiceModels", isDirectory: true) + .appendingPathComponent(manifest.bundleName, isDirectory: true) + } + + public func installedNativeModelFileNames() throws -> Set { + var names: Set = [] + for manifest in NativeVoiceModelBundle.all { + let directory = try Self.nativeModelURL(modelID: manifest.modelID) + guard fileManager.fileExists(atPath: directory.path) else { continue } + do { + try manifest.validate(at: directory) + names.insert(manifest.bundleName) + } catch { + NSLog("Cortexa native model validation failed: %@ (%@)", manifest.modelID, error.localizedDescription) + } + } + return names + } + + public func installedVoiceModelFileNames() throws -> Set { + let directory = try BundledWhisperRuntimeInstaller.defaultInstallDirectory(fileManager: fileManager) + .appendingPathComponent("models", isDirectory: true) + let whisperFiles: Set + if fileManager.fileExists(atPath: directory.path) { + whisperFiles = Set(try fileManager.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .filter { $0.pathExtension == "bin" }.map(\.lastPathComponent)) + } else { + whisperFiles = [] + } + return try whisperFiles.union(installedParakeetModelFileNames()).union(installedNativeModelFileNames()) + } + + public func installNativeModel( + _ descriptor: VoiceModelDescriptor, + progressHandler: (@Sendable (VoiceModelInstallProgress) -> Void)? + ) async throws -> URL { + guard #available(macOS 15, *) else { + throw VoiceModelInstallerError.runtimeInstallFailed("Native voice models require macOS 15 or later.") + } + guard LocalVoiceModelCatalog.nativeModelsAvailable else { + throw VoiceModelInstallerError.runtimeInstallFailed("Native voice models require an Apple Silicon Mac running macOS 15 or later.") + } + let manifest = try NativeVoiceModelBundle.manifest(modelID: descriptor.id) + let destination = try Self.nativeModelURL(modelID: descriptor.id) + let parent = destination.deletingLastPathComponent() + try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) + let staging = parent.appendingPathComponent(".install-" + UUID().uuidString, isDirectory: true) + try fileManager.createDirectory(at: staging, withIntermediateDirectories: true) + defer { try? fileManager.removeItem(at: staging) } + let client = VoiceModelDownloadClient(fileManager: fileManager) + var completed: Int64 = 0 + for file in manifest.files { + try Task.checkCancellation() + let target = staging.appendingPathComponent(file.path) + try fileManager.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + let source = URL(string: "https://huggingface.co/\(manifest.repository)/resolve/\(manifest.revision)/\(file.path)")! + let previous = completed + try await client.download(from: source, to: target, expectedDownloadBytes: file.bytes) { progress in + let received = previous + Int64(Double(file.bytes) * progress.fractionCompleted) + progressHandler?(VoiceModelInstallProgress(phase: .downloading, fractionCompleted: Double(received) / Double(manifest.expectedBytes), receivedBytes: received, totalBytes: manifest.expectedBytes)) + } + completed += file.bytes + } + try Task.checkCancellation() + progressHandler?(VoiceModelInstallProgress(phase: .finalizing, fractionCompleted: 0, isIndeterminate: true)) + try manifest.validate(at: staging) + try Task.checkCancellation() + if fileManager.fileExists(atPath: destination.path) { + _ = try fileManager.replaceItemAt(destination, withItemAt: staging) + } else { + try fileManager.moveItem(at: staging, to: destination) + } + progressHandler?(VoiceModelInstallProgress(phase: .finalizing, fractionCompleted: 1)) + return destination + } + + public func removeNativeModel(_ descriptor: VoiceModelDescriptor) throws { + let destination = try Self.nativeModelURL(modelID: descriptor.id) + if fileManager.fileExists(atPath: destination.path) { + try fileManager.removeItem(at: destination) + } + } +} diff --git a/Sources/FluidASRMac/FluidSpeechEngine.swift b/Sources/FluidASRMac/FluidSpeechEngine.swift new file mode 100644 index 0000000..2660e38 --- /dev/null +++ b/Sources/FluidASRMac/FluidSpeechEngine.swift @@ -0,0 +1,167 @@ +#if os(macOS) +import ASRCore +import FluidAudio +import Foundation + +func makeStreamingWindowUpdate( + words: [WordTiming], committed: [WordTiming], baseTime: TimeInterval +) -> (committed: [WordTiming], mutable: [WordTiming], offset: Int) { + let boundary = baseTime + 12 + let lastStableWord = words.last { $0.endTime <= boundary } + let newBoundary = lastStableWord?.endTime ?? boundary + let newlyCommitted = words.filter { word in + // The next window may emit the previously committed final token at time zero again. + let repeatsBoundaryWord = word.startTime <= baseTime + 0.05 && committed.last?.word == word.word + return word.endTime > baseTime && word.endTime <= newBoundary && !repeatsBoundaryWord + } + return (committed + newlyCommitted, words.filter { $0.endTime > newBoundary }, Int(newBoundary * 16_000)) +} + +public final class FluidSpeechEngine: WhisperEngine, @unchecked Sendable { + public var onPartial: ((PartialTranscript) -> Void)? + public var onFinalSegment: ((FinalSegment) -> Void)? + public var onFailure: ((Error) -> Void)? + private let lock = NSLock() + private var manager: AsrManager? + private var samples: [Float] = [] + private var generation: UInt64 = 0 + private var running = false + private var decoding = false + private var lastDecodedCount = 0 + private var offset = 0 + private var committed: [WordTiming] = [] + private var language: Language? + private var sequence = 0 + private var partialTask: Task? + + public init() {} + + public func loadModel(at path: URL, config: ASRCore.ASRConfig) throws { + let version: AsrModelVersion + switch config.modelID { + case "fermion.phonon2": version = .phonon2 + case "moondream.parakeet-redux": version = .redux + default: throw WhisperEngineError.modelNotLoaded + } + let models = try AsrModels.loadLocal(from: path, version: version) + let loaded = AsrManager(models: models) + locked { manager = loaded; language = config.languageHint.flatMap(Language.init(rawValue:)) } + } + + public func prepareForInference() async throws { + let snapshot = try locked { () throws -> (AsrManager, Language?) in + guard let manager else { throw WhisperEngineError.modelNotLoaded } + return (manager, language) + } + var state = try TdtDecoderState() + _ = try await snapshot.0.transcribe( + [Float](repeating: 0, count: 16_000), decoderState: &state, language: snapshot.1) + } + + public func startStreaming() throws { + try locked { + guard manager != nil else { throw WhisperEngineError.modelNotLoaded } + guard !running else { throw WhisperEngineError.engineAlreadyRunning } + generation &+= 1 + running = true + samples = []; committed = []; offset = 0; lastDecodedCount = 0; sequence = 0 + } + } + + public func pushAudioPCM16kMono(_ buffer: UnsafePointer, frameCount: Int) throws { + guard frameCount > 0 else { throw WhisperEngineError.invalidAudioBuffer } + let shouldDecode = try locked { + guard running else { throw WhisperEngineError.engineNotRunning } + samples.append(contentsOf: UnsafeBufferPointer(start: buffer, count: frameCount)) + return !decoding && samples.count - offset >= 16_000 && samples.count - lastDecodedCount >= 8_000 + } + if shouldDecode { schedulePartial() } + } + + private func schedulePartial() { + let snapshot = locked { () -> (AsrManager, [Float], UInt64, Int, Language?)? in + guard running, !decoding, let manager else { return nil } + decoding = true + lastDecodedCount = samples.count + let end = min(samples.count, offset + 240_000) + return (manager, Array(samples[offset.. PartialTranscript? in + guard running, generation == snapshot.2 else { return nil } + let baseTime = Double(snapshot.3) / 16_000 + let words = buildWordTimings(from: result.tokenTimings ?? []).map { + WordTiming(word: $0.word, startTime: $0.startTime + baseTime, + endTime: $0.endTime + baseTime) + } + let mutableWords: [WordTiming] + if snapshot.1.count == 240_000 { + let update = makeStreamingWindowUpdate( + words: words, committed: committed, baseTime: baseTime) + committed = update.committed + offset = update.offset + mutableWords = update.mutable + } else { + mutableWords = words + } + let text = (committed.map(\.word) + mutableWords.map(\.word)).joined(separator: " ") + sequence += 1 + return PartialTranscript(text: text, sequenceNumber: sequence) + } + if let partial { onPartial?(partial) } + } catch { + if locked({ running && generation == snapshot.2 }) { onFailure?(error) } + } + let shouldScheduleNext = locked { + decoding = false + return running && samples.count - offset >= 16_000 + && samples.count - lastDecodedCount >= 8_000 + } + if shouldScheduleNext { schedulePartial() } + } + locked { partialTask = task } + } + + public func stopStreaming() async throws -> FinalTranscript { + let snapshot = try locked { () -> (AsrManager, [Float], UInt64, Language?) in + guard running, let manager else { throw WhisperEngineError.engineNotRunning } + running = false + return (manager, samples, generation, language) + } + let pending = locked { partialTask } + await pending?.value + guard locked({ generation == snapshot.2 }) else { throw CancellationError() } + var state = try TdtDecoderState() + let result = try await snapshot.0.transcribe(snapshot.1, decoderState: &state, language: snapshot.3) + guard locked({ generation == snapshot.2 }) else { throw CancellationError() } + return transcript(result) + } + + public func resetStreaming() { + locked { generation &+= 1; running = false; samples = []; committed = [] } + } + + public func transcribeFile(url: URL) async throws -> FinalTranscript { + guard let manager = locked({ manager }) else { throw WhisperEngineError.modelNotLoaded } + var state = try TdtDecoderState() + return transcript(try await manager.transcribe(url, decoderState: &state, language: locked({ language }))) + } + + private func transcript(_ result: ASRResult) -> FinalTranscript { + let segments = buildWordTimings(from: result.tokenTimings ?? []).map { + FinalSegment(text: $0.word, startTime: $0.startTime, endTime: $0.endTime) + } + return FinalTranscript(text: result.text, segments: segments, language: locked({ language?.rawValue })) + } + + private func locked(_ body: () throws -> T) rethrows -> T { + lock.lock() + defer { lock.unlock() } + return try body() + } +} +#endif diff --git a/Tests/ASRCoreTests/NativeVoiceModelBundleTests.swift b/Tests/ASRCoreTests/NativeVoiceModelBundleTests.swift new file mode 100644 index 0000000..692f509 --- /dev/null +++ b/Tests/ASRCoreTests/NativeVoiceModelBundleTests.swift @@ -0,0 +1,37 @@ +import CryptoKit +import XCTest +@testable import ASRCore + +final class NativeVoiceModelBundleTests: XCTestCase { + func testPinnedManifestAndCapabilities() throws { + for manifest in NativeVoiceModelBundle.all { + XCTAssertEqual(manifest.revision.count, 40) + XCTAssertGreaterThan(manifest.expectedBytes, 200_000_000) + XCTAssertEqual(Set(manifest.files.map(\.path)).count, manifest.files.count) + XCTAssertTrue(manifest.files.allSatisfy { !$0.path.contains("..") && !$0.path.hasPrefix("/") && $0.sha256.count == 64 }) + let descriptor = try XCTUnwrap(LocalVoiceModelCatalog.model(id: manifest.modelID)) + XCTAssertEqual(descriptor.expectedDownloadBytes, manifest.expectedBytes) + XCTAssertTrue(descriptor.supportsLiveTranscription) + XCTAssertFalse(descriptor.supportsTranslationToEnglish) + } + XCTAssertEqual(LocalVoiceModelCatalog.model(id: "fermion.phonon2")?.supportedLanguageCodes, ["en"]) + XCTAssertTrue(LocalVoiceModelCatalog.model(id: "moondream.parakeet-redux")!.supportedLanguageCodes!.contains("de")) + } + + func testBundleRejectsMissingCorruptAndWrongSizedFiles() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let bytes = Data("valid".utf8) + let digest = SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + let bundle = NativeVoiceModelBundle(modelID: "test", repository: "test", revision: "test", bundleName: "test", files: [NativeVoiceModelFile(path: "data", bytes: 5, sha256: digest)]) + XCTAssertThrowsError(try bundle.validate(at: directory)) + let file = directory.appendingPathComponent("data") + try bytes.write(to: file) + XCTAssertNoThrow(try bundle.validate(at: directory)) + try Data("wrong".utf8).write(to: file) + XCTAssertThrowsError(try bundle.validate(at: directory)) + try Data("shorter".utf8).write(to: file) + XCTAssertThrowsError(try bundle.validate(at: directory)) + } +} diff --git a/Tests/AppShellSupportTests/AppShellCharacterizationTests.swift b/Tests/AppShellSupportTests/AppShellCharacterizationTests.swift index 6fa8c0c..a45525d 100644 --- a/Tests/AppShellSupportTests/AppShellCharacterizationTests.swift +++ b/Tests/AppShellSupportTests/AppShellCharacterizationTests.swift @@ -484,6 +484,7 @@ private final class StubDictationRuntime: DictationRuntimeControlling { private(set) var openAccessibilitySettingsCallCount = 0 private(set) var promptAccessibilityTrustCallCount = 0 + func releaseModelBeforeRemoval(modelID: String) throws {} func prepareRuntime() { prepareRuntimeCallCount += 1 } diff --git a/Tests/AppShellSupportTests/AppShellSupportTestFixtures.swift b/Tests/AppShellSupportTests/AppShellSupportTestFixtures.swift index 9f4b5d4..5d5f7ab 100644 --- a/Tests/AppShellSupportTests/AppShellSupportTestFixtures.swift +++ b/Tests/AppShellSupportTests/AppShellSupportTestFixtures.swift @@ -40,6 +40,7 @@ final class AppShellTestDictationRuntime: DictationRuntimeControlling { private(set) var promptAccessibilityTrustCallCount = 0 private(set) var aiProcessingServiceCount = 0 + func releaseModelBeforeRemoval(modelID: String) throws {} func prepareRuntime() { prepareRuntimeCallCount += 1 } diff --git a/Tests/AppShellSupportTests/AppUninstallerTests.swift b/Tests/AppShellSupportTests/AppUninstallerTests.swift index 51f9701..7d9023c 100644 --- a/Tests/AppShellSupportTests/AppUninstallerTests.swift +++ b/Tests/AppShellSupportTests/AppUninstallerTests.swift @@ -39,6 +39,9 @@ final class AppUninstallerTests: XCTestCase { let runtimeRoot = root.appendingPathComponent( "Application Support/Cortexa/Runtime/NeMo-Speech", isDirectory: true ) + let nativeModels = root.appendingPathComponent( + "Application Support/Cortexa/NativeVoiceModels", isDirectory: true + ) let temporary = root.appendingPathComponent("Temporary", isDirectory: true) let unmanagedRuntime = root.appendingPathComponent("External/nemo-speech", isDirectory: true) try FileManager.default.createDirectory(at: support, withIntermediateDirectories: true) @@ -46,6 +49,7 @@ final class AppUninstallerTests: XCTestCase { try FileManager.default.createDirectory(at: modelCache, withIntermediateDirectories: true) try FileManager.default.createDirectory(at: temporary, withIntermediateDirectories: true) try FileManager.default.createDirectory(at: unmanagedRuntime, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: nativeModels, withIntermediateDirectories: true) defer { try? FileManager.default.removeItem(at: root) } let transcript = support.appendingPathComponent("transcript-history.json") @@ -89,6 +93,7 @@ final class AppUninstallerTests: XCTestCase { try Data("keep".utf8).write(to: unrelatedRuntime.appendingPathComponent("keep.txt")) let externalMarker = unmanagedRuntime.appendingPathComponent("keep.txt") try Data("keep".utf8).write(to: externalMarker) + try Data("model bundle".utf8).write(to: nativeModels.appendingPathComponent("phonon-model")) let preferencesDomain = "AppUninstallerTests.\(UUID().uuidString)" let defaults = try XCTUnwrap(UserDefaults(suiteName: preferencesDomain)) @@ -107,6 +112,7 @@ final class AppUninstallerTests: XCTestCase { appSupportDirectory: support, appCacheDirectory: appCache, nemoModelsDirectory: modelCache, + nativeModelsDirectory: nativeModels, managedNemoRuntimeDirectory: runtimeRoot, temporaryDirectory: temporary, preferencesDomain: preferencesDomain @@ -119,6 +125,7 @@ final class AppUninstallerTests: XCTestCase { XCTAssertFalse(FileManager.default.fileExists(atPath: appCache.path)) XCTAssertFalse(FileManager.default.fileExists(atPath: parakeetModel.path)) XCTAssertFalse(FileManager.default.fileExists(atPath: partialDownload.path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: nativeModels.path)) XCTAssertFalse(FileManager.default.fileExists(atPath: managedRuntime.path)) XCTAssertFalse(FileManager.default.fileExists(atPath: interruptedRuntime.path)) XCTAssertFalse(FileManager.default.fileExists(atPath: cortexaTemporary.path)) @@ -166,6 +173,7 @@ final class AppUninstallerTests: XCTestCase { appSupportDirectory: support, appCacheDirectory: root.appendingPathComponent("missing-cache", isDirectory: true), nemoModelsDirectory: root.appendingPathComponent("missing-models", isDirectory: true), + nativeModelsDirectory: root.appendingPathComponent("missing-native-models", isDirectory: true), managedNemoRuntimeDirectory: root.appendingPathComponent("missing-runtime", isDirectory: true), temporaryDirectory: root.appendingPathComponent("missing-temp", isDirectory: true), preferencesDomain: preferencesDomain @@ -186,6 +194,7 @@ final class AppUninstallerTests: XCTestCase { appSupportDirectory: URL, appCacheDirectory: URL, nemoModelsDirectory: URL, + nativeModelsDirectory: URL, managedNemoRuntimeDirectory: URL, temporaryDirectory: URL, preferencesDomain: String @@ -194,6 +203,7 @@ final class AppUninstallerTests: XCTestCase { appSupportDirectory: appSupportDirectory, appCacheDirectory: appCacheDirectory, nemoModelsDirectory: nemoModelsDirectory, + nativeModelsDirectory: nativeModelsDirectory, managedNemoRuntimeDirectory: managedNemoRuntimeDirectory, temporaryDirectory: temporaryDirectory, appBundleURL: root.appendingPathComponent("Cortexa.app", isDirectory: true), diff --git a/Tests/AppShellSupportTests/DictationRuntimeEventBridgeTests.swift b/Tests/AppShellSupportTests/DictationRuntimeEventBridgeTests.swift index b454949..d12637d 100644 --- a/Tests/AppShellSupportTests/DictationRuntimeEventBridgeTests.swift +++ b/Tests/AppShellSupportTests/DictationRuntimeEventBridgeTests.swift @@ -144,6 +144,7 @@ private final class BridgeTestRuntime: DictationRuntimeControlling { var onSessionActivityChanged: ((Bool) -> Void)? var onPermissionInteractionFinished: (() -> Void)? + func releaseModelBeforeRemoval(modelID: String) throws {} func prepareRuntime() {} func prepareRuntimeIfModelAvailable() {} func setVoiceModelActiveDuration(_ duration: VoiceModelActiveDuration) {} diff --git a/Tests/AppShellSupportTests/SpeechModelControllerTests.swift b/Tests/AppShellSupportTests/SpeechModelControllerTests.swift index 284601c..e0764a1 100644 --- a/Tests/AppShellSupportTests/SpeechModelControllerTests.swift +++ b/Tests/AppShellSupportTests/SpeechModelControllerTests.swift @@ -6,6 +6,58 @@ @MainActor final class SpeechModelControllerTests: XCTestCase { + func testNativeEnglishSelectionPreservesIncompatibleLanguageForExplicitChoice() { + let phonon = LocalVoiceModelCatalog.model(id: "fermion.phonon2", includeParakeet: true)! + let state = SpeechModelControllerState( + selectedLanguage: .german, + translationOutputMode: .original, + selectedVoiceProviderID: phonon.providerID, + selectedVoiceModelID: phonon.id, + voiceLanguageOverrides: [], + voiceProviders: LocalVoiceModelCatalog.availableProviders(), + voiceModels: LocalVoiceModelCatalog.availableModels(includeParakeet: true), + installedVoiceModelFileNames: [phonon.localFileName!], + voiceModelOperationStates: [:]) + let controller = makeController(state: state) + controller.sanitizeSpeechModelSelections() + XCTAssertEqual(state.selectedLanguage, .german) + XCTAssertEqual(state.selectedVoiceModelID, phonon.id) + XCTAssertFalse(controller.canUseVoiceModel(phonon, for: .german)) + XCTAssertTrue(controller.canUseVoiceModel(phonon, for: .english)) + } + + func testSelectingPhononSwitchesLanguageToEnglish() { + let phonon = LocalVoiceModelCatalog.model(id: "fermion.phonon2", includeParakeet: true)! + let state = SpeechModelControllerState( + selectedLanguage: .german, + translationOutputMode: .original, + selectedVoiceProviderID: LocalVoiceModelCatalog.defaultProviderID, + selectedVoiceModelID: LocalVoiceModelCatalog.defaultModelID, + voiceLanguageOverrides: [], + voiceProviders: LocalVoiceModelCatalog.availableProviders(), + voiceModels: LocalVoiceModelCatalog.availableModels(includeParakeet: true), + installedVoiceModelFileNames: [phonon.localFileName!], + voiceModelOperationStates: [:]) + + makeController(state: state).setSelectedVoiceModel(phonon) + + XCTAssertEqual(state.selectedVoiceModelID, phonon.id) + XCTAssertEqual(state.selectedLanguage, .english) + } + + func testFixedLanguageSelectionKeepsOverridesForThePreviousLanguage() { + XCTAssertFalse(shouldUpdateVoiceModelLanguageOverride( + previousLanguage: .german, + selectedLanguage: .english, + hadOverride: true + )) + XCTAssertTrue(shouldUpdateVoiceModelLanguageOverride( + previousLanguage: .german, + selectedLanguage: .german, + hadOverride: true + )) + } + func testVoiceModelOperationProgressPublishesOnlyMeaningfulDownloadUpdates() { let current = VoiceModelOperationKind.installing( VoiceModelInstallProgress( diff --git a/Tests/AppShellSupportTests/VoiceModelCapabilityPolicyTests.swift b/Tests/AppShellSupportTests/VoiceModelCapabilityPolicyTests.swift index fdb9b47..ccec3ec 100644 --- a/Tests/AppShellSupportTests/VoiceModelCapabilityPolicyTests.swift +++ b/Tests/AppShellSupportTests/VoiceModelCapabilityPolicyTests.swift @@ -73,6 +73,46 @@ final class VoiceModelCapabilityPolicyTests: XCTestCase { catalog: models, installedFiles: files)?.id, standard.id) } + func testNativeEnglishModelRequiresEnglishAndNeverSilentlyFallsBack() { + let phonon = VoiceModelDescriptor( + id: "fermion.phonon2", providerID: "fermion.phonon2", + displayName: "Phonon-2", languageCode: "en", languageScope: .english, + supportsTranslationToEnglish: false, + supportsAutomaticLanguageDetection: false, + supportsLanguageSelection: false, + speedScore: 9, accuracyScore: 8, sizeLabel: "360 MB", + installState: .downloadable, localFileName: "phonon-2", + runtimeID: "fluidAudio") + let standard = model("whisper.standard") + let catalog = [phonon, standard] + let installed: Set = ["phonon-2", standard.localFileName!] + XCTAssertTrue(policy.supports( + .language(.english), model: phonon, currentLanguage: .german)) + XCTAssertFalse(policy.supports( + .language(.auto), model: phonon, currentLanguage: .german)) + XCTAssertTrue(policy.supports(language: .english, model: phonon)) + for language in [DictationLanguage.german, .auto] { + XCTAssertFalse(policy.supports(language: language, model: phonon)) + XCTAssertNil(policy.effectiveModel( + for: language, selectedModelID: phonon.id, + overrides: [VoiceLanguageOverride(languageCode: language.rawValue, modelID: standard.id)], + catalog: catalog, installedFiles: installed)) + } + XCTAssertEqual(policy.effectiveModel( + for: .english, selectedModelID: phonon.id, overrides: [], + catalog: catalog, installedFiles: installed)?.id, phonon.id) + } + + func testNativeReduxSupportsExplicitGermanAndAutomaticDetection() { + let redux = model("moondream.parakeet-redux") + + XCTAssertTrue(policy.supports(language: .german, model: redux)) + XCTAssertTrue(policy.supports(language: .auto, model: redux)) + XCTAssertEqual(policy.effectiveModel( + for: .german, selectedModelID: redux.id, overrides: [], + catalog: [redux], installedFiles: [redux.localFileName!])?.id, redux.id) + } + private func model(_ id: String) -> VoiceModelDescriptor { LocalVoiceModelCatalog.model(id: id)! } diff --git a/Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift b/Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift new file mode 100644 index 0000000..90a7222 --- /dev/null +++ b/Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift @@ -0,0 +1,220 @@ +#if os(macOS) +import AVFoundation +import ASRCore +import Foundation +import FluidAudio +import XCTest +@testable import FluidASRMac + +private final class LivePartialRecorder: @unchecked Sendable { + private let lock = NSLock() + private var timestamps: [TimeInterval] = [] + + func record() { + lock.lock() + timestamps.append(ProcessInfo.processInfo.systemUptime) + lock.unlock() + } + + func snapshot() -> [TimeInterval] { + lock.lock() + defer { lock.unlock() } + return timestamps + } +} + +final class FluidSpeechEngineSmokeTests: XCTestCase { + func testUnknownModelIDIsRejectedInsteadOfSelectingRedux() { + let engine = FluidSpeechEngine() + let config = ASRConfig( + languageHint: "en", + modelID: "unrecognized.phonon-compatible", + backend: .fluidAudio, + latencyProfile: .quality + ) + + XCTAssertThrowsError(try engine.loadModel(at: URL(fileURLWithPath: "/missing"), config: config)) { + guard case WhisperEngineError.modelNotLoaded = $0 else { + XCTFail("Expected an explicit rejection for an unknown native model ID.") + return + } + } + } + + func testOverlappingWindowsCommitEachWordOnlyOnce() { + let first = makeStreamingWindowUpdate(words: [ + WordTiming(word: "one", startTime: 0.2, endTime: 0.5), + WordTiming(word: "two", startTime: 10.8, endTime: 11.3), + WordTiming(word: "three", startTime: 11.4, endTime: 12.2), + WordTiming(word: "four", startTime: 13, endTime: 13.5), + ], committed: [], baseTime: 0) + let second = makeStreamingWindowUpdate(words: [ + WordTiming(word: "two", startTime: 11.3, endTime: 11.6), + WordTiming(word: "three", startTime: 11.7, endTime: 12.2), + WordTiming(word: "two", startTime: 12.3, endTime: 12.6), + WordTiming(word: "four", startTime: 13, endTime: 13.5), + WordTiming(word: "five", startTime: 23, endTime: 23.4), + ], committed: first.committed, baseTime: Double(first.offset) / 16_000) + + XCTAssertEqual(first.committed.map(\.word), ["one", "two"]) + XCTAssertEqual(first.mutable.map(\.word), ["three", "four"]) + XCTAssertEqual(second.committed.map(\.word), ["one", "two", "three", "two", "four"]) + XCTAssertEqual(second.mutable.map(\.word), ["five"]) + } + + func testBothManagedModelsTranscribeThirtyMinuteRecordingOffline() async throws { + guard let audio = ProcessInfo.processInfo.environment["CORTEXA_NATIVE_LONG_SMOKE_AUDIO"] else { + throw XCTSkip("Provide a 30-minute 16-kHz mono WAV to verify long recording transcription.") + } + let file = try AVAudioFile(forReading: URL(fileURLWithPath: audio)) + XCTAssertEqual(file.processingFormat.sampleRate, 16_000) + XCTAssertEqual(file.processingFormat.channelCount, 1) + let duration = Double(file.length) / file.processingFormat.sampleRate + XCTAssertGreaterThanOrEqual(duration, 1_800) + for modelID in ["fermion.phonon2", "moondream.parakeet-redux"] { + let manifest = try NativeVoiceModelBundle.manifest(modelID: modelID) + let root = try XCTUnwrap(ProcessInfo.processInfo.environment["CORTEXA_NATIVE_SMOKE_MODEL_ROOT"]) + let url = URL(fileURLWithPath: root).appendingPathComponent(manifest.bundleName) + try manifest.validate(at: url) + let engine = FluidSpeechEngine() + let config = ASRConfig(languageHint: "en", initialPrompt: nil, translationMode: .original, + modelID: modelID, backend: .fluidAudio, latencyProfile: .quality) + try engine.loadModel(at: url, config: config) + let transcript = try await engine.transcribeFile(url: URL(fileURLWithPath: audio)) + XCTAssertFalse(transcript.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + XCTAssertFalse(transcript.segments.isEmpty) + XCTAssertLessThanOrEqual(transcript.segments.last?.endTime ?? 0, duration + 1) + } + } + + func testReduxTranscribesGermanOffline() async throws { + let environment = ProcessInfo.processInfo.environment + guard let audio = environment["CORTEXA_NATIVE_GERMAN_SMOKE_AUDIO"], + let expected = environment["CORTEXA_NATIVE_GERMAN_SMOKE_EXPECTED"] else { + throw XCTSkip("Provide a German 16-kHz mono WAV and expected phrase to verify Redux language hinting.") + } + let modelID = "moondream.parakeet-redux" + let manifest = try NativeVoiceModelBundle.manifest(modelID: modelID) + let url: URL + if let root = environment["CORTEXA_NATIVE_SMOKE_MODEL_ROOT"] { + url = URL(fileURLWithPath: root).appendingPathComponent(manifest.bundleName) + } else { + url = try VoiceModelInstaller.nativeModelURL(modelID: modelID) + } + try manifest.validate(at: url) + let engine = FluidSpeechEngine() + let config = ASRConfig(languageHint: "de", initialPrompt: nil, translationMode: .original, + modelID: modelID, backend: .fluidAudio, latencyProfile: .quality) + try engine.loadModel(at: url, config: config) + + let transcript = try await engine.transcribeFile(url: URL(fileURLWithPath: audio)) + XCTAssertTrue(transcript.text.localizedCaseInsensitiveContains(expected), transcript.text) + XCTAssertFalse(transcript.segments.isEmpty) + XCTAssertEqual(transcript.language, "de") + } + + func testBothManagedModelsTranscribeAndResetOffline() async throws { + let environment = ProcessInfo.processInfo.environment + guard let audio = environment["CORTEXA_NATIVE_SMOKE_AUDIO"], + let expected = environment["CORTEXA_NATIVE_SMOKE_EXPECTED"] else { + throw XCTSkip("Provide CORTEXA_NATIVE_SMOKE_AUDIO (English 16-kHz mono WAV) and CORTEXA_NATIVE_SMOKE_EXPECTED.") + } + for modelID in ["fermion.phonon2", "moondream.parakeet-redux"] { + let manifest = try NativeVoiceModelBundle.manifest(modelID: modelID) + let url: URL + if let root = environment["CORTEXA_NATIVE_SMOKE_MODEL_ROOT"] { + url = URL(fileURLWithPath: root).appendingPathComponent(manifest.bundleName) + } else { + url = try VoiceModelInstaller.nativeModelURL(modelID: modelID) + } + try manifest.validate(at: url) + let engine = FluidSpeechEngine() + let config = ASRConfig(languageHint: "en", initialPrompt: nil, translationMode: .original, + modelID: modelID, backend: .fluidAudio, latencyProfile: .streaming) + try engine.loadModel(at: url, config: config) + try await engine.prepareForInference() + let file = try AVAudioFile(forReading: URL(fileURLWithPath: audio)) + XCTAssertEqual(file.processingFormat.sampleRate, 16_000) + XCTAssertEqual(file.processingFormat.channelCount, 1) + let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: AVAudioFrameCount(file.length))) + try file.read(into: buffer) + let samples = try XCTUnwrap(buffer.floatChannelData)[0] + let live = expectation(description: "Live text for \(modelID)") + live.assertForOverFulfill = false + let partialRecorder = LivePartialRecorder() + engine.onPartial = { partial in + guard !partial.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + partialRecorder.record() + live.fulfill() + } + try engine.startStreaming() + let firstLiveStarted = ProcessInfo.processInfo.systemUptime + var audioOffset = 0 + while audioOffset < Int(buffer.frameLength) { + let chunkCount = min(8_000, Int(buffer.frameLength) - audioOffset) + try engine.pushAudioPCM16kMono(samples.advanced(by: audioOffset), frameCount: chunkCount) + audioOffset += chunkCount + try await Task.sleep(for: .milliseconds(500)) + } + await fulfillment(of: [live], timeout: 30) + try await Task.sleep(for: .milliseconds(750)) + let final = try await engine.stopStreaming() + XCTAssertTrue(final.text.localizedCaseInsensitiveContains(expected), final.text) + XCTAssertFalse(final.segments.isEmpty) + let timestamps = partialRecorder.snapshot() + let intervals = zip(timestamps, timestamps.dropFirst()).map { $1 - $0 }.sorted() + XCTAssertGreaterThanOrEqual(timestamps.count, 4, + "Expected repeated live updates for \(modelID).") + let firstLatency = (timestamps.first ?? .infinity) - firstLiveStarted + XCTAssertLessThanOrEqual(firstLatency, 2, + "First live text exceeded two seconds after model preparation for \(modelID).") + guard !intervals.isEmpty else { + XCTFail("Expected multiple live updates for \(modelID).") + engine.resetStreaming() + continue + } + let updateP95 = intervals[Int(ceil(Double(intervals.count) * 0.95)) - 1] + XCTAssertLessThanOrEqual(updateP95, 2, + "Live update p95 exceeded two seconds for \(modelID).") + print("\(modelID) live cadence: partials=\(timestamps.count), first=\(String(format: "%.3f", firstLatency))s, update p95=\(String(format: "%.3f", updateP95))s") + engine.resetStreaming() + let transcript = try await engine.transcribeFile(url: URL(fileURLWithPath: audio)) + XCTAssertTrue(transcript.text.localizedCaseInsensitiveContains(expected), transcript.text) + XCTAssertFalse(transcript.segments.isEmpty) + for segment in transcript.segments { + XCTAssertTrue(segment.startTime.isFinite) + XCTAssertGreaterThanOrEqual(segment.endTime, segment.startTime) + } + var warmLatencies: [TimeInterval] = [] + for _ in 0..<21 { + let firstPartial = expectation(description: "First partial for \(modelID)") + firstPartial.assertForOverFulfill = false + engine.onPartial = { partial in + if !partial.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + firstPartial.fulfill() + } + } + try engine.startStreaming() + let startedAt = Date() + try engine.pushAudioPCM16kMono(samples, frameCount: min(16_000, Int(buffer.frameLength))) + await fulfillment(of: [firstPartial], timeout: 2.5) + let latency = Date().timeIntervalSince(startedAt) + if warmLatencies.count < 20 { warmLatencies.append(latency) } + engine.resetStreaming() + } + let p95 = warmLatencies.sorted()[Int(ceil(Double(warmLatencies.count) * 0.95)) - 1] + print("\(modelID) first-partial warm p95: \(String(format: "%.3f", p95))s") + XCTAssertLessThanOrEqual(p95, 2, "Warm first-partial p95 exceeded two seconds for \(modelID).") + let stale = expectation(description: "No text after reset for \(modelID)") + stale.isInverted = true + engine.onPartial = { _ in stale.fulfill() } + try engine.startStreaming() + try engine.pushAudioPCM16kMono(samples, frameCount: Int(buffer.frameLength)) + engine.resetStreaming() + await fulfillment(of: [stale], timeout: 2) + try engine.startStreaming() + engine.resetStreaming() + } + } +} +#endif diff --git a/VERSION b/VERSION index e85205d..5c4503b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.48.2 +0.49.0 diff --git a/apps/macos/AppShell/AppUninstaller.swift b/apps/macos/AppShell/AppUninstaller.swift index 013d3ab..6cdf226 100644 --- a/apps/macos/AppShell/AppUninstaller.swift +++ b/apps/macos/AppShell/AppUninstaller.swift @@ -9,6 +9,7 @@ struct AppUninstallPaths { let appSupportDirectory: URL let appCacheDirectory: URL let nemoModelsDirectory: URL + let nativeModelsDirectory: URL let managedNemoRuntimeDirectory: URL let temporaryDirectory: URL let appBundleURL: URL @@ -47,6 +48,8 @@ struct AppUninstallPaths { nemoModelsDirectory: caches .appendingPathComponent("NeMoSpeech", isDirectory: true) .appendingPathComponent("models", isDirectory: true), + nativeModelsDirectory: applicationSupport + .appendingPathComponent("Cortexa/NativeVoiceModels", isDirectory: true), managedNemoRuntimeDirectory: applicationSupport .appendingPathComponent("Cortexa/Runtime/NeMo-Speech", isDirectory: true), temporaryDirectory: temporaryDirectory, @@ -59,6 +62,7 @@ struct AppUninstallPaths { appSupportDirectory: URL, appCacheDirectory: URL, nemoModelsDirectory: URL, + nativeModelsDirectory: URL, managedNemoRuntimeDirectory: URL, temporaryDirectory: URL, appBundleURL: URL, @@ -67,6 +71,7 @@ struct AppUninstallPaths { self.appSupportDirectory = appSupportDirectory self.appCacheDirectory = appCacheDirectory self.nemoModelsDirectory = nemoModelsDirectory + self.nativeModelsDirectory = nativeModelsDirectory self.managedNemoRuntimeDirectory = managedNemoRuntimeDirectory self.temporaryDirectory = temporaryDirectory self.appBundleURL = appBundleURL @@ -156,6 +161,9 @@ struct AppUninstallCleanup { attempt("NeMo-Modellcache", failures: &failures) { try removeKnownParakeetModels(in: paths.nemoModelsDirectory) } + attempt("native Core-ML-Modelle", failures: &failures) { + try removeIfPresent(paths.nativeModelsDirectory) + } attempt("Cortexa-verwaltete NeMo-Runtime", failures: &failures) { try removeManagedNemoRuntimes(in: paths.managedNemoRuntimeDirectory) } diff --git a/apps/macos/AppShell/DictationRuntime.swift b/apps/macos/AppShell/DictationRuntime.swift index 447684e..a4c9b75 100644 --- a/apps/macos/AppShell/DictationRuntime.swift +++ b/apps/macos/AppShell/DictationRuntime.swift @@ -7,6 +7,7 @@ import AudioCore import CapabilityCore import Carbon import Foundation +import FluidASRMac import SessionCore import SnippetCore import TextTargetMac @@ -305,7 +306,7 @@ final class DictationRuntime: @unchecked Sendable { /// Mikrofon-/AX-Dialoge oder TCC-Updates: UI soll `authorizationStatus` / AX erneut lesen. var onPermissionInteractionFinished: (() -> Void)? - private let whisperEngine = WhisperCppEngine() + private var whisperEngine: WhisperEngine = WhisperCppEngine() private let processQueue = DispatchQueue(label: "wispr.dictation.process", qos: .userInitiated) private let insertionQueue = DispatchQueue(label: "wispr.dictation.insert", qos: .userInitiated) private let capabilityProfiler = CapabilityProfiler() @@ -416,7 +417,7 @@ final class DictationRuntime: @unchecked Sendable { whisperEngine.onPartial = { [weak self] partial in self?.handlePartialText(partial.text) } - whisperEngine.onDebugEvent = { [weak self] event in + (whisperEngine as? WhisperCppEngine)?.onDebugEvent = { [weak self] event in self?.publishDebug(event) } } catch { @@ -511,8 +512,9 @@ final class DictationRuntime: @unchecked Sendable { return } - prepareRuntimeIfModelAvailable() - guard runtimePrepared else { + let usesFluid = options.selectedVoiceModelID == "fermion.phonon2" || options.selectedVoiceModelID == "moondream.parakeet-redux" + if !usesFluid { prepareRuntimeIfModelAvailable() } + guard usesFluid || runtimePrepared else { withSessionLock { isStarting = false } @@ -558,7 +560,7 @@ final class DictationRuntime: @unchecked Sendable { let requestedMode: DictationMode = modelRequiresOfflineTranscription ? .finalize : options.mode let effectiveMode: DictationMode = accessibilityGranted ? requestedMode : .finalize - try configureEngine(for: options, runtimeMode: effectiveMode) + try await configureEngine(for: options, runtimeMode: effectiveMode) if accessibilityGranted { var lockedTarget = try? captureFocusedTextTarget() @@ -1381,8 +1383,12 @@ final class DictationRuntime: @unchecked Sendable { } private func configureEngine(for options: DictationStartOptions, runtimeMode: DictationMode) - throws + async throws { + if options.selectedVoiceModelID == "fermion.phonon2" || options.selectedVoiceModelID == "moondream.parakeet-redux" { + try await configureFluidEngine(options: options, runtimeMode: runtimeMode) + return + } let runtime = try BundledWhisperRuntimeInstaller.syncRuntimeCLIOnly( bundle: .main, appName: "WisprLocal") let preset = selectEnginePreset(options: options, runtimeMode: runtimeMode) @@ -1419,6 +1425,11 @@ final class DictationRuntime: @unchecked Sendable { return } + if !(whisperEngine is WhisperCppEngine) { + whisperEngine.resetStreaming() + whisperEngine = WhisperCppEngine() + bindEngineCallbacks() + } try whisperEngine.loadModel(at: modelURL, config: config) loadedModelPath = modelURL loadedConfig = config @@ -1427,6 +1438,55 @@ final class DictationRuntime: @unchecked Sendable { ) } + private func bindEngineCallbacks() { + whisperEngine.onPartial = { [weak self] partial in self?.handlePartialText(partial.text) } + (whisperEngine as? WhisperCppEngine)?.onDebugEvent = { [weak self] event in self?.publishDebug(event) } + (whisperEngine as? FluidSpeechEngine)?.onFailure = { [weak self] error in + self?.publishDiagnostic("Live transcription failed: \(error.localizedDescription)") + } + } + + func releaseModelBeforeRemoval(modelID: String) throws { + guard loadedConfig?.modelID == modelID else { return } + guard !withSessionLock({ isRunning || isStarting }) else { + throw WhisperEngineError.backendUnavailable("Stop dictation before removing the active model.") + } + cancelRuntimeUnloadTask() + whisperEngine.resetStreaming() + whisperEngine = WhisperCppEngine() + bindEngineCallbacks() + loadedModelPath = nil + loadedConfig = nil + runtimePrepared = false + } + + private func configureFluidEngine(options: DictationStartOptions, runtimeMode: DictationMode) async throws { + guard let descriptor = LocalVoiceModelCatalog.model(id: options.selectedVoiceModelID, includeParakeet: true), + descriptor.localFileName != nil else { throw WhisperEngineError.modelNotLoaded } + guard model(descriptor, supports: options.language) else { + throw WhisperEngineError.backendUnavailable("Choose a supported language or explicitly select another installed model.") + } + let url = try VoiceModelInstaller.nativeModelURL(modelID: descriptor.id) + let config = ASRConfig(languageHint: options.language.asrHint, initialPrompt: nil, + translationMode: .original, modelID: descriptor.id, backend: .fluidAudio, + latencyProfile: runtimeMode == .streaming ? .streaming : .quality) + if loadedModelPath == url, loadedConfig == config { return } + publishStatus("Preparing speech model") + publishDiagnostic("Preparing speech model for the Neural Engine. The first preparation can take several minutes.") + let engine = FluidSpeechEngine() + try await Task.detached { + try engine.loadModel(at: url, config: config) + try await engine.prepareForInference() + }.value + try Task.checkCancellation() + whisperEngine.resetStreaming() + whisperEngine = engine + bindEngineCallbacks() + loadedModelPath = url + loadedConfig = config + runtimePrepared = true + } + private func selectEnginePreset( options: DictationStartOptions, runtimeMode: DictationMode ) -> EnginePreset { @@ -1584,6 +1644,7 @@ final class DictationRuntime: @unchecked Sendable { private func model(_ descriptor: VoiceModelDescriptor, supports language: DictationLanguage) -> Bool { if language == .auto { return descriptor.supportsAutomaticLanguageDetection } + if descriptor.languageCode == language.rawValue { return true } guard descriptor.supportsLanguageSelection else { return false } if let fixedCode = descriptor.languageCode, fixedCode != language.rawValue { return false } return descriptor.supportedLanguageCodes?.contains(language.rawValue) ?? true @@ -1927,6 +1988,8 @@ final class DictationRuntime: @unchecked Sendable { } whisperEngine.resetStreaming() + whisperEngine = WhisperCppEngine() + bindEngineCallbacks() publishDiagnostic( "ASR runtime war im Leerlauf zu lange inaktiv und wird beim nächsten Start neu geladen." ) diff --git a/apps/macos/AppShell/DictationRuntimeServices.swift b/apps/macos/AppShell/DictationRuntimeServices.swift index 09562d8..1d2396b 100644 --- a/apps/macos/AppShell/DictationRuntimeServices.swift +++ b/apps/macos/AppShell/DictationRuntimeServices.swift @@ -52,7 +52,7 @@ internal struct RuntimePermissionService { promptIfNeeded ? [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary : [:] as CFDictionary - return AXIsProcessTrustedWithOptions(options) + return runOnMainThread { AXIsProcessTrustedWithOptions(options) } } } diff --git a/apps/macos/AppShell/MacAppState.swift b/apps/macos/AppShell/MacAppState.swift index 95c46b7..0e1724a 100644 --- a/apps/macos/AppShell/MacAppState.swift +++ b/apps/macos/AppShell/MacAppState.swift @@ -95,6 +95,7 @@ protocol DictationRuntimeControlling: AnyObject { var onSessionActivityChanged: ((Bool) -> Void)? { get set } var onPermissionInteractionFinished: (() -> Void)? { get set } + func releaseModelBeforeRemoval(modelID: String) throws func prepareRuntime() func prepareRuntimeIfModelAvailable() func setVoiceModelActiveDuration(_ duration: VoiceModelActiveDuration) @@ -1008,7 +1009,7 @@ final class MacAppState: ObservableObject { } var selectedVoiceModelLanguageHintText: String? { - guard let selectedVoiceModel = activeVoiceModel else { return nil } + guard let selectedVoiceModel else { return nil } if let languageCode = selectedVoiceModel.languageCode { let languageName = DictationLanguage(rawValue: languageCode)?.displayName ?? languageCode.uppercased() @@ -1709,9 +1710,11 @@ final class MacAppState: ObservableObject { rebuildAIProcessingStack(reason: "initial-load") updateUpdaterState() dictationRuntime.setVoiceModelActiveDuration(voiceModelActiveDuration) - dictationRuntime.prepareRuntime() - if onboardingStore.isComplete { - dictationRuntime.prepareRuntimeIfModelAvailable() + if selectedVoiceModel?.runtimeID != "fluidAudio" { + dictationRuntime.prepareRuntime() + if onboardingStore.isComplete { + dictationRuntime.prepareRuntimeIfModelAvailable() + } } refreshPermissionStates() permissionCoordinator.beginLaunchPermissionStabilization() @@ -1730,7 +1733,9 @@ final class MacAppState: ObservableObject { } func onboardingDidComplete() { - dictationRuntime.prepareRuntimeIfModelAvailable() + if selectedVoiceModel?.runtimeID != "fluidAudio" { + dictationRuntime.prepareRuntimeIfModelAvailable() + } appendAudit("onboarding.completed") } @@ -1938,10 +1943,12 @@ final class MacAppState: ObservableObject { updateCapabilitySummary() rebuildAIProcessingStack(reason: reason) permissionCoordinator.refreshPermissionsAfterExternalEvent(reason: reason) - if onboardingStore.isComplete { - dictationRuntime.prepareRuntimeIfModelAvailable() - } else { - dictationRuntime.prepareRuntime() + if selectedVoiceModel?.runtimeID != "fluidAudio" { + if onboardingStore.isComplete { + dictationRuntime.prepareRuntimeIfModelAvailable() + } else { + dictationRuntime.prepareRuntime() + } } updateUpdaterState() appendAudit("lifecycle.refresh reason=\(reason)") diff --git a/apps/macos/AppShell/MacAppStateFacadeActions.swift b/apps/macos/AppShell/MacAppStateFacadeActions.swift index 5719b92..1ef141c 100644 --- a/apps/macos/AppShell/MacAppStateFacadeActions.swift +++ b/apps/macos/AppShell/MacAppStateFacadeActions.swift @@ -126,7 +126,12 @@ extension MacAppState { } func removeVoiceModel(_ descriptor: VoiceModelDescriptor) { - speechModelController.removeVoiceModel(descriptor) + do { + try dictationRuntime.releaseModelBeforeRemoval(modelID: descriptor.id) + speechModelController.removeVoiceModel(descriptor) + } catch { + appendDiagnostic("Speech-Modell kann derzeit nicht entfernt werden: \(error.localizedDescription)") + } } func retryUnusedNemoRuntimeCleanup() { diff --git a/apps/macos/AppShell/MacAppStateModelCapabilities.swift b/apps/macos/AppShell/MacAppStateModelCapabilities.swift index b17bbc9..0f3e045 100644 --- a/apps/macos/AppShell/MacAppStateModelCapabilities.swift +++ b/apps/macos/AppShell/MacAppStateModelCapabilities.swift @@ -14,16 +14,23 @@ extension MacAppState { } func supportsCapability(_ choice: VoiceModelCapabilityChoice) -> Bool { - if case .language(let language) = choice, - language != .auto, - let override = voiceLanguageOverrides.first(where: { - $0.languageCode == language.rawValue - }), - let model = voiceModels.first(where: { $0.id == override.modelID }), - isVoiceModelInstalled(model), - voiceProviders.contains(where: { $0.id == model.providerID && $0.isAvailable }), - VoiceModelCapabilityPolicy().supports(language: language, model: model) { - return true + if case .language(let language) = choice { + if language != .auto, + let override = voiceLanguageOverrides.first(where: { + $0.languageCode == language.rawValue + }), + let model = voiceModels.first(where: { $0.id == override.modelID }), + isVoiceModelInstalled(model), + voiceProviders.contains(where: { $0.id == model.providerID && $0.isAvailable }), + VoiceModelCapabilityPolicy().supports(language: language, model: model) { + return true + } + guard let selectedVoiceModel, + isVoiceModelInstalled(selectedVoiceModel), + voiceProviders.contains(where: { + $0.id == selectedVoiceModel.providerID && $0.isAvailable + }) else { return false } + return VoiceModelCapabilityPolicy().supports(language: language, model: selectedVoiceModel) } guard let activeVoiceModel else { return false } return VoiceModelCapabilityPolicy().supports( @@ -67,7 +74,11 @@ extension MacAppState { let language = selectedLanguage let hadOverride = selectedLanguageVoiceOverride != nil setSelectedVoiceModel(model) - if hadOverride, language != .auto { + if shouldUpdateVoiceModelLanguageOverride( + previousLanguage: language, + selectedLanguage: selectedLanguage, + hadOverride: hadOverride + ) { voiceLanguageOverrides.removeAll { $0.languageCode == language.rawValue } if VoiceModelCapabilityPolicy().supports(language: language, model: model) { voiceLanguageOverrides.append(VoiceLanguageOverride( @@ -78,7 +89,7 @@ extension MacAppState { } func requestVoiceCapability(_ choice: VoiceModelCapabilityChoice) { - guard let source = activeVoiceModel else { + guard let source = activeVoiceModel ?? selectedVoiceModel else { Task { @MainActor [weak self] in await Task.yield() self?.showCapabilityModelManagementAlert() @@ -126,7 +137,7 @@ extension MacAppState { NSApplication.shared.activate(ignoringOtherApps: true) guard alert.runModal() == .alertFirstButtonReturn, self.voiceSelectionRevision == selectionRevision, - self.activeVoiceModel?.id == source.id else { return } + (self.activeVoiceModel ?? self.selectedVoiceModel)?.id == source.id else { return } if installed { self.applyVoiceCapability(choice, model: suggestion) @@ -134,7 +145,7 @@ extension MacAppState { self.speechModelController.installVoiceModelForCapability(suggestion) { [weak self] in guard let self, self.voiceSelectionRevision == selectionRevision, - self.activeVoiceModel?.id == source.id else { return } + (self.activeVoiceModel ?? self.selectedVoiceModel)?.id == source.id else { return } self.applyVoiceCapability(choice, model: suggestion) } } @@ -198,3 +209,11 @@ extension MacAppState { return (AppLanguage(rawValue: raw) ?? .system).text(german, english) } } + +func shouldUpdateVoiceModelLanguageOverride( + previousLanguage: DictationLanguage, + selectedLanguage: DictationLanguage, + hadOverride: Bool +) -> Bool { + hadOverride && previousLanguage != .auto && previousLanguage == selectedLanguage +} diff --git a/apps/macos/AppShell/SettingsViewSpeechDictationSections.swift b/apps/macos/AppShell/SettingsViewSpeechDictationSections.swift index 4f3d617..522ddf0 100644 --- a/apps/macos/AppShell/SettingsViewSpeechDictationSections.swift +++ b/apps/macos/AppShell/SettingsViewSpeechDictationSections.swift @@ -91,9 +91,12 @@ enum SpeechModelManagementMatcher { var details: [String] = [ model.displayName, model.id, model.providerID, provider?.displayName ?? "", model.sizeLabel, model.localFileName ?? "", model.downloadIdentifier ?? "", - model.runtimeID ?? "", "\(model.speedScore)/10", "\(model.accuracyScore)/10", - "geschwindigkeit", "speed", "genauigkeit", "accuracy", "größe", "size" + model.runtimeID ?? "", "größe", "size" ] + if model.runtimeID != "fluidAudio" { + details += ["\(model.speedScore)/10", "\(model.accuracyScore)/10", + "geschwindigkeit", "speed", "genauigkeit", "accuracy"] + } details.append(contentsOf: languageCodes) details.append(contentsOf: languageNames) @@ -275,9 +278,13 @@ extension SettingsView { Text( "\(text("Übersetzung", "Translation")): \(selectedModel.supportsTranslationToEnglish ? text("Ja", "Yes") : text("Nein", "No"))" ) - Text( - "\(text("Geschwindigkeit", "Speed")) \(selectedModel.speedScore)/10 \(text("Genauigkeit", "Accuracy")) \(selectedModel.accuracyScore)/10 \(selectedModel.sizeLabel)" - ) + if selectedModel.runtimeID == "fluidAudio" { + Text(selectedModel.sizeLabel) + } else { + Text( + "\(text("Geschwindigkeit", "Speed")) \(selectedModel.speedScore)/10 \(text("Genauigkeit", "Accuracy")) \(selectedModel.accuracyScore)/10 \(selectedModel.sizeLabel)" + ) + } } .font(.footnote) .foregroundStyle(.secondary) @@ -554,7 +561,9 @@ extension SettingsView { } return text("Mehrsprachig", "Multilingual") }() - Text("\(providerName) • \(languageSummary) • Speed \(model.speedScore)/10 • Accuracy \(model.accuracyScore)/10 • \(model.sizeLabel)") + Text(model.runtimeID == "fluidAudio" + ? "\(providerName) • \(languageSummary) • \(model.sizeLabel)" + : "\(providerName) • \(languageSummary) • Speed \(model.speedScore)/10 • Accuracy \(model.accuracyScore)/10 • \(model.sizeLabel)") .font(.footnote) .foregroundStyle(.secondary) .lineLimit(2) diff --git a/apps/macos/AppShell/SpeechModelController.swift b/apps/macos/AppShell/SpeechModelController.swift index 652f7c4..8234ad8 100644 --- a/apps/macos/AppShell/SpeechModelController.swift +++ b/apps/macos/AppShell/SpeechModelController.swift @@ -99,9 +99,7 @@ final class SpeechModelController { guard let self else { return } var installedFiles: Set = [] do { - installedFiles = try await self.voiceModelInstaller - .installedWhisperModelFileNames() - installedFiles.formUnion(try await self.voiceModelInstaller.installedParakeetModelFileNames()) + installedFiles = try await self.voiceModelInstaller.installedVoiceModelFileNames() self.setInstalledVoiceModelFileNames(installedFiles) } catch { self.setInstalledVoiceModelFileNames([]) @@ -176,7 +174,8 @@ final class SpeechModelController { ) { guard voiceModelOperationState(for: descriptor) == nil else { return } guard descriptor.providerID == VoiceProviderID.whisperCpp.rawValue - || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue else { + || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue + || descriptor.runtimeID == "fluidAudio" else { appendDiagnostic("Dieser Speech-Anbieter ist lokal aktuell nicht installierbar.") return } @@ -195,17 +194,21 @@ final class SpeechModelController { do { let installer = self.voiceModelInstaller - let runtime = try await installer.install(descriptor) { [weak self] progress in + let progressHandler: @Sendable (VoiceModelInstallProgress) -> Void = { [weak self] progress in Task { @MainActor [weak self] in self?.setOperation(.installing(progress), for: descriptor.id) } } - var installedFileNames = Set(runtime.availableModelFileNames) - installedFileNames.formUnion( - try await self.voiceModelInstaller.installedParakeetModelFileNames()) + if descriptor.runtimeID == "fluidAudio" { + _ = try await installer.installNativeModel(descriptor, progressHandler: progressHandler) + } else { + _ = try await installer.install(descriptor, progressHandler: progressHandler) + } + let installedFileNames = try await installer.installedVoiceModelFileNames() self.setInstalledVoiceModelFileNames(installedFileNames) self.setVoiceProviders(LocalVoiceModelCatalog.availableProviders( parakeetBinaryURL: VoiceModelInstaller.resolveNemoSpeechCLI())) + self.clearOperation(for: descriptor.id) onInstalled() self.appendDiagnostic("Speech-Modell \(descriptor.displayName) wurde installiert.") } catch { @@ -229,7 +232,8 @@ final class SpeechModelController { func removeVoiceModel(_ descriptor: VoiceModelDescriptor) { guard descriptor.providerID == VoiceProviderID.whisperCpp.rawValue - || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue else { + || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue + || descriptor.runtimeID == "fluidAudio" else { appendDiagnostic("Dieser Speech-Anbieter ist lokal aktuell nicht entfernbar.") return } @@ -253,10 +257,12 @@ final class SpeechModelController { } do { - let runtime = try await self.voiceModelInstaller.remove(descriptor) - var installedFileNames = Set(runtime.availableModelFileNames) - installedFileNames.formUnion( - try await self.voiceModelInstaller.installedParakeetModelFileNames()) + if descriptor.runtimeID == "fluidAudio" { + try await self.voiceModelInstaller.removeNativeModel(descriptor) + } else { + _ = try await self.voiceModelInstaller.remove(descriptor) + } + let installedFileNames = try await self.voiceModelInstaller.installedVoiceModelFileNames() self.setInstalledVoiceModelFileNames(installedFileNames) self.setVoiceLanguageOverrides( self.currentVoiceLanguageOverrides().filter { $0.modelID != descriptor.id }) @@ -264,8 +270,7 @@ final class SpeechModelController { self.appendDiagnostic("Speech-Modell \(descriptor.displayName) wurde entfernt.") } catch { do { - var actualFiles = Set(try await self.voiceModelInstaller.installedWhisperModelFileNames()) - actualFiles.formUnion(try await self.voiceModelInstaller.installedParakeetModelFileNames()) + let actualFiles = try await self.voiceModelInstaller.installedVoiceModelFileNames() self.setInstalledVoiceModelFileNames(actualFiles) } catch { self.setInstalledVoiceModelFileNames(previousInstalledFileNames) @@ -298,6 +303,12 @@ final class SpeechModelController { ) return } + if descriptor.runtimeID == "fluidAudio", + let languageCode = descriptor.languageCode, + let fixedLanguage = DictationLanguage(rawValue: languageCode), + currentSelectedLanguage() != fixedLanguage { + setSelectedLanguage(fixedLanguage) + } if currentSelectedVoiceProviderID() != descriptor.providerID { setSelectedVoiceProviderID(descriptor.providerID) } @@ -351,11 +362,6 @@ final class SpeechModelController { return false } - if descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue { - guard let fileName = descriptor.localFileName else { return false } - return currentInstalledVoiceModelFileNames().contains(fileName) - } - guard descriptor.providerID == VoiceProviderID.whisperCpp.rawValue else { return false } guard let localFileName = descriptor.localFileName else { return false } return currentInstalledVoiceModelFileNames().contains(localFileName) } @@ -368,17 +374,13 @@ final class SpeechModelController { -> Bool { guard descriptor.providerID == VoiceProviderID.whisperCpp.rawValue - || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue else { return false } + || descriptor.providerID == VoiceProviderID.nvidiaParakeet.rawValue + || descriptor.runtimeID == "fluidAudio" else { return false } guard isVoiceProviderAvailable(descriptor.providerID) else { return false } if !isVoiceModelInstalled(descriptor) { return false } - if language == .auto { return descriptor.supportsAutomaticLanguageDetection } - guard descriptor.supportsLanguageSelection else { return false } - if let languageCode = descriptor.languageCode, languageCode != language.rawValue { - return false - } - return descriptor.supportedLanguageCodes?.contains(language.rawValue) ?? true + return VoiceModelCapabilityPolicy().supports(language: language, model: descriptor) } func voiceLanguageOptions(for descriptor: VoiceModelDescriptor?) -> [DictationLanguage] { @@ -386,12 +388,7 @@ final class SpeechModelController { return Self.autoFirstLanguageOptions(DictationLanguage.allCases) } var options = DictationLanguage.allCases.filter { language in - if language == .auto { return descriptor.supportsAutomaticLanguageDetection } - guard descriptor.supportsLanguageSelection else { return false } - if let languageCode = descriptor.languageCode, languageCode != language.rawValue { - return false - } - return descriptor.supportedLanguageCodes?.contains(language.rawValue) ?? true + VoiceModelCapabilityPolicy().supports(language: language, model: descriptor) } if options.isEmpty, let fixedCode = descriptor.languageCode, let fixedLanguage = DictationLanguage(rawValue: fixedCode) { @@ -469,7 +466,8 @@ final class SpeechModelController { else { return false } return canUseVoiceModel(model, for: language) } - if !availableLanguages.contains(language.rawValue) && !hasUsableOverride { + if selectedVoiceModel.runtimeID != "fluidAudio", + !availableLanguages.contains(language.rawValue) && !hasUsableOverride { setSelectedLanguage(voiceLanguageOptions(for: selectedVoiceModel).first ?? .english) } } diff --git a/apps/macos/AppShell/VoiceModelCapabilityPolicy.swift b/apps/macos/AppShell/VoiceModelCapabilityPolicy.swift index 48bb0ee..6472f3e 100644 --- a/apps/macos/AppShell/VoiceModelCapabilityPolicy.swift +++ b/apps/macos/AppShell/VoiceModelCapabilityPolicy.swift @@ -21,6 +21,11 @@ struct VoiceModelCapabilityPolicy { catalog: [VoiceModelDescriptor], installedFiles: Set ) -> VoiceModelDescriptor? { + if let selected = catalog.first(where: { $0.id == selectedModelID }), + selected.runtimeID == "fluidAudio" { + return isInstalled(selected, files: installedFiles) + && supports(language: language, model: selected) ? selected : nil + } let overrideID = language == .auto ? nil : overrides.first { $0.languageCode == language.rawValue }?.modelID @@ -63,8 +68,8 @@ struct VoiceModelCapabilityPolicy { func supports(language: DictationLanguage, model: VoiceModelDescriptor) -> Bool { if language == .auto { return model.supportsAutomaticLanguageDetection } - guard model.supportsLanguageSelection else { return false } - if let fixed = model.languageCode, fixed != language.rawValue { return false } + if let fixed = model.languageCode { return fixed == language.rawValue } + guard model.supportsLanguageSelection || model.runtimeID == "fluidAudio" else { return false } return model.supportedLanguageCodes?.contains(language.rawValue) ?? true } diff --git a/apps/macos/README.md b/apps/macos/README.md index f7074b4..53d5d07 100644 --- a/apps/macos/README.md +++ b/apps/macos/README.md @@ -26,7 +26,9 @@ The dictation settings also include a live rewrite scope that limits how far bac Translation remains part of the dictation settings and is separate from language recognition: `Default` keeps the spoken language, `English` enables Whisper translation. NVIDIA Parakeet TDT v3 is available in the model catalog and downloads its official Q8 GGUF directly from NVIDIA's Hugging Face repository into `~/Library/Caches/NeMoSpeech/models`. Its descriptor declares the `nemo-speech` runtime dependency. When no compatible read-only external runtime is detected, installation downloads the pinned NVIDIA NeMo-Speech.cpp 0.1.0 Apple Silicon Metal release, verifies its SHA-256, and atomically installs it under `~/Library/Application Support/Cortexa/Runtime/NeMo-Speech/0.1.0`. Cortexa never modifies or removes an external runtime. The runtime tree retains NVIDIA's `LICENSE`, `NOTICE`, and third-party notices. Removing the last installed model that declares this runtime dependency removes only the Cortexa-managed runtime; a shared process lock waits for an active NeMo transcription to finish. If runtime cleanup fails, the model list reflects the files actually on disk and Settings offers a retry. -The `Advanced` settings page provides a confirmed full uninstall. A short-lived second app process waits for Cortexa to exit, then removes all contents of its `~/Library/Application Support/WisprLocal` and `~/Library/Caches/WisprLocal` directories (including Whisper models and app logs), the catalogued Parakeet model and its partial download, Cortexa-managed NeMo runtime versions and interrupted install/upgrade transaction directories, and Cortexa-prefixed model-download temporary folders. It also removes preferences and remote-provider Keychain items, resets only Cortexa's Microphone and Accessibility privacy decisions, and moves the app bundle to the Trash after cleanup succeeds. Unrecognized files in the shared NeMo model cache, unrelated temporary files, and external NeMo runtimes are retained. If a cleanup or Trash operation fails, the user sees the failed steps and the app stays available for another attempt or manual removal. +Phonon-2 and Parakeet Redux are optional, local Core ML models provided through the macOS-only `FluidASRMac` adapter and FluidAudio 0.17.7. They require Apple Silicon and macOS 15 or later; the app itself retains its macOS 14 minimum, and these models are unavailable on older or Intel Macs. Their pinned model bundles are about 378.1 MB (Phonon-2) and 220.3 MB (Redux). Phonon-2 is English-only; Redux supports Bulgarian, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Hungarian, Italian, Latvian, Lithuanian, Maltese, Polish, Portuguese, Romanian, Russian, Slovak, Slovenian, Spanish, Swedish, and Ukrainian. Neither model supports translation, prompts, hotwords, or quality profiles. Both provide local live partials and final transcription; on a loaded model, 20 warm first-partial trials on the development Mac measured p95 0.041 s for Phonon-2 and 0.116 s for Redux. The first Core ML preparation runs visibly during loading and can take longer; it is not part of that warm latency measurement. Models are downloaded only after the user selects Install, stored under `~/Library/Application Support/Cortexa/NativeVoiceModels`, and then work offline. Downloads use pinned revisions, per-file sizes and SHA-256 checks, and an atomic install. Model attribution and license links are listed in `docs/third-party-notices.md`. + +The `Advanced` settings page provides a confirmed full uninstall. A short-lived second app process waits for Cortexa to exit, then removes all contents of its `~/Library/Application Support/WisprLocal` and `~/Library/Caches/WisprLocal` directories (including Whisper models and app logs), the catalogued Parakeet model and its partial download, Cortexa-managed NeMo runtime versions and interrupted install/upgrade transaction directories, the Cortexa-owned `~/Library/Application Support/Cortexa/NativeVoiceModels` bundles, and Cortexa-prefixed model-download temporary folders. It also removes preferences and remote-provider Keychain items, resets only Cortexa's Microphone and Accessibility privacy decisions, and moves the app bundle to the Trash after cleanup succeeds. Unrecognized files in the shared NeMo model cache, unrelated temporary files, and external NeMo runtimes are retained. If a cleanup or Trash operation fails, the user sees the failed steps and the app stays available for another attempt or manual removal. Parakeet supports 25 European languages with automatic language detection, but does not accept a fixed language prompt. Transcription runs over the full recording after stop, with a 24-minute maximum supported recording; exceeding a model's duration limit stops the session with an explicit error instead of silently dropping earlier audio. Live partial text and translation are unavailable. The model selector includes catalog models from all providers independently of the active transcription provider. Model management can be filtered by provider, installation status, language scope, specific recognition language, or automatic detection. Its search accepts combined terms for provider, model name or ID, localized language names and codes, size, runtime, file name, and supported capabilities; unsupported languages and capabilities are excluded from a model's search terms. In-progress model downloads can be canceled; temporary download files are removed, and a NeMo runtime installed only for that canceled Parakeet install is cleaned up. The quality profile changes Whisper engine presets; Parakeet does not consume those parameters. The quality selector is shown in Settings and the menu bar only when the effective model supports profiles. For models with fixed recognition settings, Settings explains the limitation and the saved profile is retained for a later compatible model. Unsupported language, translation, and live-text choices offer a confirmed switch to a compatible model and a download when needed. Compatible languages and Auto can be selected directly. The preferred counterpart of an English-only Whisper model is its multilingual variant. Saved live-text, translation, and quality preferences are paused for an incompatible model and resume when a compatible model is active. A canceled or failed download leaves the model and choice unchanged, and a manual model change during download cancels the pending automatic switch. Parakeet weights are attributed to NVIDIA under CC BY 4.0; runtime source code is Apache-2.0 with separate third-party notices. diff --git a/apps/macos/WisprLocalMac/project.yml b/apps/macos/WisprLocalMac/project.yml index 73adca0..dd8e5c3 100644 --- a/apps/macos/WisprLocalMac/project.yml +++ b/apps/macos/WisprLocalMac/project.yml @@ -58,6 +58,8 @@ targets: product: AIProcessingCore - package: WisprLocalPackage product: ASRCore + - package: WisprLocalPackage + product: FluidASRMac - package: WisprLocalPackage product: AudioCore - package: WisprLocalPackage diff --git a/changelog.md b/changelog.md index cae9c4e..fbc2c36 100644 --- a/changelog.md +++ b/changelog.md @@ -1,7 +1,25 @@ # Changelog +## Features + +- 2026-10-29: Phonon-2 und Parakeet Redux als lokale Core-ML-Spracherkennungsmodelle ergänzt. + - Dateien: `Package.swift`, `Package.resolved`, `Sources/ASRCore/ASRTypes.swift`, `Sources/ASRCore/NativeVoiceModelBundle.swift`, `Sources/ASRCore/VoiceModelCatalog.swift`, `Sources/ASRCore/VoiceModelInstaller.swift`, `Sources/FluidASRMac/FluidSpeechEngine.swift`, `apps/macos/AppShell/DictationRuntime.swift`, `apps/macos/AppShell/AppUninstaller.swift`, `apps/macos/AppShell/MacAppState.swift`, `apps/macos/AppShell/MacAppStateFacadeActions.swift`, `apps/macos/AppShell/MacAppStateModelCapabilities.swift`, `apps/macos/AppShell/SettingsViewSpeechDictationSections.swift`, `apps/macos/AppShell/SpeechModelController.swift`, `apps/macos/AppShell/VoiceModelCapabilityPolicy.swift`, `apps/macos/WisprLocalMac/project.yml`, `Tests/ASRCoreTests/NativeVoiceModelBundleTests.swift`, `Tests/AppShellSupportTests/AppShellCharacterizationTests.swift`, `Tests/AppShellSupportTests/AppShellSupportTestFixtures.swift`, `Tests/AppShellSupportTests/AppUninstallerTests.swift`, `Tests/AppShellSupportTests/DictationRuntimeEventBridgeTests.swift`, `Tests/AppShellSupportTests/SpeechModelControllerTests.swift`, `Tests/AppShellSupportTests/VoiceModelCapabilityPolicyTests.swift`, `Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift`, `apps/macos/README.md`, `docs/features-and-implementation.md`, `docs/third-party-notices.md`, `VERSION`, `changelog.md`. + - Funktionalität: Auswahl und geprüfter Download der angepinnten Core-ML-Bundles, Offline-Transkription, Live-Teilergebnisse und Finalisierung über FluidAudio v0.17.7. Phonon-2 wird als Englischmodell geführt; Redux als mehrsprachig. Inkompatible Sprachen lösen keinen stillen Modellwechsel aus. Auch bei bestehendem Sprachkonflikt bleibt die feste Sprachgrenze des gewählten Modells sichtbar; eine kompatible Sprache oder ein passendes Modell wird nur nach bewusster Auswahl verwendet. Neue Modelle benötigen Apple Silicon und macOS 15 oder neuer; vorhandene Mindestversionen und Modellstandards bleiben bestehen. Nicht erhobene Speed-/Accuracy-Werte werden für diese Modelle nicht angezeigt. Die Deinstallation entfernt die beiden Cortexa-eigenen Modellbundles gezielt. Feature-Version `0.49.0`. + ## Fixes +- 2026-10-29: Phonon-2 übernimmt bei expliziter Modellauswahl automatisch Englisch; Bedienungshilfen-Abfragen laufen serialisiert auf dem Main Thread. + - Dateien: `apps/macos/AppShell/SpeechModelController.swift`, `apps/macos/AppShell/MacAppStateModelCapabilities.swift`, `apps/macos/AppShell/DictationRuntimeServices.swift`, `Tests/AppShellSupportTests/SpeechModelControllerTests.swift`, `changelog.md`. + - Funktionalität: Die bewusste Auswahl von Phonon-2 stellt die erforderliche englische Diktatsprache direkt mit ein und bewahrt Modell-Overrides für die vorherige Sprache. Der Absturzbericht vom 29. Oktober zeigte `AXIsProcessTrustedWithOptions` beim Diktatstart auf einem Hintergrundthread; der Aufruf wird nun mit den Berechtigungsaktualisierungen auf dem Main Thread ausgeführt. Patch-Fix innerhalb des geplanten Feature-Releases `0.49.0`. + +- 2026-10-29: Unbekannte FluidAudio-Modell-IDs werden explizit abgelehnt statt stillschweigend als Parakeet Redux geladen zu werden. + - Dateien: `Sources/FluidASRMac/FluidSpeechEngine.swift`, `Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift`, `changelog.md`. + - Funktionalität: Die Engine ordnet nur die beiden bekannten Modell-IDs ihren FluidAudio-Modellversionen zu. Eine unbekannte ID liefert `modelNotLoaded`; Redux wird nicht als impliziter Fallback ausgewählt. Feature-Version `0.49.0`. + +- 2026-10-29: Native Modellvorbereitung schließt die Core-ML-Erstkompilierung vor dem ersten Diktat ab. + - Dateien: `Sources/FluidASRMac/FluidSpeechEngine.swift`, `apps/macos/AppShell/DictationRuntime.swift`, `Tests/FluidASRMacTests/FluidSpeechEngineSmokeTests.swift`, `changelog.md`. + - Funktionalität: Die Engine führt während des sichtbaren Vorbereitungszustands eine lokale Stille-Inferenz aus, damit die erste echte Live-Ausgabe nicht durch die Core-ML-Kompilierung verzögert wird. Der Echtmodell-Smoke prüft 500-ms-Eingabeintervalle, wiederholte Teilergebnisse, erste Ausgabe unter zwei Sekunden und Update-p95 unter zwei Sekunden. Im letzten Lauf auf dem Test-Mac lag die erste Ausgabe bei 587/671 ms und Update-p95 bei 548/536 ms für Phonon-2/Redux. Feature-Version `0.49.0`. + - 2026-09-29: Vollständige Entfernung abgebrochener Cortexa-NeMo-Runtime-Installationen bei der Deinstallation. - Dateien: `apps/macos/AppShell/AppUninstaller.swift`, `Tests/AppShellSupportTests/AppUninstallerTests.swift`, `apps/macos/README.md`, `docs/system-design.md`, `VERSION`, `changelog.md`. - Funktionalität: Die Deinstallation entfernt jetzt neben bestätigten Runtime-Versionen auch versteckte Cortexa-Staging- und Backup-Ordner, wenn Versionsmuster und UUID exakt zur Installationsroutine passen. Nicht passende Ordner sowie externe Runtimes und unbekannte Modellcache-Dateien bleiben erhalten. Patch-Version `0.48.2`. diff --git a/docs/features-and-implementation.md b/docs/features-and-implementation.md index 72754c4..9d8571f 100644 --- a/docs/features-and-implementation.md +++ b/docs/features-and-implementation.md @@ -6,7 +6,7 @@ Stand: 2026-04-04. Diese Seite ordnet **sichtbare Produktfunktionen** den **Swif | Bereich | macOS | iOS / iPadOS | | ------------------------ | --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| Lokale ASR (whisper.cpp) | Vollständig über gebündelten Runtime-Installer + Katalog | Host/Extension-Blueprint; Parität mit macOS-Laufzeitpfad laut [apps/ios/README.md](../apps/ios/README.md) noch nicht vollständig | +| Lokale ASR (Whisper.cpp, NeMo-Speech, FluidAudio) | Whisper.cpp und NeMo-Speech; optional installierbare FluidAudio-Modelle Phonon-2 und Parakeet Redux | Host/Extension-Blueprint; Parität mit macOS-Laufzeitpfad laut [apps/ios/README.md](../apps/ios/README.md) noch nicht vollständig | | Text einfügen | Accessibility (`TextTargetMac`) + optional Paste-Fallback | Nur im Extension-Kontext über `textDocumentProxy` | | Globales Diktat-Hotkey | Ja (`GlobalHotkeyManager`, konfigurierbar) | Nicht anwendbar | | Snippets | `SnippetCore` + UI, Import/Export JSON | Geteilt über App Group | @@ -18,7 +18,8 @@ Stand: 2026-04-04. Diese Seite ordnet **sichtbare Produktfunktionen** den **Swif | Modul | Rolle | Wesentliche Umsetzung | | -------------------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| **ASRCore** | Whisper-Engine, CLI-Ausführung, Modell-Registry, gebündelter Runtime | `WhisperCppEngine`, `BundledWhisperRuntime`, Katalog für Provider/Modelle, Checksummen | +| **ASRCore** | ASR-Verträge, Modellkatalog und verwaltete Laufzeit-/Modell-Installer | `WhisperCppEngine`, `BundledWhisperRuntime`, NeMo- und Core-ML-Bundle-Prüfsummen | +| **FluidASRMac** | Lokale Core-ML-Erkennung für Phonon-2 und Parakeet Redux auf macOS 15+ | FluidAudio v0.17.7; Neural Engine; Modelle werden bei Bedarf in Application Support geladen | | **AudioCore** | Aufnahme 16 kHz mono, Unterbrechungen | `AVAudioEngine`, optional Preprocessing (siehe unten) | | **SessionCore** | Zustandsmaschine, Streaming-Stabilisator, Commit-Regeln | Koordiniert mit Snippets und Insert-Pfad | | **SnippetCore** | Regeln, Matcher, Persistenz, sichere Dateirechte | Longest-match, Streaming + Final | diff --git a/docs/system-design.md b/docs/system-design.md index a0130c9..96982c6 100644 --- a/docs/system-design.md +++ b/docs/system-design.md @@ -21,6 +21,7 @@ Cortexa is an offline-first dictation/transcription product for Apple platforms. - local speech-model catalog with explicit provider/model descriptors, install state, and per-language selection - optional NVIDIA NeMo-Speech.cpp Parakeet TDT v3 backend; model descriptors declare runtime dependencies, and the pinned 0.1.0 Apple Silicon Metal runtime is SHA-256 verified and installed atomically in Cortexa Application Support when no compatible external runtime is found. External runtimes are read-only and never removed. GGUF model files stay in the separate NeMo model cache. A runtime dependency scan removes the Cortexa-owned runtime after the final dependent model is removed, synchronized with active NeMo subprocesses. Model capabilities drive language, live-mode, translation, and maximum-recording-duration compatibility; the engine fails explicitly at model duration limits instead of truncating the oldest captured audio. Parakeet uses automatic language detection and full-utterance offline transcription. - model capability selection is shared by Settings, the menu bar, and session configuration. It resolves a valid installed language override before the global model, then checks the descriptor's language, automatic detection, translation, live-text, and quality flags. An unsupported user choice proposes the declared counterpart or a compatible installed/downloadable model. The requested model and option are applied only after confirmation and successful installation; a changed selection revision invalidates pending switches. Incompatible live-text, translation, and quality preferences remain stored while the session configuration uses safe effective values. +- Explicitly selecting the English-only FluidAudio Phonon-2 model also selects English; startup still preserves a previously stored language conflict for the user to resolve. 2. `AudioCore` diff --git a/docs/third-party-notices.md b/docs/third-party-notices.md index 30578d9..2569611 100644 --- a/docs/third-party-notices.md +++ b/docs/third-party-notices.md @@ -187,3 +187,21 @@ Modell-Attribution: **Parakeet TDT 0.6B v3**, NVIDIA. Cortexa lädt das unverän - Modellkarte und Lizenz: - Lizenztext: + +## Phonon-2 — CC BY 4.0 + +Phonon-2 von Fermion Research ist ein quantisierter, aus NVIDIA Parakeet TDT v3 abgeleiteter Spracherkenner. Cortexa bezieht die Apple-Core-ML-Konvertierung von Fluid Inference unter einer festgelegten Modellrevision. Gewichte unter CC BY 4.0; FluidAudio-Laufzeit unter MIT. Die jeweiligen Modell- und Laufzeitlizenzen sowie Hinweise werden mit den lokal bezogenen Artefakten geführt. + +- Modell und Lizenz: +- Core-ML-Konvertierung: +- Runtime und Quellcode: +- Lizenztext: + +## Moondream Parakeet Redux — CC BY 4.0 + +Parakeet Redux von Moondream ist eine ternäre, mehrsprachige Ableitung von NVIDIA Parakeet TDT v3. Cortexa bezieht die Core-ML-Konvertierung von Fluid Inference unter einer festgelegten Modellrevision. Gewichte unter CC BY 4.0; FluidAudio-Laufzeit unter MIT. Änderungen durch Modellkonvertierung sind in den Anbieterhinweisen dokumentiert. + +- Modell und Lizenz: +- Core-ML-Konvertierung: +- Runtime und Quellcode: +- Lizenztext: