diff --git a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts b/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts index 8360472980..a5b12287a3 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts @@ -8,9 +8,24 @@ import { scalePoint } from './points'; * @category Types */ export type BoxMap = { - xyxy: { xmin: number; ymin: number; xmax: number; ymax: number }; - xywh: { xmin: number; ymin: number; w: number; h: number }; - cxcywh: { cx: number; cy: number; w: number; h: number }; + xyxy: { + readonly xmin: number; + readonly ymin: number; + readonly xmax: number; + readonly ymax: number; + }; + xywh: { + readonly xmin: number; + readonly ymin: number; + readonly w: number; + readonly h: number; + }; + cxcywh: { + readonly cx: number; + readonly cy: number; + readonly w: number; + readonly h: number; + }; }; /** @@ -130,9 +145,16 @@ export function scaleBox( * @category Types */ export type NmsOptions = { + /** Bounding box format (`xyxy`, `cxcywh`, etc.). */ readonly boxFormat: BoxFormat; + /** IoU threshold for suppressing overlapping boxes. */ readonly iouThreshold: number; + /** Minimum confidence score threshold for filtering candidate boxes. */ readonly confidenceThreshold: number; + /** + * NMS algorithm variant (`standard` for hard suppression, `weighted` for soft + * coordinate averaging). + */ readonly nmsType: 'standard' | 'weighted'; }; @@ -142,7 +164,13 @@ export type NmsOptions = { * @category Utils * @param boxes Bounding boxes coordinate tensor. * @param scores Bounding boxes confidence scores tensor. - * @param opts Options configure NMS thresholds and execution mode. + * @param opts Options configuring NMS thresholds and execution mode. + * @param opts.boxFormat Bounding box format (`xyxy`, `cxcywh`, etc.). + * @param opts.iouThreshold Intersection over Union (IoU) threshold for + * suppression. + * @param opts.confidenceThreshold Minimum confidence score for candidate + * selection. + * @param opts.nmsType Suppression algorithm mode (`standard` or `weighted`). * @returns The resulting indices of the non-suppressed boxes: * - For `standard` NMS: A 1D array of indices (`number[]`) representing the * selected boxes. diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts index 1d98b73c41..b484332dad 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts @@ -74,8 +74,11 @@ export type InterpolationMethod = 'nearest' | 'area' | 'cubic' | 'lanczos' | 'li * @category Types */ export type ResizeOptions = { + /** Resize algorithm mode (`stretch`, `letterbox`, `crop`). */ readonly mode?: ResizeMode; + /** Background fill value used when letterboxing. */ readonly padValue?: number; + /** Pixel interpolation method. */ readonly interpolation?: InterpolationMethod; }; @@ -84,7 +87,9 @@ export type ResizeOptions = { * @category Types */ export type NormalizeOptions = { + /** Multiplicative scaling coefficient(s). */ readonly alpha?: number | readonly number[]; + /** Additive offset coefficient(s). */ readonly beta?: number | readonly number[]; }; @@ -99,6 +104,11 @@ export type NormalizeOptions = { * to. `dst` must be in HWC layout and its number of channels must match `src`. * Shape [H',W',C]. * @param opts Configuration options for resizing. + * @param opts.mode Resize algorithm mode (`stretch`, `letterbox`, `crop`). + * Defaults to `'stretch'`. + * @param opts.interpolation Pixel interpolation method (`linear`, `lanczos`, + * etc.). Defaults to `'lanczos'`. + * @param opts.padValue Fill value for letterboxing. Defaults to `0`. * @returns The destination tensor containing the resized image. */ export function resize(src: Tensor, dst: Tensor, opts?: ResizeOptions): Tensor { @@ -173,6 +183,9 @@ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor { * @param dst The pre-allocated destination tensor to write the normalized * values to. `dst` must have the same shape as `src`. Shape [C,H,W]. * @param opts Normalization scaling coefficients. + * @param opts.alpha Multiplicative scaling coefficient(s). Defaults to + * `1 / 255.0`. + * @param opts.beta Additive offset coefficient(s). Defaults to `0.0`. * @returns The destination tensor containing the normalized image. */ export function normalize(src: Tensor, dst: Tensor, opts?: NormalizeOptions): Tensor { diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index 3a8b2d244c..a621888d66 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -14,15 +14,20 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro * vocabulary. * @category Types */ -export type ClassifierOptions = ImagePreprocessorOptions & { readonly labels: readonly L[] }; +export type ClassifierOptions = ImagePreprocessorOptions & { + /** Array of class labels matching the model's output vocabulary. */ + readonly labels: readonly L[]; +}; /** * Model configuration required to instantiate a classifier task runner. * @category Types */ export type ClassifierModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly classifierOpts: ClassifierOptions; + /** Image preprocessor and vocabulary options. */ + readonly opts: ClassifierOptions; }; /** @@ -30,7 +35,9 @@ export type ClassifierModel = { * @category Types */ export type Classification = { + /** Predicted class label. */ readonly label: L; + /** Confidence score of the prediction (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -74,7 +81,7 @@ export async function createClassifier( */ classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification[]; }> { - const { modelPath, classifierOpts } = config; + const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const meta = validateModelSchema( @@ -87,9 +94,9 @@ export async function createClassifier( const outShape = meta.outputTensorMeta[0]!.shape; const numLabels = outShape[outShape.length - 1]!; - if (classifierOpts.labels.length !== numLabels) { + if (opts.labels.length !== numLabels) { throw new Error( - `Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${numLabels}).` + `Classifier labels length (${opts.labels.length}) must match model output dimension (${numLabels}).` ); } @@ -100,7 +107,7 @@ export async function createClassifier( ] as const; const [tLogits, tProbas] = tensors; - const preprocessor = createImagePreprocessor(classifierOpts, inpShape); + const preprocessor = createImagePreprocessor(opts, inpShape); const dispose = () => { preprocessor.dispose(); @@ -125,7 +132,7 @@ export async function createClassifier( .getData(new Float32Array(tProbas.numel)); return Array.from(probas) - .map((confidence, index) => ({ confidence, label: classifierOpts.labels[index]! })) + .map((confidence, index) => ({ confidence, label: opts.labels[index]! })) .sort((a, b) => b.confidence - a.confidence) .slice(0, options?.topk); }; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts index afd4c63383..0aaca44933 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts @@ -13,7 +13,9 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro * @category Types */ export type ImageEmbedderModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Image preprocessor options. */ readonly opts: ImagePreprocessorOptions; }; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts index d26407ea39..9440a8f081 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts @@ -31,11 +31,17 @@ export type InstanceSegmenterOptions = Omit< ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for input images. */ readonly resizeMode: 'stretch'; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; + /** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */ readonly boxFormat: F; + /** Default IoU threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default probability threshold for mask values. */ readonly defaultMaskThreshold: number; + /** Default minimum confidence score threshold for detected instances. */ readonly defaultConfidenceThreshold: number; }; @@ -46,7 +52,9 @@ export type InstanceSegmenterOptions = Omit< * @typeParam L The label type. */ export type InstanceSegmenterModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Instance segmenter options. */ readonly opts: InstanceSegmenterOptions; }; @@ -58,9 +66,13 @@ export type InstanceSegmenterModel = { * @typeParam L The label type. */ export type InstanceSegmentationResult = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Binary segmentation mask buffer cropped to the instance bounding box. */ readonly mask: ImageBuffer; + /** Predicted instance class label. */ readonly label: L; + /** Confidence score of the instance detection (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -93,10 +105,12 @@ export async function createInstanceSegmenter( * Performs asynchronous instance segmentation on the given input image. * @param input The input image buffer. * @param options Execution override options. - * @param options.confidenceThreshold Override for the minimum confidence - * threshold. - * @param options.iouThreshold Override for the IoU threshold in NMS. - * @param options.maskThreshold Override for the mask binarization threshold. + * @param options.confidenceThreshold Minimum confidence threshold. If + * omitted, uses `opts.defaultConfidenceThreshold`. + * @param options.iouThreshold IoU threshold in NMS. If omitted, uses + * `opts.defaultIouThreshold`. + * @param options.maskThreshold Mask binarization threshold. If omitted, + * uses `opts.defaultMaskThreshold`. * @returns A promise resolving to a list of detected instances. */ segmentInstances: ( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index d6ea03a294..242cffccbb 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -22,10 +22,15 @@ export type KeypointDetectorOptions ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for preprocessing input images (`stretch` or `letterbox`). */ readonly resizeMode: Exclude; + /** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */ readonly boxFormat: F; + /** Array of landmark names matching the model output keypoint locations. */ readonly landmarks: readonly L[]; + /** Default IoU threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default minimum confidence score threshold for keypoint detections. */ readonly defaultConfidenceThreshold: number; }; @@ -34,7 +39,9 @@ export type KeypointDetectorOptions * @category Types */ export type KeypointDetectorModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Keypoint detector options. */ readonly opts: KeypointDetectorOptions; }; @@ -51,8 +58,11 @@ export type Landmarks = Record = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Overall confidence score of the detection (between 0.0 and 1.0). */ readonly confidence: number; + /** Map of landmark names to their scaled pixel coordinates and individual confidence scores. */ readonly landmarks: Landmarks; }; @@ -160,9 +170,9 @@ export async function createKeypointDetector = Omit< ImagePreprocessorOptions, 'resizeMode' > & { + /** Resize mode for preprocessing input images (`stretch` or `letterbox`). */ readonly resizeMode: Exclude; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; + /** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */ readonly boxFormat: F; + /** Default IoU threshold for Non-Maximum Suppression (NMS). */ readonly defaultIouThreshold: number; + /** Default minimum confidence score threshold for detections. */ readonly defaultConfidenceThreshold: number; }; @@ -33,7 +38,9 @@ export type ObjectDetectorOptions = Omit< * @category Types */ export type ObjectDetectorModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Object detector preprocessor and threshold options. */ readonly opts: ObjectDetectorOptions; }; @@ -42,8 +49,11 @@ export type ObjectDetectorModel = { * @category Types */ export type ObjectDetection = { + /** Scaled bounding box coordinates matching the input image resolution. */ readonly box: BoundingBox; + /** Predicted object class label. */ readonly label: L; + /** Confidence score of the detection (between 0.0 and 1.0). */ readonly confidence: number; }; @@ -72,12 +82,12 @@ export async function createObjectDetector( */ dispose: () => void; /** - * Performs asynchronous object detection on the given input image. * @param input The input image buffer. * @param options Configuration options for object detection. - * @param options.confidenceThreshold Minimum confidence score for returned - * detections. - * @param options.iouThreshold Non-maximum suppression IoU threshold. + * @param options.confidenceThreshold Minimum confidence score threshold. If + * omitted, uses `opts.defaultConfidenceThreshold`. + * @param options.iouThreshold Non-maximum suppression IoU threshold. If + * omitted, uses `opts.defaultIouThreshold`. * @returns A promise resolving to the list of object detections. */ detectObjects: ( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts index 3569210481..df9f438d72 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts @@ -5,6 +5,7 @@ import type { ImageBuffer } from '../image'; import { type ResizeMode, type InterpolationMethod, + type NormalizeOptions, FORMAT_CONVERSION, FORMAT_CHANNELS, resize, @@ -18,10 +19,13 @@ import { * @category Types */ export type ImagePreprocessorOptions = { + /** Resize algorithm mode (`stretch`, `letterbox`, `crop`). */ readonly resizeMode: ResizeMode; + /** Pixel interpolation method (`linear`, `lanczos`, etc.). */ readonly interpolation: InterpolationMethod; - readonly alpha: number | readonly number[]; - readonly beta: number | readonly number[]; + /** Normalization scaling coefficients. */ + readonly normalizeOpts: NormalizeOptions; + /** Optional background fill value used when letterboxing. */ readonly padValue?: number; }; @@ -84,7 +88,7 @@ export function createImagePreprocessor( ] as const; const [tColor, tChanFirst, tNorm, tOutput] = tensors; - const { resizeMode, interpolation, alpha, beta, padValue } = opts; + const { resizeMode, interpolation, normalizeOpts, padValue } = opts; const dispose = () => tensors.forEach((t) => t.dispose()); const process = (input: ImageBuffer): Tensor => { @@ -105,7 +109,7 @@ export function createImagePreprocessor( }) .throughIf(colorCode !== null, cvtColor, tColor, colorCode!) .through(toChannelsFirst, tChanFirst) - .through(normalize, tNorm, { alpha, beta }) + .through(normalize, tNorm, normalizeOpts) .copyTo(tOutput); } finally { tInput.dispose(); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index 148cf9b267..fd08d34bbf 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -22,9 +22,12 @@ import { sigmoid, argmax } from '../../math'; * vocabulary. * @category Types */ -export type SemanticSegmentationOptions = Omit & { +export type SemanticSegmenterOptions = Omit & { + /** Resize mode for input images. */ readonly resizeMode: 'stretch'; + /** Interpolation method used when resizing output masks back to input image dimensions. */ readonly outInterpolation: InterpolationMethod; + /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; }; @@ -32,9 +35,11 @@ export type SemanticSegmentationOptions = Omit = { +export type SemanticSegmenterModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; - readonly opts: SemanticSegmentationOptions; + /** Semantic segmenter options. */ + readonly opts: SemanticSegmenterOptions; }; /** @@ -48,8 +53,10 @@ export type ColorMap = Record = { - buffer: ImageBuffer; - colormap?: ColorMap; + /** Generated output RGBA image buffer containing the colored segmentation mask. */ + readonly buffer: ImageBuffer; + /** Applied color map mapping each class label to its RGBA tuple. */ + readonly colormap?: ColorMap; }; function hslToRgb(h: number, s: number, l: number): [number, number, number] { @@ -77,7 +84,7 @@ function hslToRgb(h: number, s: number, l: number): [number, number, number] { * disposal controls. */ export async function createSemanticSegmenter( - config: SemanticSegmentationModel, + config: SemanticSegmenterModel, runtime?: WorkletRuntime ): Promise<{ /** diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index 1c1a89cbe5..7e2d97caca 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -13,6 +13,7 @@ import { cvtColor, resize, type InterpolationMethod, + type NormalizeOptions, } from '../ops/image'; /** @@ -20,9 +21,11 @@ import { * @category Types */ export type StyleTransferOptions = Omit & { + /** Resize mode for input images. */ readonly resizeMode: 'stretch'; - readonly outAlpha: number | number[]; - readonly outBeta: number | number[]; + /** Normalization options for postprocessing output tensors back to uint8 pixel values. */ + readonly outNormalizeOpts: NormalizeOptions; + /** Interpolation method used when resizing output styled images to input dimensions. */ readonly outInterpolation: InterpolationMethod; }; @@ -31,7 +34,9 @@ export type StyleTransferOptions = Omit * @category Types */ export type StyleTransferModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Style transfer preprocessor and postprocessor options. */ readonly opts: StyleTransferOptions; }; @@ -108,7 +113,7 @@ export async function createStyleTransfer( try { tOutput .copyTo(tReshape) - .through(normalize, tUint8, { alpha: opts.outAlpha, beta: opts.outBeta }) + .through(normalize, tUint8, opts.outNormalizeOpts) .through(toChannelsLast, tChanLast) .through(cvtColor, tRgba, 'RGB2RGBA') .through(resize, tResize, { mode: 'stretch', interpolation: opts.outInterpolation }) diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index 857457d89f..9b5859b540 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -12,8 +12,11 @@ import { loadTokenizer } from '../tokenizer'; * @category Types */ export type TextEmbedderModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; + /** Optional default prompt prefix added to input text before embedding. */ readonly defaultPrompt?: string; }; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 9225c743ff..8e1dc354c2 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -33,35 +33,35 @@ const DEFAULT_DETECTION_MARGIN_MS = 100; * Tunable thresholds controlling how per-frame speech probabilities are turned * into speech {@link Segment}s. * @category Types - * @property {number} [speechThreshold] - Minimum speech probability (0-1) for a - * frame to count as speech. Defaults to `0.6`. - * @property {number} [minSpeechDurationMs] - Minimum duration a region must stay - * above the threshold to open a segment. Defaults to `250`. - * @property {number} [minSilenceDurationMs] - Minimum duration below the - * threshold required to close a segment. Defaults to `100`. - * @property {number} [speechPadMs] - Padding added to both ends of every - * detected segment. Defaults to `30`. - * @property {number} [mergeGapMs] - Segments closer than this gap are merged - * into one. Defaults to `0`. */ export type VadOptions = { + /** Minimum speech probability (0-1) for a frame to count as speech. */ readonly speechThreshold?: number; + /** + * Minimum duration a region must stay above the threshold to open a + * segment. + */ readonly minSpeechDurationMs?: number; + /** Minimum duration below threshold required to close a segment. */ readonly minSilenceDurationMs?: number; + /** Padding added to both ends of every detected segment. */ readonly speechPadMs?: number; + /** Segments closer than this gap are merged into one. */ readonly mergeGapMs?: number; }; /** * Model configuration required to instantiate an FSMN-VAD task runner. * @category Types - * @property {string} modelPath - Local path or remote URL of the `.pte` model. - * @property {Required} defaultOptions - Detection thresholds tuned - * for this model, overridable per `detectVoice` call. Defined alongside the - * model in the `models` registry so the defaults are discoverable there. */ export type FsmnVadModel = { + /** Local path or remote URL of the `.pte` model. */ readonly modelPath: string; + /** + * Detection thresholds tuned for this model, overridable per `detectVoice` + * call. Defined alongside the model in the `models` registry so defaults + * are discoverable there. + */ readonly defaultOptions: Required; }; @@ -70,7 +70,9 @@ export type FsmnVadModel = { * @category Types */ export type Segment = { + /** Start time of the speech segment in seconds. */ readonly start: number; + /** End time of the speech segment in seconds. */ readonly end: number; }; @@ -78,11 +80,12 @@ export type Segment = { * Options controlling live detection via `detectVoiceOnStream`. Extends the * per-call detection thresholds ({@link VadOptions}). * @category Types - * @property {number} [detectionMargin] - How recent (in milliseconds) the last - * detected speech segment must reach toward the end of the window for speech to - * still be considered ongoing. Defaults to `100`. */ export type VadStreamOptions = VadOptions & { + /** + * How recent (in milliseconds) the last detected speech segment must reach + * toward the end of the window for speech to still be considered ongoing. + */ readonly detectionMargin?: number; }; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index dcd71f9bc4..f5d0db9bbe 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -55,11 +55,13 @@ export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number]; /** * Options passed to a single transcription call. * @category Types - * @property language - Whisper language code of the spoken audio. Must be one of - * the {@link WhisperLanguage} values declared in the model's - * `supportedLanguages` list. */ export type WhisperSttOptions = { + /** + * Whisper language code of the spoken audio. Must be one of + * the {@link WhisperLanguage} values declared in the model's + * `supportedLanguages` list. + */ readonly language: L; }; @@ -67,20 +69,28 @@ export type WhisperSttOptions = { * Options for the live-streaming transcription API. * Extends {@link WhisperSttOptions} with optional VAD tuning. * @category Types - * @property vadOptions - Fine-tuning knobs forwarded to the voice-activity - * detector. Omit to use the detector's built-in defaults. */ export type WhisperStreamOptions = - WhisperSttOptions & { readonly vadOptions?: VadStreamOptions }; + WhisperSttOptions & { + /** + * Fine-tuning knobs forwarded to the voice-activity detector. Omit to use + * built-in defaults. + */ + readonly vadOptions?: VadStreamOptions; + }; /** * Paths and metadata required to instantiate a Whisper speech-to-text model. * @category Types */ export type WhisperSttModel = { + /** Local path or remote URL of the `.pte` model. */ readonly modelPath: string; + /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; + /** List of supported language codes for this model. */ readonly supportedLanguages: readonly L[]; + /** VAD model configuration used for speech segmentation. */ readonly vadModel: FsmnVadModel; }; diff --git a/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts b/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts index 7b39705d64..12755c281e 100644 --- a/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts +++ b/packages/react-native-executorch/src/extensions/speech/utils/vadUtils.ts @@ -4,13 +4,13 @@ import { type Tensor } from '../../../core/tensor'; /** * Options controlling how {@link extractFrames} slices and filters the waveform. * @category Types - * @property {number} numFrames - Number of frames to write (must be `<= dst.shape[0]`). - * @property {number} hopLength - Samples between consecutive frames. - * @property {number} preemphasis - Pre-emphasis filter coefficient. */ export type ExtractFramesOptions = { + /** Number of audio frames to extract and write into the destination tensor. */ readonly numFrames: number; + /** Samples between consecutive frames. */ readonly hopLength: number; + /** Pre-emphasis filter coefficient. */ readonly preemphasis: number; }; @@ -27,6 +27,10 @@ export type ExtractFramesOptions = { * @param hann Precomputed Hann window, shape `[frameLength]`. * @param dst Pre-allocated destination, shape `[frames, fftLength]`. * @param options Framing options. + * @param options.numFrames Number of frames to write (must not exceed `dst` + * tensor's first dimension `dst.shape[0]`). + * @param options.hopLength Number of audio samples between consecutive frames. + * @param options.preemphasis Pre-emphasis filter coefficient. * @returns The `dst` tensor, for convenience. */ export function extractFrames( diff --git a/packages/react-native-executorch/src/hooks/useClassifier.ts b/packages/react-native-executorch/src/hooks/useClassifier.ts index 574b040ea8..6dbb15be4f 100644 --- a/packages/react-native-executorch/src/hooks/useClassifier.ts +++ b/packages/react-native-executorch/src/hooks/useClassifier.ts @@ -34,7 +34,7 @@ export function useClassifier(config: ClassifierModel, options?: { prevent error: downloadError || error, downloadProgress, localPath, - labels: config.classifierOpts.labels, + labels: config.opts.labels, classify: model?.classify, classifyWorklet: model?.classifyWorklet, }; diff --git a/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts index 8140204e18..59d47330c1 100644 --- a/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts +++ b/packages/react-native-executorch/src/hooks/useSemanticSegmenter.ts @@ -2,7 +2,7 @@ import { useModel } from './useModel'; import { useResourceDownload } from './useResourceDownload'; import { createSemanticSegmenter, - type SemanticSegmentationModel, + type SemanticSegmenterModel, } from '../extensions/cv/tasks/semanticSegmentation'; /** @@ -22,7 +22,7 @@ import { * progress, and segmentation functions. */ export function useSemanticSegmenter( - config: SemanticSegmentationModel, + config: SemanticSegmenterModel, options?: { preventLoad?: boolean } ) { const { localPath, downloadProgress, downloadError } = useResourceDownload( diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index fc8e7a5e7d..4400ef2f20 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1,7 +1,7 @@ import type { ClassifierModel } from './extensions/cv/tasks/classification'; import type { ObjectDetectorModel } from './extensions/cv/tasks/objectDetection'; import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer'; -import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation'; +import type { SemanticSegmenterModel } from './extensions/cv/tasks/semanticSegmentation'; import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection'; import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation'; import type { ImageEmbedderModel } from './extensions/cv/tasks/imageEmbedding'; @@ -38,21 +38,20 @@ const NEXT_VERSION_TAG = 'resolve/v0.10.0'; const EFFICIENTNET_V2_S_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, labels: IMAGENET1K_LABELS, }; const EFFICIENTNET_V2_S_XNNPACK_INT8: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/xnnpack/efficientnet_v2_s_xnnpack_int8.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + opts: EFFICIENTNET_V2_S_OPTS, }; const EFFICIENTNET_V2_S_XNNPACK_FP32: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/xnnpack/efficientnet_v2_s_xnnpack_fp32.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + opts: EFFICIENTNET_V2_S_OPTS, }; const EFFICIENTNET_V2_S_COREML_FP16: ClassifierModel = { modelPath: `${BASE_URL}-efficientnet-v2-s/${VERSION_TAG}/coreml/efficientnet_v2_s_coreml_fp16.pte`, - classifierOpts: EFFICIENTNET_V2_S_OPTS, + opts: EFFICIENTNET_V2_S_OPTS, }; // ============================================================================= @@ -61,10 +60,8 @@ const EFFICIENTNET_V2_S_COREML_FP16: ClassifierModel = { const STYLE_TRANSFER_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, - outAlpha: 255.0, - outBeta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, + outNormalizeOpts: { alpha: 255.0, beta: 0.0 }, outInterpolation: 'lanczos' as const, }; const STYLE_TRANSFER_CANDY_XNNPACK_FP32: StyleTransferModel = { @@ -135,14 +132,13 @@ const STYLE_TRANSFER_UDNIE_COREML_FP32: StyleTransferModel = { // ============================================================================= // Semantic Segmentation // ============================================================================= -const SELFIE_SEGMENTATION_XNNPACK_FP32: SemanticSegmentationModel<'background' | 'person'> = { +const SELFIE_SEGMENTATION_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'person'> = { modelPath: `${BASE_URL}-selfie-segmentation/${VERSION_TAG}/xnnpack/selfie_segmentation_xnnpack_fp32.pte`, opts: { labels: ['background', 'person'] as const, resizeMode: 'stretch', interpolation: 'linear', - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, outInterpolation: 'lanczos', }, }; @@ -152,13 +148,13 @@ const LRASPP_MOBILENET_V3_LARGE_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = { +const LRASPP_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_fp32.pte`, opts: LRASPP_MOBILENET_V3_LARGE_OPTS, }; -const LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = { +const LRASPP_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-lraspp/${VERSION_TAG}/xnnpack/lraspp_mobilenet_v3_large_xnnpack_int8.pte`, opts: LRASPP_MOBILENET_V3_LARGE_OPTS, }; @@ -168,29 +164,29 @@ const DEEPLAB_V3_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const DEEPLAB_V3_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET50_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_fp32.pte`, opts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET50_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet50_xnnpack_int8.pte`, opts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET101_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_fp32.pte`, opts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_RESNET101_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_resnet101_xnnpack_int8.pte`, opts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmentationModel = { +const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_fp32.pte`, opts: DEEPLAB_V3_OPTS, }; -const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmentationModel = { +const DEEPLAB_V3_MOBILENET_V3_LARGE_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-deeplab-v3/${NEXT_VERSION_TAG}/xnnpack/deeplab_v3_mobilenet_v3_large_xnnpack_int8.pte`, opts: DEEPLAB_V3_OPTS, }; @@ -200,21 +196,21 @@ const FCN_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, outInterpolation: 'lanczos' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, }; -const FCN_RESNET50_XNNPACK_FP32: SemanticSegmentationModel = { +const FCN_RESNET50_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_fp32.pte`, opts: FCN_OPTS, }; -const FCN_RESNET50_XNNPACK_INT8: SemanticSegmentationModel = { +const FCN_RESNET50_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet50_xnnpack_int8.pte`, opts: FCN_OPTS, }; -const FCN_RESNET101_XNNPACK_FP32: SemanticSegmentationModel = { +const FCN_RESNET101_XNNPACK_FP32: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_fp32.pte`, opts: FCN_OPTS, }; -const FCN_RESNET101_XNNPACK_INT8: SemanticSegmentationModel = { +const FCN_RESNET101_XNNPACK_INT8: SemanticSegmenterModel = { modelPath: `${BASE_URL}-fcn/${NEXT_VERSION_TAG}/xnnpack/fcn_resnet101_xnnpack_int8.pte`, opts: FCN_OPTS, }; @@ -227,8 +223,7 @@ const SSDLITE320_MOBILENET_V3_LARGE_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, }; @@ -250,7 +245,7 @@ const RFDETR_NANO_DETECTOR_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, }; @@ -268,8 +263,7 @@ const YOLO26_DETECTOR_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'letterbox' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.25, defaultIouThreshold: 0.7, }; @@ -348,8 +342,7 @@ const BLAZEFACE_XNNPACK_FP32: KeypointDetectorModel<'xyxy', BlazeFaceLandmark> = boxFormat: 'xyxy', resizeMode: 'letterbox', interpolation: 'linear', - alpha: 1 / 127.5, - beta: -1.0, + normalizeOpts: { alpha: 1 / 127.5, beta: -1.0 }, defaultIouThreshold: 0.3, defaultConfidenceThreshold: 0.75, landmarks: BLAZEFACE_LANDMARKS, @@ -360,8 +353,7 @@ const YOLO26_POSE_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'letterbox' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultIouThreshold: 0.7, defaultConfidenceThreshold: 0.25, landmarks: COCO_LANDMARKS, @@ -383,7 +375,7 @@ const RFDETR_KEYPOINT_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultIouThreshold: 0.55, defaultConfidenceThreshold: 0.5, landmarks: COCO_LANDMARKS, @@ -409,8 +401,7 @@ const FASTSAM_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.9, defaultMaskThreshold: 0.5, @@ -445,7 +436,7 @@ const RFDETR_NANO_SEG_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - ...IMAGENET_NORM, + normalizeOpts: IMAGENET_NORM, defaultConfidenceThreshold: 0.5, defaultIouThreshold: 0.55, defaultMaskThreshold: 0.5, @@ -464,8 +455,7 @@ const YOLO26_SEG_OPTS = { boxFormat: 'xyxy' as const, resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, defaultConfidenceThreshold: 0.25, defaultIouThreshold: 0.7, defaultMaskThreshold: 0.5, @@ -584,8 +574,7 @@ const LFM2_5_EMBEDDING_350M_MLX_INT4: TextEmbedderModel = { const CLIP_IMAGE_EMBEDDINGS_OPTS = { resizeMode: 'stretch' as const, interpolation: 'linear' as const, - alpha: 1 / 255.0, - beta: 0.0, + normalizeOpts: { alpha: 1 / 255.0, beta: 0.0 }, }; const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32: ImageEmbedderModel = { modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte`,