Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 32 additions & 4 deletions packages/react-native-executorch/src/extensions/cv/ops/boxes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,24 @@ import { scalePoint } from './points';
* @category Types
*/
export type BoxMap = {
xyxy: { xmin: number; ymin: number; xmax: number; ymax: number };
xywh: { xmin: number; ymin: number; w: number; h: number };
cxcywh: { cx: number; cy: number; w: number; h: number };
xyxy: {
readonly xmin: number;
readonly ymin: number;
readonly xmax: number;
readonly ymax: number;
};
xywh: {
readonly xmin: number;
readonly ymin: number;
readonly w: number;
readonly h: number;
};
cxcywh: {
readonly cx: number;
readonly cy: number;
readonly w: number;
readonly h: number;
};
};

/**
Expand Down Expand Up @@ -130,9 +145,16 @@ export function scaleBox<F extends BoxFormat>(
* @category Types
*/
export type NmsOptions = {
/** Bounding box format (`xyxy`, `cxcywh`, etc.). */
readonly boxFormat: BoxFormat;
/** IoU threshold for suppressing overlapping boxes. */
readonly iouThreshold: number;
/** Minimum confidence score threshold for filtering candidate boxes. */
readonly confidenceThreshold: number;
/**
* NMS algorithm variant (`standard` for hard suppression, `weighted` for soft
* coordinate averaging).
*/
readonly nmsType: 'standard' | 'weighted';
};

Expand All @@ -142,7 +164,13 @@ export type NmsOptions = {
* @category Utils
* @param boxes Bounding boxes coordinate tensor.
* @param scores Bounding boxes confidence scores tensor.
* @param opts Options configure NMS thresholds and execution mode.
* @param opts Options configuring NMS thresholds and execution mode.
* @param opts.boxFormat Bounding box format (`xyxy`, `cxcywh`, etc.).
* @param opts.iouThreshold Intersection over Union (IoU) threshold for
* suppression.
* @param opts.confidenceThreshold Minimum confidence score for candidate
* selection.
* @param opts.nmsType Suppression algorithm mode (`standard` or `weighted`).
* @returns The resulting indices of the non-suppressed boxes:
* - For `standard` NMS: A 1D array of indices (`number[]`) representing the
* selected boxes.
Expand Down
13 changes: 13 additions & 0 deletions packages/react-native-executorch/src/extensions/cv/ops/image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,11 @@ export type InterpolationMethod = 'nearest' | 'area' | 'cubic' | 'lanczos' | 'li
* @category Types
*/
export type ResizeOptions = {
/** Resize algorithm mode (`stretch`, `letterbox`, `crop`). */
readonly mode?: ResizeMode;
/** Background fill value used when letterboxing. */
readonly padValue?: number;
/** Pixel interpolation method. */
readonly interpolation?: InterpolationMethod;
};

Expand All @@ -84,7 +87,9 @@ export type ResizeOptions = {
* @category Types
*/
export type NormalizeOptions = {
/** Multiplicative scaling coefficient(s). */
readonly alpha?: number | readonly number[];
/** Additive offset coefficient(s). */
readonly beta?: number | readonly number[];
};

Expand All @@ -99,6 +104,11 @@ export type NormalizeOptions = {
* to. `dst` must be in HWC layout and its number of channels must match `src`.
* Shape [H',W',C].
* @param opts Configuration options for resizing.
* @param opts.mode Resize algorithm mode (`stretch`, `letterbox`, `crop`).
* Defaults to `'stretch'`.
* @param opts.interpolation Pixel interpolation method (`linear`, `lanczos`,
* etc.). Defaults to `'lanczos'`.
* @param opts.padValue Fill value for letterboxing. Defaults to `0`.
* @returns The destination tensor containing the resized image.
*/
export function resize(src: Tensor, dst: Tensor, opts?: ResizeOptions): Tensor {
Expand Down Expand Up @@ -173,6 +183,9 @@ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor {
* @param dst The pre-allocated destination tensor to write the normalized
* values to. `dst` must have the same shape as `src`. Shape [C,H,W].
* @param opts Normalization scaling coefficients.
* @param opts.alpha Multiplicative scaling coefficient(s). Defaults to
* `1 / 255.0`.
* @param opts.beta Additive offset coefficient(s). Defaults to `0.0`.
* @returns The destination tensor containing the normalized image.
*/
export function normalize(src: Tensor, dst: Tensor, opts?: NormalizeOptions): Tensor {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,30 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro
* vocabulary.
* @category Types
*/
export type ClassifierOptions<L> = ImagePreprocessorOptions & { readonly labels: readonly L[] };
export type ClassifierOptions<L> = ImagePreprocessorOptions & {
/** Array of class labels matching the model's output vocabulary. */
readonly labels: readonly L[];
};

/**
* Model configuration required to instantiate a classifier task runner.
* @category Types
*/
export type ClassifierModel<L> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
readonly classifierOpts: ClassifierOptions<L>;
/** Image preprocessor and vocabulary options. */
readonly opts: ClassifierOptions<L>;
};

/**
* Result structure representing a single classification prediction.
* @category Types
*/
export type Classification<L> = {
/** Predicted class label. */
readonly label: L;
/** Confidence score of the prediction (between 0.0 and 1.0). */
readonly confidence: number;
};

Expand Down Expand Up @@ -74,7 +81,7 @@ export async function createClassifier<L>(
*/
classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification<L>[];
}> {
const { modelPath, classifierOpts } = config;
const { modelPath, opts } = config;
const model = await wrapAsync(loadModel, runtime)(modelPath);

const meta = validateModelSchema(
Expand All @@ -87,9 +94,9 @@ export async function createClassifier<L>(
const outShape = meta.outputTensorMeta[0]!.shape;

const numLabels = outShape[outShape.length - 1]!;
if (classifierOpts.labels.length !== numLabels) {
if (opts.labels.length !== numLabels) {
throw new Error(
`Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${numLabels}).`
`Classifier labels length (${opts.labels.length}) must match model output dimension (${numLabels}).`
);
}

Expand All @@ -100,7 +107,7 @@ export async function createClassifier<L>(
] as const;

const [tLogits, tProbas] = tensors;
const preprocessor = createImagePreprocessor(classifierOpts, inpShape);
const preprocessor = createImagePreprocessor(opts, inpShape);

const dispose = () => {
preprocessor.dispose();
Expand All @@ -125,7 +132,7 @@ export async function createClassifier<L>(
.getData(new Float32Array(tProbas.numel));

return Array.from(probas)
.map((confidence, index) => ({ confidence, label: classifierOpts.labels[index]! }))
.map((confidence, index) => ({ confidence, label: opts.labels[index]! }))
.sort((a, b) => b.confidence - a.confidence)
.slice(0, options?.topk);
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from './prepro
* @category Types
*/
export type ImageEmbedderModel = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
/** Image preprocessor options. */
readonly opts: ImagePreprocessorOptions;
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,17 @@ export type InstanceSegmenterOptions<F extends BoxFormat, L> = Omit<
ImagePreprocessorOptions,
'resizeMode'
> & {
/** Resize mode for input images. */
readonly resizeMode: 'stretch';
/** Array of class labels matching the model's output vocabulary. */
readonly labels: readonly L[];
/** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */
readonly boxFormat: F;
/** Default IoU threshold for Non-Maximum Suppression (NMS). */
readonly defaultIouThreshold: number;
/** Default probability threshold for mask values. */
readonly defaultMaskThreshold: number;
/** Default minimum confidence score threshold for detected instances. */
readonly defaultConfidenceThreshold: number;
};

Expand All @@ -46,7 +52,9 @@ export type InstanceSegmenterOptions<F extends BoxFormat, L> = Omit<
* @typeParam L The label type.
*/
export type InstanceSegmenterModel<F extends BoxFormat, L> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
/** Instance segmenter options. */
readonly opts: InstanceSegmenterOptions<F, L>;
};

Expand All @@ -58,9 +66,13 @@ export type InstanceSegmenterModel<F extends BoxFormat, L> = {
* @typeParam L The label type.
*/
export type InstanceSegmentationResult<F extends BoxFormat, L> = {
/** Scaled bounding box coordinates matching the input image resolution. */
readonly box: BoundingBox<F>;
/** Binary segmentation mask buffer cropped to the instance bounding box. */
readonly mask: ImageBuffer;
/** Predicted instance class label. */
readonly label: L;
/** Confidence score of the instance detection (between 0.0 and 1.0). */
readonly confidence: number;
};

Expand Down Expand Up @@ -93,10 +105,12 @@ export async function createInstanceSegmenter<F extends BoxFormat, L>(
* Performs asynchronous instance segmentation on the given input image.
* @param input The input image buffer.
* @param options Execution override options.
* @param options.confidenceThreshold Override for the minimum confidence
* threshold.
* @param options.iouThreshold Override for the IoU threshold in NMS.
* @param options.maskThreshold Override for the mask binarization threshold.
* @param options.confidenceThreshold Minimum confidence threshold. If
* omitted, uses `opts.defaultConfidenceThreshold`.
* @param options.iouThreshold IoU threshold in NMS. If omitted, uses
* `opts.defaultIouThreshold`.
* @param options.maskThreshold Mask binarization threshold. If omitted,
* uses `opts.defaultMaskThreshold`.
* @returns A promise resolving to a list of detected instances.
*/
segmentInstances: (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,15 @@ export type KeypointDetectorOptions<F extends BoxFormat, L extends PropertyKey>
ImagePreprocessorOptions,
'resizeMode'
> & {
/** Resize mode for preprocessing input images (`stretch` or `letterbox`). */
readonly resizeMode: Exclude<ResizeMode, 'crop'>;
/** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */
readonly boxFormat: F;
/** Array of landmark names matching the model output keypoint locations. */
readonly landmarks: readonly L[];
/** Default IoU threshold for Non-Maximum Suppression (NMS). */
readonly defaultIouThreshold: number;
/** Default minimum confidence score threshold for keypoint detections. */
readonly defaultConfidenceThreshold: number;
};

Expand All @@ -34,7 +39,9 @@ export type KeypointDetectorOptions<F extends BoxFormat, L extends PropertyKey>
* @category Types
*/
export type KeypointDetectorModel<F extends BoxFormat, L extends PropertyKey> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
/** Keypoint detector options. */
readonly opts: KeypointDetectorOptions<F, L>;
};

Expand All @@ -51,8 +58,11 @@ export type Landmarks<L extends PropertyKey> = Record<L, Point & { readonly conf
* @category Types
*/
export type KeypointDetection<F extends BoxFormat, L extends PropertyKey> = {
/** Scaled bounding box coordinates matching the input image resolution. */
readonly box: BoundingBox<F>;
/** Overall confidence score of the detection (between 0.0 and 1.0). */
readonly confidence: number;
/** Map of landmark names to their scaled pixel coordinates and individual confidence scores. */
readonly landmarks: Landmarks<L>;
};

Expand Down Expand Up @@ -160,9 +170,9 @@ export async function createKeypointDetector<F extends BoxFormat, L extends Prop
* @param input The input image buffer.
* @param options Configuration options for keypoint detection.
* @param options.confidenceThreshold Minimum confidence score for a
* detection. If omitted, uses the model default.
* detection. If omitted, uses `opts.defaultConfidenceThreshold`.
* @param options.iouThreshold Intersection over Union (IoU) threshold for
* NMS. If omitted, uses the model default.
* NMS. If omitted, uses `opts.defaultIouThreshold`.
* @returns A promise resolving to the list of keypoint detections.
*/
detectKeypoints: (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,15 @@ export type ObjectDetectorOptions<F extends BoxFormat, L> = Omit<
ImagePreprocessorOptions,
'resizeMode'
> & {
/** Resize mode for preprocessing input images (`stretch` or `letterbox`). */
readonly resizeMode: Exclude<ResizeMode, 'crop'>;
/** Array of class labels matching the model's output vocabulary. */
readonly labels: readonly L[];
/** Bounding box format exported by the model (`xyxy`, `cxcywh`, etc.). */
readonly boxFormat: F;
/** Default IoU threshold for Non-Maximum Suppression (NMS). */
readonly defaultIouThreshold: number;
/** Default minimum confidence score threshold for detections. */
readonly defaultConfidenceThreshold: number;
};

Expand All @@ -33,7 +38,9 @@ export type ObjectDetectorOptions<F extends BoxFormat, L> = Omit<
* @category Types
*/
export type ObjectDetectorModel<F extends BoxFormat, L> = {
/** Local path or remote URL of the `.pte` model file. */
readonly modelPath: string;
/** Object detector preprocessor and threshold options. */
readonly opts: ObjectDetectorOptions<F, L>;
};

Expand All @@ -42,8 +49,11 @@ export type ObjectDetectorModel<F extends BoxFormat, L> = {
* @category Types
*/
export type ObjectDetection<F extends BoxFormat, L> = {
/** Scaled bounding box coordinates matching the input image resolution. */
readonly box: BoundingBox<F>;
/** Predicted object class label. */
readonly label: L;
/** Confidence score of the detection (between 0.0 and 1.0). */
readonly confidence: number;
};

Expand Down Expand Up @@ -72,12 +82,12 @@ export async function createObjectDetector<F extends BoxFormat, L>(
*/
dispose: () => void;
/**
* Performs asynchronous object detection on the given input image.
* @param input The input image buffer.
* @param options Configuration options for object detection.
* @param options.confidenceThreshold Minimum confidence score for returned
* detections.
* @param options.iouThreshold Non-maximum suppression IoU threshold.
* @param options.confidenceThreshold Minimum confidence score threshold. If

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

options and opts, I don't know if this won't cause a confusion.

* omitted, uses `opts.defaultConfidenceThreshold`.
* @param options.iouThreshold Non-maximum suppression IoU threshold. If
* omitted, uses `opts.defaultIouThreshold`.
* @returns A promise resolving to the list of object detections.
*/
detectObjects: (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { ImageBuffer } from '../image';
import {
type ResizeMode,
type InterpolationMethod,
type NormalizeOptions,
FORMAT_CONVERSION,
FORMAT_CHANNELS,
resize,
Expand All @@ -18,10 +19,13 @@ import {
* @category Types
*/
export type ImagePreprocessorOptions = {
/** Resize algorithm mode (`stretch`, `letterbox`, `crop`). */
readonly resizeMode: ResizeMode;
/** Pixel interpolation method (`linear`, `lanczos`, etc.). */
readonly interpolation: InterpolationMethod;
readonly alpha: number | readonly number[];
readonly beta: number | readonly number[];
/** Normalization scaling coefficients. */
readonly normalizeOpts: NormalizeOptions;
/** Optional background fill value used when letterboxing. */
readonly padValue?: number;
};

Expand Down Expand Up @@ -84,7 +88,7 @@ export function createImagePreprocessor(
] as const;

const [tColor, tChanFirst, tNorm, tOutput] = tensors;
const { resizeMode, interpolation, alpha, beta, padValue } = opts;
const { resizeMode, interpolation, normalizeOpts, padValue } = opts;

const dispose = () => tensors.forEach((t) => t.dispose());
const process = (input: ImageBuffer): Tensor => {
Expand All @@ -105,7 +109,7 @@ export function createImagePreprocessor(
})
.throughIf(colorCode !== null, cvtColor, tColor, colorCode!)
.through(toChannelsFirst, tChanFirst)
.through(normalize, tNorm, { alpha, beta })
.through(normalize, tNorm, normalizeOpts)
.copyTo(tOutput);
} finally {
tInput.dispose();
Expand Down
Loading