diff --git a/.storybook/main.ts b/.storybook/main.ts index 38b3b201..58c8a98c 100644 --- a/.storybook/main.ts +++ b/.storybook/main.ts @@ -170,7 +170,10 @@ const config: StorybookConfig = { typescript: { reactDocgen: 'react-docgen-typescript', }, - staticDirs: ['./public', { from: '../node_modules/@kerebron/wasm/assets', to: '/kerebron-wasm' }], + staticDirs: ['./public', { from: '../node_modules/@kerebron/wasm/assets', to: '/kerebron-wasm' },{ + from: '../node_modules/onnxruntime-web/dist', + to: '/ort-wasm', + },], async viteFinal(config) { const optimizeDepNames = [ ...datavisDependencyNames, diff --git a/.storybook/public/models/id-card-detector-v1.onnx b/.storybook/public/models/id-card-detector-v1.onnx new file mode 100644 index 00000000..4bc68cc2 Binary files /dev/null and b/.storybook/public/models/id-card-detector-v1.onnx differ diff --git a/eslint.config.js b/eslint.config.js index 11f0a092..af51d994 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -53,6 +53,8 @@ export default [ HTMLFormElement: 'readonly', HTMLSelectElement: 'readonly', HTMLAnchorElement: 'readonly', + CanvasRenderingContext2D: 'readonly', + CanvasImageSource: 'readonly', SVGSVGElement: 'readonly', Node: 'readonly', // Events diff --git a/package.json b/package.json index baafbebb..e8f08741 100644 --- a/package.json +++ b/package.json @@ -199,6 +199,7 @@ "datavis-ace": "=4.0.0-PRE.2", "js-yaml": ">=4.0.0", "mermaid": ">=10.0.0", + "onnxruntime-web": ">=1.26.0", "papaparse": ">=5.0.0", "react": ">=18.0.0", "react-dom": ">=18.0.0", @@ -214,6 +215,9 @@ "@kerebron/wasm": { "optional": true }, + "onnxruntime-web": { + "optional": true + }, "@mieweb/datavis": { "optional": true }, @@ -272,6 +276,7 @@ "@kerebron/editor": "0.7.9", "@kerebron/editor-kits": "0.7.9", "@kerebron/wasm": "0.7.9", + "@mieweb/datavis": "=0.0.0-PRE.4", "@monaco-editor/react": "^4.7.0", "@playwright/test": "^1.58.2", "@storybook/addon-a11y": "^10.2.11", @@ -309,7 +314,6 @@ "d3-selection": "^3.0.0", "d3-shape": "^3.2.0", "d3-zoom": "^3.0.0", - "@mieweb/datavis": "=0.0.0-PRE.4", "eslint": "^9.39.3", "eslint-plugin-jsx-a11y": "^6.10.2", "eslint-plugin-react": "^7.37.5", @@ -318,6 +322,7 @@ "js-yaml": "^4.1.1", "jsdom": "^26.1.0", "mermaid": "^11.12.3", + "onnxruntime-web": "1.26.0", "papaparse": "^5.5.3", "postcss": "^8.5.10", "prettier": "^3.8.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66ce2636..74961f7b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -236,6 +236,9 @@ importers: mermaid: specifier: ^11.12.3 version: 11.15.0 + onnxruntime-web: + specifier: 1.26.0 + version: 1.26.0 papaparse: specifier: ^5.5.3 version: 5.5.3 @@ -1147,6 +1150,33 @@ packages: '@popperjs/core@2.11.8': resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.1': + resolution: {integrity: sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==} + '@reduxjs/toolkit@2.12.0': resolution: {integrity: sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==} peerDependencies: @@ -3109,6 +3139,9 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + flatted@3.4.2: resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} @@ -3246,6 +3279,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + hachure-fill@0.5.2: resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} @@ -3960,6 +3996,9 @@ packages: resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} engines: {node: '>= 0.6.0'} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -4185,6 +4224,12 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + onnxruntime-common@1.26.0: + resolution: {integrity: sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==} + + onnxruntime-web@1.26.0: + resolution: {integrity: sha512-LbRr/8zZt2xilI2smrVQGGKINo0U46i8qJp+UXyMBGfqN7KjnH1BiwCwLwyNIVV4i9CKFv7Sf4PwLKWnT8/bEA==} + open@10.2.0: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} @@ -4311,6 +4356,9 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.58.2: resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} engines: {node: '>=18'} @@ -4469,6 +4517,10 @@ packages: prosemirror-view@1.40.0: resolution: {integrity: sha512-2G3svX0Cr1sJjkD/DYWSe3cfV5VPVTBOxI9XQEGWJDFEpsZb/gh4MV29ctv+OJx2RFX4BLt09i+6zaGM/ldkCw==} + protobufjs@7.6.4: + resolution: {integrity: sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==} + engines: {node: '>=12.0.0'} + proxy-from-env@2.1.0: resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} engines: {node: '>=10'} @@ -6463,6 +6515,26 @@ snapshots: '@popperjs/core@2.11.8': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.1': {} + '@reduxjs/toolkit@2.12.0(react-redux@9.3.0(@types/react@19.2.14)(react@19.2.4)(redux@5.0.1))(react@19.2.4)': dependencies: '@standard-schema/spec': 1.1.0 @@ -8609,6 +8681,8 @@ snapshots: flatted: 3.4.2 keyv: 4.5.4 + flatbuffers@25.9.23: {} + flatted@3.4.2: {} follow-redirects@1.16.0: {} @@ -8749,6 +8823,8 @@ snapshots: graceful-fs@4.2.11: {} + guid-typescript@1.0.9: {} + hachure-fill@0.5.2: {} has-bigints@1.1.0: {} @@ -9635,6 +9711,8 @@ snapshots: loglevel@1.9.2: {} + long@5.3.2: {} + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -9884,6 +9962,17 @@ snapshots: dependencies: mimic-fn: 2.1.0 + onnxruntime-common@1.26.0: {} + + onnxruntime-web@1.26.0: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.26.0 + platform: 1.3.6 + protobufjs: 7.6.4 + open@10.2.0: dependencies: default-browser: 5.5.0 @@ -10004,6 +10093,8 @@ snapshots: mlly: 1.8.0 pathe: 2.0.3 + platform@1.3.6: {} + playwright-core@1.58.2: {} playwright@1.58.2: @@ -10112,6 +10203,20 @@ snapshots: prosemirror-state: 1.4.3 prosemirror-transform: 1.10.4 + protobufjs@7.6.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.1 + '@types/node': 22.19.11 + long: 5.3.2 + proxy-from-env@2.1.0: {} punycode@2.3.1: {} diff --git a/src/components/CardCapture/CardCapture.stories.tsx b/src/components/CardCapture/CardCapture.stories.tsx new file mode 100644 index 00000000..f5ac805a --- /dev/null +++ b/src/components/CardCapture/CardCapture.stories.tsx @@ -0,0 +1,220 @@ +import type { Meta, StoryObj } from '@storybook/react-vite'; +import * as React from 'react'; + +import { Button } from '../Button'; +import { Text } from '../Text'; +import { CardCapture, type CardCaptureProps } from './CardCapture'; + +const meta: Meta = { + title: 'Components/Images & Media/CardCapture', + component: CardCapture, + tags: ['autodocs'], + parameters: { + layout: 'fullscreen', + docs: { + description: { + component: ` +CardCapture provides model-assisted ID-card capture directly in the browser. + +The component combines the existing MIE camera and document-quality utilities +with an ONNX ID-card detector. Automatic capture begins only after the image is +clear, stable, properly lit, and the model has consistently detected an ID card. + +The ONNX model is supplied separately through \`modelUrl\`, allowing the model +to be versioned or improved independently from the UI component. + +Manual capture remains available when automatic detection is disabled, +unavailable, or uncertain. + +### MVP scope + +- Single-class \`id_card\` detection +- Browser inference through ONNX Runtime Web +- WASM execution provider +- Stable consecutive detections +- Automatic countdown and capture +- Manual capture fallback +- Captured result returned as a \`File\` + +The initial detector demonstrates the complete browser workflow. Model +calibration across additional card types, cameras, and lighting conditions can +be performed independently without changing the component API. + `, + }, + }, + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], + argTypes: { + open: { + description: 'Whether the card-capture modal is open', + control: false, + }, + onOpenChange: { + description: 'Called when the modal open state changes', + control: false, + }, + onCapture: { + description: 'Called with the captured image after the user confirms it', + control: false, + table: { + type: { + summary: '(file: File) => void', + }, + }, + }, + modelUrl: { + description: + 'Browser-accessible URL containing the ONNX ID-card detector', + control: 'text', + }, + wasmPaths: { + description: 'Public directory containing ONNX Runtime Web WASM assets', + control: 'text', + }, + enableAutoCapture: { + description: + 'Enable model-assisted automatic capture after stable detection', + control: 'boolean', + table: { + defaultValue: { + summary: 'true', + }, + }, + }, + confidenceThreshold: { + description: 'Minimum model confidence accepted as an ID-card detection', + control: { + type: 'range', + min: 0, + max: 1, + step: 0.05, + }, + table: { + defaultValue: { + summary: '0.7', + }, + }, + }, + countdownSeconds: { + description: 'Number of seconds before automatic capture begins', + control: { + type: 'number', + min: 1, + max: 5, + }, + table: { + defaultValue: { + summary: '2', + }, + }, + }, + title: { + description: 'Title displayed in the capture modal', + control: 'text', + }, + description: { + description: 'Instructions displayed above the camera preview', + control: 'text', + }, + }, +}; + +export default meta; + +type Story = StoryObj; + +function CardCaptureDemo(props: CardCaptureProps) { + const [open, setOpen] = React.useState(false); + const [capturedFile, setCapturedFile] = React.useState(null); + + const handleCapture = (file: File) => { + setCapturedFile(file); + props.onCapture(file); + }; + + return ( +
+
+ + Browser ID-card detection + + + + Open the camera and position an ID card within the preview. + +
+ + + + {capturedFile && ( +
+ Capture completed + + + {capturedFile.name} · {Math.round(capturedFile.size / 1024)} KB + +
+ )} + + +
+ ); +} + +export const LiveBrowserModel: Story = { + args: { + open: false, + onOpenChange: () => undefined, + onCapture: () => undefined, + modelUrl: '/models/id-card-detector-v1.onnx', + wasmPaths: '/ort-wasm/', + enableAutoCapture: true, + confidenceThreshold: 0.7, + countdownSeconds: 2, + title: 'Capture ID card', + description: 'Position your ID card within the frame and hold it steady.', + }, + render: (args) => , + parameters: { + docs: { + description: { + story: ` +This live story loads the ONNX detector from +\`/models/id-card-detector-v1.onnx\` and performs inference directly inside the +browser. + +For local development, place the approved model at: + +\`\`\`text +.storybook/public/models/id-card-detector-v1.onnx +\`\`\` + +The model is intentionally not bundled with the component. Manual capture +remains available throughout the flow. + `, + }, + }, + }, +}; diff --git a/src/components/CardCapture/CardCapture.test.tsx b/src/components/CardCapture/CardCapture.test.tsx new file mode 100644 index 00000000..b9ebd5f5 --- /dev/null +++ b/src/components/CardCapture/CardCapture.test.tsx @@ -0,0 +1,335 @@ +import * as React from 'react'; +import { + act, + fireEvent, + render, + screen, + waitFor, + within, +} from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useCamera } from '../DocumentScanner/useCamera'; +import { useDocumentDetection } from '../DocumentScanner/useDocumentDetection'; +import { CardCapture } from './CardCapture'; +import { useCardDetection } from './useCardDetection'; + +vi.mock('../DocumentScanner/useCamera', () => ({ + useCamera: vi.fn(), +})); + +vi.mock('../DocumentScanner/useDocumentDetection', () => ({ + useDocumentDetection: vi.fn(), +})); + +vi.mock('./useCardDetection', () => ({ + useCardDetection: vi.fn(), +})); + +const mockUseCamera = vi.mocked(useCamera); +const mockUseDocumentDetection = vi.mocked(useDocumentDetection); +const mockUseCardDetection = vi.mocked(useCardDetection); + +const mockStartCamera = vi.fn().mockResolvedValue(undefined); +const mockStopCamera = vi.fn(); +const mockCapturePhoto = vi.fn(); +const mockSwitchCamera = vi.fn(); + +const mockStartQualityDetection = vi.fn(); +const mockStopQualityDetection = vi.fn(); +const mockResetQualityDetection = vi.fn(); + +const mockStartCardDetection = vi.fn(); +const mockStopCardDetection = vi.fn(); +const mockResetCardDetection = vi.fn(); + +const mockCreateObjectUrl = vi.fn(() => 'blob:card-preview'); +const mockRevokeObjectUrl = vi.fn(); + +function createCapturedFile(): File { + return new File(['captured-card'], 'captured-card.jpg', { + type: 'image/jpeg', + }); +} + +function setupCameraMock( + overrides: Partial> = {} +): void { + mockUseCamera.mockReturnValue({ + permission: 'granted', + stream: null, + videoRef: React.createRef(), + isReady: true, + error: null, + startCamera: mockStartCamera, + stopCamera: mockStopCamera, + capturePhoto: mockCapturePhoto, + switchCamera: mockSwitchCamera, + currentFacingMode: 'environment', + ...overrides, + }); +} + +function setupQualityDetectionMock( + overrides: Partial> = {} +): void { + mockUseDocumentDetection.mockReturnValue({ + isDetecting: true, + metrics: { + focusScore: 100, + isInFocus: true, + brightness: 120, + isBrightnessOk: true, + boundary: null, + isDocumentDetected: true, + documentCoverage: 50, + isStable: true, + stabilityDuration: 500, + }, + isReadyForCapture: true, + captureCountdown: 0, + error: null, + startDetection: mockStartQualityDetection, + stopDetection: mockStopQualityDetection, + resetDetection: mockResetQualityDetection, + ...overrides, + }); +} + +function setupCardDetectionMock( + overrides: Partial> = {} +): void { + mockUseCardDetection.mockReturnValue({ + status: 'detected', + isModelReady: true, + isDetecting: true, + isCardDetected: true, + confidence: 0.91, + consecutiveDetections: 2, + error: null, + startDetection: mockStartCardDetection, + stopDetection: mockStopCardDetection, + resetDetection: mockResetCardDetection, + ...overrides, + }); +} + +function renderCardCapture( + props: Partial> = {} +) { + const onOpenChange = vi.fn(); + const onCapture = vi.fn(); + + render( + + ); + + return { + onOpenChange, + onCapture, + }; +} + +describe('CardCapture', () => { + beforeEach(() => { + vi.clearAllMocks(); + + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + writable: true, + value: mockCreateObjectUrl, + }); + + Object.defineProperty(URL, 'revokeObjectURL', { + configurable: true, + writable: true, + value: mockRevokeObjectUrl, + }); + + mockCapturePhoto.mockReturnValue(createCapturedFile()); + + setupCameraMock(); + setupQualityDetectionMock(); + setupCardDetectionMock(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('starts the camera and detection hooks when opened', async () => { + renderCardCapture(); + + await waitFor(() => { + expect(mockStartCamera).toHaveBeenCalled(); + }); + + expect(mockStartQualityDetection).toHaveBeenCalled(); + expect(mockStartCardDetection).toHaveBeenCalled(); + + expect(mockUseDocumentDetection).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + enableAutoCapture: false, + stabilityThreshold: 80, + stabilityDuration: 400, + }) + ); + }); + + it('captures manually and returns the confirmed File', async () => { + const capturedFile = createCapturedFile(); + mockCapturePhoto.mockReturnValue(capturedFile); + + const { onCapture, onOpenChange } = renderCardCapture(); + + fireEvent.click( + screen.getByRole('button', { + name: /manual capture/i, + }) + ); + + expect(await screen.findByAltText('Captured ID card')).toBeInTheDocument(); + + expect(mockStopCamera).toHaveBeenCalled(); + expect(mockStopCardDetection).toHaveBeenCalled(); + expect(mockStopQualityDetection).toHaveBeenCalled(); + + fireEvent.click( + screen.getByRole('button', { + name: /use this photo/i, + }) + ); + + expect(onCapture).toHaveBeenCalledWith(capturedFile); + expect(onOpenChange).toHaveBeenCalledWith(false); + }); + + it('restarts the camera and detectors when retaking', async () => { + renderCardCapture(); + + fireEvent.click( + screen.getByRole('button', { + name: /manual capture/i, + }) + ); + + expect(await screen.findByAltText('Captured ID card')).toBeInTheDocument(); + + fireEvent.click( + screen.getByRole('button', { + name: /retake/i, + }) + ); + + expect(mockResetCardDetection).toHaveBeenCalled(); + expect(mockResetQualityDetection).toHaveBeenCalled(); + + await waitFor(() => { + expect(mockStartCamera).toHaveBeenCalledTimes(2); + }); + + expect( + screen.getByRole('button', { + name: /manual capture/i, + }) + ).toBeInTheDocument(); + }); + + it('shows a permission message and allows retrying the camera', () => { + setupCameraMock({ + permission: 'denied', + isReady: false, + }); + + renderCardCapture(); + + expect(screen.getByText('Camera access denied')).toBeInTheDocument(); + + fireEvent.click( + screen.getByRole('button', { + name: /try again/i, + }) + ); + + expect(mockStartCamera).toHaveBeenCalledTimes(1); + }); + + it('keeps manual capture available when model detection fails', () => { + setupCardDetectionMock({ + status: 'error', + isModelReady: false, + isDetecting: false, + isCardDetected: false, + confidence: 0, + consecutiveDetections: 0, + error: 'Unable to load the card detector.', + }); + + renderCardCapture(); + + const modelWarning = screen.getByRole('alert'); + + expect( + within(modelWarning).getByText('Automatic detection unavailable') + ).toBeInTheDocument(); + + expect( + within(modelWarning).getByText(/unable to load the card detector/i) + ).toBeInTheDocument(); + + expect( + screen.getByRole('button', { + name: /manual capture/i, + }) + ).toBeEnabled(); + }); + + it('automatically captures after the readiness countdown', async () => { + vi.useFakeTimers(); + + const capturedFile = createCapturedFile(); + mockCapturePhoto.mockReturnValue(capturedFile); + + renderCardCapture({ + countdownSeconds: 2, + }); + + expect(screen.getByText('Capturing in 2...')).toBeInTheDocument(); + + act(() => { + vi.advanceTimersByTime(1000); + }); + + expect(screen.getByText('Capturing in 1...')).toBeInTheDocument(); + + act(() => { + vi.advanceTimersByTime(1000); + }); + + expect(mockCapturePhoto).toHaveBeenCalled(); + + expect(screen.getByAltText('Captured ID card')).toBeInTheDocument(); + }); + + it('closes and cleans up when cancel is selected', () => { + const { onOpenChange } = renderCardCapture(); + + fireEvent.click( + screen.getByRole('button', { + name: /cancel/i, + }) + ); + + expect(mockStopCamera).toHaveBeenCalled(); + expect(mockStopCardDetection).toHaveBeenCalled(); + expect(mockStopQualityDetection).toHaveBeenCalled(); + expect(onOpenChange).toHaveBeenCalledWith(false); + }); +}); diff --git a/src/components/CardCapture/CardCapture.tsx b/src/components/CardCapture/CardCapture.tsx new file mode 100644 index 00000000..632e18da --- /dev/null +++ b/src/components/CardCapture/CardCapture.tsx @@ -0,0 +1,624 @@ +import * as React from 'react'; + +import { cn } from '../../utils/cn'; +import { Alert, AlertDescription, AlertTitle } from '../Alert'; +import { Button } from '../Button'; +import { + AlertCircleIcon, + CameraIcon, + CheckIcon, + RefreshIcon, + ScanLineIcon, +} from '../Icons'; +import { + Modal, + ModalBody, + ModalClose, + ModalFooter, + ModalHeader, + ModalTitle, +} from '../Modal'; +import { Spinner } from '../Spinner'; +import { Text } from '../Text'; +import { useCamera } from '../DocumentScanner/useCamera'; +import { useDocumentDetection } from '../DocumentScanner/useDocumentDetection'; +import { useCardDetection } from './useCardDetection'; + +export interface CardCaptureProps { + /** Whether the card-capture modal is open. */ + open: boolean; + + /** Called when the modal should open or close. */ + onOpenChange: (open: boolean) => void; + + /** Called after the user confirms the captured image. */ + onCapture: (file: File) => void; + + /** Browser-accessible URL for the ONNX ID-card model. */ + modelUrl: string; + + /** Public path containing ONNX Runtime Web's WASM assets. */ + wasmPaths?: string; + + /** Enable automatic capture after a stable ID-card detection. */ + enableAutoCapture?: boolean; + + /** Minimum model confidence required for an ID-card prediction. */ + confidenceThreshold?: number; + + /** Number of seconds shown before automatic capture. */ + countdownSeconds?: number; + + /** Modal title. */ + title?: string; + + /** Instructions displayed above the camera. */ + description?: string; +} + +interface CaptureStatusOverlayProps { + isCameraReady: boolean; + isModelReady: boolean; + modelError: string | null; + isInFocus: boolean; + isBrightnessOk: boolean; + isStable: boolean; + isCardDetected: boolean; + countdown: number; + enableAutoCapture: boolean; +} + +function getCaptureStatus({ + isCameraReady, + isModelReady, + modelError, + isInFocus, + isBrightnessOk, + isStable, + isCardDetected, + countdown, + enableAutoCapture, +}: CaptureStatusOverlayProps): string { + if (!isCameraReady) { + return 'Starting camera...'; + } + + if (!enableAutoCapture) { + return 'Position your ID card within the frame'; + } + + if (modelError) { + return 'Automatic detection unavailable'; + } + + if (!isModelReady) { + return 'Loading ID-card detector...'; + } + + if (countdown > 0) { + return `Capturing in ${countdown}...`; + } + + if (!isInFocus) { + return 'Hold the camera steady'; + } + + if (!isBrightnessOk) { + return 'Adjust the lighting'; + } + + if (!isStable) { + return 'Hold the card steady'; + } + + if (!isCardDetected) { + return 'Position your ID card within the frame'; + } + + return 'ID card detected'; +} + +function CaptureStatusOverlay(props: CaptureStatusOverlayProps) { + const ready = + props.isCardDetected && + props.isInFocus && + props.isBrightnessOk && + props.isStable; + + const message = getCaptureStatus(props); + + return ( +
+
+
+
+
+
+
+ +
+
+ {props.isModelReady && !props.modelError ? ( + + ) : ( + + )} + + {message} +
+
+ + {props.countdown > 0 && ( +
+
+ {props.countdown} +
+
+ )} +
+ ); +} + +function PhotoPreview({ + previewUrl, + onConfirm, + onRetake, +}: { + previewUrl: string; + onConfirm: () => void; + onRetake: () => void; +}) { + return ( +
+
+ Captured ID card +
+ +
+ + + +
+
+ ); +} + +function PermissionDeniedMessage({ onRetry }: { onRetry: () => void }) { + return ( +
+ + + Camera access denied + + Allow camera access in your browser settings, then try again. + + + +
+ +
+
+ ); +} + +function CameraUnavailableMessage() { + return ( + + + Camera unavailable + + No available camera was detected on this device. + + + ); +} + +export function CardCapture({ + open, + onOpenChange, + onCapture, + modelUrl, + wasmPaths, + enableAutoCapture = true, + confidenceThreshold = 0.7, + countdownSeconds = 2, + title = 'Capture ID card', + description = 'Position your ID card within the frame and hold it steady.', +}: CardCaptureProps) { + const { + permission, + videoRef, + isReady, + error: cameraError, + startCamera, + stopCamera, + capturePhoto, + switchCamera, + currentFacingMode, + } = useCamera({ + facingMode: 'environment', + width: 1920, + height: 1080, + }); + + const { + metrics: qualityMetrics, + isReadyForCapture: isQualityReady, + startDetection: startQualityDetection, + stopDetection: stopQualityDetection, + resetDetection: resetQualityDetection, + } = useDocumentDetection(videoRef, { + enableAutoCapture: false, + minFocusScore: 15, + stabilityThreshold: 80, + stabilityDuration: 400, + }); + + const { + isModelReady, + isCardDetected, + error: cardDetectionError, + startDetection: startCardDetection, + stopDetection: stopCardDetection, + resetDetection: resetCardDetection, + } = useCardDetection(videoRef, { + modelUrl, + wasmPaths, + confidenceThreshold, + detectionIntervalMs: 500, + stableDetectionsRequired: 2, + allowedMisses: 1, + }); + + const [capturedFile, setCapturedFile] = React.useState(null); + const [previewUrl, setPreviewUrl] = React.useState(null); + const [countdown, setCountdown] = React.useState(0); + + const hasStartedCameraRef = React.useRef(false); + const autoCaptureStartedRef = React.useRef(false); + + const clearPreview = React.useCallback(() => { + setPreviewUrl((currentPreviewUrl) => { + if (currentPreviewUrl) { + URL.revokeObjectURL(currentPreviewUrl); + } + + return null; + }); + }, []); + + const resetCaptureState = React.useCallback(() => { + setCapturedFile(null); + clearPreview(); + setCountdown(0); + autoCaptureStartedRef.current = false; + }, [clearPreview]); + + const handleCapture = React.useCallback(() => { + const file = capturePhoto(); + + if (!file) { + autoCaptureStartedRef.current = false; + setCountdown(0); + return; + } + + stopCardDetection(); + stopQualityDetection(); + + setCapturedFile(file); + setPreviewUrl(URL.createObjectURL(file)); + setCountdown(0); + + stopCamera(); + }, [capturePhoto, stopCamera, stopCardDetection, stopQualityDetection]); + + const handleClose = React.useCallback(() => { + stopCamera(); + stopCardDetection(); + stopQualityDetection(); + resetCaptureState(); + hasStartedCameraRef.current = false; + onOpenChange(false); + }, [ + onOpenChange, + resetCaptureState, + stopCamera, + stopCardDetection, + stopQualityDetection, + ]); + + const handleRetake = React.useCallback(() => { + resetCaptureState(); + resetCardDetection(); + resetQualityDetection(); + void startCamera(); + }, [ + resetCaptureState, + resetCardDetection, + resetQualityDetection, + startCamera, + ]); + + const handleConfirm = React.useCallback(() => { + if (!capturedFile) { + return; + } + + onCapture(capturedFile); + handleClose(); + }, [capturedFile, handleClose, onCapture]); + + React.useEffect(() => { + if ( + open && + permission !== 'denied' && + permission !== 'unavailable' && + !hasStartedCameraRef.current + ) { + hasStartedCameraRef.current = true; + void startCamera(); + } + }, [open, permission, startCamera]); + + React.useEffect(() => { + if (!open) { + stopCamera(); + stopCardDetection(); + stopQualityDetection(); + resetCaptureState(); + hasStartedCameraRef.current = false; + } + }, [ + open, + resetCaptureState, + stopCamera, + stopCardDetection, + stopQualityDetection, + ]); + + React.useEffect(() => { + if (open && isReady && !capturedFile && enableAutoCapture) { + startQualityDetection(); + startCardDetection(); + + return () => { + stopQualityDetection(); + stopCardDetection(); + }; + } + + stopQualityDetection(); + stopCardDetection(); + + return undefined; + }, [ + capturedFile, + enableAutoCapture, + isReady, + open, + startCardDetection, + startQualityDetection, + stopCardDetection, + stopQualityDetection, + ]); + + const isReadyForAutoCapture = + enableAutoCapture && isQualityReady && isCardDetected && !capturedFile; + + React.useEffect(() => { + if (!isReadyForAutoCapture) { + setCountdown(0); + autoCaptureStartedRef.current = false; + return; + } + + if (!autoCaptureStartedRef.current) { + autoCaptureStartedRef.current = true; + setCountdown(countdownSeconds); + } + }, [countdownSeconds, isReadyForAutoCapture]); + + React.useEffect(() => { + if (countdown <= 0) { + return undefined; + } + + const countdownTimer = window.setTimeout(() => { + if (countdown === 1) { + setCountdown(0); + handleCapture(); + return; + } + + setCountdown((currentCountdown) => Math.max(0, currentCountdown - 1)); + }, 1000); + + return () => { + window.clearTimeout(countdownTimer); + }; + }, [countdown, handleCapture]); + + React.useEffect(() => { + return () => { + clearPreview(); + }; + }, [clearPreview]); + + const renderContent = () => { + if (permission === 'denied') { + return ( + { + void startCamera(); + }} + /> + ); + } + + if (permission === 'unavailable') { + return ; + } + + if (capturedFile && previewUrl) { + return ( + + ); + } + + return ( +
+
+ + {description} + +
+ +
+
+ + {enableAutoCapture && cardDetectionError && ( + + + Automatic detection unavailable + + {cardDetectionError} You can still capture the image manually. + + + )} + + {cameraError && ( + + + Camera error + {cameraError.message} + + )} + +
+ + + +
+
+ ); + }; + + return ( + { + if (!nextOpen) { + handleClose(); + } + }} + size="lg" + aria-label="ID-card capture" + > + + {title} + + + + {renderContent()} + + {permission !== 'denied' && + permission !== 'unavailable' && + !capturedFile && ( + + + + )} + + ); +} + +CardCapture.displayName = 'CardCapture'; diff --git a/src/components/CardCapture/MAINTAINERS.md b/src/components/CardCapture/MAINTAINERS.md new file mode 100644 index 00000000..edff96a8 --- /dev/null +++ b/src/components/CardCapture/MAINTAINERS.md @@ -0,0 +1,262 @@ +# CardCapture — Maintainer Notes + +> **Provider notes:** Implementation details for maintainers changing the +> `CardCapture` component. Consumers should use the Storybook documentation. + +## Purpose + +`CardCapture` extends the existing MIE camera workflow with browser-based +semantic ID-card detection. + +The component combines: + +- `useCamera` for webcam access and image capture +- `useDocumentDetection` for focus, brightness, and frame stability +- `useCardDetection` for repeated browser ONNX inference +- Existing MIE UI components for the modal, controls, status messages, errors, + and captured-image preview + +Automatic capture begins only after both image-quality checks and stable +ID-card detections pass. Manual capture remains available as a fallback. + +## Optional Runtime Dependency + +`CardCapture` depends on `onnxruntime-web`, which is declared as an optional +peer dependency. + +The component is available through its individual subpath entry: + +```ts +import { CardCapture } from '@mieweb/ui/components/CardCapture'; +``` + +```` + +Do not export it from the main package barrel unless the optional dependency +and bundle-size impact are deliberately reconsidered. + +`onnxruntime-web` must remain listed in the `tsup` external configuration so +that it is not bundled into the main UI package. + +## Model Delivery + +The ONNX model is not bundled with `@mieweb/ui`. + +Consumers provide a browser-accessible model URL: + +```tsx + +``` + +Keeping the model external allows model versions, hosting, licensing, and +calibration to evolve independently from the component API. + +Any model loaded by a browser can be downloaded by a user who has access to +the application. Do not treat a browser-loaded model as confidential. + +## ONNX Runtime Assets + +ONNX Runtime Web requires its `.mjs` and `.wasm` files to be served separately. + +The `wasmPaths` prop must point to the public directory containing those files: + +```tsx + +``` + +Storybook exposes `node_modules/onnxruntime-web/dist` at `/ort-wasm/`. + +Applications consuming `CardCapture` must provide an equivalent static-asset +mapping. + +## Current Model Contract + +The MVP was developed against a single-class YOLO ONNX model with the following +contract: + +```text +Input name: images +Input type: float32 +Input shape: [1, 3, 640, 640] +Color format: RGB +Value range: 0–1 +Tensor order: NCHW + +Output name: output0 +Output type: float32 +Output shape: [1, 5, 8400] + +Class 0: id_card +``` + +The output contains center X, center Y, width, height, and confidence values for +each candidate prediction. + +The model does not include non-maximum suppression, so post-processing is +performed inside the component implementation. + +Do not assume that a replacement model is compatible only because it is an +ONNX file. Verify its: + +- Input and output names +- Tensor dimensions +- Preprocessing requirements +- Class definitions +- Output layout +- Confidence behavior + +## Detection Pipeline + +```text +Camera frame +→ letterbox resize to 640 × 640 +→ RGBA-to-RGB conversion +→ normalize values to 0–1 +→ convert to NCHW tensor +→ execute ONNX session +→ parse YOLO predictions +→ apply confidence filtering +→ apply non-maximum suppression +→ select the strongest card prediction +→ require consecutive detections +→ combine with image-quality readiness +→ start the automatic-capture countdown +``` + +The model session is loaded once and reused while the component is mounted. + +Only one inference request may run at a time. Preserve the inference lock when +changing polling behavior. + +## Stability Behavior + +`CardCapture` uses two complementary safeguards: + +1. Document-quality checks for focus, brightness, and frame stability +2. Consecutive semantic ID-card predictions + +The shared document-detection hook retains its existing default stability +threshold. + +`CardCapture` uses a slightly relaxed configuration to tolerate normal handheld +movement while still requiring the card to remain sufficiently stable. + +When tuning detection, test both responsiveness and false captures. Avoid +lowering confidence or stability thresholds without retesting: + +- Phones +- Blank paper +- Notebooks +- Screens +- Credit cards +- Other card-like objects + +## Model Scope + +The current MVP model targets one `id_card` class. + +It should not automatically be described as supporting every: + +- Student ID +- Employee badge +- Insurance card +- Business card +- Regional ID format + +Those formats should only be claimed as supported after they have been +evaluated. + +Model coverage and accuracy improvements should generally be handled by +publishing a new model version rather than changing the `CardCapture` public +API. + +## Privacy + +Sampled camera frames are processed in browser memory and are not sent to a +server by `CardCapture`. + +Do not: + +- Log image pixels, canvas data, or Base64 frames +- Persist sampled detection frames +- Include real identity documents in tests or Storybook fixtures +- Add OCR, visitor records, or application-specific processing to this component + +The consuming application decides what happens to the confirmed `File` returned +by `onCapture`. + +## Testing + +Unit tests mock the webcam and ONNX runtime. They must not require: + +- A physical camera +- The real ONNX model +- Network access +- Real identity-document images + +The live Storybook story is used for manual end-to-end testing with an approved +model. + +Before changing inference logic, verify: + +- Model loading +- WASM asset loading +- Supported ID-card detection +- Phones, paper, and notebooks remain rejected +- Countdown cancellation when readiness is lost +- Manual capture fallback +- Retake and camera restart +- Camera cleanup +- Timer cleanup +- Model-session cleanup +- Object URL cleanup + +## Key Implementation Files + +### `CardCapture.tsx` + +Contains the visible MIE UI camera and capture experience. + +### `loadCardModel.ts` + +Loads the ONNX model and configures ONNX Runtime Web WASM assets. + +### `preprocessCardFrame.ts` + +Converts browser image data into the tensor format expected by the model. + +### `postprocessCardDetections.ts` + +Parses YOLO output, filters predictions, maps boxes, and applies non-maximum +suppression. + +### `runCardInference.ts` + +Runs one complete browser inference pass. + +### `useCardDetection.ts` + +Manages the model lifecycle, repeated inference, detection stability, and +runtime errors. + +## Out of Scope + +`CardCapture` intentionally does not contain: + +- OCR +- Meteor methods +- Vistamate visitor logic +- Server-side inference +- Roboflow integration +- Model training code +- Dataset management +- Model-hosting credentials + +```` diff --git a/src/components/CardCapture/index.ts b/src/components/CardCapture/index.ts new file mode 100644 index 00000000..5dfa9fbc --- /dev/null +++ b/src/components/CardCapture/index.ts @@ -0,0 +1,8 @@ +export { CardCapture, type CardCaptureProps } from './CardCapture'; + +export { + useCardDetection, + type CardDetectionStatus, + type UseCardDetectionOptions, + type UseCardDetectionReturn, +} from './useCardDetection'; diff --git a/src/components/CardCapture/loadCardModel.test.ts b/src/components/CardCapture/loadCardModel.test.ts new file mode 100644 index 00000000..d4699e23 --- /dev/null +++ b/src/components/CardCapture/loadCardModel.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import * as ort from 'onnxruntime-web'; + +import { loadCardModel } from './loadCardModel'; + +vi.mock('onnxruntime-web', () => ({ + env: { + wasm: { + wasmPaths: undefined, + }, + }, + InferenceSession: { + create: vi.fn(), + }, +})); + +const mockCreateSession = vi.mocked(ort.InferenceSession.create); + +function createMockSession(): ort.InferenceSession { + return { + inputNames: ['images'], + outputNames: ['output0'], + } as unknown as ort.InferenceSession; +} + +describe('loadCardModel', () => { + beforeEach(() => { + mockCreateSession.mockReset(); + ort.env.wasm.wasmPaths = undefined; + }); + + it('loads the model with the WebAssembly execution provider', async () => { + const session = createMockSession(); + + mockCreateSession.mockResolvedValue(session); + + await expect( + loadCardModel('/models/id-card-detector-v1.onnx', { + wasmPaths: 'https://assets.example.test/ort-wasm/', + }) + ).resolves.toBe(session); + + expect(ort.env.wasm.wasmPaths).toBe( + 'https://assets.example.test/ort-wasm/' + ); + + expect(mockCreateSession).toHaveBeenCalledWith( + '/models/id-card-detector-v1.onnx', + { + executionProviders: ['wasm'], + } + ); + }); + + it('trims whitespace from the model URL', async () => { + const session = createMockSession(); + + mockCreateSession.mockResolvedValue(session); + + await loadCardModel(' /models/id-card-detector-v1.onnx ', { + wasmPaths: 'https://assets.example.test/ort-wasm/', + }); + + expect(mockCreateSession).toHaveBeenCalledWith( + '/models/id-card-detector-v1.onnx', + { + executionProviders: ['wasm'], + } + ); + }); + + it('rejects an empty model URL before creating a session', async () => { + await expect(loadCardModel(' ')).rejects.toThrow( + 'A model URL is required to load the card detector.' + ); + + expect(mockCreateSession).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/CardCapture/loadCardModel.ts b/src/components/CardCapture/loadCardModel.ts new file mode 100644 index 00000000..d2d8362b --- /dev/null +++ b/src/components/CardCapture/loadCardModel.ts @@ -0,0 +1,43 @@ +import * as ort from 'onnxruntime-web'; + +const DEFAULT_WASM_PATHS = '/ort-wasm/'; + +export interface LoadCardModelOptions { + /** + * Public directory containing ONNX Runtime Web's .mjs and .wasm files. + * + * Consumers can override this when their application hosts the runtime + * assets at a different location. + */ + wasmPaths?: string; +} + +/** + * Loads an ONNX card-detection model for browser inference. + * + * The model and ONNX Runtime assets are downloaded by the browser, while + * inference runs locally through the WebAssembly execution provider. + */ +export async function loadCardModel( + modelUrl: string, + options: LoadCardModelOptions = {} +): Promise { + const normalizedModelUrl = modelUrl.trim(); + + if (!normalizedModelUrl) { + throw new Error('A model URL is required to load the card detector.'); + } + + const normalizedWasmPathsInput = + options.wasmPaths?.trim() || DEFAULT_WASM_PATHS; + const normalizedWasmPaths = normalizedWasmPathsInput.endsWith('/') + ? normalizedWasmPathsInput + : `${normalizedWasmPathsInput}/`; + ort.env.wasm.wasmPaths = + typeof window === 'undefined' + ? normalizedWasmPaths + : new URL(normalizedWasmPaths, window.location.origin).href; + return ort.InferenceSession.create(normalizedModelUrl, { + executionProviders: ['wasm'], + }); +} diff --git a/src/components/CardCapture/postprocessCardDetections.test.ts b/src/components/CardCapture/postprocessCardDetections.test.ts new file mode 100644 index 00000000..b7f465d2 --- /dev/null +++ b/src/components/CardCapture/postprocessCardDetections.test.ts @@ -0,0 +1,208 @@ +import * as ort from 'onnxruntime-web'; +import { describe, expect, it } from 'vitest'; + +import type { LetterboxMetadata } from './preprocessCardFrame'; +import { + applyNonMaximumSuppression, + calculateIntersectionOverUnion, + mapModelBoxToSource, + parseCardDetections, + type CardPrediction, +} from './postprocessCardDetections'; + +const LANDSCAPE_METADATA: LetterboxMetadata = { + sourceWidth: 1280, + sourceHeight: 720, + inputSize: 640, + scale: 0.5, + resizedWidth: 640, + resizedHeight: 360, + padX: 0, + padY: 140, +}; + +function createPrediction( + confidence: number, + x1: number, + y1: number, + x2: number, + y2: number +): CardPrediction { + const width = x2 - x1; + const height = y2 - y1; + + return { + classId: 0, + label: 'id_card', + confidence, + x1, + y1, + x2, + y2, + x: x1 + width / 2, + y: y1 + height / 2, + width, + height, + }; +} + +describe('mapModelBoxToSource', () => { + it('removes letterbox padding and maps coordinates to the source frame', () => { + const result = mapModelBoxToSource( + { + x1: 160, + y1: 230, + x2: 480, + y2: 410, + }, + LANDSCAPE_METADATA + ); + + expect(result).toEqual({ + x1: 320, + y1: 180, + x2: 960, + y2: 540, + x: 640, + y: 360, + width: 640, + height: 360, + }); + }); + + it('supports normalized model coordinates', () => { + const squareMetadata: LetterboxMetadata = { + sourceWidth: 640, + sourceHeight: 640, + inputSize: 640, + scale: 1, + resizedWidth: 640, + resizedHeight: 640, + padX: 0, + padY: 0, + }; + + const result = mapModelBoxToSource( + { + x1: 0.25, + y1: 0.25, + x2: 0.75, + y2: 0.75, + }, + squareMetadata + ); + + expect(result).toEqual({ + x1: 160, + y1: 160, + x2: 480, + y2: 480, + x: 320, + y: 320, + width: 320, + height: 320, + }); + }); +}); + +describe('calculateIntersectionOverUnion', () => { + it('returns zero for boxes that do not overlap', () => { + const first = createPrediction(0.9, 0, 0, 100, 100); + const second = createPrediction(0.8, 200, 200, 300, 300); + + expect(calculateIntersectionOverUnion(first, second)).toBe(0); + }); + + it('returns one for identical boxes', () => { + const first = createPrediction(0.9, 0, 0, 100, 100); + const second = createPrediction(0.8, 0, 0, 100, 100); + + expect(calculateIntersectionOverUnion(first, second)).toBe(1); + }); +}); + +describe('applyNonMaximumSuppression', () => { + it('keeps the strongest prediction when boxes overlap', () => { + const strongest = createPrediction(0.9, 100, 100, 300, 300); + + const overlapping = createPrediction(0.8, 110, 110, 295, 295); + + const separate = createPrediction(0.75, 400, 100, 550, 250); + + expect( + applyNonMaximumSuppression([overlapping, separate, strongest], 0.45) + ).toEqual([strongest, separate]); + }); +}); + +describe('parseCardDetections', () => { + it('parses [1, attributes, boxes] model output and filters weak predictions', () => { + const output = new ort.Tensor( + 'float32', + new Float32Array([ + // center x + 320, 500, + // center y + 320, 500, + // width + 320, 50, + // height + 180, 50, + // confidence + 0.9, 0.4, + ]), + [1, 5, 2] + ); + + const predictions = parseCardDetections(output, LANDSCAPE_METADATA, { + confidenceThreshold: 0.7, + }); + + expect(predictions).toHaveLength(1); + expect(predictions[0].confidence).toBeCloseTo(0.9); + expect(predictions[0].label).toBe('id_card'); + expect(predictions[0].x1).toBeCloseTo(320); + expect(predictions[0].y1).toBeCloseTo(180); + expect(predictions[0].width).toBeCloseTo(640); + expect(predictions[0].height).toBeCloseTo(360); + }); + + it('supports [1, boxes, attributes] model output', () => { + const output = new ort.Tensor( + 'float32', + new Float32Array([320, 320, 320, 180, 0.9, 500, 500, 50, 50, 0.4]), + [1, 2, 5] + ); + + const predictions = parseCardDetections(output, LANDSCAPE_METADATA, { + confidenceThreshold: 0.7, + }); + + expect(predictions).toHaveLength(1); + expect(predictions[0].confidence).toBeCloseTo(0.9); + }); + + it('rejects an unsupported output shape', () => { + const output = new ort.Tensor('float32', new Float32Array(24), [1, 6, 4]); + + expect(() => parseCardDetections(output, LANDSCAPE_METADATA)).toThrow( + 'Expected 5 YOLO attributes per box' + ); + }); + + it('validates confidence and IoU thresholds', () => { + const output = new ort.Tensor('float32', new Float32Array(5), [1, 5, 1]); + + expect(() => + parseCardDetections(output, LANDSCAPE_METADATA, { + confidenceThreshold: 1.2, + }) + ).toThrow('Confidence threshold must be between 0 and 1.'); + + expect(() => + parseCardDetections(output, LANDSCAPE_METADATA, { + iouThreshold: -0.1, + }) + ).toThrow('IoU threshold must be between 0 and 1.'); + }); +}); diff --git a/src/components/CardCapture/postprocessCardDetections.ts b/src/components/CardCapture/postprocessCardDetections.ts new file mode 100644 index 00000000..17e49d20 --- /dev/null +++ b/src/components/CardCapture/postprocessCardDetections.ts @@ -0,0 +1,273 @@ +import type { Tensor } from 'onnxruntime-web'; + +import type { LetterboxMetadata } from './preprocessCardFrame'; + +export interface CardBoundingBox { + x1: number; + y1: number; + x2: number; + y2: number; + x: number; + y: number; + width: number; + height: number; +} + +export interface CardPrediction extends CardBoundingBox { + classId: 0; + label: 'id_card'; + confidence: number; +} + +export interface PostprocessCardDetectionsOptions { + confidenceThreshold?: number; + iouThreshold?: number; +} + +interface ModelBoundingBox { + x1: number; + y1: number; + x2: number; + y2: number; +} + +const DEFAULT_CONFIDENCE_THRESHOLD = 0.7; +const DEFAULT_IOU_THRESHOLD = 0.45; +const EXPECTED_ATTRIBUTE_COUNT = 5; + +function clamp(value: number, minimum: number, maximum: number): number { + return Math.max(minimum, Math.min(maximum, value)); +} + +function validateThreshold(value: number, name: string): void { + if (value < 0 || value > 1) { + throw new Error(`${name} must be between 0 and 1.`); + } +} + +/** + * Converts a box from the 640 × 640 letterboxed model coordinates back to + * coordinates in the original webcam frame. + */ +export function mapModelBoxToSource( + box: ModelBoundingBox, + metadata: LetterboxMetadata +): CardBoundingBox { + let { x1, y1, x2, y2 } = box; + + // Some exported models return normalized values between 0 and 1. + const coordinatesAreNormalized = Math.max(x1, y1, x2, y2) <= 1.5; + + if (coordinatesAreNormalized) { + x1 *= metadata.inputSize; + y1 *= metadata.inputSize; + x2 *= metadata.inputSize; + y2 *= metadata.inputSize; + } + + const sourceX1 = clamp( + (x1 - metadata.padX) / metadata.scale, + 0, + metadata.sourceWidth + ); + + const sourceY1 = clamp( + (y1 - metadata.padY) / metadata.scale, + 0, + metadata.sourceHeight + ); + + const sourceX2 = clamp( + (x2 - metadata.padX) / metadata.scale, + 0, + metadata.sourceWidth + ); + + const sourceY2 = clamp( + (y2 - metadata.padY) / metadata.scale, + 0, + metadata.sourceHeight + ); + + const width = Math.max(0, sourceX2 - sourceX1); + const height = Math.max(0, sourceY2 - sourceY1); + + return { + x1: sourceX1, + y1: sourceY1, + x2: sourceX2, + y2: sourceY2, + x: sourceX1 + width / 2, + y: sourceY1 + height / 2, + width, + height, + }; +} + +/** + * Measures how much two bounding boxes overlap. + * + * 0 means no overlap. + * 1 means the boxes are identical. + */ +export function calculateIntersectionOverUnion( + first: CardBoundingBox, + second: CardBoundingBox +): number { + const intersectionX1 = Math.max(first.x1, second.x1); + const intersectionY1 = Math.max(first.y1, second.y1); + const intersectionX2 = Math.min(first.x2, second.x2); + const intersectionY2 = Math.min(first.y2, second.y2); + + const intersectionWidth = Math.max(0, intersectionX2 - intersectionX1); + + const intersectionHeight = Math.max(0, intersectionY2 - intersectionY1); + + const intersectionArea = intersectionWidth * intersectionHeight; + + const firstArea = + Math.max(0, first.x2 - first.x1) * Math.max(0, first.y2 - first.y1); + + const secondArea = + Math.max(0, second.x2 - second.x1) * Math.max(0, second.y2 - second.y1); + + const unionArea = firstArea + secondArea - intersectionArea; + + return unionArea > 0 ? intersectionArea / unionArea : 0; +} + +/** + * Removes duplicate predictions that describe the same detected card. + */ +export function applyNonMaximumSuppression( + predictions: CardPrediction[], + iouThreshold = DEFAULT_IOU_THRESHOLD +): CardPrediction[] { + validateThreshold(iouThreshold, 'IoU threshold'); + + const sortedPredictions = [...predictions].sort( + (first, second) => second.confidence - first.confidence + ); + + const selectedPredictions: CardPrediction[] = []; + + for (const prediction of sortedPredictions) { + const overlapsSelectedPrediction = selectedPredictions.some( + (selectedPrediction) => + calculateIntersectionOverUnion(selectedPrediction, prediction) > + iouThreshold + ); + + if (!overlapsSelectedPrediction) { + selectedPredictions.push(prediction); + } + } + + return selectedPredictions; +} + +function createOutputAccessor(output: Tensor): { + boxCount: number; + getValue: (boxIndex: number, attributeIndex: number) => number; +} { + if (output.type !== 'float32') { + throw new Error( + `Expected a float32 model output, received ${output.type}.` + ); + } + + const dimensions = [...output.dims]; + + if (dimensions.length !== 3 || dimensions[0] !== 1) { + throw new Error( + `Expected YOLO output dimensions [1, 5, boxes] or [1, boxes, 5], received [${dimensions.join( + ', ' + )}].` + ); + } + + const data = output.data as Float32Array; + const [, firstDimension, secondDimension] = dimensions; + + // Model format: [1, attributes, boxes] + if (firstDimension === EXPECTED_ATTRIBUTE_COUNT) { + return { + boxCount: secondDimension, + getValue: (boxIndex, attributeIndex) => + Number(data[attributeIndex * secondDimension + boxIndex]), + }; + } + + // Alternative export format: [1, boxes, attributes] + if (secondDimension === EXPECTED_ATTRIBUTE_COUNT) { + return { + boxCount: firstDimension, + getValue: (boxIndex, attributeIndex) => + Number(data[boxIndex * EXPECTED_ATTRIBUTE_COUNT + attributeIndex]), + }; + } + + throw new Error( + `Expected 5 YOLO attributes per box, received dimensions [${dimensions.join( + ', ' + )}].` + ); +} + +/** + * Converts the one-class YOLO model output into card predictions in the + * coordinates of the original webcam frame. + */ +export function parseCardDetections( + output: Tensor, + metadata: LetterboxMetadata, + options: PostprocessCardDetectionsOptions = {} +): CardPrediction[] { + const confidenceThreshold = + options.confidenceThreshold ?? DEFAULT_CONFIDENCE_THRESHOLD; + + const iouThreshold = options.iouThreshold ?? DEFAULT_IOU_THRESHOLD; + + validateThreshold(confidenceThreshold, 'Confidence threshold'); + + validateThreshold(iouThreshold, 'IoU threshold'); + + const { boxCount, getValue } = createOutputAccessor(output); + + const predictions: CardPrediction[] = []; + + for (let boxIndex = 0; boxIndex < boxCount; boxIndex += 1) { + const centerX = getValue(boxIndex, 0); + const centerY = getValue(boxIndex, 1); + const width = getValue(boxIndex, 2); + const height = getValue(boxIndex, 3); + const confidence = getValue(boxIndex, 4); + + if (confidence < confidenceThreshold) { + continue; + } + + const boundingBox = mapModelBoxToSource( + { + x1: centerX - width / 2, + y1: centerY - height / 2, + x2: centerX + width / 2, + y2: centerY + height / 2, + }, + metadata + ); + + if (boundingBox.width === 0 || boundingBox.height === 0) { + continue; + } + + predictions.push({ + classId: 0, + label: 'id_card', + confidence, + ...boundingBox, + }); + } + + return applyNonMaximumSuppression(predictions, iouThreshold); +} diff --git a/src/components/CardCapture/preprocessCardFrame.test.ts b/src/components/CardCapture/preprocessCardFrame.test.ts new file mode 100644 index 00000000..dbd530f3 --- /dev/null +++ b/src/components/CardCapture/preprocessCardFrame.test.ts @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + calculateLetterboxMetadata, + convertRgbaToNchw, + preprocessCardFrame, +} from './preprocessCardFrame'; + +describe('calculateLetterboxMetadata', () => { + it('letterboxes a landscape frame', () => { + expect(calculateLetterboxMetadata(1280, 720, 640)).toEqual({ + sourceWidth: 1280, + sourceHeight: 720, + inputSize: 640, + scale: 0.5, + resizedWidth: 640, + resizedHeight: 360, + padX: 0, + padY: 140, + }); + }); + + it('letterboxes a portrait frame', () => { + expect(calculateLetterboxMetadata(720, 1280, 640)).toEqual({ + sourceWidth: 720, + sourceHeight: 1280, + inputSize: 640, + scale: 0.5, + resizedWidth: 360, + resizedHeight: 640, + padX: 140, + padY: 0, + }); + }); + + it('rejects invalid dimensions', () => { + expect(() => calculateLetterboxMetadata(0, 720, 640)).toThrow( + 'Source dimensions must be greater than zero.' + ); + + expect(() => calculateLetterboxMetadata(1280, 720, 0)).toThrow( + 'Model input size must be greater than zero.' + ); + }); +}); + +describe('convertRgbaToNchw', () => { + it('normalizes RGB pixels and converts them to channel-first layout', () => { + const rgba = new Uint8ClampedArray([ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 255, 255, + ]); + + const result = convertRgbaToNchw(rgba, 2, 2); + + expect(Array.from(result)).toEqual([1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1]); + }); + + it('rejects pixel data with an unexpected length', () => { + expect(() => + convertRgbaToNchw(new Uint8ClampedArray([255, 0, 0]), 1, 1) + ).toThrow('Expected 4 RGBA values, received 3.'); + }); +}); + +describe('preprocessCardFrame', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('creates a float32 ONNX tensor from a browser image source', () => { + const rgba = new Uint8ClampedArray([ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 255, 255, + ]); + + const context = { + fillStyle: '', + fillRect: vi.fn(), + drawImage: vi.fn(), + getImageData: vi.fn().mockReturnValue({ + data: rgba, + width: 2, + height: 2, + colorSpace: 'srgb', + }), + } as unknown as CanvasRenderingContext2D; + + const canvas = { + width: 0, + height: 0, + getContext: vi.fn().mockReturnValue(context), + } as unknown as HTMLCanvasElement; + + vi.spyOn(document, 'createElement').mockReturnValue(canvas); + + const source = {} as CanvasImageSource; + + const result = preprocessCardFrame(source, 2, 2, 2); + + expect(canvas.width).toBe(2); + expect(canvas.height).toBe(2); + + expect(context.fillRect).toHaveBeenCalledWith(0, 0, 2, 2); + expect(context.drawImage).toHaveBeenCalledWith(source, 0, 0, 2, 2); + + expect(result.tensor.type).toBe('float32'); + expect(result.tensor.dims).toEqual([1, 3, 2, 2]); + + expect(Array.from(result.tensor.data as Float32Array)).toEqual([ + 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, + ]); + }); +}); diff --git a/src/components/CardCapture/preprocessCardFrame.ts b/src/components/CardCapture/preprocessCardFrame.ts new file mode 100644 index 00000000..493fee88 --- /dev/null +++ b/src/components/CardCapture/preprocessCardFrame.ts @@ -0,0 +1,125 @@ +import * as ort from 'onnxruntime-web'; + +export interface LetterboxMetadata { + sourceWidth: number; + sourceHeight: number; + inputSize: number; + scale: number; + resizedWidth: number; + resizedHeight: number; + padX: number; + padY: number; +} + +export interface PreprocessedCardFrame { + tensor: ort.Tensor; + metadata: LetterboxMetadata; +} + +const LETTERBOX_BACKGROUND = 'rgb(114, 114, 114)'; + +export function calculateLetterboxMetadata( + sourceWidth: number, + sourceHeight: number, + inputSize: number +): LetterboxMetadata { + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw new Error('Source dimensions must be greater than zero.'); + } + + if (inputSize <= 0) { + throw new Error('Model input size must be greater than zero.'); + } + + const scale = Math.min(inputSize / sourceWidth, inputSize / sourceHeight); + + const resizedWidth = Math.round(sourceWidth * scale); + const resizedHeight = Math.round(sourceHeight * scale); + + const padX = Math.floor((inputSize - resizedWidth) / 2); + const padY = Math.floor((inputSize - resizedHeight) / 2); + + return { + sourceWidth, + sourceHeight, + inputSize, + scale, + resizedWidth, + resizedHeight, + padX, + padY, + }; +} + +export function convertRgbaToNchw( + rgba: Uint8ClampedArray, + width: number, + height: number +): Float32Array { + const pixelCount = width * height; + const expectedLength = pixelCount * 4; + + if (rgba.length !== expectedLength) { + throw new Error( + `Expected ${expectedLength} RGBA values, received ${rgba.length}.` + ); + } + + const tensorData = new Float32Array(pixelCount * 3); + + for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) { + const rgbaIndex = pixelIndex * 4; + + tensorData[pixelIndex] = rgba[rgbaIndex] / 255; + tensorData[pixelCount + pixelIndex] = rgba[rgbaIndex + 1] / 255; + tensorData[pixelCount * 2 + pixelIndex] = rgba[rgbaIndex + 2] / 255; + } + + return tensorData; +} + +export function preprocessCardFrame( + source: CanvasImageSource, + sourceWidth: number, + sourceHeight: number, + inputSize = 640, + reusableCanvas?: HTMLCanvasElement +): PreprocessedCardFrame { + const metadata = calculateLetterboxMetadata( + sourceWidth, + sourceHeight, + inputSize + ); + + const canvas = reusableCanvas ?? document.createElement('canvas'); + canvas.width = inputSize; + canvas.height = inputSize; + + const context = canvas.getContext('2d', { + willReadFrequently: true, + }); + + if (!context) { + throw new Error('Unable to create a canvas rendering context.'); + } + + context.fillStyle = LETTERBOX_BACKGROUND; + context.fillRect(0, 0, inputSize, inputSize); + + context.drawImage( + source, + metadata.padX, + metadata.padY, + metadata.resizedWidth, + metadata.resizedHeight + ); + + const imageData = context.getImageData(0, 0, inputSize, inputSize); + + const tensorData = convertRgbaToNchw(imageData.data, inputSize, inputSize); + + return { + tensor: new ort.Tensor('float32', tensorData, [1, 3, inputSize, inputSize]), + metadata, + }; +} diff --git a/src/components/CardCapture/runCardInference.test.ts b/src/components/CardCapture/runCardInference.test.ts new file mode 100644 index 00000000..bf2473d2 --- /dev/null +++ b/src/components/CardCapture/runCardInference.test.ts @@ -0,0 +1,220 @@ +import * as ort from 'onnxruntime-web'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + parseCardDetections, + type CardPrediction, +} from './postprocessCardDetections'; +import { preprocessCardFrame } from './preprocessCardFrame'; +import { runCardInference } from './runCardInference'; + +vi.mock('./preprocessCardFrame', () => ({ + preprocessCardFrame: vi.fn(), +})); + +vi.mock('./postprocessCardDetections', () => ({ + parseCardDetections: vi.fn(), +})); + +const mockPreprocessCardFrame = vi.mocked(preprocessCardFrame); + +const mockParseCardDetections = vi.mocked(parseCardDetections); + +const MODEL_METADATA = { + sourceWidth: 1280, + sourceHeight: 720, + inputSize: 640, + scale: 0.5, + resizedWidth: 640, + resizedHeight: 360, + padX: 0, + padY: 140, +}; + +function createPrediction(confidence: number): CardPrediction { + return { + classId: 0, + label: 'id_card', + confidence, + x1: 100, + y1: 100, + x2: 300, + y2: 200, + x: 200, + y: 150, + width: 200, + height: 100, + }; +} + +function createMockSession(outputTensor: ort.Tensor): { + session: ort.InferenceSession; + run: ReturnType; +} { + const run = vi.fn().mockResolvedValue({ + output0: outputTensor, + }); + + const session = { + inputNames: ['images'], + outputNames: ['output0'], + run, + } as unknown as ort.InferenceSession; + + return { + session, + run, + }; +} + +describe('runCardInference', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('preprocesses the frame, runs the model, and returns the strongest prediction', async () => { + const inputTensor = new ort.Tensor( + 'float32', + new Float32Array(12), + [1, 3, 2, 2] + ); + + const outputTensor = new ort.Tensor( + 'float32', + new Float32Array(5), + [1, 5, 1] + ); + + mockPreprocessCardFrame.mockReturnValue({ + tensor: inputTensor, + metadata: MODEL_METADATA, + }); + + const weakerPrediction = createPrediction(0.76); + const strongerPrediction = createPrediction(0.91); + + mockParseCardDetections.mockReturnValue([ + weakerPrediction, + strongerPrediction, + ]); + + const { session, run } = createMockSession(outputTensor); + + const source = {} as CanvasImageSource; + const preprocessingCanvas = document.createElement('canvas'); + const result = await runCardInference(session, source, 1280, 720, { + inputSize: 640, + confidenceThreshold: 0.7, + iouThreshold: 0.45, + preprocessingCanvas, + }); + + expect(mockPreprocessCardFrame).toHaveBeenCalledWith( + source, + 1280, + 720, + 640, + preprocessingCanvas + ); + + expect(run).toHaveBeenCalledWith({ + images: inputTensor, + }); + + expect(mockParseCardDetections).toHaveBeenCalledWith( + outputTensor, + MODEL_METADATA, + { + confidenceThreshold: 0.7, + iouThreshold: 0.45, + } + ); + + expect(result).toEqual({ + detected: true, + confidence: 0.91, + prediction: strongerPrediction, + }); + }); + + it('returns a clear no-detection result when no card prediction is present', async () => { + const inputTensor = new ort.Tensor( + 'float32', + new Float32Array(12), + [1, 3, 2, 2] + ); + + const outputTensor = new ort.Tensor( + 'float32', + new Float32Array(5), + [1, 5, 1] + ); + + mockPreprocessCardFrame.mockReturnValue({ + tensor: inputTensor, + metadata: MODEL_METADATA, + }); + + mockParseCardDetections.mockReturnValue([]); + + const { session } = createMockSession(outputTensor); + + const result = await runCardInference( + session, + {} as CanvasImageSource, + 1280, + 720 + ); + + expect(result).toEqual({ + detected: false, + confidence: 0, + prediction: null, + }); + }); + + it('throws a useful error when the expected model output is missing', async () => { + const inputTensor = new ort.Tensor( + 'float32', + new Float32Array(12), + [1, 3, 2, 2] + ); + + mockPreprocessCardFrame.mockReturnValue({ + tensor: inputTensor, + metadata: MODEL_METADATA, + }); + + const session = { + inputNames: ['images'], + outputNames: ['output0'], + run: vi.fn().mockResolvedValue({}), + } as unknown as ort.InferenceSession; + + await expect( + runCardInference(session, {} as CanvasImageSource, 1280, 720) + ).rejects.toThrow( + 'The card-detection model did not return output "output0".' + ); + }); + + it('rejects a session without model input or output names', async () => { + const noInputSession = { + inputNames: [], + outputNames: ['output0'], + } as unknown as ort.InferenceSession; + + await expect( + runCardInference(noInputSession, {} as CanvasImageSource, 1280, 720) + ).rejects.toThrow('The card-detection model has no input.'); + + const noOutputSession = { + inputNames: ['images'], + outputNames: [], + } as unknown as ort.InferenceSession; + + await expect( + runCardInference(noOutputSession, {} as CanvasImageSource, 1280, 720) + ).rejects.toThrow('The card-detection model has no output.'); + }); +}); diff --git a/src/components/CardCapture/runCardInference.ts b/src/components/CardCapture/runCardInference.ts new file mode 100644 index 00000000..87c75468 --- /dev/null +++ b/src/components/CardCapture/runCardInference.ts @@ -0,0 +1,100 @@ +import type * as ort from 'onnxruntime-web'; + +import { + parseCardDetections, + type CardPrediction, + type PostprocessCardDetectionsOptions, +} from './postprocessCardDetections'; +import { preprocessCardFrame } from './preprocessCardFrame'; + +export interface RunCardInferenceOptions extends PostprocessCardDetectionsOptions { + inputSize?: number; + preprocessingCanvas?: HTMLCanvasElement; +} +export interface CardInferenceResult { + detected: boolean; + confidence: number; + + /** + * Internal information about the strongest model prediction. + * + * The capture component may use this in the future for alignment or + * diagnostics, but it does not need to display a bounding box. + */ + prediction: CardPrediction | null; +} + +/** + * Runs one ID-card detection pass against a browser image source. + * + * The camera frame is preprocessed into the tensor shape expected by the + * model, executed through the supplied ONNX session, and converted into a + * simple detection result. + */ +export async function runCardInference( + session: ort.InferenceSession, + source: CanvasImageSource, + sourceWidth: number, + sourceHeight: number, + options: RunCardInferenceOptions = {} +): Promise { + const inputName = session.inputNames[0]; + const outputName = session.outputNames[0]; + + if (!inputName) { + throw new Error('The card-detection model has no input.'); + } + + if (!outputName) { + throw new Error('The card-detection model has no output.'); + } + + const { + inputSize = 640, + preprocessingCanvas, + ...postprocessOptions + } = options; + + const { tensor, metadata } = preprocessCardFrame( + source, + sourceWidth, + sourceHeight, + inputSize, + preprocessingCanvas + ); + + const outputs = await session.run({ + [inputName]: tensor, + }); + + const outputTensor = outputs[outputName] as ort.Tensor | undefined; + + if (!outputTensor) { + throw new Error( + `The card-detection model did not return output "${outputName}".` + ); + } + + const predictions = parseCardDetections( + outputTensor, + metadata, + postprocessOptions + ); + + const strongestPrediction = predictions.reduce( + (strongest, prediction) => { + if (!strongest || prediction.confidence > strongest.confidence) { + return prediction; + } + + return strongest; + }, + null + ); + + return { + detected: strongestPrediction !== null, + confidence: strongestPrediction?.confidence ?? 0, + prediction: strongestPrediction, + }; +} diff --git a/src/components/CardCapture/useCardDetection.test.ts b/src/components/CardCapture/useCardDetection.test.ts new file mode 100644 index 00000000..a60a6b6f --- /dev/null +++ b/src/components/CardCapture/useCardDetection.test.ts @@ -0,0 +1,281 @@ +import * as React from 'react'; +import { act, renderHook, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { InferenceSession } from 'onnxruntime-web'; + +import { loadCardModel } from './loadCardModel'; +import { CardInferenceResult, runCardInference } from './runCardInference'; +import { useCardDetection } from './useCardDetection'; +import type { CardPrediction } from './postprocessCardDetections'; + +vi.mock('./loadCardModel', () => ({ + loadCardModel: vi.fn(), +})); + +vi.mock('./runCardInference', () => ({ + runCardInference: vi.fn(), +})); + +const mockLoadCardModel = vi.mocked(loadCardModel); +const mockRunCardInference = vi.mocked(runCardInference); + +function createMockSession(): InferenceSession { + return { + inputNames: ['images'], + outputNames: ['output0'], + run: vi.fn(), + release: vi.fn().mockResolvedValue(undefined), + } as unknown as InferenceSession; +} + +function createVideoRef(): React.RefObject { + return { + current: { + readyState: 4, + videoWidth: 1280, + videoHeight: 720, + } as HTMLVideoElement, + }; +} +function createPrediction(confidence: number): CardPrediction { + return { + classId: 0, + label: 'id_card', + confidence, + x1: 100, + y1: 100, + x2: 300, + y2: 200, + x: 200, + y: 150, + width: 200, + height: 100, + }; +} +function createDeferred(): { + promise: Promise; + resolve: (value: T | PromiseLike) => void; +} { + let resolvePromise!: (value: T | PromiseLike) => void; + + const promise = new Promise((resolve) => { + resolvePromise = resolve; + }); + + return { + promise, + resolve: resolvePromise, + }; +} +describe('useCardDetection', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('loads the model and reports when it is ready', async () => { + const session = createMockSession(); + mockLoadCardModel.mockResolvedValue(session); + + const { result, unmount } = renderHook(() => + useCardDetection(createVideoRef(), { + modelUrl: '/models/id-card-detector-v1.onnx', + }) + ); + + expect(result.current.status).toBe('loading'); + + await waitFor(() => { + expect(result.current.status).toBe('ready'); + }); + + expect(result.current.isModelReady).toBe(true); + expect(mockLoadCardModel).toHaveBeenCalledWith( + '/models/id-card-detector-v1.onnx', + { + wasmPaths: undefined, + } + ); + + unmount(); + + expect(session.release).toHaveBeenCalled(); + }); + + it('requires consecutive positive detections before reporting a stable card', async () => { + mockLoadCardModel.mockResolvedValue(createMockSession()); + + mockRunCardInference.mockResolvedValue({ + detected: true, + confidence: 0.91, + prediction: createPrediction(0.91), + } satisfies CardInferenceResult); + + const videoRef = createVideoRef(); + + const { result, unmount } = renderHook(() => + useCardDetection(videoRef, { + modelUrl: '/models/id-card-detector-v1.onnx', + detectionIntervalMs: 5, + stableDetectionsRequired: 2, + }) + ); + + await waitFor(() => { + expect(result.current.isModelReady).toBe(true); + }); + + act(() => { + result.current.startDetection(); + }); + + await waitFor(() => { + expect(result.current.isCardDetected).toBe(true); + }); + + expect(result.current.status).toBe('detected'); + expect(result.current.confidence).toBe(0.91); + expect(result.current.consecutiveDetections).toBeGreaterThanOrEqual(2); + + act(() => { + result.current.stopDetection(); + }); + + unmount(); + }); + + it('resets a stable detection after the allowed misses are exceeded', async () => { + mockLoadCardModel.mockResolvedValue(createMockSession()); + + const missedDetection = createDeferred(); + + const positiveResult = { + detected: true, + confidence: 0.88, + prediction: createPrediction(0.88), + } satisfies CardInferenceResult; + + const negativeResult = { + detected: false, + confidence: 0, + prediction: null, + } satisfies CardInferenceResult; + + mockRunCardInference + .mockResolvedValueOnce(positiveResult) + .mockReturnValueOnce(missedDetection.promise) + .mockResolvedValue(negativeResult); + + const { result, unmount } = renderHook(() => + useCardDetection(createVideoRef(), { + modelUrl: '/models/id-card-detector-v1.onnx', + detectionIntervalMs: 20, + stableDetectionsRequired: 1, + allowedMisses: 0, + }) + ); + + await waitFor(() => { + expect(result.current.isModelReady).toBe(true); + }); + + act(() => { + result.current.startDetection(); + }); + + await waitFor(() => { + expect(result.current.isCardDetected).toBe(true); + }); + + expect(result.current.confidence).toBe(0.88); + expect(result.current.consecutiveDetections).toBe(1); + + await act(async () => { + missedDetection.resolve(negativeResult); + await missedDetection.promise; + }); + + await waitFor(() => { + expect(result.current.isCardDetected).toBe(false); + }); + + expect(result.current.consecutiveDetections).toBe(0); + expect(result.current.confidence).toBe(0); + + act(() => { + result.current.stopDetection(); + }); + + unmount(); + }); + it('does not start a second inference while one is still running', async () => { + mockLoadCardModel.mockResolvedValue(createMockSession()); + + let resolveInference: + | (( + value: CardInferenceResult | PromiseLike + ) => void) + | undefined; + + mockRunCardInference.mockImplementation( + () => + new Promise((resolve) => { + resolveInference = resolve; + }) + ); + + const { result, unmount } = renderHook(() => + useCardDetection(createVideoRef(), { + modelUrl: '/models/id-card-detector-v1.onnx', + detectionIntervalMs: 5, + }) + ); + + await waitFor(() => { + expect(result.current.isModelReady).toBe(true); + }); + + act(() => { + result.current.startDetection(); + result.current.startDetection(); + }); + + await waitFor(() => { + expect(mockRunCardInference).toHaveBeenCalledTimes(1); + }); + + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(mockRunCardInference).toHaveBeenCalledTimes(1); + + await act(async () => { + resolveInference?.({ + detected: false, + confidence: 0, + prediction: null, + }); + }); + + act(() => { + result.current.stopDetection(); + }); + + unmount(); + }); + + it('reports a model-loading error', async () => { + mockLoadCardModel.mockRejectedValue(new Error('Model file was not found.')); + + const { result } = renderHook(() => + useCardDetection(createVideoRef(), { + modelUrl: '/missing-model.onnx', + }) + ); + + await waitFor(() => { + expect(result.current.status).toBe('error'); + }); + + expect(result.current.error).toBe('Model file was not found.'); + expect(result.current.isModelReady).toBe(false); + }); +}); diff --git a/src/components/CardCapture/useCardDetection.ts b/src/components/CardCapture/useCardDetection.ts new file mode 100644 index 00000000..f8028a5a --- /dev/null +++ b/src/components/CardCapture/useCardDetection.ts @@ -0,0 +1,318 @@ +import * as React from 'react'; +import type { InferenceSession } from 'onnxruntime-web'; + +import { loadCardModel } from './loadCardModel'; +import { runCardInference } from './runCardInference'; + +export type CardDetectionStatus = + | 'idle' + | 'loading' + | 'ready' + | 'searching' + | 'detected' + | 'error'; + +export interface UseCardDetectionOptions { + enabled?: boolean; + modelUrl: string; + wasmPaths?: string; + confidenceThreshold?: number; + inputSize?: number; + iouThreshold?: number; + detectionIntervalMs?: number; + stableDetectionsRequired?: number; + allowedMisses?: number; +} + +export interface UseCardDetectionReturn { + status: CardDetectionStatus; + isModelReady: boolean; + isDetecting: boolean; + isCardDetected: boolean; + confidence: number; + consecutiveDetections: number; + error: string | null; + startDetection: () => void; + stopDetection: () => void; + resetDetection: () => void; +} + +const DEFAULT_CONFIDENCE_THRESHOLD = 0.7; +const DEFAULT_INPUT_SIZE = 640; +const DEFAULT_IOU_THRESHOLD = 0.45; +const DEFAULT_DETECTION_INTERVAL_MS = 800; +const DEFAULT_STABLE_DETECTIONS_REQUIRED = 2; +const DEFAULT_ALLOWED_MISSES = 1; + +export function useCardDetection( + videoRef: React.RefObject, + { + enabled = true, + modelUrl, + wasmPaths, + confidenceThreshold = DEFAULT_CONFIDENCE_THRESHOLD, + inputSize = DEFAULT_INPUT_SIZE, + iouThreshold = DEFAULT_IOU_THRESHOLD, + detectionIntervalMs = DEFAULT_DETECTION_INTERVAL_MS, + stableDetectionsRequired = DEFAULT_STABLE_DETECTIONS_REQUIRED, + allowedMisses = DEFAULT_ALLOWED_MISSES, + }: UseCardDetectionOptions +): UseCardDetectionReturn { + const [status, setStatus] = React.useState('idle'); + const [isModelReady, setIsModelReady] = React.useState(false); + const [isDetecting, setIsDetecting] = React.useState(false); + const [isCardDetected, setIsCardDetected] = React.useState(false); + const [confidence, setConfidence] = React.useState(0); + const [consecutiveDetections, setConsecutiveDetections] = React.useState(0); + const [error, setError] = React.useState(null); + + const sessionRef = React.useRef(null); + const timerRef = React.useRef | null>(null); + const isDetectingRef = React.useRef(false); + const isInferenceRunningRef = React.useRef(false); + const consecutiveDetectionsRef = React.useRef(0); + const missedDetectionsRef = React.useRef(0); + const mountedRef = React.useRef(true); + const processFrameRef = React.useRef<(() => Promise) | null>(null); + const preprocessingCanvasRef = React.useRef(null); + const clearScheduledDetection = React.useCallback(() => { + if (timerRef.current) { + clearTimeout(timerRef.current); + timerRef.current = null; + } + }, []); + + const scheduleNextDetection = React.useCallback(() => { + clearScheduledDetection(); + + if (!isDetectingRef.current) { + return; + } + + timerRef.current = setTimeout(() => { + void processFrameRef.current?.(); + }, detectionIntervalMs); + }, [clearScheduledDetection, detectionIntervalMs]); + + const processFrame = React.useCallback(async () => { + if (!isDetectingRef.current) { + return; + } + + const session = sessionRef.current; + const video = videoRef.current; + + if ( + !session || + !video || + video.readyState < 2 || + video.videoWidth <= 0 || + video.videoHeight <= 0 + ) { + scheduleNextDetection(); + return; + } + if (!preprocessingCanvasRef.current) { + preprocessingCanvasRef.current = document.createElement('canvas'); + } + if (isInferenceRunningRef.current) { + return; + } + + isInferenceRunningRef.current = true; + + try { + const result = await runCardInference( + session, + video, + video.videoWidth, + video.videoHeight, + { + confidenceThreshold, + inputSize, + iouThreshold, + preprocessingCanvas: preprocessingCanvasRef.current, + } + ); + + if (!mountedRef.current || !isDetectingRef.current) { + return; + } + + if (result.detected) { + missedDetectionsRef.current = 0; + consecutiveDetectionsRef.current += 1; + + const stable = + consecutiveDetectionsRef.current >= stableDetectionsRequired; + + setConfidence(result.confidence); + setConsecutiveDetections(consecutiveDetectionsRef.current); + setIsCardDetected(stable); + setStatus(stable ? 'detected' : 'searching'); + } else { + missedDetectionsRef.current += 1; + + if (missedDetectionsRef.current > allowedMisses) { + consecutiveDetectionsRef.current = 0; + + setConfidence(0); + setConsecutiveDetections(0); + setIsCardDetected(false); + setStatus('searching'); + } + } + } catch (inferenceError) { + if (!mountedRef.current) { + return; + } + + isDetectingRef.current = false; + clearScheduledDetection(); + + setIsDetecting(false); + setIsCardDetected(false); + setStatus('error'); + setError( + inferenceError instanceof Error + ? inferenceError.message + : 'Card detection failed.' + ); + } finally { + isInferenceRunningRef.current = false; + + if (isDetectingRef.current) { + scheduleNextDetection(); + } + } + }, [ + allowedMisses, + clearScheduledDetection, + confidenceThreshold, + inputSize, + iouThreshold, + scheduleNextDetection, + stableDetectionsRequired, + videoRef, + ]); + + React.useEffect(() => { + processFrameRef.current = processFrame; + }, [processFrame]); + + React.useEffect(() => { + if (!enabled) { + setStatus('idle'); + setIsModelReady(false); + setError(null); + return; + } + let cancelled = false; + let loadedSession: InferenceSession | null = null; + + setStatus('loading'); + setIsModelReady(false); + setError(null); + void loadCardModel(modelUrl, { + wasmPaths, + }) + .then((session) => { + loadedSession = session; + + if (cancelled) { + void session.release(); + return; + } + + sessionRef.current = session; + setIsModelReady(true); + setStatus(isDetectingRef.current ? 'searching' : 'ready'); + + if (isDetectingRef.current) { + void processFrameRef.current?.(); + } + }) + .catch((modelError) => { + if (cancelled) { + return; + } + + isDetectingRef.current = false; + setIsDetecting(false); + setStatus('error'); + setError( + modelError instanceof Error + ? modelError.message + : 'Unable to load the card-detection model.' + ); + }); + + return () => { + cancelled = true; + + if (sessionRef.current === loadedSession) { + sessionRef.current = null; + } + + if (loadedSession) { + void loadedSession.release(); + } + }; + }, [enabled, modelUrl, wasmPaths]); + + React.useEffect(() => { + mountedRef.current = true; + + return () => { + mountedRef.current = false; + isDetectingRef.current = false; + clearScheduledDetection(); + }; + }, [clearScheduledDetection]); + + const startDetection = React.useCallback(() => { + if (!enabled) { + return; + } + isDetectingRef.current = true; + setIsDetecting(true); + setError(null); + setStatus(sessionRef.current ? 'searching' : 'loading'); + + if (sessionRef.current) { + void processFrameRef.current?.(); + } + }, [enabled]); + + const stopDetection = React.useCallback(() => { + isDetectingRef.current = false; + clearScheduledDetection(); + + consecutiveDetectionsRef.current = 0; + missedDetectionsRef.current = 0; + + setIsDetecting(false); + setIsCardDetected(false); + setConfidence(0); + setConsecutiveDetections(0); + setStatus(sessionRef.current ? 'ready' : 'idle'); + }, [clearScheduledDetection]); + + const resetDetection = React.useCallback(() => { + stopDetection(); + setError(null); + }, [stopDetection]); + + return { + status, + isModelReady, + isDetecting, + isCardDetected, + confidence, + consecutiveDetections, + error, + startDetection, + stopDetection, + resetDetection, + }; +} diff --git a/src/components/DocumentScanner/useDocumentDetection.test.ts b/src/components/DocumentScanner/useDocumentDetection.test.ts index bb06d00d..808ebf2a 100644 --- a/src/components/DocumentScanner/useDocumentDetection.test.ts +++ b/src/components/DocumentScanner/useDocumentDetection.test.ts @@ -144,6 +144,15 @@ describe('Stability Detection Functions', () => { // First few values (top-left) should differ expect(fp1.slice(0, 16)).not.toEqual(fp2.slice(0, 16)); }); + it('allows mild handheld movement with a relaxed threshold', () => { + const firstFrame = Array(64).fill(100); + const slightlyMovedFrame = Array(64).fill(109); + + const similarity = compareFingerprints(firstFrame, slightlyMovedFrame); + + expect(similarity).toBeLessThan(85); + expect(similarity).toBeGreaterThanOrEqual(80); + }); }); describe('compareFingerprints', () => { diff --git a/src/components/DocumentScanner/useDocumentDetection.ts b/src/components/DocumentScanner/useDocumentDetection.ts index f62edd68..8b817bd1 100644 --- a/src/components/DocumentScanner/useDocumentDetection.ts +++ b/src/components/DocumentScanner/useDocumentDetection.ts @@ -81,6 +81,8 @@ export interface DetectionConfig { minDocumentCoverage?: number; /** Maximum document coverage percentage (default: 95) - not used in simplified mode */ maxDocumentCoverage?: number; + /** Minimum frame similarity required for stability (0-100, default: 85) */ + stabilityThreshold?: number; /** Stability duration required before capture (ms, default: 500) */ stabilityDuration?: number; /** Auto-capture countdown duration (seconds, default: 2) */ @@ -97,6 +99,7 @@ const DEFAULT_CONFIG: Required = { maxBrightness: 240, minDocumentCoverage: 10, maxDocumentCoverage: 95, + stabilityThreshold: 85, stabilityDuration: 500, captureCountdown: 2, detectionFps: 5, @@ -268,6 +271,7 @@ export function useDocumentDetection( config.maxBrightness, config.minDocumentCoverage, config.maxDocumentCoverage, + config.stabilityThreshold, config.stabilityDuration, config.captureCountdown, config.detectionFps, @@ -285,9 +289,6 @@ export function useDocumentDetection( const countdownIntervalRef = useRef(null); const isDetectingRef = useRef(false); - // Stability threshold - how similar frames need to be (0-100) - const STABILITY_THRESHOLD = 85; - // Initialize canvas useEffect(() => { canvasRef.current = document.createElement('canvas'); @@ -343,7 +344,7 @@ export function useDocumentDetection( fingerprint, lastFingerprintRef.current ); - const isSimilarFrame = similarity >= STABILITY_THRESHOLD; + const isSimilarFrame = similarity >= mergedConfig.stabilityThreshold; if (isSimilarFrame && lastFingerprintRef.current.length > 0) { if (!stabilityStartRef.current) { diff --git a/tsup.config.ts b/tsup.config.ts index cb0d9348..d1c81c8e 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -4,9 +4,9 @@ export default defineConfig({ entry: { index: 'src/index.ts', 'ag-grid': 'src/ag-grid.ts', - 'datavis': 'src/datavis.ts', - 'esheet': 'src/esheet.ts', - 'kerebron': 'src/kerebron.ts', + datavis: 'src/datavis.ts', + esheet: 'src/esheet.ts', + kerebron: 'src/kerebron.ts', 'hooks/index': 'src/hooks/index.ts', 'utils/index': 'src/utils/index.ts', 'tailwind-preset': 'src/tailwind-preset.ts', @@ -21,9 +21,11 @@ export default defineConfig({ 'components/Breadcrumb/index': 'src/components/Breadcrumb/index.ts', 'components/Button/index': 'src/components/Button/index.ts', 'components/Card/index': 'src/components/Card/index.ts', + 'components/CardCapture/index': 'src/components/CardCapture/index.ts', 'components/Checkbox/index': 'src/components/Checkbox/index.ts', 'components/Collapsible/index': 'src/components/Collapsible/index.ts', - 'components/CountryCodeDropdown/index': 'src/components/CountryCodeDropdown/index.ts', + 'components/CountryCodeDropdown/index': + 'src/components/CountryCodeDropdown/index.ts', 'components/DateInput/index': 'src/components/DateInput/index.ts', 'components/Dropdown/index': 'src/components/Dropdown/index.ts', 'components/FloatingWindow/index': 'src/components/FloatingWindow/index.ts', @@ -71,6 +73,7 @@ export default defineConfig({ external: [ 'react', 'react-dom', + 'onnxruntime-web', 'ag-grid-community', 'ag-grid-react', '@mieweb/ui',