From 4758e65ff4f7853bdaccc99fa48843f317a8fd70 Mon Sep 17 00:00:00 2001 From: Ohm Date: Mon, 17 Aug 2026 10:52:57 -0500 Subject: [PATCH 01/38] refactor: migrate PhishingController data fetching to PhishingDataService Extract all remote data fetching from PhishingController into a new PhishingDataService built on the BaseDataService pattern. Removes CacheManager in favor of the service's built-in caching, persistence, and request coalescing. Co-Authored-By: Claude Fable 5 --- eslint-suppressions.json | 19 +- packages/phishing-controller/CHANGELOG.md | 23 + packages/phishing-controller/package.json | 5 + .../src/BulkTokenScan.test.ts | 77 +- .../src/CacheManager.test.ts | 200 ----- .../phishing-controller/src/CacheManager.ts | 210 ----- .../src/PhishingController.test.ts | 242 ++++-- .../src/PhishingController.ts | 820 +++++------------- ...PhishingDataService-method-action-types.ts | 140 +++ .../src/PhishingDataService.test.ts | 712 +++++++++++++++ .../src/PhishingDataService.ts | 789 +++++++++++++++++ packages/phishing-controller/src/index.ts | 28 +- packages/phishing-controller/src/types.ts | 174 ++++ .../phishing-controller/src/utils.test.ts | 167 ---- packages/phishing-controller/src/utils.ts | 69 +- .../phishing-controller/tsconfig.build.json | 12 +- packages/phishing-controller/tsconfig.json | 12 +- yarn.lock | 5 + 18 files changed, 2306 insertions(+), 1398 deletions(-) delete mode 100644 packages/phishing-controller/src/CacheManager.test.ts delete mode 100644 packages/phishing-controller/src/CacheManager.ts create mode 100644 packages/phishing-controller/src/PhishingDataService-method-action-types.ts create mode 100644 packages/phishing-controller/src/PhishingDataService.test.ts create mode 100644 packages/phishing-controller/src/PhishingDataService.ts diff --git a/eslint-suppressions.json b/eslint-suppressions.json index c32201a64ed..bc1d68890ba 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1559,16 +1559,6 @@ "count": 1 } }, - "packages/phishing-controller/src/CacheManager.test.ts": { - "@typescript-eslint/explicit-function-return-type": { - "count": 2 - } - }, - "packages/phishing-controller/src/CacheManager.ts": { - "@typescript-eslint/naming-convention": { - "count": 3 - } - }, "packages/phishing-controller/src/PathTrie.ts": { "@typescript-eslint/explicit-function-return-type": { "count": 2 @@ -1576,10 +1566,7 @@ }, "packages/phishing-controller/src/PhishingController.ts": { "@typescript-eslint/explicit-function-return-type": { - "count": 14 - }, - "@typescript-eslint/naming-convention": { - "count": 1 + "count": 11 }, "@typescript-eslint/prefer-nullish-coalescing": { "count": 6 @@ -1610,7 +1597,7 @@ }, "packages/phishing-controller/src/utils.ts": { "@typescript-eslint/explicit-function-return-type": { - "count": 5 + "count": 4 }, "@typescript-eslint/prefer-nullish-coalescing": { "count": 1 @@ -2343,4 +2330,4 @@ "count": 10 } } -} +} \ No newline at end of file diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 66ab0fe4ea1..32d4547c1a5 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -7,10 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) + - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` + - Requests are wrapped in a shared retry policy, configurable via the `policyOptions` constructor option; circuit breaking is disabled by default because the service spans four independent API hosts and a broken circuit caused by one host would pause phishing-list updates from the others + - Scan results are cached per URL hostname, token, and address for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs); bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached + - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions and an `init` call during client initialization (automatic with `@metamask/wallet`); pass `persistenceConfig: null` to disable + +- Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys + ### Changed +- **BREAKING:** `PhishingController` no longer performs network requests directly; a `PhishingDataService` must be registered and its method actions delegated to the controller's messenger + - `PhishingControllerMessenger` now requires the `PhishingDataService` method actions listed above as allowed actions +- **BREAKING:** Remove the `urlScanCache`, `tokenScanCache`, and `addressScanCache` properties from `PhishingControllerState`; scan results are now cached (and persisted) by `PhishingDataService`'s query cache + - Client state migrations should remove these properties from persisted `PhishingController` state +- **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` +- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call +- `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` +- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through - Bump `@metamask/transaction-controller` from `^69.4.0` to `^69.5.2` ([#9780](https://github.com/MetaMask/core/pull/9780), [#9798](https://github.com/MetaMask/core/pull/9798), [#9823](https://github.com/MetaMask/core/pull/9823)) +### Removed + +- **BREAKING:** Remove the `CacheEntry` type; the custom cache manager has been replaced by `PhishingDataService`'s query cache +- **BREAKING:** Remove the `DEFAULT_URL_SCAN_CACHE_TTL`, `DEFAULT_URL_SCAN_CACHE_MAX_SIZE`, `DEFAULT_TOKEN_SCAN_CACHE_TTL`, `DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE`, `DEFAULT_ADDRESS_SCAN_CACHE_TTL`, and `DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE` constants + ## [17.3.1] ### Changed diff --git a/packages/phishing-controller/package.json b/packages/phishing-controller/package.json index 179d6998abb..ed266366c2d 100644 --- a/packages/phishing-controller/package.json +++ b/packages/phishing-controller/package.json @@ -57,10 +57,15 @@ "dependencies": { "@metamask/address-book-controller": "^7.1.2", "@metamask/base-controller": "^9.1.0", + "@metamask/base-data-service": "^0.1.3", "@metamask/controller-utils": "^12.3.0", "@metamask/messenger": "^2.0.0", + "@metamask/storage-service": "^1.0.2", + "@metamask/superstruct": "^3.4.1", "@metamask/transaction-controller": "^69.5.2", + "@metamask/utils": "^11.11.0", "@noble/hashes": "^1.8.0", + "@tanstack/query-core": "^4.43.0", "@types/punycode": "^2.1.0", "ethereum-cryptography": "^2.1.2", "fastest-levenshtein": "^1.0.16", diff --git a/packages/phishing-controller/src/BulkTokenScan.test.ts b/packages/phishing-controller/src/BulkTokenScan.test.ts index 7f2ab12fc82..b012582d680 100644 --- a/packages/phishing-controller/src/BulkTokenScan.test.ts +++ b/packages/phishing-controller/src/BulkTokenScan.test.ts @@ -1,4 +1,3 @@ -import { safelyExecuteWithTimeout } from '@metamask/controller-utils'; import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; import type { MessengerActions, @@ -16,19 +15,11 @@ import type { PhishingControllerMessenger, PhishingControllerOptions, } from './PhishingController.js'; +import { PhishingDataService } from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; import { TokenScanResultType } from './types.js'; import type { BulkTokenScanRequest, TokenScanApiResponse } from './types.js'; -jest.mock('@metamask/controller-utils', () => ({ - ...jest.requireActual('@metamask/controller-utils'), - safelyExecuteWithTimeout: jest.fn(), -})); - -const mockSafelyExecuteWithTimeout = - safelyExecuteWithTimeout as jest.MockedFunction< - typeof safelyExecuteWithTimeout - >; - const controllerName = 'PhishingController'; type AllPhishingControllerActions = @@ -38,11 +29,13 @@ type AllPhishingControllerEvents = MessengerEvents; type RootMessenger = Messenger< MockAnyNamespace, - AllPhishingControllerActions, - AllPhishingControllerEvents, + AllPhishingControllerActions | MessengerActions, + AllPhishingControllerEvents | MessengerEvents, RootMessenger >; +const createdDataServices: PhishingDataService[] = []; + /** * Creates and returns a root messenger for testing * @@ -55,7 +48,8 @@ function getRootMessenger(): RootMessenger { } /** - * Constructs a messenger with transaction events enabled. + * Constructs a messenger with transaction events enabled, plus a real + * PhishingDataService so that tests exercise the full request path via nock. * * @returns A restricted messenger that can listen to TransactionController events. */ @@ -72,8 +66,34 @@ function getMessengerWithTransactionEvents() { parent: rootMessenger, }); + const dataServiceMessenger = new Messenger< + 'PhishingDataService', + MessengerActions, + MessengerEvents, + RootMessenger + >({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + createdDataServices.push( + new PhishingDataService({ + messenger: dataServiceMessenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + }), + ); + rootMessenger.delegate({ - actions: [], + actions: [ + 'PhishingDataService:getStalelist', + 'PhishingDataService:getHotlistDiffs', + 'PhishingDataService:getC2DomainBlocklist', + 'PhishingDataService:scanUrl', + 'PhishingDataService:bulkScanUrls', + 'PhishingDataService:bulkScanTokens', + 'PhishingDataService:scanAddress', + 'PhishingDataService:getApprovals', + ], events: ['TransactionController:stateChange'], messenger, }); @@ -105,21 +125,15 @@ describe('PhishingController - Bulk Token Scanning', () => { controller = getPhishingController(); consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(); - - // Reset the mock to its default behavior (pass through to real implementation) - mockSafelyExecuteWithTimeout.mockImplementation( - (fn, throwOnTimeout, timeout) => { - return jest - .requireActual('@metamask/controller-utils') - .safelyExecuteWithTimeout(fn, throwOnTimeout, timeout); - }, - ); }); afterEach(() => { cleanAll(); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); + while (createdDataServices.length > 0) { + createdDataServices.pop()?.destroy(); + } }); describe('bulkScanTokens', () => { @@ -420,22 +434,31 @@ describe('PhishingController - Bulk Token Scanning', () => { }); it('should handle API timeout and return empty results', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); const tokens = ['0x1234567890123456789012345678901234567890']; - // Mock safelyExecuteWithTimeout to return null (simulating a timeout) - mockSafelyExecuteWithTimeout.mockResolvedValueOnce(null); + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .delayConnection(10000) + .reply(200, { results: {} }); const request: BulkTokenScanRequest = { chainId: '0x1', tokens, }; - const result = await controller.bulkScanTokens(request); + const promise = controller.bulkScanTokens(request); + jest.advanceTimersByTime(8000); + const result = await promise; expect(result).toStrictEqual({}); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Error scanning tokens: timeout of 8000ms exceeded', ); + jest.useRealTimers(); }); }); diff --git a/packages/phishing-controller/src/CacheManager.test.ts b/packages/phishing-controller/src/CacheManager.test.ts deleted file mode 100644 index 5bbf8c92cb5..00000000000 --- a/packages/phishing-controller/src/CacheManager.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { CacheManager } from './CacheManager.js'; -import * as utils from './utils.js'; - -describe('CacheManager', () => { - let updateStateSpy: jest.Mock; - let cache: CacheManager<{ value: string }>; - - beforeEach(() => { - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); - jest - .spyOn(utils, 'fetchTimeNow') - .mockImplementation(() => Math.floor(Date.now() / 1000)); - updateStateSpy = jest.fn(); - cache = new CacheManager<{ value: string }>({ - cacheTTL: 300, // 5 minutes - maxCacheSize: 3, - updateState: updateStateSpy, - }); - }); - - afterEach(() => { - jest.useRealTimers(); - jest.restoreAllMocks(); - }); - - describe('constructor', () => { - it('should initialize with empty cache when no initialCache provided', () => { - const emptyCache = new CacheManager<{ value: string }>({ - // eslint-disable-next-line no-empty-function - updateState: () => {}, - }); - expect(emptyCache.get('test-key')).toBeUndefined(); - }); - - it('should initialize with provided initialCache data', () => { - const now = Math.floor(Date.now() / 1000); - const initialCache = { - 'test-key': { - data: { value: 'test-value' }, - timestamp: now, - }, - }; - - const cacheWithInitialData = new CacheManager<{ value: string }>({ - initialCache, - // eslint-disable-next-line no-empty-function - updateState: () => {}, - }); - - expect(cacheWithInitialData.get('test-key')).toStrictEqual({ - value: 'test-value', - }); - }); - }); - - describe('get', () => { - it('should return undefined for non-existent keys', () => { - expect(cache.get('non-existent')).toBeUndefined(); - }); - - it('should return data for existing keys', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.get('key1')).toStrictEqual({ value: 'value1' }); - }); - - it('should return undefined for expired entries', () => { - cache.set('key1', { value: 'value1' }); - - // Fast forward time past TTL - jest.advanceTimersByTime(301 * 1000); - - expect(cache.get('key1')).toBeUndefined(); - }); - }); - - describe('set', () => { - it('should add new entries', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.get('key1')).toStrictEqual({ value: 'value1' }); - }); - - it('should update existing entries', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key1', { value: 'updated-value' }); - expect(cache.get('key1')).toStrictEqual({ value: 'updated-value' }); - }); - - it('should call updateState when adding entries', () => { - cache.set('key1', { value: 'value1' }); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - - it('should evict oldest entries when cache exceeds max size', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.set('key3', { value: 'value3' }); - cache.set('key4', { value: 'value4' }); // This should evict key1 - - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toStrictEqual({ value: 'value2' }); - expect(cache.get('key3')).toStrictEqual({ value: 'value3' }); - expect(cache.get('key4')).toStrictEqual({ value: 'value4' }); - }); - }); - - describe('delete', () => { - it('should remove entries', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.delete('key1')).toBe(true); - expect(cache.get('key1')).toBeUndefined(); - }); - - it('should return false when deleting non-existent keys', () => { - expect(cache.delete('non-existent')).toBe(false); - }); - - it('should call updateState when deleting entries', () => { - cache.set('key1', { value: 'value1' }); - updateStateSpy.mockClear(); - cache.delete('key1'); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - }); - - describe('clear', () => { - it('should remove all entries', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.clear(); - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toBeUndefined(); - }); - - it('should call updateState', () => { - cache.set('key1', { value: 'value1' }); - updateStateSpy.mockClear(); - cache.clear(); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - }); - - describe('setTTL', () => { - it('should update the TTL', () => { - cache.setTTL(600); - expect(cache.getTTL()).toBe(600); - }); - }); - - describe('setMaxSize', () => { - it('should update the max size', () => { - cache.setMaxSize(5); - expect(cache.getMaxSize()).toBe(5); - }); - - it('should evict entries if new size is smaller than current cache size', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.set('key3', { value: 'value3' }); - cache.setMaxSize(2); // This should evict key1 - - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toStrictEqual({ value: 'value2' }); - expect(cache.get('key3')).toStrictEqual({ value: 'value3' }); - }); - }); - - describe('getSize', () => { - it('should return the current cache size', () => { - expect(cache.getSize()).toBe(0); - cache.set('key1', { value: 'value1' }); - expect(cache.getSize()).toBe(1); - cache.set('key2', { value: 'value2' }); - expect(cache.getSize()).toBe(2); - cache.delete('key1'); - expect(cache.getSize()).toBe(1); - }); - }); - - describe('keys', () => { - it('should return all cache keys', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - expect(cache.keys()).toStrictEqual(['key1', 'key2']); - }); - }); - - describe('getAllEntries', () => { - it('should return all cache entries', () => { - const now = Math.floor(Date.now() / 1000); - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - const entries = cache.getAllEntries(); - expect(Object.keys(entries)).toStrictEqual(['key1', 'key2']); - expect(entries.key1.data).toStrictEqual({ value: 'value1' }); - expect(entries.key2.data).toStrictEqual({ value: 'value2' }); - expect(entries.key1.timestamp).toBeGreaterThanOrEqual(now); - expect(entries.key2.timestamp).toBeGreaterThanOrEqual(now); - }); - }); -}); diff --git a/packages/phishing-controller/src/CacheManager.ts b/packages/phishing-controller/src/CacheManager.ts deleted file mode 100644 index 9dd4b256353..00000000000 --- a/packages/phishing-controller/src/CacheManager.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { fetchTimeNow } from './utils.js'; - -/** - * Generic cache entry type that wraps the data with a timestamp - */ -export type CacheEntry = { - data: T; - timestamp: number; -}; - -/** - * Configuration options for CacheManager - */ -export type CacheManagerOptions = { - cacheTTL?: number; - maxCacheSize?: number; - initialCache?: Record>; - updateState: (cache: Record>) => void; -}; - -/** - * Generic cache manager with TTL and size limit support - * - * @template T - The type of data to cache - */ -export class CacheManager { - #cacheTTL: number; - - #maxCacheSize: number; - - readonly #cache: Map>; - - readonly #updateState: (cache: Record>) => void; - - /** - * Constructor for CacheManager - * - * @param options - Cache configuration options - * @param options.cacheTTL - Time to live in seconds for cached entries - * @param options.maxCacheSize - Maximum number of entries in the cache - * @param options.initialCache - Initial cache state - * @param options.updateState - Function to update the state when cache changes - */ - constructor({ - cacheTTL = 300, // 5 minutes default - maxCacheSize = 100, - initialCache = {}, - updateState, - }: CacheManagerOptions) { - this.#cacheTTL = cacheTTL; - this.#maxCacheSize = maxCacheSize; - this.#cache = new Map(Object.entries(initialCache)); - this.#updateState = updateState; - this.#evictEntries(); - } - - /** - * Set the time-to-live for cached entries - * - * @param ttl - The TTL in seconds - */ - setTTL(ttl: number): void { - this.#cacheTTL = ttl; - } - - /** - * Get the current TTL setting - * - * @returns The TTL in seconds - */ - getTTL(): number { - return this.#cacheTTL; - } - - /** - * Set the maximum cache size - * - * @param maxSize - The maximum cache size - */ - setMaxSize(maxSize: number): void { - this.#maxCacheSize = maxSize; - this.#evictEntries(); - } - - /** - * Get the current maximum cache size - * - * @returns The maximum cache size - */ - getMaxSize(): number { - return this.#maxCacheSize; - } - - /** - * Get the current cache size - * - * @returns The current number of entries in the cache - */ - getSize(): number { - return this.#cache.size; - } - - /** - * Clear the cache - */ - clear(): void { - this.#cache.clear(); - this.#persistCache(); - } - - /** - * Get a cached result if it exists and is not expired - * - * @param key - The cache key - * @returns The cached data or undefined if not found or expired - */ - get(key: string): T | undefined { - const cacheEntry = this.#cache.get(key); - if (!cacheEntry) { - return undefined; - } - - // Check if the entry is expired - const now = fetchTimeNow(); - if (now - cacheEntry.timestamp > this.#cacheTTL) { - // Entry expired, remove it from cache - this.#cache.delete(key); - this.#persistCache(); - return undefined; - } - - return cacheEntry.data; - } - - /** - * Add an entry to the cache, evicting oldest entries if necessary - * - * @param key - The cache key - * @param data - The data to cache - */ - set(key: string, data: T): void { - this.#cache.set(key, { - data, - timestamp: fetchTimeNow(), - }); - - this.#evictEntries(); - this.#persistCache(); - } - - /** - * Delete a specific entry from the cache - * - * @param key - The cache key - * @returns True if an entry was deleted - */ - delete(key: string): boolean { - const result = this.#cache.delete(key); - if (result) { - this.#persistCache(); - } - return result; - } - - /** - * Get all keys in the cache - * - * @returns Array of cache keys - */ - keys(): string[] { - return Array.from(this.#cache.keys()); - } - - /** - * Get all entries in the cache (including expired ones) - * Useful for debugging or persistence - * - * @returns Record of all cache entries - */ - getAllEntries(): Record> { - return Object.fromEntries(this.#cache); - } - - /** - * Persist the current cache state - */ - #persistCache(): void { - this.#updateState(Object.fromEntries(this.#cache)); - } - - /** - * Evict oldest entries if cache exceeds max size - */ - #evictEntries(): void { - if (this.#cache.size <= this.#maxCacheSize) { - return; - } - - const entriesToRemove = this.#cache.size - this.#maxCacheSize; - let count = 0; - // Delete the oldest entries (Map maintains insertion order) - for (const key of this.#cache.keys()) { - if (count >= entriesToRemove) { - break; - } - this.#cache.delete(key); - count += 1; - } - } -} diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index d7437aa8a7c..80734ed321d 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -18,6 +18,10 @@ import { ListNames, METAMASK_HOTLIST_DIFF_FILE, METAMASK_STALELIST_FILE, + METAMASK_HOTLIST_DIFF_URL, + METAMASK_STALELIST_URL, + C2_DOMAIN_BLOCKLIST_URL, + phishingListKeyNameMap, PhishingController, PHISHING_CONFIG_BASE_URL, CLIENT_SIDE_DETECION_BASE_URL, @@ -34,6 +38,11 @@ import type { BulkPhishingDetectionScanResponse, PhishingControllerMessenger, } from './PhishingController.js'; +import { + PhishingDataService, + SCAN_RESULT_STALE_TIME, +} from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; import { createMockStateChangePayload, createMockTransaction, @@ -62,11 +71,61 @@ type AllPhishingControllerEvents = MessengerEvents; type RootMessenger = Messenger< MockAnyNamespace, - AllPhishingControllerActions, - AllPhishingControllerEvents, + AllPhishingControllerActions | MessengerActions, + AllPhishingControllerEvents | MessengerEvents, RootMessenger >; +const PHISHING_DATA_SERVICE_ACTIONS = [ + 'PhishingDataService:getStalelist', + 'PhishingDataService:getHotlistDiffs', + 'PhishingDataService:getC2DomainBlocklist', + 'PhishingDataService:scanUrl', + 'PhishingDataService:bulkScanUrls', + 'PhishingDataService:bulkScanTokens', + 'PhishingDataService:scanAddress', + 'PhishingDataService:getApprovals', +] as const; + +const createdDataServices: PhishingDataService[] = []; + +/** + * Destroys all data services created during the current test, releasing their + * query cache resources. + */ +function destroyDataServices(): void { + while (createdDataServices.length > 0) { + createdDataServices.pop()?.destroy(); + } +} + +/** + * Constructs a real PhishingDataService wired to the given root messenger, so + * that controller tests exercise the full request path via nock. + * + * @param rootMessenger - The root messenger. + * @returns The data service. + */ +function setupDataService(rootMessenger: RootMessenger): PhishingDataService { + const dataServiceMessenger = new Messenger< + 'PhishingDataService', + MessengerActions, + MessengerEvents, + RootMessenger + >({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + + const dataService = new PhishingDataService({ + messenger: dataServiceMessenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + }); + createdDataServices.push(dataService); + return dataService; +} + type SetupMessengerOptions = { transactionControllerState?: TransactionControllerState; addressBookControllerState?: AddressBookControllerState; @@ -133,10 +192,13 @@ function setupMessenger(options: SetupMessengerOptions = {}): { parent: rootMessenger, }); + setupDataService(rootMessenger); + rootMessenger.delegate({ actions: [ 'AddressBookController:getState', 'TransactionController:getState', + ...PHISHING_DATA_SERVICE_ACTIONS, ], events: [ // eslint-disable-next-line no-restricted-syntax @@ -194,6 +256,7 @@ describe('PhishingController', () => { afterEach(() => { jest.useRealTimers(); cleanAll(); + destroyDataServices(); }); it('should have no default phishing lists', () => { @@ -201,6 +264,21 @@ describe('PhishingController', () => { expect(controller.state.phishingLists).toStrictEqual([]); }); + it('re-exports API URLs and list mappings for backwards compatibility', () => { + expect(METAMASK_STALELIST_URL).toBe( + `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`, + ); + expect(METAMASK_HOTLIST_DIFF_URL).toBe( + `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`, + ); + expect(C2_DOMAIN_BLOCKLIST_URL).toBe( + `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`, + ); + expect(phishingListKeyNameMap.eth_phishing_detect_config).toBe( + ListNames.MetaMask, + ); + }); + it('should default to an empty whitelist', () => { const { controller } = getPhishingController(); expect(controller.state.whitelist).toStrictEqual([]); @@ -2833,7 +2911,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); it('should return the scan result', async () => { @@ -3088,7 +3171,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { @@ -3438,6 +3526,7 @@ describe('PhishingController', () => { // eslint-disable-next-line import-x/no-named-as-default-member expect(nock.pendingMocks()).toHaveLength(0); }); + it('should handle invalid URLs properly when mixed with valid URLs and cache results correctly', async () => { const validUrl = 'https://valid-example.com'; const invalidUrl = 'not-a-url'; @@ -3558,7 +3647,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { @@ -3981,11 +4075,17 @@ describe('PhishingController', () => { describe('URL Scan Cache', () => { beforeEach(() => { - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { jest.useRealTimers(); cleanAll(); + destroyDataServices(); }); it('should cache scan results and return them on subsequent calls', async () => { @@ -4030,9 +4130,8 @@ describe('URL Scan Cache', () => { fetchSpy.mockRestore(); }); - it('should expire cache entries after TTL', async () => { + it('should expire cache entries after the scan result stale time', async () => { const testDomain = 'example.com'; - const cacheTTL = 300; // 5 minutes nock(PHISHING_DETECTION_BASE_URL) .get( @@ -4052,25 +4151,23 @@ describe('URL Scan Cache', () => { recommendedAction: RecommendedAction.None, }); - const { rootMessenger } = getPhishingController({ - urlScanCacheTTL: cacheTTL, - }); + const { rootMessenger } = getPhishingController(); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, ); - // Before TTL expires, should use cache - jest.advanceTimersByTime((cacheTTL - 10) * 1000); + // Before the stale time elapses, should use cache + jest.advanceTimersByTime(SCAN_RESULT_STALE_TIME - 10_000); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, ); expect(pendingMocks()).toHaveLength(1); // One mock remaining - // After TTL expires, should fetch again - jest.advanceTimersByTime(11 * 1000); + // After the stale time elapses, should fetch again + jest.advanceTimersByTime(11_000); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, @@ -4078,66 +4175,6 @@ describe('URL Scan Cache', () => { expect(pendingMocks()).toHaveLength(0); // All mocks used }); - it('should evict oldest entries when cache exceeds max size', async () => { - const maxCacheSize = 2; - const domains = ['domain1.com', 'domain2.com', 'domain3.com']; - - // Setup nock to respond to all three domains - domains.forEach((domain) => { - nock(PHISHING_DETECTION_BASE_URL) - .get( - `/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent( - domain, - )}`, - ) - .reply(200, { - recommendedAction: RecommendedAction.None, - }); - }); - - // Setup a second request for the first domain - nock(PHISHING_DETECTION_BASE_URL) - .get( - `/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent( - domains[0], - )}`, - ) - .reply(200, { - recommendedAction: RecommendedAction.Warn, - }); - - const { rootMessenger } = getPhishingController({ - urlScanCacheMaxSize: maxCacheSize, - }); - - // Fill the cache - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[0]}`, - ); - jest.advanceTimersByTime(1000); // Ensure different timestamps - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[1]}`, - ); - - // This should evict the oldest entry (domain1) - jest.advanceTimersByTime(1000); - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[2]}`, - ); - - // Now domain1 should not be in cache and require a new fetch - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[0]}`, - ); - - // All mocks should be used - expect(isDone()).toBe(true); - }); - it('should handle fetch errors and not cache them', async () => { const testDomain = 'example.com'; @@ -4281,13 +4318,10 @@ describe('URL Scan Cache', () => { ), ).toMatchInlineSnapshot(` { - "addressScanCache": {}, "c2DomainBlocklistLastFetched": 0, "hotlistLastFetched": 0, "phishingLists": [], "stalelistLastFetched": 0, - "tokenScanCache": {}, - "urlScanCache": {}, "whitelist": [], "whitelistPaths": {}, } @@ -4303,13 +4337,7 @@ describe('URL Scan Cache', () => { controller.metadata, 'usedInUi', ), - ).toMatchInlineSnapshot(` - { - "addressScanCache": {}, - "tokenScanCache": {}, - "urlScanCache": {}, - } - `); + ).toMatchInlineSnapshot(`{}`); }); }); }); @@ -4335,6 +4363,7 @@ describe('Transaction Controller State Change Integration', () => { afterEach(() => { bulkScanTokensSpy.mockRestore(); + destroyDataServices(); }); it('triggers bulk token scanning when transaction with token balance changes is added', async () => { @@ -4420,6 +4449,47 @@ describe('Transaction Controller State Change Integration', () => { expect(bulkScanTokensSpy).not.toHaveBeenCalled(); }); + it('groups tokens from multiple transactions on the same chain into one scan', async () => { + const transaction1 = createMockTransaction('test-tx-1', [ + TEST_ADDRESSES.USDC, + ]); + const transaction2 = createMockTransaction('test-tx-2', [ + TEST_ADDRESSES.MOCK_TOKEN_1, + ]); + const stateChangePayload = createMockStateChangePayload([ + transaction1, + transaction2, + ]); + + globalMessenger.publish( + 'TransactionController:stateChange', + stateChangePayload, + [ + { + op: 'add' as const, + path: ['transactions', 0], + value: transaction1, + }, + { + op: 'add' as const, + path: ['transactions', 1], + value: transaction2, + }, + ], + ); + + await new Promise((resolve) => process.nextTick(resolve)); + + expect(bulkScanTokensSpy).toHaveBeenCalledTimes(1); + expect(bulkScanTokensSpy).toHaveBeenCalledWith({ + chainId: transaction1.chainId.toLowerCase(), + tokens: [ + TEST_ADDRESSES.USDC.toLowerCase(), + TEST_ADDRESSES.MOCK_TOKEN_1.toLowerCase(), + ], + }); + }); + it('does not trigger bulk token scanning when transaction has no token balance changes', async () => { const mockTransaction = createMockTransaction('test-tx-1', []); @@ -4571,6 +4641,8 @@ describe('Transaction Controller State Change Integration', () => { }); describe('Address poisoning detection', () => { + afterEach(destroyDataServices); + const ADDRESS_BOOK_RECIPIENT = '0x1234bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb5678' as `0x${string}`; const CONFIRMED_TX_RECIPIENT = diff --git a/packages/phishing-controller/src/PhishingController.ts b/packages/phishing-controller/src/PhishingController.ts index 3ec4df29c1c..e8827edc1ce 100644 --- a/packages/phishing-controller/src/PhishingController.ts +++ b/packages/phishing-controller/src/PhishingController.ts @@ -9,12 +9,9 @@ import type { ControllerGetStateAction, ControllerStateChangeEvent, } from '@metamask/base-controller'; -import { - isValidHexAddress, - safelyExecute, - safelyExecuteWithTimeout, -} from '@metamask/controller-utils'; +import { HttpError, isValidHexAddress } from '@metamask/controller-utils'; import type { Messenger } from '@metamask/messenger'; +import { getErrorMessage } from '@metamask/utils'; import type { TransactionControllerGetStateAction, TransactionControllerState, @@ -29,8 +26,6 @@ import type { Patch } from 'immer'; import { toASCII } from 'punycode/punycode.js'; import { findSimilarAddresses } from './address-poisoning.js'; -import { CacheManager } from './CacheManager.js'; -import type { CacheEntry } from './CacheManager.js'; import { convertListToTrie, insertToTrie, @@ -42,23 +37,31 @@ import type { PhishingControllerMethodActions, PhishingControllerTestOriginAction, } from './PhishingController-method-action-types.js'; +import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; import { PhishingDetector } from './PhishingDetector.js'; import { PhishingDetectorResultType, RecommendedAction, AddressScanResultType, + ListKeys, + phishingListNameKeyMap, + phishingListKeyNameMap, } from './types.js'; import type { PhishingDetectorResult, PhishingDetectionScanResult, - TokenScanCacheData, BulkTokenScanResponse, BulkTokenScanRequest, TokenScanApiResponse, - AddressScanCacheData, AddressScanResult, SimilarAddressMatch, ApprovalsResponse, + BulkPhishingDetectionScanResponse, + C2DomainBlocklistResponse, + DataResultWrapper, + Hotlist, + PhishingListState, + PhishingStalelist, } from './types.js'; import { applyDiffs, @@ -67,8 +70,6 @@ import { roundToNearestMinute, getHostnameFromWebUrl, getPhishingDetectionScanUrlParam, - buildCacheKey, - splitCacheHits, resolveChainName, getPathnameFromUrl, isAddressScanSupportedChain, @@ -76,200 +77,46 @@ import { isTokenScanSupportedChain, } from './utils.js'; -export const PHISHING_CONFIG_BASE_URL = - 'https://phishing-detection.api.cx.metamask.io'; -export const METAMASK_STALELIST_FILE = '/v1/stalelist'; -export const METAMASK_HOTLIST_DIFF_FILE = '/v2/diffsSince'; - -export const CLIENT_SIDE_DETECION_BASE_URL = - 'https://client-side-detection.api.cx.metamask.io'; -export const C2_DOMAIN_BLOCKLIST_ENDPOINT = '/v1/request-blocklist'; - -export const PHISHING_DETECTION_BASE_URL = - 'https://dapp-scanning.api.cx.metamask.io'; -export const PHISHING_DETECTION_SCAN_ENDPOINT = 'v2/scan'; -export const PHISHING_DETECTION_BULK_SCAN_ENDPOINT = 'bulk-scan'; - -export const SECURITY_ALERTS_BASE_URL = - 'https://security-alerts.api.cx.metamask.io'; -export const TOKEN_BULK_SCANNING_ENDPOINT = '/token/scan-bulk'; -export const ADDRESS_SCAN_ENDPOINT = '/address/evm/scan'; -export const APPROVALS_ENDPOINT = '/address/evm/approvals'; - -// Cache configuration defaults -export const DEFAULT_URL_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_URL_SCAN_CACHE_MAX_SIZE = 250; -export const DEFAULT_TOKEN_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE = 1000; -export const DEFAULT_ADDRESS_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE = 1000; +export { + PHISHING_CONFIG_BASE_URL, + METAMASK_STALELIST_FILE, + METAMASK_HOTLIST_DIFF_FILE, + CLIENT_SIDE_DETECION_BASE_URL, + C2_DOMAIN_BLOCKLIST_ENDPOINT, + PHISHING_DETECTION_BASE_URL, + PHISHING_DETECTION_SCAN_ENDPOINT, + PHISHING_DETECTION_BULK_SCAN_ENDPOINT, + SECURITY_ALERTS_BASE_URL, + TOKEN_BULK_SCANNING_ENDPOINT, + ADDRESS_SCAN_ENDPOINT, + APPROVALS_ENDPOINT, + METAMASK_STALELIST_URL, + METAMASK_HOTLIST_DIFF_URL, + C2_DOMAIN_BLOCKLIST_URL, +} from './PhishingDataService.js'; +export { ListKeys, ListNames, phishingListKeyNameMap } from './types.js'; +export type { + ListTypes, + EthPhishingResponse, + C2DomainBlocklistResponse, + PhishingStalelist, + PhishingListState, + HotlistDiff, + DataResultWrapper, + Hotlist, + BulkPhishingDetectionScanResponse, +} from './types.js'; export const C2_DOMAIN_BLOCKLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const HOTLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const STALELIST_REFRESH_INTERVAL = 30 * 24 * 60 * 60; // 30 days in seconds -export const METAMASK_STALELIST_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`; -export const METAMASK_HOTLIST_DIFF_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`; -export const C2_DOMAIN_BLOCKLIST_URL = `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`; - -/** - * @type ListTypes - * - * Type outlining the types of lists provided by aggregating different source lists - */ -export type ListTypes = - | 'fuzzylist' - | 'blocklist' - | 'blocklistPaths' - | 'allowlist' - | 'c2DomainBlocklist'; - -/** - * @type EthPhishingResponse - * - * Configuration response from the eth-phishing-detect package - * consisting of approved and unapproved website origins - * - * @property blacklist - List of unapproved origins - * @property fuzzylist - List of fuzzy-matched unapproved origins - * @property tolerance - Fuzzy match tolerance level - * @property version - Version number of this configuration - * @property whitelist - List of approved origins - */ -export type EthPhishingResponse = { - blacklist: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - whitelist: string[]; -}; - -/** - * @type C2DomainBlocklistResponse - * - * Response for blocklist update requests - * - * @property recentlyAdded - List of c2 domains recently added to the blocklist - * @property recentlyRemoved - List of c2 domains recently removed from the blocklist - * @property lastFetchedAt - Timestamp of the last fetch request - */ -export type C2DomainBlocklistResponse = { - recentlyAdded: string[]; - recentlyRemoved: string[]; - lastFetchedAt: string; -}; - -/** - * PhishingStalelist defines the expected type of the stalelist from the API. - * - * allowlist - List of approved origins. - * blocklist - List of unapproved origins (hostname-only entries). - * blocklistPaths - Trie of unapproved origins with paths (hostname + path entries). - * fuzzylist - List of fuzzy-matched unapproved origins. - * tolerance - Fuzzy match tolerance level - * lastUpdated - Timestamp of last update. - * version - Stalelist data structure iteration. - */ -export type PhishingStalelist = { - allowlist: string[]; - blocklist: string[]; - blocklistPaths: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - lastUpdated: number; -}; - -/** - * @type PhishingListState - * - * type defining the persisted list state. This is the persisted state that is updated frequently with `this.maybeUpdateState()`. - * - * @property allowlist - List of approved origins (legacy naming "whitelist") - * @property blocklist - List of unapproved origins (legacy naming "blacklist") - * @property blocklistPaths - Trie of unapproved origins with paths (hostname + path, no query params). - * @property c2DomainBlocklist - List of hashed hostnames that C2 requests are blocked against. - * @property fuzzylist - List of fuzzy-matched unapproved origins - * @property tolerance - Fuzzy match tolerance level - * @property lastUpdated - Timestamp of last update. - * @property version - Version of the phishing list state. - * @property name - Name of the list. Used for attribution. - */ -export type PhishingListState = { - allowlist: string[]; - blocklist: string[]; - blocklistPaths: PathTrie; - c2DomainBlocklist: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - lastUpdated: number; - name: ListNames; -}; - -/** - * @type HotlistDiff - * - * type defining the expected type of the diffs in hotlist.json file. - * - * @property url - Url of the diff entry. - * @property timestamp - Timestamp at which the diff was identified. - * @property targetList - The list name where the diff was identified. - * @property isRemoval - Was the diff identified a removal type. - */ -export type HotlistDiff = { - url: string; - timestamp: number; - targetList: `${ListKeys}.${ListTypes}`; - isRemoval?: boolean; -}; - -export type DataResultWrapper = { - data: T; -}; - -/** - * @type Hotlist - * - * Type defining expected hotlist.json file. - * - * @property url - Url of the diff entry. - * @property timestamp - Timestamp at which the diff was identified. - * @property targetList - The list name where the diff was identified. - * @property isRemoval - Was the diff identified a removal type. - */ -export type Hotlist = HotlistDiff[]; - -/** - * Enum containing upstream data provider source list keys. - * These are the keys denoting lists consumed by the upstream data provider. - */ -export enum ListKeys { - EthPhishingDetectConfig = 'eth_phishing_detect_config', -} - -/** - * Enum containing downstream client attribution names. - */ -export enum ListNames { - MetaMask = 'MetaMask', -} - -/** - * Maps from downstream client attribution name - * to list key sourced from upstream data provider. - */ -const phishingListNameKeyMap = { - [ListNames.MetaMask]: ListKeys.EthPhishingDetectConfig, -}; - -/** - * Maps from list key sourced from upstream data - * provider to downstream client attribution name. - */ -export const phishingListKeyNameMap = { - [ListKeys.EthPhishingDetectConfig]: ListNames.MetaMask, -}; +// Request timeouts, in milliseconds. +const URL_SCAN_TIMEOUT = 8000; +const BULK_URL_SCAN_TIMEOUT = 15000; +const TOKEN_SCAN_TIMEOUT = 8000; +const ADDRESS_SCAN_TIMEOUT = 5000; +const APPROVALS_TIMEOUT = 5000; const controllerName = 'PhishingController'; @@ -310,24 +157,6 @@ const metadata: StateMetadata = { includeInDebugSnapshot: false, usedInUi: false, }, - urlScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, - tokenScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, - addressScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, }; /** @@ -343,9 +172,6 @@ const getDefaultState = (): PhishingControllerState => { hotlistLastFetched: 0, stalelistLastFetched: 0, c2DomainBlocklistLastFetched: 0, - urlScanCache: {}, - tokenScanCache: {}, - addressScanCache: {}, }; }; @@ -359,9 +185,6 @@ const getDefaultState = (): PhishingControllerState => { * hotlistLastFetched - timestamp of the last hotlist fetch * stalelistLastFetched - timestamp of the last stalelist fetch * c2DomainBlocklistLastFetched - timestamp of the last c2 domain blocklist fetch - * urlScanCache - cache of URL scan results - * tokenScanCache - cache of token scan results - * addressScanCache - cache of address scan results */ export type PhishingControllerState = { phishingLists: PhishingListState[]; @@ -370,9 +193,6 @@ export type PhishingControllerState = { hotlistLastFetched: number; stalelistLastFetched: number; c2DomainBlocklistLastFetched: number; - urlScanCache: Record>; - tokenScanCache: Record>; - addressScanCache: Record>; }; /** @@ -382,23 +202,11 @@ export type PhishingControllerState = { * stalelistRefreshInterval - Polling interval used to fetch stale list. * hotlistRefreshInterval - Polling interval used to fetch hotlist diff list. * c2DomainBlocklistRefreshInterval - Polling interval used to fetch c2 domain blocklist. - * urlScanCacheTTL - Time to live in seconds for cached scan results. - * urlScanCacheMaxSize - Maximum number of entries in the scan cache. - * tokenScanCacheTTL - Time to live in seconds for cached token scan results. - * tokenScanCacheMaxSize - Maximum number of entries in the token scan cache. - * addressScanCacheTTL - Time to live in seconds for cached address scan results. - * addressScanCacheMaxSize - Maximum number of entries in the address scan cache. */ export type PhishingControllerOptions = { stalelistRefreshInterval?: number; hotlistRefreshInterval?: number; c2DomainBlocklistRefreshInterval?: number; - urlScanCacheTTL?: number; - urlScanCacheMaxSize?: number; - tokenScanCacheTTL?: number; - tokenScanCacheMaxSize?: number; - addressScanCacheTTL?: number; - addressScanCacheMaxSize?: number; messenger: PhishingControllerMessenger; state?: Partial; }; @@ -447,7 +255,8 @@ export type PhishingControllerEvents = PhishingControllerStateChangeEvent; */ type AllowedActions = | AddressBookControllerGetStateAction - | TransactionControllerGetStateAction; + | TransactionControllerGetStateAction + | PhishingDataServiceMethodActions; /** * The external events available to the PhishingController. @@ -462,19 +271,6 @@ export type PhishingControllerMessenger = Messenger< PhishingControllerEvents | AllowedEvents >; -/** - * BulkPhishingDetectionScanResponse - * - * Response for bulk phishing detection scan requests - * results - Record of domain names and their corresponding phishing detection scan results - * - * errors - Record of domain names and their corresponding errors - */ -export type BulkPhishingDetectionScanResponse = { - results: Record; - errors: Record; -}; - /** * Controller that manages community-maintained lists of approved and unapproved website origins. */ @@ -493,12 +289,6 @@ export class PhishingController extends BaseController< readonly #c2DomainBlocklistRefreshInterval: number; - readonly #urlScanCache: CacheManager; - - readonly #tokenScanCache: CacheManager; - - readonly #addressScanCache: CacheManager; - readonly #knownRecipients: Set; readonly #transactionRecipients: Set; @@ -531,12 +321,6 @@ export class PhishingController extends BaseController< * @param config.stalelistRefreshInterval - Polling interval used to fetch stale list. * @param config.hotlistRefreshInterval - Polling interval used to fetch hotlist diff list. * @param config.c2DomainBlocklistRefreshInterval - Polling interval used to fetch c2 domain blocklist. - * @param config.urlScanCacheTTL - Time to live in seconds for cached scan results. - * @param config.urlScanCacheMaxSize - Maximum number of entries in the scan cache. - * @param config.tokenScanCacheTTL - Time to live in seconds for cached token scan results. - * @param config.tokenScanCacheMaxSize - Maximum number of entries in the token scan cache. - * @param config.addressScanCacheTTL - Time to live in seconds for cached address scan results. - * @param config.addressScanCacheMaxSize - Maximum number of entries in the address scan cache. * @param config.messenger - The controller restricted messenger. * @param config.state - Initial state to set on this controller. */ @@ -544,12 +328,6 @@ export class PhishingController extends BaseController< stalelistRefreshInterval = STALELIST_REFRESH_INTERVAL, hotlistRefreshInterval = HOTLIST_REFRESH_INTERVAL, c2DomainBlocklistRefreshInterval = C2_DOMAIN_BLOCKLIST_REFRESH_INTERVAL, - urlScanCacheTTL = DEFAULT_URL_SCAN_CACHE_TTL, - urlScanCacheMaxSize = DEFAULT_URL_SCAN_CACHE_MAX_SIZE, - tokenScanCacheTTL = DEFAULT_TOKEN_SCAN_CACHE_TTL, - tokenScanCacheMaxSize = DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE, - addressScanCacheTTL = DEFAULT_ADDRESS_SCAN_CACHE_TTL, - addressScanCacheMaxSize = DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE, messenger, state = {}, }: PhishingControllerOptions) { @@ -575,36 +353,6 @@ export class PhishingController extends BaseController< this.#onTransactionControllerStateChange.bind(this); this.#addressBookControllerStateChangeHandler = this.#onAddressBookControllerStateChange.bind(this); - this.#urlScanCache = new CacheManager({ - cacheTTL: urlScanCacheTTL, - maxCacheSize: urlScanCacheMaxSize, - initialCache: this.state.urlScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.urlScanCache = cache; - }); - }, - }); - this.#tokenScanCache = new CacheManager({ - cacheTTL: tokenScanCacheTTL, - maxCacheSize: tokenScanCacheMaxSize, - initialCache: this.state.tokenScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.tokenScanCache = cache; - }); - }, - }); - this.#addressScanCache = new CacheManager({ - cacheTTL: addressScanCacheTTL, - maxCacheSize: addressScanCacheMaxSize, - initialCache: this.state.addressScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.addressScanCache = cache; - }); - }, - }); this.messenger.registerMethodActionHandlers( this, @@ -1231,58 +979,24 @@ export class PhishingController extends BaseController< const [hostname] = getHostnameFromWebUrl(url); - const cachedResult = this.#urlScanCache.get(scanUrlParam); - if (cachedResult) { - return cachedResult; - } - - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent(scanUrlParam)}`, - { - method: 'GET', - headers: { - Accept: 'application/json', - }, - }, - ); - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - }; - } - const data = await res.json(); - return data; - }, - true, - 8000, - ); - - // Need to do it this way because safelyExecuteWithTimeout returns undefined for both timeouts and errors. - if (!apiResponse) { - return { - hostname: '', - recommendedAction: RecommendedAction.None, - fetchError: 'timeout of 8000ms exceeded', - }; - } else if ((apiResponse as { error?: string }).error) { + let scanResult: PhishingDetectionScanResult; + try { + scanResult = await this.#callWithTimeout( + this.messenger.call('PhishingDataService:scanUrl', scanUrlParam), + URL_SCAN_TIMEOUT, + ); + } catch (error) { return { hostname: '', recommendedAction: RecommendedAction.None, - fetchError: (apiResponse as { error: string }).error, + fetchError: getErrorMessage(error), }; } - const scanResult = apiResponse as PhishingDetectionScanResult; - const result = { + return { hostname, recommendedAction: scanResult.recommendedAction, }; - - this.#urlScanCache.set(scanUrlParam, result); - - return result; } /** @@ -1321,8 +1035,7 @@ export class PhishingController extends BaseController< errors: {}, }; - // Extract hostnames from URLs and check for validity and length constraints - const urlsToHostnames: Record = {}; + // Check URLs for validity and length constraints const urlsToFetch: string[] = []; for (const url of urls) { @@ -1333,22 +1046,13 @@ export class PhishingController extends BaseController< continue; } - const [hostname, ok] = getHostnameFromWebUrl(url); + const [, ok] = getHostnameFromWebUrl(url); if (!ok) { combinedResponse.errors[url] = ['url is not a valid web URL']; continue; } - // Check if result is already in cache - const cachedResult = this.#urlScanCache.get(hostname); - if (cachedResult) { - // Use cached result - combinedResponse.results[url] = cachedResult; - } else { - // Add to list of URLs to fetch - urlsToHostnames[url] = hostname; - urlsToFetch.push(url); - } + urlsToFetch.push(url); } // If there are URLs to fetch, process them in batches @@ -1367,12 +1071,7 @@ export class PhishingController extends BaseController< // Merge results and errors from all batches batchResults.forEach((batchResponse) => { - // Add results to cache and combine with response Object.entries(batchResponse.results).forEach(([url, result]) => { - const hostname = urlsToHostnames[url]; - if (hostname) { - this.#urlScanCache.set(hostname, result); - } combinedResponse.results[url] = result; }); @@ -1400,55 +1099,23 @@ export class PhishingController extends BaseController< chain: string, tokens: string[], ): Promise => { - const timeout = 8000; // 8 seconds - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const response = await fetch( - `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - tokens, - }), - }, - ); - - if (!response.ok) { - return { - error: `${response.status} ${response.statusText}`, - status: response.status, - statusText: response.statusText, - }; - } - - const data = await response.json(); - return data; - }, - true, - timeout, - ); - - if (!apiResponse) { - console.error(`Error scanning tokens: timeout of ${timeout}ms exceeded`); - return null; - } - - if ((apiResponse as { error?: string }).error) { - const { status, statusText } = apiResponse as { - status: number; - statusText: string; - }; - - console.warn(`Token bulk screening API error: ${status} ${statusText}`); + try { + return await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:bulkScanTokens', + chain, + tokens, + ), + TOKEN_SCAN_TIMEOUT, + ); + } catch (error) { + if (error instanceof HttpError) { + console.warn(`Token bulk screening API error: ${error.message}`); + } else { + console.error(`Error scanning tokens: ${getErrorMessage(error)}`); + } return null; } - - return apiResponse as TokenScanApiResponse; }; /** @@ -1480,67 +1147,25 @@ export class PhishingController extends BaseController< }; } - const cacheKey = buildCacheKey(normalizedChainId, normalizedAddress); - const cachedResult = this.#addressScanCache.get(cacheKey); - if (cachedResult) { - return { - result_type: cachedResult.result_type, - label: cachedResult.label, - }; - } - - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - address: normalizedAddress, - }), - }, - ); - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - }; - } - const data: AddressScanResult = await res.json(); - return data; - }, - true, - 5000, - ); - - if (!apiResponse) { + try { + const scanResult = await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:scanAddress', + chain, + normalizedAddress, + ), + ADDRESS_SCAN_TIMEOUT, + ); return { - result_type: AddressScanResultType.ErrorResult, - label: '', + result_type: scanResult.result_type, + label: scanResult.label, }; - } else if ((apiResponse as { error?: string }).error) { + } catch { return { result_type: AddressScanResultType.ErrorResult, label: '', }; } - - const scanResult = apiResponse as AddressScanResult; - const result: AddressScanCacheData = { - result_type: scanResult.result_type, - label: scanResult.label, - }; - - this.#addressScanCache.set(cacheKey, result); - - return { - result_type: scanResult.result_type, - label: scanResult.label, - }; } /** @@ -1566,44 +1191,18 @@ export class PhishingController extends BaseController< return { approvals: [] }; } - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - address: normalizedAddress, - }), - }, - ); - if (!res.ok) { - return { error: `${res.status} ${res.statusText}` }; - } - const data: ApprovalsResponse = await res.json(); - return data; - }, - true, - 5000, - ); - - if (!apiResponse) { - return { approvals: [] }; - } - - if ( - (apiResponse as { error?: string }).error || - !Array.isArray((apiResponse as Partial).approvals) - ) { + try { + return await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:getApprovals', + chain, + normalizedAddress, + ), + APPROVALS_TIMEOUT, + ); + } catch { return { approvals: [] }; } - - return apiResponse as ApprovalsResponse; }; /** @@ -1646,50 +1245,26 @@ export class PhishingController extends BaseController< // EVM addresses are case-insensitive; non-EVM addresses (e.g. Solana // base58) are case-sensitive and must not be lowercased. const caseSensitive = !normalizedChainId.startsWith('0x'); + const normalizedTokens = caseSensitive + ? tokens + : tokens.map((tokenAddress) => tokenAddress.toLowerCase()); - // Split tokens into cached results and tokens that need to be fetched - const { cachedResults, tokensToFetch } = splitCacheHits( - this.#tokenScanCache, - normalizedChainId, - tokens, - caseSensitive, - ); + const results: BulkTokenScanResponse = {}; - const results: BulkTokenScanResponse = { ...cachedResults }; - - // If there are tokens to fetch, call the bulk token scan API - if (tokensToFetch.length > 0) { - const apiResponse = await this.#fetchTokenScanBulkResults( - chain, - tokensToFetch, - ); - if (apiResponse?.results) { - // Process API results and update cache - for (const tokenAddress of tokensToFetch) { - const normalizedAddress = caseSensitive - ? tokenAddress - : tokenAddress.toLowerCase(); - const tokenResult = apiResponse.results[normalizedAddress]; - - if (tokenResult?.result_type) { - const result = { - result_type: tokenResult.result_type, - chain: tokenResult.chain || normalizedChainId, - address: tokenResult.address || normalizedAddress, - }; - - // Update cache - const cacheKey = buildCacheKey( - normalizedChainId, - normalizedAddress, - caseSensitive, - ); - this.#tokenScanCache.set(cacheKey, { - result_type: tokenResult.result_type, - }); - - results[normalizedAddress] = result; - } + const apiResponse = await this.#fetchTokenScanBulkResults( + chain, + normalizedTokens, + ); + if (apiResponse?.results) { + for (const normalizedAddress of normalizedTokens) { + const tokenResult = apiResponse.results[normalizedAddress]; + + if (tokenResult?.result_type) { + results[normalizedAddress] = { + result_type: tokenResult.result_type, + chain: tokenResult.chain || normalizedChainId, + address: tokenResult.address || normalizedAddress, + }; } } } @@ -1706,61 +1281,27 @@ export class PhishingController extends BaseController< readonly #processBatch = async ( urls: string[], ): Promise => { - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ urls }), + try { + return await this.#callWithTimeout( + this.messenger.call('PhishingDataService:bulkScanUrls', urls), + BULK_URL_SCAN_TIMEOUT, + ); + } catch (error) { + if (error instanceof HttpError) { + return { + results: {}, + errors: { + api_error: [error.message], }, - ); - - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - status: res.status, - statusText: res.statusText, - }; - } - - const data = await res.json(); - return data; - }, - true, - 15000, - ); - - // Handle timeout or network errors - if (!apiResponse) { - return { - results: {}, - errors: { - network_error: ['timeout of 15000ms exceeded'], - }, - }; - } - - // Handle HTTP error responses - if ((apiResponse as { error?: string }).error) { - const { status, statusText } = apiResponse as { - status: number; - statusText: string; - }; - + }; + } return { results: {}, errors: { - api_error: [`${status} ${statusText}`], + network_error: [getErrorMessage(error)], }, }; } - - return apiResponse as BulkPhishingDetectionScanResponse; }; /** @@ -1774,12 +1315,13 @@ export class PhishingController extends BaseController< let hotlistDiffsResponse: DataResultWrapper | null = null; let c2DomainBlocklistResponse: C2DomainBlocklistResponse | null = null; try { - const stalelistPromise = this.#queryConfig< - DataResultWrapper - >(METAMASK_STALELIST_URL); + const stalelistPromise = this.#safelyCallService(() => + this.messenger.call('PhishingDataService:getStalelist'), + ); - const c2DomainBlocklistPromise = - this.#queryConfig(C2_DOMAIN_BLOCKLIST_URL); + const c2DomainBlocklistPromise = this.#safelyCallService(() => + this.messenger.call('PhishingDataService:getC2DomainBlocklist'), + ); [stalelistResponse, c2DomainBlocklistResponse] = await Promise.all([ stalelistPromise, @@ -1787,10 +1329,14 @@ export class PhishingController extends BaseController< ]); // Fetching hotlist diffs relies on having a lastUpdated timestamp to do `GET /v1/diffsSince/:timestamp`, // so it doesn't make sense to call if there is not a timestamp to begin with. - if (stalelistResponse?.data && stalelistResponse.data.lastUpdated > 0) { - hotlistDiffsResponse = await this.#queryConfig< - DataResultWrapper - >(`${METAMASK_HOTLIST_DIFF_URL}/${stalelistResponse.data.lastUpdated}`); + const stalelistData = stalelistResponse?.data; + if (stalelistData && stalelistData.lastUpdated > 0) { + hotlistDiffsResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getHotlistDiffs', + stalelistData.lastUpdated, + ), + ); } } finally { // Set `stalelistLastFetched` and `hotlistLastFetched` even for failed requests to prevent server @@ -1853,8 +1399,11 @@ export class PhishingController extends BaseController< ...this.state.phishingLists.map(({ lastUpdated }) => lastUpdated), ); - hotlistResponse = await this.#queryConfig>( - `${METAMASK_HOTLIST_DIFF_URL}/${lastDiffTimestamp}`, + hotlistResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getHotlistDiffs', + lastDiffTimestamp, + ), ); } finally { // Set `hotlistLastFetched` even for failed requests to prevent server from being overwhelmed with @@ -1893,12 +1442,12 @@ export class PhishingController extends BaseController< * this function that prevents redundant configuration updates. */ async #updateC2DomainBlocklist() { - const c2DomainBlocklistResponse = - await this.#queryConfig( - `${C2_DOMAIN_BLOCKLIST_URL}?timestamp=${roundToNearestMinute( - this.state.c2DomainBlocklistLastFetched, - )}`, - ); + const c2DomainBlocklistResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getC2DomainBlocklist', + roundToNearestMinute(this.state.c2DomainBlocklistLastFetched), + ), + ); if (!c2DomainBlocklistResponse) { return; @@ -1929,22 +1478,51 @@ export class PhishingController extends BaseController< this.updatePhishingDetector(); } - async #queryConfig( - input: RequestInfo, - ): Promise { - const response = await safelyExecute( - () => fetch(input, { cache: 'no-cache' }), - true, - ); - - switch (response?.status) { - case 200: { - return await response.json(); - } + /** + * Calls the data service, returning `null` instead of throwing if the call + * fails for any reason (network error, non-2xx response, or malformed + * response). + * + * @param call - The service call to execute. + * @returns The result of the call, or `null` if it failed. + */ + async #safelyCallService( + call: () => Promise, + ): Promise { + try { + return await call(); + } catch (error) { + console.error(error); + return null; + } + } - default: { - return null; - } + /** + * Awaits a promise, rejecting if it does not settle within the given + * timeout. On timeout, any eventual rejection of the original promise is + * suppressed to avoid unhandled rejections. + * + * @param promise - The promise to await. + * @param timeout - The timeout in milliseconds. + * @returns The result of the promise. + */ + async #callWithTimeout( + promise: Promise, + timeout: number, + ): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + promise.catch(() => undefined); + reject(new Error(`timeout of ${timeout}ms exceeded`)); + }, timeout); + }), + ]); + } finally { + clearTimeout(timer); } } } diff --git a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts new file mode 100644 index 00000000000..5390cda5ed5 --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts @@ -0,0 +1,140 @@ +/** + * This file is auto generated. + * Do not edit manually. + */ + +import type { PhishingDataService } from './PhishingDataService.js'; + +/** + * Fetches the full phishing detection stalelist. + * + * @returns The stalelist response. + */ +export type PhishingDataServiceGetStalelistAction = { + type: `PhishingDataService:getStalelist`; + handler: PhishingDataService['getStalelist']; +}; + +/** + * Fetches the hotlist diffs recorded since the given timestamp. + * + * @param timestamp - The timestamp (in seconds) to fetch diffs since. + * @returns The hotlist diffs response. + */ +export type PhishingDataServiceGetHotlistDiffsAction = { + type: `PhishingDataService:getHotlistDiffs`; + handler: PhishingDataService['getHotlistDiffs']; +}; + +/** + * Fetches the C2 domain blocklist changes recorded since the given + * timestamp, or the current blocklist if no timestamp is given. + * + * @param timestamp - The timestamp (in seconds) to fetch changes since. + * @returns The C2 domain blocklist response. + */ +export type PhishingDataServiceGetC2DomainBlocklistAction = { + type: `PhishingDataService:getC2DomainBlocklist`; + handler: PhishingDataService['getC2DomainBlocklist']; +}; + +/** + * Scans a URL for phishing via the dapp-scanning API. + * + * @param url - The prepared URL parameter to scan (hostname, or hostname + * plus path for shared gateways). + * @returns The phishing detection scan result. + */ +export type PhishingDataServiceScanUrlAction = { + type: `PhishingDataService:scanUrl`; + handler: PhishingDataService['scanUrl']; +}; + +/** + * Scans a batch of URLs for phishing via the dapp-scanning API. + * + * Results are cached per hostname using the same query keys as + * {@link PhishingDataService.scanUrl}, so results are shared between single + * and bulk scans. Only hostnames without a fresh cached result are sent to + * the API, in requests of up to 50 URLs. + * + * @param urls - The URLs to scan. + * @returns The scan results, keyed by URL, and any batch-level errors. + */ +export type PhishingDataServiceBulkScanUrlsAction = { + type: `PhishingDataService:bulkScanUrls`; + handler: PhishingDataService['bulkScanUrls']; +}; + +/** + * Scans a token for malicious activity via the security-alerts API. + * + * Requests made while a bulk scan is being assembled are coalesced into a + * single request to the bulk scanning endpoint. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result + * for the token. + */ +export type PhishingDataServiceScanTokenAction = { + type: `PhishingDataService:scanToken`; + handler: PhishingDataService['scanToken']; +}; + +/** + * Scans a batch of tokens for malicious activity via the security-alerts + * API. + * + * Results are cached per token; only tokens without a fresh cached result + * are sent to the API, in requests of up to 100 tokens. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param tokens - The token addresses to scan. + * @returns The token scan results, keyed by token address. Tokens for which + * the API returned no result are omitted. + */ +export type PhishingDataServiceBulkScanTokensAction = { + type: `PhishingDataService:bulkScanTokens`; + handler: PhishingDataService['bulkScanTokens']; +}; + +/** + * Scans an address for security alerts via the security-alerts API. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to scan. + * @returns The address scan result. + */ +export type PhishingDataServiceScanAddressAction = { + type: `PhishingDataService:scanAddress`; + handler: PhishingDataService['scanAddress']; +}; + +/** + * Gets token approvals for an address with security enrichments via the + * security-alerts API. Approvals reflect live account state and are never + * cached. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to get approvals for. + * @returns The approvals response. + */ +export type PhishingDataServiceGetApprovalsAction = { + type: `PhishingDataService:getApprovals`; + handler: PhishingDataService['getApprovals']; +}; + +/** + * Union of all PhishingDataService action types. + */ +export type PhishingDataServiceMethodActions = + | PhishingDataServiceGetStalelistAction + | PhishingDataServiceGetHotlistDiffsAction + | PhishingDataServiceGetC2DomainBlocklistAction + | PhishingDataServiceScanUrlAction + | PhishingDataServiceBulkScanUrlsAction + | PhishingDataServiceScanTokenAction + | PhishingDataServiceBulkScanTokensAction + | PhishingDataServiceScanAddressAction + | PhishingDataServiceGetApprovalsAction; diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts new file mode 100644 index 00000000000..bc1f76993ee --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -0,0 +1,712 @@ +import { ConstantBackoff } from '@metamask/base-data-service'; +import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; +import type { + MessengerActions, + MessengerEvents, + MockAnyNamespace, +} from '@metamask/messenger'; +import nock, { cleanAll } from 'nock'; + +import { flushPromises } from '../../../tests/helpers.js'; +import { + PhishingDataService, + C2_DOMAIN_BLOCKLIST_ENDPOINT, + CLIENT_SIDE_DETECION_BASE_URL, + METAMASK_HOTLIST_DIFF_FILE, + METAMASK_STALELIST_FILE, + PHISHING_CONFIG_BASE_URL, + PHISHING_DETECTION_BASE_URL, + PHISHING_DETECTION_BULK_SCAN_ENDPOINT, + PHISHING_DETECTION_SCAN_ENDPOINT, + SECURITY_ALERTS_BASE_URL, + TOKEN_BULK_SCANNING_ENDPOINT, + ADDRESS_SCAN_ENDPOINT, + APPROVALS_ENDPOINT, + SCAN_RESULT_STALE_TIME, +} from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; +import { TokenScanResultType } from './types.js'; +import type { TokenScanApiResponse } from './types.js'; + +const createdServices: PhishingDataService[] = []; + +const STALELIST_RESPONSE = { + data: { + allowlist: [], + blocklist: ['phishing.example.com'], + blocklistPaths: [], + fuzzylist: [], + tolerance: 2, + version: 1, + lastUpdated: 1700000000, + }, +}; + +describe('PhishingDataService', () => { + afterEach(() => { + jest.useRealTimers(); + cleanAll(); + while (createdServices.length > 0) { + createdServices.pop()?.destroy(); + } + }); + + describe('constructor', () => { + it('applies default options when only a messenger is given', () => { + const rootMessenger = createRootMessenger(); + const messenger: PhishingDataServiceMessenger = new Messenger({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + const service = new PhishingDataService({ messenger }); + createdServices.push(service); + + expect(service.name).toBe('PhishingDataService'); + }); + }); + + describe('getStalelist', () => { + it('returns the stalelist from the API', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getStalelist', + ); + + expect(response).toStrictEqual(STALELIST_RESPONSE); + }); + + it('throws if the API returns a non-200 status', async () => { + nock(PHISHING_CONFIG_BASE_URL).get(METAMASK_STALELIST_FILE).reply(500); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getStalelist'), + ).rejects.toThrow('500 Internal Server Error'); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, { data: { lastUpdated: 'not a number' } }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getStalelist'), + ).rejects.toThrow('Malformed response received from stalelist endpoint'); + }); + }); + + describe('getHotlistDiffs', () => { + it('returns the hotlist diffs recorded since the given timestamp', async () => { + const diffs = { + data: [ + { + url: 'phishing.example.com', + timestamp: 1700000001, + targetList: 'eth_phishing_detect_config.blocklist', + }, + ], + }; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, diffs); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getHotlistDiffs', + 1700000000, + ); + + expect(response).toStrictEqual(diffs); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, { data: 'not an array' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getHotlistDiffs', 1700000000), + ).rejects.toThrow( + 'Malformed response received from hotlist diffs endpoint', + ); + }); + }); + + describe('getC2DomainBlocklist', () => { + it('returns the C2 domain blocklist when no timestamp is given', async () => { + const blocklist = { + recentlyAdded: ['0415f1f1'], + recentlyRemoved: [], + lastFetchedAt: '2024-01-01T00:00:00Z', + }; + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, blocklist); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getC2DomainBlocklist', + ); + + expect(response).toStrictEqual(blocklist); + }); + + it('passes the given timestamp to the API', async () => { + const blocklist = { + recentlyAdded: [], + recentlyRemoved: ['0415f1f1'], + lastFetchedAt: '2024-01-01T00:00:00Z', + }; + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .query({ timestamp: 1700000000 }) + .reply(200, blocklist); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getC2DomainBlocklist', + 1700000000, + ); + + expect(response).toStrictEqual(blocklist); + }); + + it('throws if the API returns a malformed response', async () => { + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, { recentlyAdded: 'not an array' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), + ).rejects.toThrow( + 'Malformed response received from C2 domain blocklist endpoint', + ); + }); + }); + + describe('scanUrl', () => { + it('returns the scan result from the API', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { hostname: 'example.com', recommendedAction: 'NONE' }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + + expect(response).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'NONE', + }); + }); + + it('serves a repeated scan of the same URL from the cache within the stale time', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + const response1 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + const response2 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + expect(response1).toStrictEqual(response2); + + // Once the result goes stale, the URL is scanned again. + jest.advanceTimersByTime(SCAN_RESULT_STALE_TIME + 1); + const response3 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + expect(response3).toStrictEqual({ recommendedAction: 'BLOCK' }); + }); + + it('throws if the API returns a non-200 status', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(404); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('404 Not Found'); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, {}); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('Malformed response received from URL scan endpoint'); + }); + }); + + describe('bulkScanUrls', () => { + it('returns the scan results from the API', async () => { + const urls = ['https://example1.com', 'https://example2.com']; + const apiResponse = { + results: { + 'https://example1.com': { + hostname: 'example1.com', + recommendedAction: 'NONE', + }, + 'https://example2.com': { + hostname: 'example2.com', + recommendedAction: 'BLOCK', + }, + }, + errors: {}, + }; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + urls, + ); + + expect(response).toStrictEqual(apiResponse); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { results: {} }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).rejects.toThrow( + 'Malformed response received from bulk URL scan endpoint', + ); + }); + }); + + describe('bulkScanTokens', () => { + it('returns the scan results from the API', async () => { + const tokens = ['0x1234567890123456789012345678901234567890']; + const apiResponse = { + results: { + [tokens[0]]: { result_type: 'Benign' }, + }, + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { chain: 'ethereum', tokens }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + tokens, + ); + + expect(response).toStrictEqual(apiResponse); + }); + + it('accepts a response without a results field', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, {}); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + ['0x1234567890123456789012345678901234567890'], + ); + + expect(response).toStrictEqual({ results: {} }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { results: 'not a record' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + ]), + ).rejects.toThrow( + 'Malformed response received from bulk token scan endpoint', + ); + }); + }); + + describe('scanToken', () => { + it('returns the scan result for a single token from the bulk API', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, { + results: { + [token]: { result_type: 'Benign' }, + }, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toStrictEqual({ result_type: 'Benign' }); + }); + + it('returns null if the API returned no result for the token', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { results: {} }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toBeNull(); + }); + + it('shares cached results with bulkScanTokens', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, { + results: { + [token]: { result_type: 'Malicious' }, + }, + }); + const { rootMessenger } = createService(); + + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [token], + ); + + // Served from the cache; there is no remaining nock interceptor, so a + // fetch would throw. + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toStrictEqual({ result_type: 'Malicious' }); + }); + }); + + describe('batching', () => { + it('splits large token batches into requests of up to 100 tokens', async () => { + const tokens = Array.from( + { length: 120 }, + (_, index) => `0x${index.toString().padStart(40, '0')}`, + ); + const firstChunk = tokens.slice(0, 100); + const secondChunk = tokens.slice(100); + const buildResults = (chunk: string[]): TokenScanApiResponse['results'] => + Object.fromEntries( + chunk.map((token) => [ + token, + { result_type: TokenScanResultType.Benign }, + ]), + ); + + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: firstChunk, + }) + .reply(200, { results: buildResults(firstChunk) }) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: secondChunk, + }) + .reply(200, { results: buildResults(secondChunk) }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + tokens, + ); + + expect(scope.isDone()).toBe(true); + expect(Object.keys(response.results ?? {})).toHaveLength(120); + }); + + it('coalesces retried queries into a new batched request', async () => { + const token = '0x1234567890123456789012345678901234567890'; + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(500) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { + results: { + [token]: { result_type: 'Benign' }, + }, + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 1, backoff: new ConstantBackoff(0) }, + }, + }); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(scope.isDone()).toBe(true); + expect(response).toStrictEqual({ result_type: 'Benign' }); + }); + }); + + describe('scanAddress', () => { + it('returns the scan result from the API', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, { result_type: 'Benign', label: '' }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + + expect(response).toStrictEqual({ result_type: 'Benign', label: '' }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL).post(ADDRESS_SCAN_ENDPOINT).reply(200, {}); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).rejects.toThrow( + 'Malformed response received from address scan endpoint', + ); + }); + }); + + describe('getApprovals', () => { + it('returns the approvals from the API without caching them', async () => { + const firstResponse = { approvals: [] }; + const secondResponse = { + approvals: [ + { + allowance: {}, + asset: {}, + exposure: {}, + spender: {}, + verdict: 'Benign', + }, + ], + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, firstResponse) + .post(APPROVALS_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, secondResponse); + const { rootMessenger } = createService(); + + const response1 = await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + const response2 = await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + + expect(response1).toStrictEqual(firstResponse); + expect(response2).toStrictEqual(secondResponse); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(200, { approvals: 'not an array' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).rejects.toThrow('Malformed response received from approvals endpoint'); + }); + }); + + describe('persistence', () => { + it('persists the query cache using the StorageService by default', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + const setItem = jest.fn(); + const { rootMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + // The persistence write is debounced; advance past the write delay. + jest.advanceTimersByTime(15_000); + await flushPromises(); + + expect(setItem).toHaveBeenCalledWith( + 'PhishingDataService', + 'cache', + expect.objectContaining({ + timestamp: expect.any(Number), + state: expect.any(Object), + }), + ); + }); + }); + + describe('direct method calls', () => { + it('does the same thing as the messenger action', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + const { service } = createService(); + + const response = await service.getStalelist(); + + expect(response).toStrictEqual(STALELIST_RESPONSE); + }); + }); +}); + +/** + * The type of the messenger populated with all external actions and events + * required by the service under test. + */ +type RootMessenger = Messenger< + MockAnyNamespace, + MessengerActions, + MessengerEvents +>; + +/** + * Constructs the messenger populated with all external actions and events + * required by the service under test. + * + * @returns The root messenger. + */ +function createRootMessenger(): RootMessenger { + return new Messenger({ namespace: MOCK_ANY_NAMESPACE }); +} + +/** + * Constructs the service under test. + * + * @param args - The arguments to this function. + * @param args.options - The options that the service constructor takes. All + * are optional and will be filled in with defaults as needed (including + * `messenger`). + * @param args.setItemMock - Optional mock `StorageService:setItem` handler to + * register and delegate to the service messenger, enabling persistence. + * @returns The new service, root messenger, and service messenger. + */ +function createService({ + options = {}, + setItemMock, +}: { + options?: Partial[0]>; + setItemMock?: jest.Mock; +} = {}): { + service: PhishingDataService; + rootMessenger: RootMessenger; + messenger: PhishingDataServiceMessenger; +} { + const rootMessenger = createRootMessenger(); + const messenger: PhishingDataServiceMessenger = new Messenger({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + if (setItemMock) { + rootMessenger.registerActionHandler('StorageService:setItem', setItemMock); + rootMessenger.delegate({ + actions: ['StorageService:setItem'], + messenger, + }); + } + const service = new PhishingDataService({ + messenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + ...options, + }); + createdServices.push(service); + + return { service, rootMessenger, messenger }; +} diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts new file mode 100644 index 00000000000..51f098b372f --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -0,0 +1,789 @@ +import { BaseDataService } from '@metamask/base-data-service'; +import type { + CreateServicePolicyOptions, + DataServiceCacheUpdatedEvent, + DataServiceGranularCacheUpdatedEvent, + DataServiceInvalidateQueriesAction, + PersistenceConfiguration, +} from '@metamask/base-data-service'; +import { HttpError } from '@metamask/controller-utils'; +import type { Messenger } from '@metamask/messenger'; +import type { Infer, Struct } from '@metamask/superstruct'; +import { + array, + is, + number, + optional, + record, + string, + type, + unknown, +} from '@metamask/superstruct'; +import type { + StorageServiceGetItemAction, + StorageServiceRemoveItemAction, + StorageServiceSetItemAction, +} from '@metamask/storage-service'; +import { Duration, inMilliseconds } from '@metamask/utils'; +import type { Json } from '@metamask/utils'; +import type { QueryClientConfig } from '@tanstack/query-core'; + +import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; +import type { + AddressScanResult, + ApprovalsResponse, + BulkPhishingDetectionScanResponse, + C2DomainBlocklistResponse, + DataResultWrapper, + Hotlist, + PhishingDetectionScanResult, + PhishingStalelist, + TokenScanApiResponse, +} from './types.js'; +import { getHostnameFromWebUrl } from './utils.js'; + +/** + * A single token's scan result as returned by the bulk token scanning + * endpoint. + */ +export type TokenScanResultResponse = TokenScanApiResponse['results'][string]; + +// === GENERAL === + +/** + * The name of the {@link PhishingDataService}, used to namespace the service's + * actions and events. + */ +export const serviceName = 'PhishingDataService'; + +export const PHISHING_CONFIG_BASE_URL = + 'https://phishing-detection.api.cx.metamask.io'; +export const METAMASK_STALELIST_FILE = '/v1/stalelist'; +export const METAMASK_HOTLIST_DIFF_FILE = '/v2/diffsSince'; + +export const CLIENT_SIDE_DETECION_BASE_URL = + 'https://client-side-detection.api.cx.metamask.io'; +export const C2_DOMAIN_BLOCKLIST_ENDPOINT = '/v1/request-blocklist'; + +export const PHISHING_DETECTION_BASE_URL = + 'https://dapp-scanning.api.cx.metamask.io'; +export const PHISHING_DETECTION_SCAN_ENDPOINT = 'v2/scan'; +export const PHISHING_DETECTION_BULK_SCAN_ENDPOINT = 'bulk-scan'; + +export const SECURITY_ALERTS_BASE_URL = + 'https://security-alerts.api.cx.metamask.io'; +export const TOKEN_BULK_SCANNING_ENDPOINT = '/token/scan-bulk'; +export const ADDRESS_SCAN_ENDPOINT = '/address/evm/scan'; +export const APPROVALS_ENDPOINT = '/address/evm/approvals'; + +export const METAMASK_STALELIST_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`; +export const METAMASK_HOTLIST_DIFF_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`; +export const C2_DOMAIN_BLOCKLIST_URL = `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`; + +/** + * The maximum number of URLs sent to the bulk dapp-scanning endpoint in one + * request. + */ +const MAX_URLS_PER_SCAN_REQUEST = 50; + +/** + * The maximum number of tokens sent to the bulk token scanning endpoint in + * one request. + */ +const MAX_TOKENS_PER_SCAN_REQUEST = 100; + +/** + * How long scan results (URL, bulk URL, token, and address scans) are + * considered fresh by the query cache. Mirrors the 1-minute TTL previously + * enforced by the controller's scan caches; scan verdicts can change quickly, + * so this value is a security parameter and should not be raised casually. + */ +export const SCAN_RESULT_STALE_TIME = inMilliseconds(1, Duration.Minute); + +/** + * Default persistence configuration for the service's query cache. The max + * age matches the longest useful lifetime of any cached entry: scan results + * go stale after {@link SCAN_RESULT_STALE_TIME} and list queries are always + * refetched, so a persisted cache older than this holds nothing usable. + */ +export const DEFAULT_PHISHING_PERSISTENCE_CONFIG: PersistenceConfiguration = { + maxAge: inMilliseconds(5, Duration.Minute), +}; + +// === MESSENGER === + +/** + * All of the methods within {@link PhishingDataService} that are exposed via + * the messenger. + */ +const MESSENGER_EXPOSED_METHODS = [ + 'getStalelist', + 'getHotlistDiffs', + 'getC2DomainBlocklist', + 'scanUrl', + 'bulkScanUrls', + 'scanToken', + 'bulkScanTokens', + 'scanAddress', + 'getApprovals', +] as const; + +/** + * Invalidates cached queries for {@link PhishingDataService}. + */ +export type PhishingDataServiceInvalidateQueriesAction = + DataServiceInvalidateQueriesAction; + +/** + * Actions that {@link PhishingDataService} exposes to other consumers. + */ +export type PhishingDataServiceActions = + | PhishingDataServiceMethodActions + | PhishingDataServiceInvalidateQueriesAction; + +/** + * Actions from other messengers that {@link PhishingDataService} calls. + * The StorageService actions are required for query cache persistence. + */ +type AllowedActions = + | StorageServiceGetItemAction + | StorageServiceSetItemAction + | StorageServiceRemoveItemAction; + +/** + * Published when {@link PhishingDataService}'s cache is updated. + */ +export type PhishingDataServiceCacheUpdatedEvent = DataServiceCacheUpdatedEvent< + typeof serviceName +>; + +/** + * Published when a key within {@link PhishingDataService}'s cache is updated. + */ +export type PhishingDataServiceGranularCacheUpdatedEvent = + DataServiceGranularCacheUpdatedEvent; + +/** + * Events that {@link PhishingDataService} exposes to other consumers. + */ +export type PhishingDataServiceEvents = + | PhishingDataServiceCacheUpdatedEvent + | PhishingDataServiceGranularCacheUpdatedEvent; + +/** + * Events from other messengers that {@link PhishingDataService} subscribes to. + */ +type AllowedEvents = never; + +/** + * The messenger which is restricted to actions and events accessed by + * {@link PhishingDataService}. + */ +export type PhishingDataServiceMessenger = Messenger< + typeof serviceName, + PhishingDataServiceActions | AllowedActions, + PhishingDataServiceEvents | AllowedEvents +>; + +// === RESPONSE VALIDATION === + +// The structs below intentionally validate only the shape that the consuming +// code depends on for control flow, mirroring the tolerance of the previous +// in-controller fetching: a response that is missing auxiliary fields is +// passed through rather than rejected. +const StalelistResponseStruct = type({ + data: type({ + lastUpdated: number(), + }), +}); + +const HotlistDiffsResponseStruct = type({ + data: array(unknown()), +}); + +const C2DomainBlocklistResponseStruct = type({ + recentlyAdded: array(string()), + recentlyRemoved: array(string()), +}); + +const ScanUrlResponseStruct = type({ + recommendedAction: string(), +}); + +const BulkScanUrlsResponseStruct = type({ + results: record(string(), unknown()), + errors: record(string(), array(string())), +}); + +const BulkScanTokensResponseStruct = type({ + results: optional(record(string(), unknown())), +}); + +const ScanAddressResponseStruct = type({ + result_type: string(), +}); + +const ApprovalsResponseStruct = type({ + approvals: array(unknown()), +}); + +// === BATCH LOADING === + +type BatchLoader = { + /** + * Registers an item to be resolved by the next executed batch. + * + * @param key - The item key, as understood by the batch endpoint. + * @returns The item's result, or `null` if the batch response did not + * include it. + */ + load: (key: string) => Promise; + /** + * Executes all pending items, in requests of up to the configured batch + * size. Items registered after a flush (e.g. by a retry) are scheduled for + * a later flush automatically. + */ + flush: () => void; +}; + +/** + * Creates a loader that coalesces individual item lookups into batched + * requests. This preserves the per-item caching granularity of the query + * cache while keeping the batched network behavior of the bulk endpoints. + * + * @param options - The loader options. + * @param options.maxBatchSize - The maximum number of items per request. + * @param options.executeBatch - Executes one batched request, returning + * results keyed by item. + * @returns The batch loader. + */ +function createBatchLoader({ + maxBatchSize, + executeBatch, +}: { + maxBatchSize: number; + executeBatch: (keys: string[]) => Promise>; +}): BatchLoader { + type PendingItem = { + key: string; + resolve: (value: Json | null) => void; + reject: (error: unknown) => void; + }; + let pending: PendingItem[] = []; + let flushScheduled = false; + + const executeChunk = async (chunk: PendingItem[]): Promise => { + try { + const results = await executeBatch(chunk.map((item) => item.key)); + for (const item of chunk) { + item.resolve(results[item.key] ?? null); + } + } catch (error) { + for (const item of chunk) { + item.reject(error); + } + } + }; + + const flush = (): void => { + flushScheduled = false; + const batch = pending; + pending = []; + for (let index = 0; index < batch.length; index += maxBatchSize) { + // Errors are routed to the chunk's items, so this promise never + // rejects. + executeChunk(batch.slice(index, index + maxBatchSize)).catch( + /* istanbul ignore next */ + () => undefined, + ); + } + }; + + return { + async load(key: string): Promise { + return new Promise((resolve, reject) => { + pending.push({ key, resolve, reject }); + // Items registered outside an explicit flush (e.g. by the retry + // policy re-running a query) are coalesced via the microtask queue. + if (!flushScheduled) { + flushScheduled = true; + queueMicrotask(() => { + if (flushScheduled) { + flush(); + } + }); + } + }); + }, + flush, + }; +} + +// === SERVICE DEFINITION === + +/** + * This service is responsible for all network requests made on behalf of + * `PhishingController`: fetching the phishing configuration lists (stalelist, + * hotlist diffs, and C2 domain blocklist) and calling the dapp-scanning and + * security-alerts APIs (URL, token, and address scans). + * + * Scan results are cached by the underlying query cache for + * {@link SCAN_RESULT_STALE_TIME} and persisted between sessions when + * `persistenceConfig` is enabled (the default), which requires the + * `StorageService:getItem`, `StorageService:setItem`, and + * `StorageService:removeItem` messenger actions to be delegated to this + * service's messenger, plus a call to `init` during client initialization. + * + * List queries are always refetched when requested; the controller remains + * responsible for deciding when the lists are out of date. + * + * Note that a single retry/circuit-breaker policy is shared across all + * endpoints of this service. The policy only counts consecutive failures, so + * an outage of one API is unlikely to pause requests to the others unless + * failures arrive without any interleaved successes. + */ +export class PhishingDataService extends BaseDataService< + typeof serviceName, + PhishingDataServiceMessenger +> { + /** + * Constructs a new PhishingDataService object. + * + * @param args - The constructor arguments. + * @param args.messenger - The messenger suited for this service. + * @param args.queryClientConfig - Configuration for the underlying TanStack + * Query client. + * @param args.policyOptions - Options to pass to `createServicePolicy`, + * which is used to wrap each request. See + * {@link CreateServicePolicyOptions}. + * @param args.persistenceConfig - Configuration for persisting the query + * cache between sessions. Defaults to + * {@link DEFAULT_PHISHING_PERSISTENCE_CONFIG}; pass `null` to disable + * persistence. + */ + constructor({ + messenger, + queryClientConfig = {}, + policyOptions = {}, + persistenceConfig = DEFAULT_PHISHING_PERSISTENCE_CONFIG, + }: { + messenger: PhishingDataServiceMessenger; + queryClientConfig?: QueryClientConfig; + policyOptions?: CreateServicePolicyOptions; + persistenceConfig?: PersistenceConfiguration | null; + }) { + super({ + name: serviceName, + messenger, + queryClientConfig, + // Circuit breaking is disabled by default: this service talks to four + // independent API hosts through a single shared policy, so a broken + // circuit caused by one host's outage would also pause phishing-list + // updates from the others. Protection against hammering a failing host + // comes from the controller's refresh-interval bookkeeping and the scan + // result stale times, matching the previous in-controller behavior. + policyOptions: { + maxConsecutiveFailures: Number.MAX_SAFE_INTEGER, + ...policyOptions, + }, + persistenceConfig: persistenceConfig ?? undefined, + }); + + this.messenger.registerMethodActionHandlers( + this, + MESSENGER_EXPOSED_METHODS, + ); + } + + /** + * Fetches the full phishing detection stalelist. + * + * @returns The stalelist response. + */ + async getStalelist(): Promise> { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getStalelist`], + queryFn: async () => this.#getJson(METAMASK_STALELIST_URL), + staleTime: 0, + }); + + return this.#validate( + jsonResponse, + StalelistResponseStruct, + 'stalelist', + ) as DataResultWrapper; + } + + /** + * Fetches the hotlist diffs recorded since the given timestamp. + * + * @param timestamp - The timestamp (in seconds) to fetch diffs since. + * @returns The hotlist diffs response. + */ + async getHotlistDiffs( + timestamp: number, + ): Promise> { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getHotlistDiffs`, timestamp], + queryFn: async () => + this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`), + staleTime: 0, + }); + + return this.#validate( + jsonResponse, + HotlistDiffsResponseStruct, + 'hotlist diffs', + ) as DataResultWrapper; + } + + /** + * Fetches the C2 domain blocklist changes recorded since the given + * timestamp, or the current blocklist if no timestamp is given. + * + * @param timestamp - The timestamp (in seconds) to fetch changes since. + * @returns The C2 domain blocklist response. + */ + async getC2DomainBlocklist( + timestamp?: number, + ): Promise { + const url = + timestamp === undefined + ? C2_DOMAIN_BLOCKLIST_URL + : `${C2_DOMAIN_BLOCKLIST_URL}?timestamp=${timestamp}`; + + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getC2DomainBlocklist`, timestamp ?? null], + queryFn: async () => this.#getJson(url), + staleTime: 0, + }); + + return this.#validate( + jsonResponse, + C2DomainBlocklistResponseStruct, + 'C2 domain blocklist', + ) as C2DomainBlocklistResponse; + } + + /** + * Scans a URL for phishing via the dapp-scanning API. + * + * @param url - The prepared URL parameter to scan (hostname, or hostname + * plus path for shared gateways). + * @returns The phishing detection scan result. + */ + async scanUrl(url: string): Promise { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:scanUrl`, url], + queryFn: async () => { + const response = await fetch( + `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent(url)}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + }, + }, + ); + return this.#toJson(response); + }, + staleTime: SCAN_RESULT_STALE_TIME, + }); + + return this.#validate( + jsonResponse, + ScanUrlResponseStruct, + 'URL scan', + ) as PhishingDetectionScanResult; + } + + /** + * Scans a batch of URLs for phishing via the dapp-scanning API. + * + * Results are cached per hostname using the same query keys as + * {@link PhishingDataService.scanUrl}, so results are shared between single + * and bulk scans. Only hostnames without a fresh cached result are sent to + * the API, in requests of up to 50 URLs. + * + * @param urls - The URLs to scan. + * @returns The scan results, keyed by URL, and any batch-level errors. + */ + async bulkScanUrls( + urls: string[], + ): Promise { + const errors: Record = {}; + const loader = createBatchLoader({ + maxBatchSize: MAX_URLS_PER_SCAN_REQUEST, + executeBatch: async (batchUrls) => { + const jsonResponse = await this.#postJson( + `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, + { urls: batchUrls }, + ); + const response = this.#validate( + jsonResponse, + BulkScanUrlsResponseStruct, + 'bulk URL scan', + ) as BulkPhishingDetectionScanResponse; + for (const [key, messages] of Object.entries(response.errors)) { + errors[key] = [...(errors[key] ?? []), ...messages]; + } + return response.results as Record; + }, + }); + + const entries = urls.map((url) => { + const [hostname] = getHostnameFromWebUrl(url); + return this.fetchQuery({ + queryKey: [`${this.name}:scanUrl`, hostname], + queryFn: async () => loader.load(url), + staleTime: SCAN_RESULT_STALE_TIME, + }).then((result) => [url, hostname, result] as const); + }); + loader.flush(); + + const results: Record = {}; + for (const [url, hostname, result] of await Promise.all(entries)) { + if (result !== null) { + const scanResult = result as PhishingDetectionScanResult; + // Entries seeded by single-URL scans hold the raw scan response, + // which may not include the hostname; fill it in from the URL. + results[url] = { + ...scanResult, + hostname: scanResult.hostname ?? hostname, + }; + } + } + + return { results, errors }; + } + + /** + * Scans a token for malicious activity via the security-alerts API. + * + * Requests made while a bulk scan is being assembled are coalesced into a + * single request to the bulk scanning endpoint. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result + * for the token. + */ + async scanToken( + chain: string, + token: string, + ): Promise { + const loader = this.#createTokenScanLoader(chain); + const result = this.#fetchTokenScanQuery(loader, chain, token); + loader.flush(); + return await result; + } + + /** + * Scans a batch of tokens for malicious activity via the security-alerts + * API. + * + * Results are cached per token; only tokens without a fresh cached result + * are sent to the API, in requests of up to 100 tokens. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param tokens - The token addresses to scan. + * @returns The token scan results, keyed by token address. Tokens for which + * the API returned no result are omitted. + */ + async bulkScanTokens( + chain: string, + tokens: string[], + ): Promise { + const loader = this.#createTokenScanLoader(chain); + const entries = tokens.map((token) => + this.#fetchTokenScanQuery(loader, chain, token).then( + (result) => [token, result] as const, + ), + ); + loader.flush(); + + const results: TokenScanApiResponse['results'] = {}; + for (const [token, result] of await Promise.all(entries)) { + if (result !== null) { + results[token] = result; + } + } + + return { results }; + } + + /** + * Creates a batch loader that resolves token scans through the bulk + * scanning endpoint. + * + * @param chain - The chain name (e.g. `ethereum`). + * @returns The batch loader. + */ + #createTokenScanLoader(chain: string): BatchLoader { + return createBatchLoader({ + maxBatchSize: MAX_TOKENS_PER_SCAN_REQUEST, + executeBatch: async (batchTokens) => { + const jsonResponse = await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, + { chain, tokens: batchTokens }, + ); + const response = this.#validate( + jsonResponse, + BulkScanTokensResponseStruct, + 'bulk token scan', + ) as TokenScanApiResponse; + return (response.results ?? {}) as Record; + }, + }); + } + + /** + * Fetches a single token scan query backed by the given batch loader. + * + * @param loader - The batch loader used to resolve cache misses. + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result. + */ + async #fetchTokenScanQuery( + loader: BatchLoader, + chain: string, + token: string, + ): Promise { + const result = await this.fetchQuery({ + queryKey: [`${this.name}:scanToken`, chain, token], + queryFn: async () => loader.load(token), + staleTime: SCAN_RESULT_STALE_TIME, + }); + return result as TokenScanResultResponse | null; + } + + /** + * Scans an address for security alerts via the security-alerts API. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to scan. + * @returns The address scan result. + */ + async scanAddress( + chain: string, + address: string, + ): Promise { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:scanAddress`, chain, address], + queryFn: async () => + this.#postJson(`${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, { + chain, + address, + }), + staleTime: SCAN_RESULT_STALE_TIME, + }); + + return this.#validate( + jsonResponse, + ScanAddressResponseStruct, + 'address scan', + ) as AddressScanResult; + } + + /** + * Gets token approvals for an address with security enrichments via the + * security-alerts API. Approvals reflect live account state and are never + * cached. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to get approvals for. + * @returns The approvals response. + */ + async getApprovals( + chain: string, + address: string, + ): Promise { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getApprovals`, chain, address], + queryFn: async () => + this.#postJson(`${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, { + chain, + address, + }), + staleTime: 0, + cacheTime: 0, + }); + + return this.#validate( + jsonResponse, + ApprovalsResponseStruct, + 'approvals', + ) as ApprovalsResponse; + } + + /** + * Performs a GET request against a phishing configuration endpoint. + * + * @param url - The URL to fetch. + * @returns The parsed JSON response. + */ + async #getJson(url: string): Promise { + const response = await fetch(url, { cache: 'no-cache' }); + return this.#toJson(response); + } + + /** + * Performs a POST request with a JSON body. + * + * @param url - The URL to fetch. + * @param body - The request body, serialized as JSON. + * @returns The parsed JSON response. + */ + async #postJson(url: string, body: Record): Promise { + const response = await fetch(url, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + return this.#toJson(response); + } + + /** + * Parses a response as JSON, throwing an {@link HttpError} for non-2xx + * responses. The error message intentionally matches the + * ` ` format historically produced by + * `PhishingController` so that consumers relying on it keep working. + * + * @param response - The response to parse. + * @returns The parsed JSON response. + */ + async #toJson(response: Response): Promise { + if (!response.ok) { + throw new HttpError( + response.status, + `${response.status} ${response.statusText}`, + ); + } + return response.json(); + } + + /** + * Validates a response against a struct, throwing if it is malformed. + * + * @param response - The response to validate. + * @param struct - The struct to validate against. + * @param endpointName - The name of the endpoint, used in error messages. + * @returns The validated response. + */ + #validate( + response: unknown, + struct: Struct, + endpointName: string, + ): Infer> { + if (!is(response, struct)) { + throw new Error( + `Malformed response received from ${endpointName} endpoint`, + ); + } + return response; + } +} diff --git a/packages/phishing-controller/src/index.ts b/packages/phishing-controller/src/index.ts index 0f963ea60c4..6dbdf8876e4 100644 --- a/packages/phishing-controller/src/index.ts +++ b/packages/phishing-controller/src/index.ts @@ -31,11 +31,11 @@ export { ApprovalResultType, ApprovalFeatureType, } from './types.js'; -export type { CacheEntry } from './CacheManager.js'; export { PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS, getPhishingDetectionScanUrlParam, isPhishingDetectionPathBasedHostname, + resolveChainName, } from './utils.js'; export type { @@ -50,3 +50,29 @@ export type { PhishingControllerGetApprovalsAction, PhishingControllerCheckAddressPoisoningAction, } from './PhishingController-method-action-types.js'; + +export { + PhishingDataService, + SCAN_RESULT_STALE_TIME, + DEFAULT_PHISHING_PERSISTENCE_CONFIG, +} from './PhishingDataService.js'; +export type { TokenScanResultResponse } from './PhishingDataService.js'; +export type { + PhishingDataServiceActions, + PhishingDataServiceEvents, + PhishingDataServiceMessenger, + PhishingDataServiceInvalidateQueriesAction, + PhishingDataServiceCacheUpdatedEvent, + PhishingDataServiceGranularCacheUpdatedEvent, +} from './PhishingDataService.js'; +export type { + PhishingDataServiceGetStalelistAction, + PhishingDataServiceGetHotlistDiffsAction, + PhishingDataServiceGetC2DomainBlocklistAction, + PhishingDataServiceScanUrlAction, + PhishingDataServiceBulkScanUrlsAction, + PhishingDataServiceScanTokenAction, + PhishingDataServiceBulkScanTokensAction, + PhishingDataServiceScanAddressAction, + PhishingDataServiceGetApprovalsAction, +} from './PhishingDataService-method-action-types.js'; diff --git a/packages/phishing-controller/src/types.ts b/packages/phishing-controller/src/types.ts index 44a9f677d02..b8ac85c7ea2 100644 --- a/packages/phishing-controller/src/types.ts +++ b/packages/phishing-controller/src/types.ts @@ -1,4 +1,178 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import type { PathTrie } from './PathTrie.js'; + +/** + * @type ListTypes + * + * Type outlining the types of lists provided by aggregating different source lists + */ +export type ListTypes = + | 'fuzzylist' + | 'blocklist' + | 'blocklistPaths' + | 'allowlist' + | 'c2DomainBlocklist'; + +/** + * @type EthPhishingResponse + * + * Configuration response from the eth-phishing-detect package + * consisting of approved and unapproved website origins + * + * @property blacklist - List of unapproved origins + * @property fuzzylist - List of fuzzy-matched unapproved origins + * @property tolerance - Fuzzy match tolerance level + * @property version - Version number of this configuration + * @property whitelist - List of approved origins + */ +export type EthPhishingResponse = { + blacklist: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + whitelist: string[]; +}; + +/** + * @type C2DomainBlocklistResponse + * + * Response for blocklist update requests + * + * @property recentlyAdded - List of c2 domains recently added to the blocklist + * @property recentlyRemoved - List of c2 domains recently removed from the blocklist + * @property lastFetchedAt - Timestamp of the last fetch request + */ +export type C2DomainBlocklistResponse = { + recentlyAdded: string[]; + recentlyRemoved: string[]; + lastFetchedAt: string; +}; + +/** + * PhishingStalelist defines the expected type of the stalelist from the API. + * + * allowlist - List of approved origins. + * blocklist - List of unapproved origins (hostname-only entries). + * blocklistPaths - Trie of unapproved origins with paths (hostname + path entries). + * fuzzylist - List of fuzzy-matched unapproved origins. + * tolerance - Fuzzy match tolerance level + * lastUpdated - Timestamp of last update. + * version - Stalelist data structure iteration. + */ +export type PhishingStalelist = { + allowlist: string[]; + blocklist: string[]; + blocklistPaths: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + lastUpdated: number; +}; + +/** + * @type PhishingListState + * + * type defining the persisted list state. This is the persisted state that is updated frequently with `this.maybeUpdateState()`. + * + * @property allowlist - List of approved origins (legacy naming "whitelist") + * @property blocklist - List of unapproved origins (legacy naming "blacklist") + * @property blocklistPaths - Trie of unapproved origins with paths (hostname + path, no query params). + * @property c2DomainBlocklist - List of hashed hostnames that C2 requests are blocked against. + * @property fuzzylist - List of fuzzy-matched unapproved origins + * @property tolerance - Fuzzy match tolerance level + * @property lastUpdated - Timestamp of last update. + * @property version - Version of the phishing list state. + * @property name - Name of the list. Used for attribution. + */ +export type PhishingListState = { + allowlist: string[]; + blocklist: string[]; + blocklistPaths: PathTrie; + c2DomainBlocklist: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + lastUpdated: number; + name: ListNames; +}; + +/** + * @type HotlistDiff + * + * type defining the expected type of the diffs in hotlist.json file. + * + * @property url - Url of the diff entry. + * @property timestamp - Timestamp at which the diff was identified. + * @property targetList - The list name where the diff was identified. + * @property isRemoval - Was the diff identified a removal type. + */ +export type HotlistDiff = { + url: string; + timestamp: number; + targetList: `${ListKeys}.${ListTypes}`; + isRemoval?: boolean; +}; + +export type DataResultWrapper = { + data: T; +}; + +/** + * @type Hotlist + * + * Type defining expected hotlist.json file. + * + * @property url - Url of the diff entry. + * @property timestamp - Timestamp at which the diff was identified. + * @property targetList - The list name where the diff was identified. + * @property isRemoval - Was the diff identified a removal type. + */ +export type Hotlist = HotlistDiff[]; + +/** + * Enum containing upstream data provider source list keys. + * These are the keys denoting lists consumed by the upstream data provider. + */ +export enum ListKeys { + EthPhishingDetectConfig = 'eth_phishing_detect_config', +} + +/** + * Enum containing downstream client attribution names. + */ +export enum ListNames { + MetaMask = 'MetaMask', +} + +/** + * Maps from downstream client attribution name + * to list key sourced from upstream data provider. + */ +export const phishingListNameKeyMap = { + [ListNames.MetaMask]: ListKeys.EthPhishingDetectConfig, +}; + +/** + * Maps from list key sourced from upstream data + * provider to downstream client attribution name. + */ +export const phishingListKeyNameMap = { + [ListKeys.EthPhishingDetectConfig]: ListNames.MetaMask, +}; + +/** + * BulkPhishingDetectionScanResponse + * + * Response for bulk phishing detection scan requests + * results - Record of domain names and their corresponding phishing detection scan results + * + * errors - Record of domain names and their corresponding errors + */ +export type BulkPhishingDetectionScanResponse = { + results: Record; + errors: Record; +}; + /** * Represents the result of checking a domain. */ diff --git a/packages/phishing-controller/src/utils.test.ts b/packages/phishing-controller/src/utils.test.ts index 14330fbdf1e..23bfa50efea 100644 --- a/packages/phishing-controller/src/utils.test.ts +++ b/packages/phishing-controller/src/utils.test.ts @@ -1,9 +1,7 @@ import { ListKeys, ListNames } from './PhishingController.js'; import type { PhishingListState } from './PhishingController.js'; -import type { TokenScanResultType } from './types.js'; import { applyDiffs, - buildCacheKey, domainToParts, fetchTimeNow, generateParentDomains, @@ -20,7 +18,6 @@ import { resolveChainName, roundToNearestMinute, sha256Hash, - splitCacheHits, validateConfig, } from './utils.js'; @@ -1183,43 +1180,6 @@ describe('generateParentDomains', () => { }); }); -describe('buildCacheKey', () => { - it('should create cache key with lowercase chainId and address', () => { - const chainId = '0x1'; - const address = '0x1234ABCD'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0x1:0x1234abcd'); - }); - - it('should handle already lowercase inputs', () => { - const chainId = '0xa'; - const address = '0xdeadbeef'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0xa:0xdeadbeef'); - }); - - it('should handle mixed case inputs', () => { - const chainId = '0X89'; - const address = '0XaBcDeF123456'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0x89:0xabcdef123456'); - }); - - it('should preserve address casing when caseSensitive is true', () => { - const chainId = 'solana'; - const address = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - const result = buildCacheKey(chainId, address, true); - expect(result).toBe('solana:Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'); - }); - - it('should lowercase address when caseSensitive is false (default)', () => { - const chainId = 'solana'; - const address = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('solana:gh9zwemdlj8dsckntktqpbnwlnnbjuszag9vp2kgtkjr'); - }); -}); - describe('resolveChainName', () => { it('should resolve known chain IDs to chain names', () => { expect(resolveChainName('0x1')).toBe('ethereum'); @@ -1287,133 +1247,6 @@ describe('isAddressScanSupportedChain', () => { }); }); -describe('splitCacheHits', () => { - const mockCache = { - get: jest.fn(), - }; - - beforeEach(() => { - mockCache.get.mockClear(); - }); - - it('should split tokens correctly when some are cached', () => { - const chainId = '0x1'; - const tokens = ['0xTOKEN1', '0xTOKEN2', '0xTOKEN3']; - - // Mock cache to return data for token1 only - const mockResponses = new Map([ - ['0x1:0xtoken1', { result_type: 'Benign' as TokenScanResultType }], - ]); - mockCache.get.mockImplementation((key: string) => mockResponses.get(key)); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({ - '0xtoken1': { - result_type: 'Benign', - chain: '0x1', - address: '0xtoken1', - }, - }); - expect(result.tokensToFetch).toStrictEqual(['0xtoken2', '0xtoken3']); - }); - - it('should handle all tokens being cached', () => { - const chainId = '0x89'; - const tokens = ['0xTOKEN1', '0xTOKEN2']; - - mockCache.get.mockReturnValue({ - result_type: 'Warning' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({ - '0xtoken1': { - result_type: 'Warning', - chain: '0x89', - address: '0xtoken1', - }, - '0xtoken2': { - result_type: 'Warning', - chain: '0x89', - address: '0xtoken2', - }, - }); - expect(result.tokensToFetch).toStrictEqual([]); - }); - - it('should handle no tokens being cached', () => { - const chainId = '0xa'; - const tokens = ['0xTOKEN1', '0xTOKEN2']; - - mockCache.get.mockReturnValue(undefined); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({}); - expect(result.tokensToFetch).toStrictEqual(['0xtoken1', '0xtoken2']); - }); - - it('should handle empty token list', () => { - const chainId = '0x1'; - const tokens: string[] = []; - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({}); - expect(result.tokensToFetch).toStrictEqual([]); - expect(mockCache.get).not.toHaveBeenCalled(); - }); - - it('should normalize addresses to lowercase', () => { - const chainId = '0X1'; - const tokens = ['0XTOKEN1']; - - mockCache.get.mockReturnValue({ - result_type: 'Malicious' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(mockCache.get).toHaveBeenCalledWith('0x1:0xtoken1'); - expect(result.cachedResults).toHaveProperty('0xtoken1'); - expect(result.cachedResults['0xtoken1'].address).toBe('0xtoken1'); - }); - - it('should preserve address casing when caseSensitive is true', () => { - const chainId = 'solana'; - const tokens = ['Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr']; - - mockCache.get.mockReturnValue(undefined); - - const result = splitCacheHits(mockCache, chainId, tokens, true); - - // tokensToFetch should preserve original casing - expect(result.tokensToFetch).toStrictEqual([ - 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr', - ]); - }); - - it('should return cached result with preserved casing when caseSensitive is true', () => { - const chainId = 'solana'; - const token = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - - mockCache.get.mockReturnValue({ - result_type: 'Benign' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, [token], true); - - expect(result.cachedResults[token]).toStrictEqual({ - result_type: 'Benign', - chain: 'solana', - address: token, - }); - expect(result.tokensToFetch).toStrictEqual([]); - }); -}); - describe('getHostnameAndPathComponents', () => { it.each([ [ diff --git a/packages/phishing-controller/src/utils.ts b/packages/phishing-controller/src/utils.ts index 57dd71ab5b1..0c59328d4d6 100644 --- a/packages/phishing-controller/src/utils.ts +++ b/packages/phishing-controller/src/utils.ts @@ -2,8 +2,6 @@ import { bytesToHex } from '@noble/hashes/utils'; import { sha256 } from 'ethereum-cryptography/sha256'; import { deleteFromTrie, insertToTrie, deepCopyPathTrie } from './PathTrie.js'; -import type { Hotlist, PhishingListState } from './PhishingController.js'; -import { ListKeys, phishingListKeyNameMap } from './PhishingController.js'; import type { PhishingDetectorList, PhishingDetectorConfiguration, @@ -12,13 +10,15 @@ import { ADDRESS_SCAN_SUPPORTED_CHAINS, APPROVAL_SUPPORTED_CHAINS, DEFAULT_CHAIN_ID_TO_NAME, + ListKeys, + phishingListKeyNameMap, TOKEN_SCAN_SUPPORTED_CHAINS, } from './types.js'; import type { AddressScanSupportedChain, ApprovalSupportedChain, - TokenScanCacheData, - TokenScanResult, + Hotlist, + PhishingListState, TokenScanSupportedChain, } from './types.js'; @@ -483,25 +483,6 @@ export const generateParentDomains = ( return domains; }; -/** - * Builds a cache key for a token scan result. - * - * @param chainId - The chain ID. - * @param address - The token address. - * @param caseSensitive - When `true`, the address is kept as-is (for chains - * like Solana where addresses are case-sensitive). When `false` (default), - * the address is lowercased (appropriate for EVM). - * @returns The cache key. - */ -export const buildCacheKey = ( - chainId: string, - address: string, - caseSensitive = false, -) => { - const normalizedAddress = caseSensitive ? address : address.toLowerCase(); - return `${chainId.toLowerCase()}:${normalizedAddress}`; -}; - /** * Determines whether a chain name is supported for token approval scanning. * @@ -548,45 +529,3 @@ export const resolveChainName = ( ): string | null => { return mapping[chainId.toLowerCase() as keyof typeof mapping] ?? null; }; - -/** - * Split tokens into cached results and tokens that need to be fetched. - * - * @param cache - Cache-like object with get method. - * @param cache.get - Method to retrieve cached data by key. - * @param chainId - The chain ID. - * @param tokens - Array of token addresses. - * @param caseSensitive - When `true`, token addresses are kept as-is (for - * chains like Solana where addresses are case-sensitive). When `false` - * (default), addresses are lowercased (appropriate for EVM). - * @returns Object containing cached results and tokens to fetch. - */ -export const splitCacheHits = ( - cache: { get: (key: string) => TokenScanCacheData | undefined }, - chainId: string, - tokens: string[], - caseSensitive = false, -): { - cachedResults: Record; - tokensToFetch: string[]; -} => { - const cachedResults: Record = {}; - const tokensToFetch: string[] = []; - - for (const address of tokens) { - const normalizedAddress = caseSensitive ? address : address.toLowerCase(); - const key = buildCacheKey(chainId, normalizedAddress, caseSensitive); - const hit = cache.get(key); - if (hit) { - cachedResults[normalizedAddress] = { - result_type: hit.result_type, - chain: chainId, - address: normalizedAddress, - }; - } else { - tokensToFetch.push(normalizedAddress); - } - } - - return { cachedResults, tokensToFetch }; -}; diff --git a/packages/phishing-controller/tsconfig.build.json b/packages/phishing-controller/tsconfig.build.json index 3f312d587cd..c4921cabbd1 100644 --- a/packages/phishing-controller/tsconfig.build.json +++ b/packages/phishing-controller/tsconfig.build.json @@ -6,20 +6,26 @@ "rootDir": "./src" }, "references": [ + { + "path": "../address-book-controller/tsconfig.build.json" + }, { "path": "../base-controller/tsconfig.build.json" }, { - "path": "../controller-utils/tsconfig.build.json" + "path": "../base-data-service/tsconfig.build.json" }, { - "path": "../transaction-controller/tsconfig.build.json" + "path": "../controller-utils/tsconfig.build.json" }, { "path": "../messenger/tsconfig.build.json" }, { - "path": "../address-book-controller/tsconfig.build.json" + "path": "../storage-service/tsconfig.build.json" + }, + { + "path": "../transaction-controller/tsconfig.build.json" } ], "include": ["../../types", "./src"] diff --git a/packages/phishing-controller/tsconfig.json b/packages/phishing-controller/tsconfig.json index 63ae98b5725..2713d5242ba 100644 --- a/packages/phishing-controller/tsconfig.json +++ b/packages/phishing-controller/tsconfig.json @@ -4,20 +4,26 @@ "baseUrl": "./" }, "references": [ + { + "path": "../address-book-controller" + }, { "path": "../base-controller" }, { - "path": "../controller-utils" + "path": "../base-data-service" }, { - "path": "../transaction-controller" + "path": "../controller-utils" }, { "path": "../messenger" }, { - "path": "../address-book-controller" + "path": "../storage-service" + }, + { + "path": "../transaction-controller" } ], "include": ["../../types", "./src", "./tests"] diff --git a/yarn.lock b/yarn.lock index fbead227c86..d6cb8860ed1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8453,10 +8453,15 @@ __metadata: "@metamask/address-book-controller": "npm:^7.1.2" "@metamask/auto-changelog": "npm:^6.1.0" "@metamask/base-controller": "npm:^9.1.0" + "@metamask/base-data-service": "npm:^0.1.3" "@metamask/controller-utils": "npm:^12.3.0" "@metamask/messenger": "npm:^2.0.0" + "@metamask/storage-service": "npm:^1.0.2" + "@metamask/superstruct": "npm:^3.4.1" "@metamask/transaction-controller": "npm:^69.5.2" + "@metamask/utils": "npm:^11.11.0" "@noble/hashes": "npm:^1.8.0" + "@tanstack/query-core": "npm:^4.43.0" "@ts-bridge/cli": "npm:^0.6.4" "@types/jest": "npm:^30.0.0" "@types/punycode": "npm:^2.1.0" From 236ea8ef3ddb7f369614b6a71bfa391da5e3a968 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 28 Aug 2026 11:53:08 -0500 Subject: [PATCH 02/38] chore: update generated README and formatting --- README.md | 2 ++ eslint-suppressions.json | 2 +- packages/phishing-controller/src/PhishingController.ts | 2 +- .../phishing-controller/src/PhishingDataService.ts | 10 +++++----- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7ea1324a138..12462fc63e9 100644 --- a/README.md +++ b/README.md @@ -533,8 +533,10 @@ linkStyle default opacity:0.5 perps_controller --> transaction_controller; phishing_controller --> address_book_controller; phishing_controller --> base_controller; + phishing_controller --> base_data_service; phishing_controller --> controller_utils; phishing_controller --> messenger; + phishing_controller --> storage_service; phishing_controller --> transaction_controller; polling_controller --> base_controller; polling_controller --> messenger; diff --git a/eslint-suppressions.json b/eslint-suppressions.json index ab6ac876861..1ccf4a81ab8 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -2300,4 +2300,4 @@ "count": 10 } } -} \ No newline at end of file +} diff --git a/packages/phishing-controller/src/PhishingController.ts b/packages/phishing-controller/src/PhishingController.ts index b32f2d2a078..368c9b1a0d5 100644 --- a/packages/phishing-controller/src/PhishingController.ts +++ b/packages/phishing-controller/src/PhishingController.ts @@ -11,7 +11,6 @@ import type { } from '@metamask/base-controller'; import { HttpError, isValidHexAddress } from '@metamask/controller-utils'; import type { Messenger } from '@metamask/messenger'; -import { getErrorMessage } from '@metamask/utils'; import type { TransactionControllerGetStateAction, TransactionControllerState, @@ -22,6 +21,7 @@ import { getEffectiveRecipient, TransactionStatus, } from '@metamask/transaction-controller'; +import { getErrorMessage } from '@metamask/utils'; import type { Patch } from 'immer'; import { toASCII } from 'punycode/punycode.js'; diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 196d80a6017..d6a2d9ee288 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -8,6 +8,11 @@ import type { } from '@metamask/base-data-service'; import { HttpError } from '@metamask/controller-utils'; import type { Messenger } from '@metamask/messenger'; +import type { + StorageServiceGetItemAction, + StorageServiceRemoveItemAction, + StorageServiceSetItemAction, +} from '@metamask/storage-service'; import type { Infer, Struct } from '@metamask/superstruct'; import { array, @@ -19,11 +24,6 @@ import { type, unknown, } from '@metamask/superstruct'; -import type { - StorageServiceGetItemAction, - StorageServiceRemoveItemAction, - StorageServiceSetItemAction, -} from '@metamask/storage-service'; import { Duration, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; import type { QueryClientConfig } from '@tanstack/query-core'; From 67a3fa88c8bd11b00a684ba8f90baf18f0ca1f5c Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Tue, 1 Sep 2026 15:42:15 -0500 Subject: [PATCH 03/38] chore: link phishing-controller changelog entries to #9914 --- packages/phishing-controller/CHANGELOG.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 96f22d89753..669410f25eb 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -9,30 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) +- Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) ([#9914](https://github.com/MetaMask/core/pull/9914)) - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` - Requests are wrapped in a shared retry policy, configurable via the `policyOptions` constructor option; circuit breaking is disabled by default because the service spans four independent API hosts and a broken circuit caused by one host would pause phishing-list updates from the others - Scan results are cached per URL hostname, token, and address for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs); bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions and an `init` call during client initialization (automatic with `@metamask/wallet`); pass `persistenceConfig: null` to disable -- Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys +- Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys ([#9914](https://github.com/MetaMask/core/pull/9914)) ### Changed -- **BREAKING:** `PhishingController` no longer performs network requests directly; a `PhishingDataService` must be registered and its method actions delegated to the controller's messenger +- **BREAKING:** `PhishingController` no longer performs network requests directly; a `PhishingDataService` must be registered and its method actions delegated to the controller's messenger ([#9914](https://github.com/MetaMask/core/pull/9914)) - `PhishingControllerMessenger` now requires the `PhishingDataService` method actions listed above as allowed actions -- **BREAKING:** Remove the `urlScanCache`, `tokenScanCache`, and `addressScanCache` properties from `PhishingControllerState`; scan results are now cached (and persisted) by `PhishingDataService`'s query cache +- **BREAKING:** Remove the `urlScanCache`, `tokenScanCache`, and `addressScanCache` properties from `PhishingControllerState`; scan results are now cached (and persisted) by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) - Client state migrations should remove these properties from persisted `PhishingController` state -- **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` -- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call -- `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` -- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through +- **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through ([#9914](https://github.com/MetaMask/core/pull/9914)) - Optimize C2 domain blocklist lookups by switching internal storage from `Array` to `Set`, reducing per-lookup complexity from O(n) to O(1) ([#6388](https://github.com/MetaMask/core/pull/6388)) - Bump `@metamask/transaction-controller` from `^69.5.2` to `^69.7.0` ([#9960](https://github.com/MetaMask/core/pull/9960), [#9969](https://github.com/MetaMask/core/pull/9969), [#10046](https://github.com/MetaMask/core/pull/10046)) ### Removed -- **BREAKING:** Remove the `CacheEntry` type; the custom cache manager has been replaced by `PhishingDataService`'s query cache -- **BREAKING:** Remove the `DEFAULT_URL_SCAN_CACHE_TTL`, `DEFAULT_URL_SCAN_CACHE_MAX_SIZE`, `DEFAULT_TOKEN_SCAN_CACHE_TTL`, `DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE`, `DEFAULT_ADDRESS_SCAN_CACHE_TTL`, and `DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE` constants +- **BREAKING:** Remove the `CacheEntry` type; the custom cache manager has been replaced by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) +- **BREAKING:** Remove the `DEFAULT_URL_SCAN_CACHE_TTL`, `DEFAULT_URL_SCAN_CACHE_MAX_SIZE`, `DEFAULT_TOKEN_SCAN_CACHE_TTL`, `DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE`, `DEFAULT_ADDRESS_SCAN_CACHE_TTL`, and `DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE` constants ([#9914](https://github.com/MetaMask/core/pull/9914)) ## [17.4.0] From 83d1c235f76d42df010e3c8279c39f0b4eca6996 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Tue, 1 Sep 2026 16:07:03 -0500 Subject: [PATCH 04/38] fix: address review findings in PhishingDataService Bulk URL scanning: - Key bulk queries by the scan URL parameter rather than the bare hostname. Path-sensitive hosts (ipfs.io, github.io, and the other entries in PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS) collapsed into one cache entry per host, so only the first path in a batch was scanned and the rest inherited its verdict. - Return the results that did resolve when some lookups fail, reporting the failures per URL, instead of discarding the whole batch. A single failed lookup previously threw away fresh cached BLOCK verdicts for unrelated URLs. The call still rejects when nothing could be resolved at all. - Stop caching a "no result" verdict for URLs the API reported an error for. Those URLs were silently skipped for the following minute. - Report invalid URLs instead of collapsing them onto a shared empty key. Request policy: - Disable retries by default. The previous in-controller implementation made a single request per call and the controller's timeouts are sized for one attempt, so inheriting maxRetries: 3 meant a timeout could fire mid-retry. Retries also amplified badly through the batch loaders: a failed batch rejects every item query in it, and each retried on its own, turning one failed request into many single-item requests against a failing host. Caching: - Validate responses inside fetchQuery via responseStruct. Validating after the fact meant a malformed 200 was committed to the cache, and persisted, before it was rejected, so every caller for the next minute got the same error with no request made. - Set gcTime explicitly on scan queries. TanStack Query defaults gcTime to Infinity when it detects a server environment, which includes the MV3 service worker, so the cache would grow unbounded; the cache this replaces was explicitly size-bounded. - Do not retain fetched lists in the query cache. The controller keeps its own copy, and retaining them meant the multi-megabyte stalelist was rewritten to disk on every scan-triggered persist. - Do not route getApprovals through the query cache. It is never cached (staleTime and gcTime are both 0), so the cache only served to publish account-specific approval data on the messenger, matching #10007. Also corrects two changelog claims: PhishingDataService is not yet one of @metamask/wallet's default instances, so init is not automatic. --- packages/phishing-controller/CHANGELOG.md | 11 +- ...PhishingDataService-method-action-types.ts | 13 +- .../src/PhishingDataService.test.ts | 320 +++++++++++++++++- .../src/PhishingDataService.ts | 217 ++++++++---- packages/phishing-controller/src/index.ts | 1 + 5 files changed, 492 insertions(+), 70 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 669410f25eb..27a87a99e9d 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -11,9 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) ([#9914](https://github.com/MetaMask/core/pull/9914)) - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` - - Requests are wrapped in a shared retry policy, configurable via the `policyOptions` constructor option; circuit breaking is disabled by default because the service spans four independent API hosts and a broken circuit caused by one host would pause phishing-list updates from the others - - Scan results are cached per URL hostname, token, and address for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs); bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached - - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions and an `init` call during client initialization (automatic with `@metamask/wallet`); pass `persistenceConfig: null` to disable + - Requests are wrapped in a shared service policy, configurable via the `policyOptions` constructor option. Retries and circuit breaking are both disabled by default: the service spans four independent API hosts, so a circuit broken by one host would pause phishing-list updates from the others, and the previous in-controller implementation made a single request per call + - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes); bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached + - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization; pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves + - Fetched lists (stalelist, hotlist diffs, and the C2 domain blocklist) are not retained by the query cache, and so are not persisted; the controller keeps its own copy in state - Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys ([#9914](https://github.com/MetaMask/core/pull/9914)) ### Changed @@ -25,7 +26,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) - Optimize C2 domain blocklist lookups by switching internal storage from `Array` to `Set`, reducing per-lookup complexity from O(n) to O(1) ([#6388](https://github.com/MetaMask/core/pull/6388)) - Bump `@metamask/transaction-controller` from `^69.5.2` to `^69.7.0` ([#9960](https://github.com/MetaMask/core/pull/9960), [#9969](https://github.com/MetaMask/core/pull/9969), [#10046](https://github.com/MetaMask/core/pull/10046)) diff --git a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts index 5390cda5ed5..6942b3b13c9 100644 --- a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts +++ b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts @@ -53,13 +53,18 @@ export type PhishingDataServiceScanUrlAction = { /** * Scans a batch of URLs for phishing via the dapp-scanning API. * - * Results are cached per hostname using the same query keys as + * Results are cached under the same query keys as * {@link PhishingDataService.scanUrl}, so results are shared between single - * and bulk scans. Only hostnames without a fresh cached result are sent to - * the API, in requests of up to 50 URLs. + * and bulk scans, including for the path-sensitive hosts listed in + * `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`. Only URLs without a fresh + * cached result are sent to the API, in requests of up to 50 URLs. + * + * If some lookups fail, the results that did resolve are still returned and + * the failures are reported per URL. The call only rejects when nothing at + * all could be resolved. * * @param urls - The URLs to scan. - * @returns The scan results, keyed by URL, and any batch-level errors. + * @returns The scan results, keyed by URL, and any per-URL errors. */ export type PhishingDataServiceBulkScanUrlsAction = { type: `PhishingDataService:bulkScanUrls`; diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index bc1f76993ee..869f610426a 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1,5 +1,6 @@ import { ConstantBackoff } from '@metamask/base-data-service'; import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; +import { Duration, inMilliseconds } from '@metamask/utils'; import type { MessengerActions, MessengerEvents, @@ -22,6 +23,7 @@ import { TOKEN_BULK_SCANNING_ENDPOINT, ADDRESS_SCAN_ENDPOINT, APPROVALS_ENDPOINT, + SCAN_RESULT_GC_TIME, SCAN_RESULT_STALE_TIME, } from './PhishingDataService.js'; import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; @@ -96,7 +98,9 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getStalelist'), - ).rejects.toThrow('Malformed response received from stalelist endpoint'); + ).rejects.toThrow( + 'Query function for "PhishingDataService:getStalelist" returned an unexpected response', + ); }); }); @@ -133,7 +137,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getHotlistDiffs', 1700000000), ).rejects.toThrow( - 'Malformed response received from hotlist diffs endpoint', + 'Query function for "PhishingDataService:getHotlistDiffs" returned an unexpected response', ); }); }); @@ -186,7 +190,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), ).rejects.toThrow( - 'Malformed response received from C2 domain blocklist endpoint', + 'Query function for "PhishingDataService:getC2DomainBlocklist" returned an unexpected response', ); }); }); @@ -264,7 +268,55 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).rejects.toThrow('Malformed response received from URL scan endpoint'); + ).rejects.toThrow( + 'Query function for "PhishingDataService:scanUrl" returned an unexpected response', + ); + }); + + it('does not cache a malformed response', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { unexpected: 'shape' }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('returned an unexpected response'); + + // The malformed body must not have been committed to the cache, so the + // next call re-requests and sees the real verdict. + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).toStrictEqual({ recommendedAction: 'BLOCK' }); + expect(scope.isDone()).toBe(true); + }); + + it('refetches once a cached result passes its garbage collection time', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .times(2) + .reply(200, { recommendedAction: 'NONE' }); + const { rootMessenger } = createService(); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1000); + await flushPromises(); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + // The entry is collected rather than being retained forever, which is + // what TanStack Query would otherwise do in a service worker. + expect(scope.isDone()).toBe(true); }); }); @@ -311,6 +363,121 @@ describe('PhishingDataService', () => { 'Malformed response received from bulk URL scan endpoint', ); }); + + it('sends each path separately for path-sensitive hosts', async () => { + const urls = ['https://ipfs.io/ipfs/AAA', 'https://ipfs.io/ipfs/BBB']; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls }) + .reply(200, { + results: { + [urls[0]]: { hostname: urls[0], recommendedAction: 'NONE' }, + [urls[1]]: { hostname: urls[1], recommendedAction: 'BLOCK' }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + urls, + ); + + // Both paths must be scanned; they must not share one verdict. + expect(response.results[urls[0]].recommendedAction).toBe('NONE'); + expect(response.results[urls[1]].recommendedAction).toBe('BLOCK'); + }); + + it('reports invalid URLs without calling the API', async () => { + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['not-a-url'], + ); + + expect(response).toStrictEqual({ + results: {}, + errors: { 'not-a-url': ['url is not a valid web URL'] }, + }); + }); + + it('does not cache URLs the API reported an error for', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(2) + .reply(200, { + results: {}, + errors: { 'https://example1.com': ['upstream failure'] }, + }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + + // The error must be reported both times rather than being cached as a + // silent "no result" for the stale time. + expect(first.errors['https://example1.com']).toStrictEqual([ + 'upstream failure', + ]); + expect(second.errors['https://example1.com']).toStrictEqual([ + 'upstream failure', + ]); + expect(scope.isDone()).toBe(true); + }); + + it('keeps fresh cached results when another lookup fails', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { + 'https://blocked.com': { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://blocked.com', + ]); + + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(500, 'boom'); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://blocked.com', 'https://uncached.com'], + ); + + expect(response.results['https://blocked.com'].recommendedAction).toBe( + 'BLOCK', + ); + expect(response.errors['https://uncached.com']).toStrictEqual([ + '500 Internal Server Error', + ]); + }); + + it('rejects when no URL could be resolved', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(500, 'boom'); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).rejects.toThrow('500 Internal Server Error'); + }); }); describe('bulkScanTokens', () => { @@ -535,7 +702,7 @@ describe('PhishingDataService', () => { '0x1234567890123456789012345678901234567890', ), ).rejects.toThrow( - 'Malformed response received from address scan endpoint', + 'Query function for "PhishingDataService:scanAddress" returned an unexpected response', ); }); }); @@ -630,6 +797,124 @@ describe('PhishingDataService', () => { }), ); }); + + it('does not persist fetched lists', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + const setItem = jest.fn(); + const { rootMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + + await rootMessenger.call('PhishingDataService:getStalelist'); + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(15_000); + await flushPromises(); + + const written = JSON.stringify(setItem.mock.calls.at(-1)?.[2]); + // The scan result is persisted, but the (multi-megabyte) list is not. + expect(written).toContain('scanUrl'); + expect(written).not.toContain('phishing.example.com'); + }); + + it('rehydrates the cache from the StorageService on init', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + // Populate a cache using one service, then hand it to a second one. + const setItem = jest.fn(); + const { rootMessenger: firstMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + await firstMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(15_000); + await flushPromises(); + + const persisted = setItem.mock.calls.at(-1)?.[2]; + expect(persisted).toBeDefined(); + + const { rootMessenger: secondMessenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: jest.fn().mockResolvedValue({ result: persisted }), + }); + service.init(); + await flushPromises(); + + // No nock interceptor is registered, so this can only succeed if the + // rehydrated entry was used. + const result = await secondMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + expect(result).toStrictEqual({ recommendedAction: 'NONE' }); + }); + + it('discards and removes a persisted cache older than maxAge', async () => { + const removeItem = jest.fn(); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: Date.now() - inMilliseconds(10, Duration.Minute), + state: { queries: [], mutations: [] }, + }, + }); + const { service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: getItem, + removeItemMock: removeItem, + }); + + service.init(); + await flushPromises(); + + expect(getItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); + expect(removeItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); + }); + }); + + describe('retry policy', () => { + it('does not retry failed requests by default', async () => { + let attempts = 0; + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .times(5) + .reply(() => { + attempts += 1; + return [500, 'boom']; + }); + // Build the service without the test helper's `maxRetries: 0` override + // so that the shipped defaults are what is exercised here. + const { rootMessenger } = createService({ + options: { policyOptions: {} }, + }); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('500 Internal Server Error'); + expect(attempts).toBe(1); + }); }); describe('direct method calls', () => { @@ -675,14 +960,22 @@ function createRootMessenger(): RootMessenger { * `messenger`). * @param args.setItemMock - Optional mock `StorageService:setItem` handler to * register and delegate to the service messenger, enabling persistence. + * @param args.getItemMock - Optional mock `StorageService:getItem` handler to + * register and delegate to the service messenger, enabling rehydration. + * @param args.removeItemMock - Optional mock `StorageService:removeItem` + * handler to register and delegate to the service messenger. * @returns The new service, root messenger, and service messenger. */ function createService({ options = {}, setItemMock, + getItemMock, + removeItemMock, }: { options?: Partial[0]>; setItemMock?: jest.Mock; + getItemMock?: jest.Mock; + removeItemMock?: jest.Mock; } = {}): { service: PhishingDataService; rootMessenger: RootMessenger; @@ -700,6 +993,23 @@ function createService({ messenger, }); } + if (getItemMock) { + rootMessenger.registerActionHandler('StorageService:getItem', getItemMock); + rootMessenger.delegate({ + actions: ['StorageService:getItem'], + messenger, + }); + } + if (removeItemMock) { + rootMessenger.registerActionHandler( + 'StorageService:removeItem', + removeItemMock, + ); + rootMessenger.delegate({ + actions: ['StorageService:removeItem'], + messenger, + }); + } const service = new PhishingDataService({ messenger, policyOptions: { maxRetries: 0 }, diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index d6a2d9ee288..609af9ae240 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -24,7 +24,7 @@ import { type, unknown, } from '@metamask/superstruct'; -import { Duration, inMilliseconds } from '@metamask/utils'; +import { Duration, getErrorMessage, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; import type { QueryClientConfig } from '@tanstack/query-core'; @@ -40,7 +40,10 @@ import type { PhishingStalelist, TokenScanApiResponse, } from './types.js'; -import { getHostnameFromWebUrl } from './utils.js'; +import { + getHostnameFromWebUrl, + getPhishingDetectionScanUrlParam, +} from './utils.js'; /** * A single token's scan result as returned by the bulk token scanning @@ -100,6 +103,25 @@ const MAX_TOKENS_PER_SCAN_REQUEST = 100; */ export const SCAN_RESULT_STALE_TIME = inMilliseconds(1, Duration.Minute); +/** + * How long a scan result is retained by the query cache before it is eligible + * for garbage collection. This is set explicitly because TanStack Query + * defaults `gcTime` to `Infinity` when it detects a server environment, which + * includes the extension's MV3 service worker (`window` is undefined there). + * Without it, the cache would grow without bound for the life of the worker, + * whereas the cache this service replaces was explicitly size-bounded. + */ +export const SCAN_RESULT_GC_TIME = inMilliseconds(5, Duration.Minute); + +/** + * How long a fetched list is retained by the query cache. The lists are always + * refetched (`staleTime: 0`) and the controller keeps its own copy in + * `phishingLists`, so retaining them here has no benefit and a large cost: the + * stalelist is several megabytes, and the persisted cache is rewritten + * whenever any query changes, including on every scan. + */ +const LIST_GC_TIME = 0; + /** * Default persistence configuration for the service's query cache. The max * age matches the longest useful lifetime of any cached entry: scan results @@ -229,6 +251,29 @@ const ApprovalsResponseStruct = type({ // === BATCH LOADING === +/** + * The outcome of one batched request: results keyed by item, plus any + * per-item errors reported by the endpoint. Items listed in `errors` are + * rejected rather than resolved, so that an endpoint-reported failure is not + * cached as a "no result" verdict. + */ +type BatchOutcome = { + results: Record; + errors?: Record; +}; + +/** + * An error reported by a batch endpoint for one specific item, as opposed to a + * failure of the request as a whole. These are surfaced to the caller per item + * and never cached, but they do not make the overall call fail. + */ +class BatchItemError extends Error { + constructor(message: string) { + super(message); + this.name = 'BatchItemError'; + } +} + type BatchLoader = { /** * Registers an item to be resolved by the next executed batch. @@ -262,7 +307,7 @@ function createBatchLoader({ executeBatch, }: { maxBatchSize: number; - executeBatch: (keys: string[]) => Promise>; + executeBatch: (keys: string[]) => Promise; }): BatchLoader { type PendingItem = { key: string; @@ -274,9 +319,16 @@ function createBatchLoader({ const executeChunk = async (chunk: PendingItem[]): Promise => { try { - const results = await executeBatch(chunk.map((item) => item.key)); + const { results, errors = {} } = await executeBatch( + chunk.map((item) => item.key), + ); for (const item of chunk) { - item.resolve(results[item.key] ?? null); + const itemError = errors[item.key]; + if (itemError) { + item.reject(itemError); + } else { + item.resolve(results[item.key] ?? null); + } } } catch (error) { for (const item of chunk) { @@ -382,8 +434,17 @@ export class PhishingDataService extends BaseDataService< // updates from the others. Protection against hammering a failing host // comes from the controller's refresh-interval bookkeeping and the scan // result stale times, matching the previous in-controller behavior. + // + // Retries are disabled by default for the same reason. The previous + // in-controller implementation made a single request per call, and the + // controller's timeouts are sized for one attempt. Retries are also + // unsafe for the batched endpoints: a failed batch rejects every item + // query in it, and each would then retry independently, turning one + // failed request into many single-item requests against a host that is + // already failing. policyOptions: { maxConsecutiveFailures: Number.MAX_SAFE_INTEGER, + maxRetries: 0, ...policyOptions, }, persistenceConfig: persistenceConfig ?? undefined, @@ -404,14 +465,12 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getStalelist`], queryFn: async () => this.#getJson(METAMASK_STALELIST_URL), + responseStruct: StalelistResponseStruct, staleTime: 0, + gcTime: LIST_GC_TIME, }); - return this.#validate( - jsonResponse, - StalelistResponseStruct, - 'stalelist', - ) as DataResultWrapper; + return jsonResponse as DataResultWrapper; } /** @@ -427,14 +486,12 @@ export class PhishingDataService extends BaseDataService< queryKey: [`${this.name}:getHotlistDiffs`, timestamp], queryFn: async () => this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`), + responseStruct: HotlistDiffsResponseStruct, staleTime: 0, + gcTime: LIST_GC_TIME, }); - return this.#validate( - jsonResponse, - HotlistDiffsResponseStruct, - 'hotlist diffs', - ) as DataResultWrapper; + return jsonResponse as DataResultWrapper; } /** @@ -455,14 +512,12 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getC2DomainBlocklist`, timestamp ?? null], queryFn: async () => this.#getJson(url), + responseStruct: C2DomainBlocklistResponseStruct, staleTime: 0, + gcTime: LIST_GC_TIME, }); - return this.#validate( - jsonResponse, - C2DomainBlocklistResponseStruct, - 'C2 domain blocklist', - ) as C2DomainBlocklistResponse; + return jsonResponse as C2DomainBlocklistResponse; } /** @@ -487,31 +542,38 @@ export class PhishingDataService extends BaseDataService< ); return this.#toJson(response); }, + responseStruct: ScanUrlResponseStruct, staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, }); - return this.#validate( - jsonResponse, - ScanUrlResponseStruct, - 'URL scan', - ) as PhishingDetectionScanResult; + return jsonResponse as PhishingDetectionScanResult; } /** * Scans a batch of URLs for phishing via the dapp-scanning API. * - * Results are cached per hostname using the same query keys as + * Results are cached under the same query keys as * {@link PhishingDataService.scanUrl}, so results are shared between single - * and bulk scans. Only hostnames without a fresh cached result are sent to - * the API, in requests of up to 50 URLs. + * and bulk scans, including for the path-sensitive hosts listed in + * `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`. Only URLs without a fresh + * cached result are sent to the API, in requests of up to 50 URLs. + * + * If some lookups fail, the results that did resolve are still returned and + * the failures are reported per URL. The call only rejects when nothing at + * all could be resolved. * * @param urls - The URLs to scan. - * @returns The scan results, keyed by URL, and any batch-level errors. + * @returns The scan results, keyed by URL, and any per-URL errors. */ async bulkScanUrls( urls: string[], ): Promise { const errors: Record = {}; + const addError = (key: string, message: string): void => { + errors[key] = [...(errors[key] ?? []), message]; + }; + const loader = createBatchLoader({ maxBatchSize: MAX_URLS_PER_SCAN_REQUEST, executeBatch: async (batchUrls) => { @@ -524,27 +586,62 @@ export class PhishingDataService extends BaseDataService< BulkScanUrlsResponseStruct, 'bulk URL scan', ) as BulkPhishingDetectionScanResponse; + // URLs the endpoint reported an error for are rejected rather than + // resolved, so that the failure is surfaced to the caller instead of + // being cached as a "no result" verdict for the stale time. + const itemErrors: Record = {}; for (const [key, messages] of Object.entries(response.errors)) { - errors[key] = [...(errors[key] ?? []), ...messages]; + itemErrors[key] = new BatchItemError(messages.join(', ')); } - return response.results as Record; + return { + results: response.results as Record, + errors: itemErrors, + }; }, }); - const entries = urls.map((url) => { + const requested: { url: string; hostname: string }[] = []; + const entries: Promise[] = []; + for (const url of urls) { + const [scanUrlParam, ok] = getPhishingDetectionScanUrlParam(url); + if (!ok) { + addError(url, 'url is not a valid web URL'); + continue; + } const [hostname] = getHostnameFromWebUrl(url); - return this.fetchQuery({ - queryKey: [`${this.name}:scanUrl`, hostname], - queryFn: async () => loader.load(url), - staleTime: SCAN_RESULT_STALE_TIME, - }).then((result) => [url, hostname, result] as const); - }); + requested.push({ url, hostname }); + entries.push( + // Keyed by the scan parameter rather than the bare hostname so that + // path-sensitive hosts (see + // `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`) get one entry per + // path, and so that entries are shared with `scanUrl`. + this.fetchQuery({ + queryKey: [`${this.name}:scanUrl`, scanUrlParam], + queryFn: async () => loader.load(url), + staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, + }), + ); + } loader.flush(); + const settled = await Promise.allSettled(entries); const results: Record = {}; - for (const [url, hostname, result] of await Promise.all(entries)) { - if (result !== null) { - const scanResult = result as PhishingDetectionScanResult; + let requestFailure: { reason: unknown } | undefined; + + for (const [index, outcome] of settled.entries()) { + const { url, hostname } = requested[index]; + + if (outcome.status === 'rejected') { + addError(url, getErrorMessage(outcome.reason)); + if (!(outcome.reason instanceof BatchItemError)) { + requestFailure ??= { reason: outcome.reason }; + } + continue; + } + + if (outcome.value !== null) { + const scanResult = outcome.value as PhishingDetectionScanResult; // Entries seeded by single-URL scans hold the raw scan response, // which may not include the hostname; fill it in from the URL. results[url] = { @@ -554,6 +651,14 @@ export class PhishingDataService extends BaseDataService< } } + // A request-level failure that produced nothing at all is surfaced to the + // caller, matching the previous behavior. If anything did resolve, keep + // it, including fresh cache hits, so that one failed lookup cannot + // discard a cached BLOCK verdict for a different URL. + if (requestFailure && Object.keys(results).length === 0) { + throw requestFailure.reason; + } + return { results, errors }; } @@ -632,7 +737,7 @@ export class PhishingDataService extends BaseDataService< BulkScanTokensResponseStruct, 'bulk token scan', ) as TokenScanApiResponse; - return (response.results ?? {}) as Record; + return { results: (response.results ?? {}) as Record }; }, }); } @@ -654,6 +759,7 @@ export class PhishingDataService extends BaseDataService< queryKey: [`${this.name}:scanToken`, chain, token], queryFn: async () => loader.load(token), staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, }); return result as TokenScanResultResponse | null; } @@ -676,14 +782,12 @@ export class PhishingDataService extends BaseDataService< chain, address, }), + responseStruct: ScanAddressResponseStruct, staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, }); - return this.#validate( - jsonResponse, - ScanAddressResponseStruct, - 'address scan', - ) as AddressScanResult; + return jsonResponse as AddressScanResult; } /** @@ -699,16 +803,15 @@ export class PhishingDataService extends BaseDataService< chain: string, address: string, ): Promise { - const jsonResponse = await this.fetchQuery({ - queryKey: [`${this.name}:getApprovals`, chain, address], - queryFn: async () => - this.#postJson(`${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, { - chain, - address, - }), - staleTime: 0, - gcTime: 0, - }); + // Deliberately not routed through `fetchQuery`. Approvals reflect live, + // account-specific state that is never cached, so the query cache would + // provide no benefit while publishing the response on the messenger as a + // `cacheUpdated` payload. This matches the handling of non-cached POSTs + // elsewhere in the monorepo. + const jsonResponse = await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, + { chain, address }, + ); return this.#validate( jsonResponse, diff --git a/packages/phishing-controller/src/index.ts b/packages/phishing-controller/src/index.ts index b078880559b..ee4218e97da 100644 --- a/packages/phishing-controller/src/index.ts +++ b/packages/phishing-controller/src/index.ts @@ -54,6 +54,7 @@ export type { export { PhishingDataService, + SCAN_RESULT_GC_TIME, SCAN_RESULT_STALE_TIME, DEFAULT_PHISHING_PERSISTENCE_CONFIG, } from './PhishingDataService.js'; From 89871134655cd782f1265907a4ef2610f4f68dae Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Tue, 1 Sep 2026 16:08:59 -0500 Subject: [PATCH 05/38] chore: apply oxfmt formatting --- packages/phishing-controller/src/PhishingDataService.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 869f610426a..8d8efc9beb9 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1,11 +1,11 @@ import { ConstantBackoff } from '@metamask/base-data-service'; import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; -import { Duration, inMilliseconds } from '@metamask/utils'; import type { MessengerActions, MessengerEvents, MockAnyNamespace, } from '@metamask/messenger'; +import { Duration, inMilliseconds } from '@metamask/utils'; import nock, { cleanAll } from 'nock'; import { flushPromises } from '../../../tests/helpers.js'; From 1cf7cc7b31f3df33bf5a0b2cacd7abc3ae10f246 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Tue, 1 Sep 2026 16:24:45 -0500 Subject: [PATCH 06/38] fix: validate inside the query function rather than via responseStruct `responseStruct` requires a `Struct`, which these structs are not (`HotlistDiffsResponseStruct` infers `unknown[]`), so the build failed. Validating inside the query function achieves the same result: a malformed response throws before TanStack Query commits it, so it is never cached or persisted, and the existing error messages are preserved. --- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 16 +++---- .../src/PhishingDataService.ts | 45 +++++++++++++------ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 27a87a99e9d..f46c2d24afd 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) - Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 8d8efc9beb9..55b04f91a8f 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -98,9 +98,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getStalelist'), - ).rejects.toThrow( - 'Query function for "PhishingDataService:getStalelist" returned an unexpected response', - ); + ).rejects.toThrow('Malformed response received from stalelist endpoint'); }); }); @@ -137,7 +135,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getHotlistDiffs', 1700000000), ).rejects.toThrow( - 'Query function for "PhishingDataService:getHotlistDiffs" returned an unexpected response', + 'Malformed response received from hotlist diffs endpoint', ); }); }); @@ -190,7 +188,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), ).rejects.toThrow( - 'Query function for "PhishingDataService:getC2DomainBlocklist" returned an unexpected response', + 'Malformed response received from C2 domain blocklist endpoint', ); }); }); @@ -268,9 +266,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).rejects.toThrow( - 'Query function for "PhishingDataService:scanUrl" returned an unexpected response', - ); + ).rejects.toThrow('Malformed response received from URL scan endpoint'); }); it('does not cache a malformed response', async () => { @@ -285,7 +281,7 @@ describe('PhishingDataService', () => { await expect( rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).rejects.toThrow('returned an unexpected response'); + ).rejects.toThrow('Malformed response received from URL scan endpoint'); // The malformed body must not have been committed to the cache, so the // next call re-requests and sees the real verdict. @@ -702,7 +698,7 @@ describe('PhishingDataService', () => { '0x1234567890123456789012345678901234567890', ), ).rejects.toThrow( - 'Query function for "PhishingDataService:scanAddress" returned an unexpected response', + 'Malformed response received from address scan endpoint', ); }); }); diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 609af9ae240..6ef2da3de0f 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -464,8 +464,14 @@ export class PhishingDataService extends BaseDataService< async getStalelist(): Promise> { const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getStalelist`], - queryFn: async () => this.#getJson(METAMASK_STALELIST_URL), - responseStruct: StalelistResponseStruct, + // Validated inside the query function so that a malformed response is + // never committed to, or persisted from, the query cache. + queryFn: async () => + this.#validate( + await this.#getJson(METAMASK_STALELIST_URL), + StalelistResponseStruct, + 'stalelist', + ) as Json, staleTime: 0, gcTime: LIST_GC_TIME, }); @@ -485,8 +491,11 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getHotlistDiffs`, timestamp], queryFn: async () => - this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`), - responseStruct: HotlistDiffsResponseStruct, + this.#validate( + await this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`), + HotlistDiffsResponseStruct, + 'hotlist diffs', + ) as Json, staleTime: 0, gcTime: LIST_GC_TIME, }); @@ -511,8 +520,12 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getC2DomainBlocklist`, timestamp ?? null], - queryFn: async () => this.#getJson(url), - responseStruct: C2DomainBlocklistResponseStruct, + queryFn: async () => + this.#validate( + await this.#getJson(url), + C2DomainBlocklistResponseStruct, + 'C2 domain blocklist', + ) as Json, staleTime: 0, gcTime: LIST_GC_TIME, }); @@ -540,9 +553,12 @@ export class PhishingDataService extends BaseDataService< }, }, ); - return this.#toJson(response); + return this.#validate( + await this.#toJson(response), + ScanUrlResponseStruct, + 'URL scan', + ) as Json; }, - responseStruct: ScanUrlResponseStruct, staleTime: SCAN_RESULT_STALE_TIME, gcTime: SCAN_RESULT_GC_TIME, }); @@ -778,11 +794,14 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:scanAddress`, chain, address], queryFn: async () => - this.#postJson(`${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, { - chain, - address, - }), - responseStruct: ScanAddressResponseStruct, + this.#validate( + await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, + { chain, address }, + ), + ScanAddressResponseStruct, + 'address scan', + ) as Json, staleTime: SCAN_RESULT_STALE_TIME, gcTime: SCAN_RESULT_GC_TIME, }); From 68646de92d948d574838135aba248759fc5aa83f Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:40:51 -0500 Subject: [PATCH 07/38] fix: preserve successful bulk token scans --- .../src/PhishingDataService.test.ts | 41 +++++++++++++++++++ .../src/PhishingDataService.ts | 13 +++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 55b04f91a8f..d7539553363 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -498,6 +498,47 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual(apiResponse); }); + it('preserves cached results when an uncached token fails', async () => { + const cachedToken = '0x1234567890123456789012345678901234567890'; + const uncachedToken = '0x0987654321098765432109876543210987654321'; + const cachedResult = { result_type: 'Malicious' }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [cachedToken], + }) + .reply(200, { + results: { + [cachedToken]: cachedResult, + }, + }); + const { rootMessenger } = createService(); + await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + cachedToken, + ); + + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [uncachedToken], + }) + .reply(500, 'boom'); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [cachedToken, uncachedToken], + ); + + expect(response).toStrictEqual({ + results: { + [cachedToken]: cachedResult, + }, + }); + }); + it('accepts a response without a results field', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(TOKEN_BULK_SCANNING_ENDPOINT) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 6ef2da3de0f..41b8818349c 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -724,12 +724,23 @@ export class PhishingDataService extends BaseDataService< loader.flush(); const results: TokenScanApiResponse['results'] = {}; - for (const [token, result] of await Promise.all(entries)) { + let firstError: unknown; + for (const outcome of await Promise.allSettled(entries)) { + if (outcome.status === 'rejected') { + firstError ??= outcome.reason; + continue; + } + + const [token, result] = outcome.value; if (result !== null) { results[token] = result; } } + if (Object.keys(results).length === 0 && firstError !== undefined) { + throw firstError; + } + return { results }; } From e4df83317ad8c9fe3199f0d2da9eab03e2cdbea4 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:43:16 -0500 Subject: [PATCH 08/38] fix: abort timed-out phishing requests --- .../src/PhishingController.ts | 14 +-- .../src/PhishingDataService.test.ts | 40 +++++++ .../src/PhishingDataService.ts | 101 +++++++++++++++--- 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/packages/phishing-controller/src/PhishingController.ts b/packages/phishing-controller/src/PhishingController.ts index fab46a2dd39..06ea99aa48c 100644 --- a/packages/phishing-controller/src/PhishingController.ts +++ b/packages/phishing-controller/src/PhishingController.ts @@ -38,6 +38,13 @@ import type { PhishingControllerTestOriginAction, } from './PhishingController-method-action-types.js'; import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; +import { + ADDRESS_SCAN_TIMEOUT, + APPROVALS_TIMEOUT, + BULK_URL_SCAN_TIMEOUT, + TOKEN_SCAN_TIMEOUT, + URL_SCAN_TIMEOUT, +} from './PhishingDataService.js'; import { PhishingDetector } from './PhishingDetector.js'; import { PhishingDetectorResultType, @@ -111,13 +118,6 @@ export const C2_DOMAIN_BLOCKLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const HOTLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const STALELIST_REFRESH_INTERVAL = 30 * 24 * 60 * 60; // 30 days in seconds -// Request timeouts, in milliseconds. -const URL_SCAN_TIMEOUT = 8000; -const BULK_URL_SCAN_TIMEOUT = 15000; -const TOKEN_SCAN_TIMEOUT = 8000; -const ADDRESS_SCAN_TIMEOUT = 5000; -const APPROVALS_TIMEOUT = 5000; - const controllerName = 'PhishingController'; const metadata: StateMetadata = { diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index d7539553363..02104551b2e 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -25,6 +25,7 @@ import { APPROVALS_ENDPOINT, SCAN_RESULT_GC_TIME, SCAN_RESULT_STALE_TIME, + URL_SCAN_TIMEOUT, } from './PhishingDataService.js'; import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; import { TokenScanResultType } from './types.js'; @@ -257,6 +258,45 @@ describe('PhishingDataService', () => { ).rejects.toThrow('404 Not Found'); }); + it('aborts a timed-out request so the next scan can retry', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + }); + const fetchMock = jest.spyOn(globalThis, 'fetch'); + fetchMock + .mockImplementationOnce( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ recommendedAction: 'BLOCK' }), { + status: 200, + }), + ); + const { rootMessenger } = createService(); + + try { + const timedOutScan = expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow(`timeout of ${URL_SCAN_TIMEOUT}ms exceeded`); + await jest.advanceTimersByTimeAsync(URL_SCAN_TIMEOUT); + await timedOutScan; + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).resolves.toStrictEqual({ recommendedAction: 'BLOCK' }); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + fetchMock.mockRestore(); + } + }); + it('throws if the API returns a malformed response', async () => { nock(PHISHING_DETECTION_BASE_URL) .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 41b8818349c..95b829235e1 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -83,6 +83,13 @@ export const METAMASK_STALELIST_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_STA export const METAMASK_HOTLIST_DIFF_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`; export const C2_DOMAIN_BLOCKLIST_URL = `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`; +// Request timeouts, in milliseconds. +export const URL_SCAN_TIMEOUT = 8000; +export const BULK_URL_SCAN_TIMEOUT = 15000; +export const TOKEN_SCAN_TIMEOUT = 8000; +export const ADDRESS_SCAN_TIMEOUT = 5000; +export const APPROVALS_TIMEOUT = 5000; + /** * The maximum number of URLs sent to the bulk dapp-scanning endpoint in one * request. @@ -543,18 +550,20 @@ export class PhishingDataService extends BaseDataService< async scanUrl(url: string): Promise { const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:scanUrl`, url], - queryFn: async () => { - const response = await fetch( + queryFn: async ({ signal }) => { + const response = await this.#fetchJson( `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent(url)}`, { method: 'GET', headers: { Accept: 'application/json', }, + signal, }, + URL_SCAN_TIMEOUT, ); return this.#validate( - await this.#toJson(response), + response, ScanUrlResponseStruct, 'URL scan', ) as Json; @@ -596,6 +605,7 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.#postJson( `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls: batchUrls }, + { timeout: BULK_URL_SCAN_TIMEOUT }, ); const response = this.#validate( jsonResponse, @@ -758,6 +768,7 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.#postJson( `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, { chain, tokens: batchTokens }, + { timeout: TOKEN_SCAN_TIMEOUT }, ); const response = this.#validate( jsonResponse, @@ -804,11 +815,12 @@ export class PhishingDataService extends BaseDataService< ): Promise { const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:scanAddress`, chain, address], - queryFn: async () => + queryFn: async ({ signal }) => this.#validate( await this.#postJson( `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, { chain, address }, + { signal, timeout: ADDRESS_SCAN_TIMEOUT }, ), ScanAddressResponseStruct, 'address scan', @@ -841,6 +853,7 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.#postJson( `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, { chain, address }, + { timeout: APPROVALS_TIMEOUT }, ); return this.#validate( @@ -857,8 +870,7 @@ export class PhishingDataService extends BaseDataService< * @returns The parsed JSON response. */ async #getJson(url: string): Promise { - const response = await fetch(url, { cache: 'no-cache' }); - return this.#toJson(response); + return this.#fetchJson(url, { cache: 'no-cache' }); } /** @@ -866,18 +878,77 @@ export class PhishingDataService extends BaseDataService< * * @param url - The URL to fetch. * @param body - The request body, serialized as JSON. + * @param options - Request cancellation options. + * @param options.signal - A signal that cancels the request. + * @param options.timeout - The request timeout, in milliseconds. * @returns The parsed JSON response. */ - async #postJson(url: string, body: Record): Promise { - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', + async #postJson( + url: string, + body: Record, + { signal, timeout }: { signal?: AbortSignal; timeout?: number } = {}, + ): Promise { + return this.#fetchJson( + url, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, }, - body: JSON.stringify(body), - }); - return this.#toJson(response); + timeout, + ); + } + + /** + * Performs a fetch request and parses its JSON response. + * + * @param url - The URL to fetch. + * @param init - The fetch request options. + * @param timeout - The optional request timeout, in milliseconds. + * @returns The parsed JSON response. + */ + async #fetchJson( + url: string, + init: RequestInit, + timeout?: number, + ): Promise { + const controller = new AbortController(); + const sourceSignal = init.signal; + let didTimeout = false; + const abort = () => controller.abort(); + const timer = + timeout === undefined + ? undefined + : setTimeout(() => { + didTimeout = true; + controller.abort(); + }, timeout); + + if (sourceSignal?.aborted) { + controller.abort(); + } else { + sourceSignal?.addEventListener('abort', abort, { once: true }); + } + + try { + const response = await fetch(url, { + ...init, + signal: controller.signal, + }); + return await this.#toJson(response); + } catch (error) { + if (didTimeout) { + throw new Error(`timeout of ${timeout}ms exceeded`, { cause: error }); + } + throw error; + } finally { + clearTimeout(timer); + sourceSignal?.removeEventListener('abort', abort); + } } /** From a46528633312877aad404bad4703725c324a6f7b Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:46:17 -0500 Subject: [PATCH 09/38] fix: validate phishing API response fields --- .../src/PhishingController.test.ts | 6 + .../src/PhishingDataService.test.ts | 74 +++++++++-- .../src/PhishingDataService.ts | 124 ++++++++++++++++-- 3 files changed, 183 insertions(+), 21 deletions(-) diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index 4bdfbceaf7b..ede6953d0fe 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -1957,6 +1957,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -1993,6 +1994,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2571,6 +2573,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2609,6 +2612,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2646,6 +2650,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2682,6 +2687,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 02104551b2e..9e1c1e4102e 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -94,7 +94,7 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(METAMASK_STALELIST_FILE) - .reply(200, { data: { lastUpdated: 'not a number' } }); + .reply(200, { data: { lastUpdated: 1700000000 } }); const { rootMessenger } = createService(); await expect( @@ -130,7 +130,7 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) - .reply(200, { data: 'not an array' }); + .reply(200, { data: [{}] }); const { rootMessenger } = createService(); await expect( @@ -301,7 +301,7 @@ describe('PhishingDataService', () => { nock(PHISHING_DETECTION_BASE_URL) .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) .query({ url: 'example.com' }) - .reply(200, {}); + .reply(200, { recommendedAction: 'INVALID' }); const { rootMessenger } = createService(); await expect( @@ -388,7 +388,10 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(PHISHING_DETECTION_BASE_URL) .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) - .reply(200, { results: {} }); + .reply(200, { + results: { 'https://example1.com': {} }, + errors: {}, + }); const { rootMessenger } = createService(); await expect( @@ -467,6 +470,31 @@ describe('PhishingDataService', () => { expect(scope.isDone()).toBe(true); }); + it('does not cache a URL omitted from the API response', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(2) + .reply(200, { results: {}, errors: {} }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + + expect(first.errors['https://example1.com']).toStrictEqual([ + 'No result returned by bulk URL scan endpoint', + ]); + expect(second.errors['https://example1.com']).toStrictEqual([ + 'No result returned by bulk URL scan endpoint', + ]); + expect(scope.isDone()).toBe(true); + }); + it('keeps fresh cached results when another lookup fails', async () => { nock(PHISHING_DETECTION_BASE_URL) .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) @@ -597,7 +625,11 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(TOKEN_BULK_SCANNING_ENDPOINT) - .reply(200, { results: 'not a record' }); + .reply(200, { + results: { + '0x1234567890123456789012345678901234567890': {}, + }, + }); const { rootMessenger } = createService(); await expect( @@ -769,7 +801,9 @@ describe('PhishingDataService', () => { }); it('throws if the API returns a malformed response', async () => { - nock(SECURITY_ALERTS_BASE_URL).post(ADDRESS_SCAN_ENDPOINT).reply(200, {}); + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT) + .reply(200, { result_type: 'Benign' }); const { rootMessenger } = createService(); await expect( @@ -791,9 +825,19 @@ describe('PhishingDataService', () => { approvals: [ { allowance: {}, - asset: {}, - exposure: {}, - spender: {}, + asset: { + address: '0xtoken', + symbol: 'TKN', + name: 'Token', + decimals: 18, + }, + exposure: { + value: '100', + raw_value: '100000000000000000000', + }, + spender: { + address: '0xspender', + }, verdict: 'Benign', }, ], @@ -829,7 +873,17 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(APPROVALS_ENDPOINT) - .reply(200, { approvals: 'not an array' }); + .reply(200, { + approvals: [ + { + allowance: {}, + asset: {}, + exposure: {}, + spender: {}, + verdict: 'Benign', + }, + ], + }); const { rootMessenger } = createService(); await expect( diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 95b829235e1..c788978483b 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -16,13 +16,15 @@ import type { import type { Infer, Struct } from '@metamask/superstruct'; import { array, + boolean, is, + literal, number, optional, record, string, type, - unknown, + union, } from '@metamask/superstruct'; import { Duration, getErrorMessage, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; @@ -40,6 +42,13 @@ import type { PhishingStalelist, TokenScanApiResponse, } from './types.js'; +import { + AddressScanResultType, + ApprovalFeatureType, + ApprovalResultType, + RecommendedAction, + TokenScanResultType, +} from './types.js'; import { getHostnameFromWebUrl, getPhishingDetectionScanUrlParam, @@ -216,18 +225,62 @@ export type PhishingDataServiceMessenger = Messenger< // === RESPONSE VALIDATION === -// The structs below intentionally validate only the shape that the consuming -// code depends on for control flow, mirroring the tolerance of the previous -// in-controller fetching: a response that is missing auxiliary fields is -// passed through rather than rejected. +const RecommendedActionStruct = union([ + literal(RecommendedAction.None), + literal(RecommendedAction.Warn), + literal(RecommendedAction.Block), + literal(RecommendedAction.Verified), +]); + +const TokenScanResultTypeStruct = union([ + literal(TokenScanResultType.Benign), + literal(TokenScanResultType.Warning), + literal(TokenScanResultType.Malicious), + literal(TokenScanResultType.Spam), +]); + +const AddressScanResultTypeStruct = union([ + literal(AddressScanResultType.Benign), + literal(AddressScanResultType.Warning), + literal(AddressScanResultType.Malicious), + literal(AddressScanResultType.ErrorResult), +]); + +const ApprovalResultTypeStruct = union([ + literal(ApprovalResultType.Benign), + literal(ApprovalResultType.Warning), + literal(ApprovalResultType.Malicious), + literal(ApprovalResultType.ErrorResult), +]); + +const ApprovalFeatureTypeStruct = union([ + literal(ApprovalFeatureType.Benign), + literal(ApprovalFeatureType.Warning), + literal(ApprovalFeatureType.Malicious), + literal(ApprovalFeatureType.Info), +]); + const StalelistResponseStruct = type({ data: type({ + allowlist: array(string()), + blocklist: array(string()), + blocklistPaths: array(string()), + fuzzylist: array(string()), + tolerance: number(), + version: number(), lastUpdated: number(), }), }); const HotlistDiffsResponseStruct = type({ - data: array(unknown()), + data: array( + type({ + url: string(), + timestamp: number(), + targetList: string(), + isRemoval: optional(boolean()), + }), + ), }); const C2DomainBlocklistResponseStruct = type({ @@ -236,24 +289,63 @@ const C2DomainBlocklistResponseStruct = type({ }); const ScanUrlResponseStruct = type({ - recommendedAction: string(), + recommendedAction: RecommendedActionStruct, }); const BulkScanUrlsResponseStruct = type({ - results: record(string(), unknown()), + results: record(string(), ScanUrlResponseStruct), errors: record(string(), array(string())), }); +const TokenScanResultStruct = type({ + result_type: TokenScanResultTypeStruct, + chain: optional(string()), + address: optional(string()), +}); + const BulkScanTokensResponseStruct = type({ - results: optional(record(string(), unknown())), + results: optional(record(string(), TokenScanResultStruct)), }); const ScanAddressResponseStruct = type({ - result_type: string(), + result_type: AddressScanResultTypeStruct, + label: string(), +}); + +const ApprovalFeatureStruct = type({ + feature_id: string(), + type: ApprovalFeatureTypeStruct, + description: string(), +}); + +const ApprovalStruct = type({ + allowance: type({ + value: optional(string()), + usd_price: optional(string()), + }), + asset: type({ + address: string(), + symbol: string(), + name: string(), + decimals: number(), + logo_url: optional(string()), + type: optional(string()), + }), + exposure: type({ + usd_price: optional(string()), + value: string(), + raw_value: string(), + }), + spender: type({ + address: string(), + label: optional(string()), + features: optional(array(ApprovalFeatureStruct)), + }), + verdict: ApprovalResultTypeStruct, }); const ApprovalsResponseStruct = type({ - approvals: array(unknown()), + approvals: array(ApprovalStruct), }); // === BATCH LOADING === @@ -619,6 +711,16 @@ export class PhishingDataService extends BaseDataService< for (const [key, messages] of Object.entries(response.errors)) { itemErrors[key] = new BatchItemError(messages.join(', ')); } + for (const url of batchUrls) { + if ( + !Object.hasOwn(response.results, url) && + !Object.hasOwn(itemErrors, url) + ) { + itemErrors[url] = new BatchItemError( + 'No result returned by bulk URL scan endpoint', + ); + } + } return { results: response.results as Record, errors: itemErrors, From 90db8dcd12501cbfb0bf33e9d25bd4a1bc062646 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:48:05 -0500 Subject: [PATCH 10/38] fix: wait for data service cache initialization --- .../src/BaseDataService.test.ts | 30 +++++++++++++++++++ .../base-data-service/src/BaseDataService.ts | 8 ++++- .../src/PhishingDataService.test.ts | 26 ++++++++++++---- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index c647ac6e7f7..390f5977a63 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -506,6 +506,36 @@ describe('BaseDataService', () => { expect(getItem).toHaveBeenCalledWith(serviceName, STORAGE_SERVICE_KEY); }); + it('waits for cache initialization before fetching a query', async () => { + cleanAll(); + const networkScope = mockAssets(); + let resolveGetItem: ((value: { result: null }) => void) | undefined; + const getItem = jest.fn( + () => + new Promise<{ result: null }>((resolve) => { + resolveGetItem = resolve; + }), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + + expect(networkScope.isDone()).toBe(false); + resolveGetItem?.({ result: null }); + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + it('discards the cache if it has expired', async () => { const getItem = jest.fn().mockResolvedValue({ result: { diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index 77ffd8981c7..dbfb3eb6bda 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -167,6 +167,8 @@ export class BaseDataService< readonly #persistenceConfig?: PersistenceConfiguration; + #initializationPromise?: Promise; + constructor({ name, messenger, @@ -284,6 +286,8 @@ export class BaseDataService< queryFn: QueryFunction; responseStruct?: TDataStruct; }): Promise { + await this.#initializationPromise; + return this.#queryClient.fetchQuery({ ...options, queryFn: async (context) => { @@ -336,6 +340,8 @@ export class BaseDataService< }, pageParam?: TPageParam, ): Promise { + await this.#initializationPromise; + const cache = this.#queryClient.getQueryCache(); const query = cache.find< @@ -408,7 +414,7 @@ export class BaseDataService< * Initialize the service, rehydrating the cache with persisted data if possible. */ init(): void { - this.#loadCache().catch( + this.#initializationPromise ??= this.#loadCache().catch( /* istanbul ignore next */ (error) => this.#messenger.captureException?.(error), ); diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 9e1c1e4102e..af608611924 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -984,21 +984,37 @@ describe('PhishingDataService', () => { const persisted = setItem.mock.calls.at(-1)?.[2]; expect(persisted).toBeDefined(); + let resolveGetItem: + | ((value: { result: typeof persisted }) => void) + | undefined; + const getItem = jest.fn( + () => + new Promise<{ result: typeof persisted }>((resolve) => { + resolveGetItem = resolve; + }), + ); const { rootMessenger: secondMessenger, service } = createService({ options: { persistenceConfig: undefined }, setItemMock: jest.fn(), - getItemMock: jest.fn().mockResolvedValue({ result: persisted }), + getItemMock: getItem, }); service.init(); - await flushPromises(); - // No nock interceptor is registered, so this can only succeed if the - // rehydrated entry was used. - const result = await secondMessenger.call( + const networkScope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(500); + const resultPromise = secondMessenger.call( 'PhishingDataService:scanUrl', 'example.com', ); + await flushPromises(); + expect(networkScope.isDone()).toBe(false); + + resolveGetItem?.({ result: persisted }); + const result = await resultPromise; expect(result).toStrictEqual({ recommendedAction: 'NONE' }); + expect(networkScope.isDone()).toBe(false); }); it('discards and removes a persisted cache older than maxAge', async () => { From 87d127c30bbbeaff1aa300ec021f9bd9caf53f66 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:48:44 -0500 Subject: [PATCH 11/38] fix: bound rehydrated phishing query lifetime --- .../src/PhishingDataService.test.ts | 10 +++++++++- .../src/PhishingDataService.ts | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index af608611924..781a1e4a962 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1003,7 +1003,7 @@ describe('PhishingDataService', () => { const networkScope = nock(PHISHING_DETECTION_BASE_URL) .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) .query({ url: 'example.com' }) - .reply(500); + .reply(200, { recommendedAction: 'BLOCK' }); const resultPromise = secondMessenger.call( 'PhishingDataService:scanUrl', 'example.com', @@ -1015,6 +1015,14 @@ describe('PhishingDataService', () => { const result = await resultPromise; expect(result).toStrictEqual({ recommendedAction: 'NONE' }); expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1); + await flushPromises(); + + await expect( + secondMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).resolves.toStrictEqual({ recommendedAction: 'BLOCK' }); + expect(networkScope.isDone()).toBe(true); }); it('discards and removes a persisted cache older than maxAge', async () => { diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index c788978483b..cb3a71e51d2 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -526,7 +526,20 @@ export class PhishingDataService extends BaseDataService< super({ name: serviceName, messenger, - queryClientConfig, + queryClientConfig: { + ...queryClientConfig, + defaultOptions: { + ...queryClientConfig.defaultOptions, + queries: { + // Hydration reconstructs queries using these defaults. Without an + // explicit value, service workers receive TanStack's server + // default of `Infinity`, which cannot later be reduced by a + // per-query option. + gcTime: SCAN_RESULT_GC_TIME, + ...queryClientConfig.defaultOptions?.queries, + }, + }, + }, // Circuit breaking is disabled by default: this service talks to four // independent API hosts through a single shared policy, so a broken // circuit caused by one host's outage would also pause phishing-list From 7c1c20c08e7ab05739e786b7e6859d1161e477e7 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:52:20 -0500 Subject: [PATCH 12/38] test: remove obsolete phishing cache state --- packages/phishing-controller/src/PhishingController.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index ede6953d0fe..cd27e9de780 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -658,7 +658,6 @@ describe('PhishingController', () => { hotlistLastFetched: 0, stalelistLastFetched: 0, c2DomainBlocklistLastFetched: 0, - urlScanCache: {}, }, }); From b23238ade841a07ee8634913951342e0c1bf90c6 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 16:52:51 -0500 Subject: [PATCH 13/38] docs: update data service changelogs --- packages/base-data-service/CHANGELOG.md | 4 ++++ packages/phishing-controller/CHANGELOG.md | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index 235f71ace6e..77b0f46380c 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Wait for cache rehydration to finish before starting a query when `init` has been called, preventing persisted results from racing the first network request ([#9914](https://github.com/MetaMask/core/pull/9914)) + ## [2.0.0] ### Changed diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 6be8fe1dee4..c746d9bd763 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -12,8 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) ([#9914](https://github.com/MetaMask/core/pull/9914)) - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` - Requests are wrapped in a shared service policy, configurable via the `policyOptions` constructor option. Retries and circuit breaking are both disabled by default: the service spans four independent API hosts, so a circuit broken by one host would pause phishing-list updates from the others, and the previous in-controller implementation made a single request per call - - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes); bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached - - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization; pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves + - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes), including after cache rehydration; bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached + - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization; queries started after `init` wait for rehydration to finish. Pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves - Fetched lists (stalelist, hotlist diffs, and the C2 domain blocklist) are not retained by the query cache, and so are not persisted; the controller keeps its own copy in state - Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys ([#9914](https://github.com/MetaMask/core/pull/9914)) @@ -27,9 +27,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Malformed API responses (e.g. a stalelist without a numeric `lastUpdated`, or scan results without a `recommendedAction`/`result_type`) are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `bulkScanTokens` now preserves successful cached verdicts when another token lookup fails, instead of discarding every result in the batch ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Timed-out URL, token, and address-security requests are now aborted so later calls can retry instead of remaining attached to the original pending query ([#9914](https://github.com/MetaMask/core/pull/9914)) ### Removed From 3478d0088a1f9f24f9ae8798eafd6d01f0d96142 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:04:13 -0500 Subject: [PATCH 14/38] test: avoid leaking timed-out HTTP requests --- .../src/BulkTokenScan.test.ts | 30 +++++++--- .../src/PhishingController.test.ts | 58 +++++++++---------- .../src/PhishingDataService.test.ts | 37 +++++++----- .../src/PhishingDataService.ts | 30 ++++------ 4 files changed, 88 insertions(+), 67 deletions(-) diff --git a/packages/phishing-controller/src/BulkTokenScan.test.ts b/packages/phishing-controller/src/BulkTokenScan.test.ts index b012582d680..d79a42495cc 100644 --- a/packages/phishing-controller/src/BulkTokenScan.test.ts +++ b/packages/phishing-controller/src/BulkTokenScan.test.ts @@ -116,6 +116,26 @@ function getPhishingController(options?: Partial) { }); } +/** + * Mock a fetch that remains pending until its abort signal fires. + * + * @returns The fetch spy. + */ +function mockPendingFetch(): jest.SpiedFunction { + return jest + .spyOn(globalThis, 'fetch') + .mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ), + ), + ); +} + describe('PhishingController - Bulk Token Scanning', () => { let controller: PhishingController; let consoleErrorSpy: jest.SpyInstance; @@ -129,8 +149,7 @@ describe('PhishingController - Bulk Token Scanning', () => { afterEach(() => { cleanAll(); - consoleErrorSpy.mockRestore(); - consoleWarnSpy.mockRestore(); + jest.restoreAllMocks(); while (createdDataServices.length > 0) { createdDataServices.pop()?.destroy(); } @@ -439,11 +458,7 @@ describe('PhishingController - Bulk Token Scanning', () => { now: 1_000_000, }); const tokens = ['0x1234567890123456789012345678901234567890']; - - nock(SECURITY_ALERTS_BASE_URL) - .post(TOKEN_BULK_SCANNING_ENDPOINT) - .delayConnection(10000) - .reply(200, { results: {} }); + const fetchMock = mockPendingFetch(); const request: BulkTokenScanRequest = { chainId: '0x1', @@ -458,6 +473,7 @@ describe('PhishingController - Bulk Token Scanning', () => { expect(consoleErrorSpy).toHaveBeenCalledWith( 'Error scanning tokens: timeout of 8000ms exceeded', ); + expect(fetchMock).toHaveBeenCalledTimes(1); jest.useRealTimers(); }); }); diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index cd27e9de780..da18cf1877d 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -251,9 +251,30 @@ function getPhishingController(options?: Partial): { return { controller, rootMessenger }; } +/** + * Mock a fetch that remains pending until its abort signal fires. + * + * @returns The fetch spy. + */ +function mockPendingFetch(): jest.SpiedFunction { + return jest + .spyOn(globalThis, 'fetch') + .mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ), + ), + ); +} + describe('PhishingController', () => { afterEach(() => { jest.useRealTimers(); + jest.restoreAllMocks(); cleanAll(); destroyDataServices(); }); @@ -2968,11 +2989,7 @@ describe('PhishingController', () => { ); it('should return a PhishingDetectionScanResult with a fetchError on timeout', async () => { - const scope = nock(PHISHING_DETECTION_BASE_URL) - .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) - .query({ url: 'example.com' }) - .delayConnection(10000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call('PhishingController:scanUrl', testUrl); jest.advanceTimersByTime(8000); @@ -2982,7 +2999,7 @@ describe('PhishingController', () => { recommendedAction: RecommendedAction.None, fetchError: 'timeout of 8000ms exceeded', }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('should only send hostname when URL contains query parameters', async () => { @@ -3274,12 +3291,7 @@ describe('PhishingController', () => { ); it('should handle timeouts correctly', async () => { - const scope = nock(PHISHING_DETECTION_BASE_URL) - .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { - urls: testUrls, - }) - .delayConnection(20000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:bulkScanUrls', @@ -3293,7 +3305,7 @@ describe('PhishingController', () => { network_error: ['timeout of 15000ms exceeded'], }, }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('should process URLs in batches when more than 50 URLs are provided', async () => { @@ -3713,13 +3725,7 @@ describe('PhishingController', () => { ); it('will return an AddressScanResult with an ErrorResult on timeout', async () => { - const scope = nock(SECURITY_ALERTS_BASE_URL) - .post(ADDRESS_SCAN_ENDPOINT, { - chain: 'ethereum', - address: testAddress.toLowerCase(), - }) - .delayConnection(10000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:scanAddress', @@ -3732,7 +3738,7 @@ describe('PhishingController', () => { result_type: AddressScanResultType.ErrorResult, label: '', }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('will return an AddressScanResult with an ErrorResult when address is missing', async () => { @@ -4020,13 +4026,7 @@ describe('PhishingController', () => { }); it('will return empty approvals on timeout', async () => { - const scope = nock(SECURITY_ALERTS_BASE_URL) - .post(APPROVALS_ENDPOINT, { - chain: 'ethereum', - address: testAddress.toLowerCase(), - }) - .delayConnection(10000) - .reply(200, mockResponse); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:getApprovals', @@ -4036,7 +4036,7 @@ describe('PhishingController', () => { jest.advanceTimersByTime(5000); const response = await promise; expect(response).toStrictEqual({ approvals: [] }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('will normalize address to lowercase before API call', async () => { diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 781a1e4a962..e3ea9e951fc 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -275,22 +275,30 @@ describe('PhishingDataService', () => { }), ) .mockResolvedValueOnce( - new Response(JSON.stringify({ recommendedAction: 'BLOCK' }), { - status: 200, - }), + new globalThis.Response( + JSON.stringify({ recommendedAction: 'BLOCK' }), + { + status: 200, + }, + ), ); const { rootMessenger } = createService(); try { - const timedOutScan = expect( - rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).rejects.toThrow(`timeout of ${URL_SCAN_TIMEOUT}ms exceeded`); + const timedOutScan = rootMessenger + .call('PhishingDataService:scanUrl', 'example.com') + .catch((error) => error); await jest.advanceTimersByTimeAsync(URL_SCAN_TIMEOUT); - await timedOutScan; + expect(await timedOutScan).toMatchObject({ + message: `timeout of ${URL_SCAN_TIMEOUT}ms exceeded`, + }); - await expect( - rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).resolves.toStrictEqual({ recommendedAction: 'BLOCK' }); + expect( + await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ), + ).toStrictEqual({ recommendedAction: 'BLOCK' }); expect(fetchMock).toHaveBeenCalledTimes(2); } finally { fetchMock.mockRestore(); @@ -1019,9 +1027,12 @@ describe('PhishingDataService', () => { jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1); await flushPromises(); - await expect( - secondMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).resolves.toStrictEqual({ recommendedAction: 'BLOCK' }); + expect( + await secondMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ), + ).toStrictEqual({ recommendedAction: 'BLOCK' }); expect(networkScope.isDone()).toBe(true); }); diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index cb3a71e51d2..2d7cbab8e18 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -781,15 +781,13 @@ export class PhishingDataService extends BaseDataService< continue; } - if (outcome.value !== null) { - const scanResult = outcome.value as PhishingDetectionScanResult; - // Entries seeded by single-URL scans hold the raw scan response, - // which may not include the hostname; fill it in from the URL. - results[url] = { - ...scanResult, - hostname: scanResult.hostname ?? hostname, - }; - } + const scanResult = outcome.value as PhishingDetectionScanResult; + // Entries seeded by single-URL scans hold the raw scan response, + // which may not include the hostname; fill it in from the URL. + results[url] = { + ...scanResult, + hostname: scanResult.hostname ?? hostname, + }; } // A request-level failure that produced nothing at all is surfaced to the @@ -849,10 +847,10 @@ export class PhishingDataService extends BaseDataService< loader.flush(); const results: TokenScanApiResponse['results'] = {}; - let firstError: unknown; + let firstError: Error | undefined; for (const outcome of await Promise.allSettled(entries)) { if (outcome.status === 'rejected') { - firstError ??= outcome.reason; + firstError ??= outcome.reason as Error; continue; } @@ -1001,7 +999,7 @@ export class PhishingDataService extends BaseDataService< async #postJson( url: string, body: Record, - { signal, timeout }: { signal?: AbortSignal; timeout?: number } = {}, + { signal, timeout }: { signal?: AbortSignal; timeout?: number }, ): Promise { return this.#fetchJson( url, @@ -1034,7 +1032,7 @@ export class PhishingDataService extends BaseDataService< const controller = new AbortController(); const sourceSignal = init.signal; let didTimeout = false; - const abort = () => controller.abort(); + const abort = (): void => controller.abort(); const timer = timeout === undefined ? undefined @@ -1043,11 +1041,7 @@ export class PhishingDataService extends BaseDataService< controller.abort(); }, timeout); - if (sourceSignal?.aborted) { - controller.abort(); - } else { - sourceSignal?.addEventListener('abort', abort, { once: true }); - } + sourceSignal?.addEventListener('abort', abort, { once: true }); try { const response = await fetch(url, { From a8bfc6bc41f7368b3fb787da6aad1a71d6f37501 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:07:36 -0500 Subject: [PATCH 15/38] fix: preserve uninitialized query timing --- packages/base-data-service/src/BaseDataService.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index dbfb3eb6bda..d1d45d0c873 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -286,7 +286,9 @@ export class BaseDataService< queryFn: QueryFunction; responseStruct?: TDataStruct; }): Promise { - await this.#initializationPromise; + if (this.#initializationPromise) { + await this.#initializationPromise; + } return this.#queryClient.fetchQuery({ ...options, @@ -340,7 +342,9 @@ export class BaseDataService< }, pageParam?: TPageParam, ): Promise { - await this.#initializationPromise; + if (this.#initializationPromise) { + await this.#initializationPromise; + } const cache = this.#queryClient.getQueryCache(); From 176dd219df82a49ec8205ba88fdbd03dcd4ef14b Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:08:02 -0500 Subject: [PATCH 16/38] fix: enforce phishing query retention bound --- .../phishing-controller/src/PhishingDataService.test.ts | 7 ++++++- packages/phishing-controller/src/PhishingDataService.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index e3ea9e951fc..31ce768a4c8 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1002,7 +1002,12 @@ describe('PhishingDataService', () => { }), ); const { rootMessenger: secondMessenger, service } = createService({ - options: { persistenceConfig: undefined }, + options: { + persistenceConfig: undefined, + queryClientConfig: { + defaultOptions: { queries: { gcTime: Infinity } }, + }, + }, setItemMock: jest.fn(), getItemMock: getItem, }); diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 2d7cbab8e18..1c84b24f6f5 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -531,12 +531,12 @@ export class PhishingDataService extends BaseDataService< defaultOptions: { ...queryClientConfig.defaultOptions, queries: { + ...queryClientConfig.defaultOptions?.queries, // Hydration reconstructs queries using these defaults. Without an // explicit value, service workers receive TanStack's server // default of `Infinity`, which cannot later be reduced by a // per-query option. gcTime: SCAN_RESULT_GC_TIME, - ...queryClientConfig.defaultOptions?.queries, }, }, }, From 0054126b5623c1193ea8b4905938f60c963ec850 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:08:37 -0500 Subject: [PATCH 17/38] fix: tighten phishing response discriminants --- .../src/PhishingDataService.test.ts | 26 ++++++++++++++++++- .../src/PhishingDataService.ts | 12 ++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 31ce768a4c8..4757506eeed 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -130,7 +130,15 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) - .reply(200, { data: [{}] }); + .reply(200, { + data: [ + { + url: 'phishing.example.com', + timestamp: 1700000001, + targetList: 'unexpected.blocklist', + }, + ], + }); const { rootMessenger } = createService(); await expect( @@ -317,6 +325,22 @@ describe('PhishingDataService', () => { ).rejects.toThrow('Malformed response received from URL scan endpoint'); }); + it('validates optional URL scan response fields when present', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { + hostname: 123, + recommendedAction: 'NONE', + fetchError: false, + }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('Malformed response received from URL scan endpoint'); + }); + it('does not cache a malformed response', async () => { const scope = nock(PHISHING_DETECTION_BASE_URL) .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 1c84b24f6f5..6c0a9165361 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -260,6 +260,14 @@ const ApprovalFeatureTypeStruct = union([ literal(ApprovalFeatureType.Info), ]); +const HotlistTargetListStruct = union([ + literal('eth_phishing_detect_config.allowlist'), + literal('eth_phishing_detect_config.blocklist'), + literal('eth_phishing_detect_config.blocklistPaths'), + literal('eth_phishing_detect_config.fuzzylist'), + literal('eth_phishing_detect_config.c2DomainBlocklist'), +]); + const StalelistResponseStruct = type({ data: type({ allowlist: array(string()), @@ -277,7 +285,7 @@ const HotlistDiffsResponseStruct = type({ type({ url: string(), timestamp: number(), - targetList: string(), + targetList: HotlistTargetListStruct, isRemoval: optional(boolean()), }), ), @@ -289,7 +297,9 @@ const C2DomainBlocklistResponseStruct = type({ }); const ScanUrlResponseStruct = type({ + hostname: optional(string()), recommendedAction: RecommendedActionStruct, + fetchError: optional(string()), }); const BulkScanUrlsResponseStruct = type({ From a5bd89ada400e729c0705eb3383276679be2b024 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:09:16 -0500 Subject: [PATCH 18/38] fix: cancel pending phishing list requests --- .../src/PhishingDataService.test.ts | 27 +++++++++++++++++++ .../src/PhishingDataService.ts | 23 ++++++++++------ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 4757506eeed..5261e2bbbad 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -101,6 +101,33 @@ describe('PhishingDataService', () => { rootMessenger.call('PhishingDataService:getStalelist'), ).rejects.toThrow('Malformed response received from stalelist endpoint'); }); + + it('aborts a pending list request when the service is destroyed', async () => { + let requestSignal: AbortSignal | null | undefined; + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + requestSignal = init?.signal; + requestSignal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + }), + ); + const { service } = createService(); + const pendingRequest = service.getStalelist().catch((error) => error); + + try { + await flushPromises(); + service.destroy(); + + expect(requestSignal?.aborted).toBe(true); + await pendingRequest; + } finally { + fetchMock.mockRestore(); + } + }); }); describe('getHotlistDiffs', () => { diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 6c0a9165361..3f4fc19108e 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -588,9 +588,9 @@ export class PhishingDataService extends BaseDataService< queryKey: [`${this.name}:getStalelist`], // Validated inside the query function so that a malformed response is // never committed to, or persisted from, the query cache. - queryFn: async () => + queryFn: async ({ signal }) => this.#validate( - await this.#getJson(METAMASK_STALELIST_URL), + await this.#getJson(METAMASK_STALELIST_URL, { signal }), StalelistResponseStruct, 'stalelist', ) as Json, @@ -612,9 +612,11 @@ export class PhishingDataService extends BaseDataService< ): Promise> { const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getHotlistDiffs`, timestamp], - queryFn: async () => + queryFn: async ({ signal }) => this.#validate( - await this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`), + await this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`, { + signal, + }), HotlistDiffsResponseStruct, 'hotlist diffs', ) as Json, @@ -642,9 +644,9 @@ export class PhishingDataService extends BaseDataService< const jsonResponse = await this.fetchQuery({ queryKey: [`${this.name}:getC2DomainBlocklist`, timestamp ?? null], - queryFn: async () => + queryFn: async ({ signal }) => this.#validate( - await this.#getJson(url), + await this.#getJson(url, { signal }), C2DomainBlocklistResponseStruct, 'C2 domain blocklist', ) as Json, @@ -990,10 +992,15 @@ export class PhishingDataService extends BaseDataService< * Performs a GET request against a phishing configuration endpoint. * * @param url - The URL to fetch. + * @param options - Request cancellation options. + * @param options.signal - A signal that cancels the request. * @returns The parsed JSON response. */ - async #getJson(url: string): Promise { - return this.#fetchJson(url, { cache: 'no-cache' }); + async #getJson( + url: string, + { signal }: { signal?: AbortSignal }, + ): Promise { + return this.#fetchJson(url, { cache: 'no-cache', signal }); } /** From a905f5c09e16fdd535f58f75789f1b2dfdc3b1f9 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 17:09:55 -0500 Subject: [PATCH 19/38] test: cover initialized infinite queries --- packages/base-data-service/src/BaseDataService.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index 390f5977a63..e2d77991116 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -509,6 +509,7 @@ describe('BaseDataService', () => { it('waits for cache initialization before fetching a query', async () => { cleanAll(); const networkScope = mockAssets(); + const activityScope = mockTransactionsPage1(); let resolveGetItem: ((value: { result: null }) => void) | undefined; const getItem = jest.fn( () => @@ -533,6 +534,9 @@ describe('BaseDataService', () => { expect(await resultPromise).toHaveLength(3); expect(networkScope.isDone()).toBe(true); + expect(await service.getActivity(TEST_ADDRESS)).toHaveProperty('data'); + expect(activityScope.isDone()).toBe(true); + service.destroy(); }); From 4a8f8e1c08274394396e9546cd7ed934acae46e6 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 18:12:04 -0500 Subject: [PATCH 20/38] fix: accept documented security alert verdicts --- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 44 ++++++++++++++++++- .../src/PhishingDataService.ts | 6 +++ packages/phishing-controller/src/types.ts | 15 +++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index c746d9bd763..3539eb76172 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) - Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanTokens` now preserves successful cached verdicts when another token lookup fails, instead of discarding every result in the batch ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 5261e2bbbad..262deec8249 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -625,6 +625,30 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual(apiResponse); }); + it('accepts the Verified verdict returned by the token API', async () => { + const token = '0x1234567890123456789012345678901234567890'; + const apiResponse = { + results: { + [token]: { result_type: 'Verified' }, + }, + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [token], + ), + ).toStrictEqual(apiResponse); + }); + it('preserves cached results when an uncached token fails', async () => { const cachedToken = '0x1234567890123456789012345678901234567890'; const uncachedToken = '0x0987654321098765432109876543210987654321'; @@ -859,6 +883,24 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual({ result_type: 'Benign', label: '' }); }); + it.each(['Verified', 'Trusted', 'Error'])( + 'accepts the %s verdict returned by the address API', + async (resultType) => { + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT) + .reply(200, { result_type: resultType, label: '' }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ result_type: resultType, label: '' }); + }, + ); + it('throws if the API returns a malformed response', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(ADDRESS_SCAN_ENDPOINT) @@ -897,7 +939,7 @@ describe('PhishingDataService', () => { spender: { address: '0xspender', }, - verdict: 'Benign', + verdict: 'Verified', }, ], }; diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 3f4fc19108e..89fde83aa06 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -233,6 +233,7 @@ const RecommendedActionStruct = union([ ]); const TokenScanResultTypeStruct = union([ + literal(TokenScanResultType.Verified), literal(TokenScanResultType.Benign), literal(TokenScanResultType.Warning), literal(TokenScanResultType.Malicious), @@ -240,13 +241,18 @@ const TokenScanResultTypeStruct = union([ ]); const AddressScanResultTypeStruct = union([ + literal(AddressScanResultType.Verified), + literal(AddressScanResultType.Trusted), literal(AddressScanResultType.Benign), literal(AddressScanResultType.Warning), literal(AddressScanResultType.Malicious), literal(AddressScanResultType.ErrorResult), + literal(AddressScanResultType.ApiError), ]); const ApprovalResultTypeStruct = union([ + literal(ApprovalResultType.Verified), + literal(ApprovalResultType.Trusted), literal(ApprovalResultType.Benign), literal(ApprovalResultType.Warning), literal(ApprovalResultType.Malicious), diff --git a/packages/phishing-controller/src/types.ts b/packages/phishing-controller/src/types.ts index b8ac85c7ea2..370db674e43 100644 --- a/packages/phishing-controller/src/types.ts +++ b/packages/phishing-controller/src/types.ts @@ -310,6 +310,7 @@ export type BulkTokenScanRequest = { * Result type of a token scan */ export enum TokenScanResultType { + Verified = 'Verified', Benign = 'Benign', Warning = 'Warning', Malicious = 'Malicious', @@ -417,6 +418,14 @@ export type ChainIdToNameMap = typeof DEFAULT_CHAIN_ID_TO_NAME; * Result type of an address scan */ export enum AddressScanResultType { + /** + * Address is verified by internal trust signals + */ + Verified = 'Verified', + /** + * Address is trusted by internal trust signals + */ + Trusted = 'Trusted', /** * Address is benign/safe */ @@ -433,6 +442,10 @@ export enum AddressScanResultType { * Error occurred during scan */ ErrorResult = 'ErrorResult', + /** + * Error returned by the security alerts API + */ + ApiError = 'Error', } /** @@ -611,6 +624,8 @@ export enum ApprovalResultType { Malicious = 'Malicious', Warning = 'Warning', Benign = 'Benign', + Trusted = 'Trusted', + Verified = 'Verified', ErrorResult = 'Error', } From 97d0161a0c6f9130260c67dfefd0fbd921262107 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 18:37:16 -0500 Subject: [PATCH 21/38] fix: avoid caching URL scan fetch errors --- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 91 +++++++++++++++++++ .../src/PhishingDataService.ts | 23 +++-- 3 files changed, 108 insertions(+), 7 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 3539eb76172..6c81d1f7aab 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) - Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) +- URL scan responses containing `fetchError` are now treated as failures and are not cached, allowing subsequent calls to retry the detector ([#9914](https://github.com/MetaMask/core/pull/9914)) - Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 262deec8249..ba1a84c2e63 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -390,6 +390,28 @@ describe('PhishingDataService', () => { expect(scope.isDone()).toBe(true); }); + it('does not cache a response containing a fetch error', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { + recommendedAction: 'NONE', + fetchError: 'detector unavailable', + }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('detector unavailable'); + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).toStrictEqual({ recommendedAction: 'BLOCK' }); + expect(scope.isDone()).toBe(true); + }); + it('refetches once a cached result passes its garbage collection time', async () => { jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], @@ -529,6 +551,75 @@ describe('PhishingDataService', () => { expect(scope.isDone()).toBe(true); }); + it('does not cache URL results containing a fetch error', async () => { + const blockedUrl = 'https://blocked.com'; + const failedUrl = 'https://failed.com'; + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { + urls: [blockedUrl, failedUrl], + }) + .reply(200, { + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'NONE', + fetchError: 'detector unavailable', + }, + }, + errors: {}, + }) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { + urls: [failedUrl], + }) + .reply(200, { + results: { + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'WARN', + }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [blockedUrl, failedUrl], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [blockedUrl, failedUrl], + ); + + expect(first).toStrictEqual({ + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + }, + errors: { [failedUrl]: ['detector unavailable'] }, + }); + expect(second).toStrictEqual({ + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'WARN', + }, + }, + errors: {}, + }); + expect(scope.isDone()).toBe(true); + }); + it('does not cache a URL omitted from the API response', async () => { const scope = nock(PHISHING_DETECTION_BASE_URL) .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 89fde83aa06..293c6c84037 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -685,11 +685,15 @@ export class PhishingDataService extends BaseDataService< }, URL_SCAN_TIMEOUT, ); - return this.#validate( + const scanResult = this.#validate( response, ScanUrlResponseStruct, 'URL scan', - ) as Json; + ); + if (scanResult.fetchError) { + throw new Error(scanResult.fetchError); + } + return scanResult as Json; }, staleTime: SCAN_RESULT_STALE_TIME, gcTime: SCAN_RESULT_GC_TIME, @@ -742,18 +746,23 @@ export class PhishingDataService extends BaseDataService< for (const [key, messages] of Object.entries(response.errors)) { itemErrors[key] = new BatchItemError(messages.join(', ')); } + const results: Record = {}; + for (const [key, result] of Object.entries(response.results)) { + if (result.fetchError) { + itemErrors[key] = new BatchItemError(result.fetchError); + } else { + results[key] = result as Json; + } + } for (const url of batchUrls) { - if ( - !Object.hasOwn(response.results, url) && - !Object.hasOwn(itemErrors, url) - ) { + if (!Object.hasOwn(results, url) && !Object.hasOwn(itemErrors, url)) { itemErrors[url] = new BatchItemError( 'No result returned by bulk URL scan endpoint', ); } } return { - results: response.results as Record, + results, errors: itemErrors, }; }, From 21f97dd89cff18acb5501da36a89b87ab76591bc Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 20:32:11 -0500 Subject: [PATCH 22/38] fix: abort requests when phishing service is destroyed --- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 44 +++++++++++++++++++ .../src/PhishingDataService.ts | 18 ++++++++ 3 files changed, 63 insertions(+) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 6c81d1f7aab..984c4c67b8b 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanTokens` now preserves successful cached verdicts when another token lookup fails, instead of discarding every result in the batch ([#9914](https://github.com/MetaMask/core/pull/9914)) - Timed-out URL, token, and address-security requests are now aborted so later calls can retry instead of remaining attached to the original pending query ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Destroying `PhishingDataService` now aborts all pending requests, including batched scans and uncached approval requests ([#9914](https://github.com/MetaMask/core/pull/9914)) ### Removed diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index ba1a84c2e63..fd744ef90a2 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -128,6 +128,50 @@ describe('PhishingDataService', () => { fetchMock.mockRestore(); } }); + + it('aborts batched and uncached POST requests when destroyed', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask', 'setImmediate'], + }); + const requestSignals: AbortSignal[] = []; + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + if (signal) { + requestSignals.push(signal); + if (signal.aborted) { + reject(new Error('aborted')); + } else { + signal.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + } + } + }), + ); + const { service } = createService(); + const pendingBulkScan = service + .bulkScanUrls(['https://example.com']) + .catch((error) => error); + const pendingApprovals = service + .getApprovals('ethereum', '0x1234567890123456789012345678901234567890') + .catch((error) => error); + + try { + await flushPromises(); + expect(requestSignals).toHaveLength(2); + + service.destroy(); + + expect(requestSignals.every((signal) => signal.aborted)).toBe(true); + await Promise.all([pendingBulkScan, pendingApprovals]); + } finally { + fetchMock.mockRestore(); + } + }); }); describe('getHotlistDiffs', () => { diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 293c6c84037..6df20377f7a 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -513,6 +513,8 @@ export class PhishingDataService extends BaseDataService< typeof serviceName, PhishingDataServiceMessenger > { + readonly #abortController = new AbortController(); + /** * Constructs a new PhishingDataService object. * @@ -584,6 +586,15 @@ export class PhishingDataService extends BaseDataService< ); } + /** + * Aborts all requests owned by this service before clearing its query cache + * and messenger registrations. + */ + override destroy(): void { + this.#abortController.abort(); + super.destroy(); + } + /** * Fetches the full phishing detection stalelist. * @@ -1063,6 +1074,7 @@ export class PhishingDataService extends BaseDataService< ): Promise { const controller = new AbortController(); const sourceSignal = init.signal; + const serviceSignal = this.#abortController.signal; let didTimeout = false; const abort = (): void => controller.abort(); const timer = @@ -1074,6 +1086,11 @@ export class PhishingDataService extends BaseDataService< }, timeout); sourceSignal?.addEventListener('abort', abort, { once: true }); + serviceSignal.addEventListener('abort', abort, { once: true }); + /* istanbul ignore next -- service actions are removed during destruction */ + if (sourceSignal?.aborted || serviceSignal.aborted) { + controller.abort(); + } try { const response = await fetch(url, { @@ -1089,6 +1106,7 @@ export class PhishingDataService extends BaseDataService< } finally { clearTimeout(timer); sourceSignal?.removeEventListener('abort', abort); + serviceSignal.removeEventListener('abort', abort); } } From e9242e8aa2002fb525d5bfcb95c4997c5417db31 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 20:43:16 -0500 Subject: [PATCH 23/38] fix: apply service policy to approval requests --- packages/base-data-service/CHANGELOG.md | 4 ++++ .../base-data-service/src/BaseDataService.ts | 17 ++++++++++++-- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 22 +++++++++++++++++++ .../src/PhishingDataService.ts | 10 +++++---- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index 77b0f46380c..fabd43e6aab 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add a protected `executeWithPolicy` helper for applying a data service's retry and circuit-breaker policy to uncached requests ([#9914](https://github.com/MetaMask/core/pull/9914)) + ### Fixed - Wait for cache rehydration to finish before starting a query when `init` has been called, preventing persisted results from racing the first network request ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index d1d45d0c873..c2534f13f65 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -293,7 +293,7 @@ export class BaseDataService< return this.#queryClient.fetchQuery({ ...options, queryFn: async (context) => { - const response = await this.#policy.execute(() => + const response = await this.executeWithPolicy(() => options.queryFn(context), ); return processQueryResponse(options.queryKey, response, responseStruct); @@ -361,7 +361,7 @@ export class BaseDataService< ...options, initialPageParam: pageParam ?? options.initialPageParam, queryFn: async (context) => { - const response = await this.#policy.execute(async () => + const response = await this.executeWithPolicy(async () => options.queryFn({ ...context, pageParam: context.meta?.pageParam ?? context.pageParam, @@ -400,6 +400,19 @@ export class BaseDataService< return result.pages[pageIndex]; } + /** + * Executes an operation using this service's retry and circuit-breaker + * policy without adding the result to the query cache. + * + * @param operation - The asynchronous operation to execute. + * @returns The operation result. + */ + protected async executeWithPolicy( + operation: () => PromiseLike | Result, + ): Promise { + return this.#policy.execute(operation); + } + /** * Invalidate queries serviced by this data service. * diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 984c4c67b8b..073c449bd25 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Uncached approval requests now use the configured retry and circuit-breaker policy consistently with cached service requests ([#9914](https://github.com/MetaMask/core/pull/9914)) - Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) - URL scan responses containing `fetchError` are now treated as failures and are not cached, allowing subsequent calls to retry the detector ([#9914](https://github.com/MetaMask/core/pull/9914)) - Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index fd744ef90a2..4b9e55dd5dc 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1106,6 +1106,28 @@ describe('PhishingDataService', () => { expect(response2).toStrictEqual(secondResponse); }); + it('applies the configured service policy', async () => { + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(500) + .post(APPROVALS_ENDPOINT) + .reply(200, { approvals: [] }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 1, backoff: new ConstantBackoff(0) }, + }, + }); + + expect( + await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ approvals: [] }); + expect(scope.isDone()).toBe(true); + }); + it('throws if the API returns a malformed response', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(APPROVALS_ENDPOINT) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 6df20377f7a..4ac7e6d1877 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -1001,10 +1001,12 @@ export class PhishingDataService extends BaseDataService< // provide no benefit while publishing the response on the messenger as a // `cacheUpdated` payload. This matches the handling of non-cached POSTs // elsewhere in the monorepo. - const jsonResponse = await this.#postJson( - `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, - { chain, address }, - { timeout: APPROVALS_TIMEOUT }, + const jsonResponse = await this.executeWithPolicy(() => + this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, + { chain, address }, + { timeout: APPROVALS_TIMEOUT }, + ), ); return this.#validate( From 53df4e3af07ec04e9935deb2ca2272690ac6f16e Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Wed, 9 Sep 2026 21:39:31 -0500 Subject: [PATCH 24/38] fix: enforce phishing service return contracts --- packages/phishing-controller/CHANGELOG.md | 1 + .../src/PhishingDataService.test.ts | 36 ++++++++++++++----- .../src/PhishingDataService.ts | 7 +++- packages/phishing-controller/src/types.ts | 4 +-- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index 073c449bd25..ce2509eee7c 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) - Uncached approval requests now use the configured retry and circuit-breaker policy consistently with cached service requests ([#9914](https://github.com/MetaMask/core/pull/9914)) - Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `PhishingDataService:scanUrl` now always returns a hostname, and C2 blocklist responses now validate `lastFetchedAt` as the numeric Unix timestamp returned by the API ([#9914](https://github.com/MetaMask/core/pull/9914)) - URL scan responses containing `fetchError` are now treated as failures and are not cached, allowing subsequent calls to retry the detector ([#9914](https://github.com/MetaMask/core/pull/9914)) - Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 4b9e55dd5dc..0033460aec0 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -225,7 +225,7 @@ describe('PhishingDataService', () => { const blocklist = { recentlyAdded: ['0415f1f1'], recentlyRemoved: [], - lastFetchedAt: '2024-01-01T00:00:00Z', + lastFetchedAt: 1700000000, }; nock(CLIENT_SIDE_DETECION_BASE_URL) .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) @@ -243,7 +243,7 @@ describe('PhishingDataService', () => { const blocklist = { recentlyAdded: [], recentlyRemoved: ['0415f1f1'], - lastFetchedAt: '2024-01-01T00:00:00Z', + lastFetchedAt: 1700000000, }; nock(CLIENT_SIDE_DETECION_BASE_URL) .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) @@ -262,7 +262,7 @@ describe('PhishingDataService', () => { it('throws if the API returns a malformed response', async () => { nock(CLIENT_SIDE_DETECION_BASE_URL) .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) - .reply(200, { recentlyAdded: 'not an array' }); + .reply(200, { recentlyAdded: [], recentlyRemoved: [] }); const { rootMessenger } = createService(); await expect( @@ -322,7 +322,10 @@ describe('PhishingDataService', () => { 'PhishingDataService:scanUrl', 'example.com', ); - expect(response3).toStrictEqual({ recommendedAction: 'BLOCK' }); + expect(response3).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); }); it('throws if the API returns a non-200 status', async () => { @@ -377,7 +380,10 @@ describe('PhishingDataService', () => { 'PhishingDataService:scanUrl', 'example.com', ), - ).toStrictEqual({ recommendedAction: 'BLOCK' }); + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); expect(fetchMock).toHaveBeenCalledTimes(2); } finally { fetchMock.mockRestore(); @@ -430,7 +436,10 @@ describe('PhishingDataService', () => { // next call re-requests and sees the real verdict. expect( await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).toStrictEqual({ recommendedAction: 'BLOCK' }); + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); expect(scope.isDone()).toBe(true); }); @@ -452,7 +461,10 @@ describe('PhishingDataService', () => { ).rejects.toThrow('detector unavailable'); expect( await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), - ).toStrictEqual({ recommendedAction: 'BLOCK' }); + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); expect(scope.isDone()).toBe(true); }); @@ -1276,7 +1288,10 @@ describe('PhishingDataService', () => { resolveGetItem?.({ result: persisted }); const result = await resultPromise; - expect(result).toStrictEqual({ recommendedAction: 'NONE' }); + expect(result).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'NONE', + }); expect(networkScope.isDone()).toBe(false); jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1); @@ -1287,7 +1302,10 @@ describe('PhishingDataService', () => { 'PhishingDataService:scanUrl', 'example.com', ), - ).toStrictEqual({ recommendedAction: 'BLOCK' }); + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); expect(networkScope.isDone()).toBe(true); }); diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 4ac7e6d1877..3dcfed4f045 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -300,6 +300,7 @@ const HotlistDiffsResponseStruct = type({ const C2DomainBlocklistResponseStruct = type({ recentlyAdded: array(string()), recentlyRemoved: array(string()), + lastFetchedAt: number(), }); const ScanUrlResponseStruct = type({ @@ -704,7 +705,11 @@ export class PhishingDataService extends BaseDataService< if (scanResult.fetchError) { throw new Error(scanResult.fetchError); } - return scanResult as Json; + const [hostname] = getHostnameFromWebUrl(`https://${url}`); + return { + ...scanResult, + hostname: scanResult.hostname ?? hostname, + } as Json; }, staleTime: SCAN_RESULT_STALE_TIME, gcTime: SCAN_RESULT_GC_TIME, diff --git a/packages/phishing-controller/src/types.ts b/packages/phishing-controller/src/types.ts index 370db674e43..5e8e1e246f9 100644 --- a/packages/phishing-controller/src/types.ts +++ b/packages/phishing-controller/src/types.ts @@ -40,12 +40,12 @@ export type EthPhishingResponse = { * * @property recentlyAdded - List of c2 domains recently added to the blocklist * @property recentlyRemoved - List of c2 domains recently removed from the blocklist - * @property lastFetchedAt - Timestamp of the last fetch request + * @property lastFetchedAt - Unix timestamp, in seconds, of the last fetch request */ export type C2DomainBlocklistResponse = { recentlyAdded: string[]; recentlyRemoved: string[]; - lastFetchedAt: string; + lastFetchedAt: number; }; /** From dec0fb79fceb2d751ae083c1677f2ac4c1a4d733 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:07:37 -0500 Subject: [PATCH 25/38] fix: bound data service cache rehydration and validate persisted caches Queries no longer wait indefinitely for StorageService:getItem after init(); the wait is bounded by a new hydrationTimeout (default 1s). Persisted caches are shape-validated before hydration and a shouldHydrateQuery filter lets services validate individual persisted queries. Co-Authored-By: Claude Opus 5 --- packages/base-data-service/CHANGELOG.md | 5 +- .../src/BaseDataService.test.ts | 147 +++++++++++++++++- .../base-data-service/src/BaseDataService.ts | 89 ++++++++++- packages/base-data-service/src/index.ts | 5 +- 4 files changed, 235 insertions(+), 11 deletions(-) diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index fabd43e6aab..b2bae662219 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -10,10 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add a protected `executeWithPolicy` helper for applying a data service's retry and circuit-breaker policy to uncached requests ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `hydrationTimeout` and `shouldHydrateQuery` options to `PersistenceConfiguration`, and export `DEFAULT_HYDRATION_TIMEOUT` ([#9914](https://github.com/MetaMask/core/pull/9914)) + - `hydrationTimeout` bounds how long a query waits for cache rehydration after `init` (default 1 second), and `shouldHydrateQuery` filters persisted queries before they are restored into the cache ### Fixed -- Wait for cache rehydration to finish before starting a query when `init` has been called, preventing persisted results from racing the first network request ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Wait for cache rehydration to finish before starting a query when `init` has been called, preventing persisted results from racing the first network request; the wait is bounded by `hydrationTimeout` so that a slow or hung storage read cannot block queries ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Discard persisted caches that fail shape validation instead of attempting to hydrate them ([#9914](https://github.com/MetaMask/core/pull/9914)) ## [2.0.0] diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index e2d77991116..455211c565d 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -26,7 +26,10 @@ import { TRANSACTIONS_PAGE_2_CURSOR, TRANSACTIONS_PAGE_3_CURSOR, } from '../tests/mocks.js'; -import { STORAGE_SERVICE_KEY } from './BaseDataService.js'; +import { + DEFAULT_HYDRATION_TIMEOUT, + STORAGE_SERVICE_KEY, +} from './BaseDataService.js'; const TEST_ADDRESS = '0x4bbeEB066eD09B7AEd07bF39EEe0460DFa261520'; @@ -540,6 +543,148 @@ describe('BaseDataService', () => { service.destroy(); }); + it('proceeds with a query when rehydration exceeds the hydration timeout', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('honors a custom hydrationTimeout', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger, { + persistenceConfig: { maxAge: 1000, hydrationTimeout: 50 }, + }); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(50); + + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('discards a persisted cache that fails shape validation', async () => { + const getItem = jest.fn().mockResolvedValue({ + result: { state: { queries: 'not-an-array' } }, + }); + const removeItem = jest.fn(); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + 'StorageService:removeItem': removeItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const publishSpy = jest.spyOn(messenger, 'publish'); + const service = new ExampleDataService(messenger); + + service.init(); + await new Promise(setImmediate); + + expect(removeItem).toHaveBeenCalledWith(serviceName, STORAGE_SERVICE_KEY); + expect(publishSpy).not.toHaveBeenCalled(); + }); + + it('skips persisted queries rejected by shouldHydrateQuery', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn().mockResolvedValue({ + result: { + state: { + queries: [ + { + queryHash: hashKey([ + 'ExampleDataService:getAssets', + MOCK_ASSETS, + ]), + queryKey: ['ExampleDataService:getAssets', MOCK_ASSETS], + state: { + data: [ + { + assetId: 'eip155:1/slip44:60', + decimals: 18, + name: 'Ethereum', + symbol: 'ETH', + }, + ], + dataUpdateCount: 1, + dataUpdatedAt: Date.now(), + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }, + ], + mutations: [], + }, + timestamp: Date.now(), + }, + }); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const shouldHydrateQuery = jest.fn(() => false); + const service = new ExampleDataService(messenger, { + persistenceConfig: { maxAge: 1000, shouldHydrateQuery }, + }); + + service.init(); + await new Promise(setImmediate); + + const result = await service.getAssets(MOCK_ASSETS); + + expect(shouldHydrateQuery).toHaveBeenCalledTimes(1); + expect(result).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + it('discards the cache if it has expired', async () => { const getItem = jest.fn().mockResolvedValue({ result: { diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index c2534f13f65..daf1730233a 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -10,7 +10,14 @@ import type { StorageServiceRemoveItemAction, StorageServiceSetItemAction, } from '@metamask/storage-service'; -import { Struct } from '@metamask/superstruct'; +import { + array, + is, + number, + Struct, + type as objectType, + unknown, +} from '@metamask/superstruct'; import { Duration, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; import { @@ -113,6 +120,13 @@ const QUERY_CLIENT_DEFAULTS: DefaultOptions = { export const STORAGE_SERVICE_KEY = 'cache'; +/** + * How long a query waits for cache rehydration to finish after `init` has been + * called before proceeding without it. Rehydration is an optimization: a slow + * or hung storage read must never block queries indefinitely. + */ +export const DEFAULT_HYDRATION_TIMEOUT = inMilliseconds(1, Duration.Second); + /** * Options for persistence configuration. */ @@ -131,6 +145,19 @@ export type PersistenceConfiguration = { * The maximum number of milliseconds to wait between persistence writes. */ maxWriteDelay?: number; + /** + * The maximum number of milliseconds a query waits for cache rehydration to + * finish after `init` has been called. Once exceeded, the query proceeds + * without the persisted cache. Defaults to {@link DEFAULT_HYDRATION_TIMEOUT}. + */ + hydrationTimeout?: number; + /** + * Decides whether a persisted query is restored into the cache during + * rehydration. Queries for which this returns `false` are discarded. Use it + * to validate persisted data before it can be served from the cache. + * Defaults to restoring every persisted query. + */ + shouldHydrateQuery?: (query: DehydratedState['queries'][number]) => boolean; }; type PersistedCache = { @@ -138,6 +165,14 @@ type PersistedCache = { timestamp: number; }; +const PersistedCacheStruct = objectType({ + timestamp: number(), + state: objectType({ + queries: array(unknown()), + mutations: array(unknown()), + }), +}); + export class BaseDataService< ServiceName extends string, ServiceMessenger extends BaseMessenger, @@ -286,9 +321,7 @@ export class BaseDataService< queryFn: QueryFunction; responseStruct?: TDataStruct; }): Promise { - if (this.#initializationPromise) { - await this.#initializationPromise; - } + await this.#waitForInitialization(); return this.#queryClient.fetchQuery({ ...options, @@ -342,9 +375,7 @@ export class BaseDataService< }, pageParam?: TPageParam, ): Promise { - if (this.#initializationPromise) { - await this.#initializationPromise; - } + await this.#waitForInitialization(); const cache = this.#queryClient.getQueryCache(); @@ -437,6 +468,31 @@ export class BaseDataService< ); } + /** + * Waits for cache rehydration to finish if `init` has been called, giving up + * after the configured hydration timeout so that a slow or hung storage read + * cannot block queries indefinitely. A late rehydration is still applied by + * TanStack, which only overwrites entries older than the persisted ones. + */ + async #waitForInitialization(): Promise { + if (!this.#initializationPromise) { + return; + } + const timeout = + this.#persistenceConfig?.hydrationTimeout ?? DEFAULT_HYDRATION_TIMEOUT; + let timer: ReturnType | undefined; + try { + await Promise.race([ + this.#initializationPromise, + new Promise((resolve) => { + timer = setTimeout(resolve, timeout); + }), + ]); + } finally { + clearTimeout(timer); + } + } + /** * Prepares the service for garbage collection. This should be extended * by any subclasses to clean up any additional connections or events. @@ -535,6 +591,14 @@ export class BaseDataService< return; } + if (!is(untypedCache, PersistedCacheStruct)) { + await this.#externalMessenger.call( + 'StorageService:removeItem', + this.name, + STORAGE_SERVICE_KEY, + ); + return; + } const cache = untypedCache as unknown as PersistedCache; if (Date.now() - cache.timestamp >= this.#persistenceConfig.maxAge) { @@ -546,6 +610,15 @@ export class BaseDataService< return; } - hydrate(this.#queryClient, cache.state); + const { shouldHydrateQuery } = this.#persistenceConfig; + hydrate( + this.#queryClient, + shouldHydrateQuery + ? { + ...cache.state, + queries: cache.state.queries.filter(shouldHydrateQuery), + } + : cache.state, + ); } } diff --git a/packages/base-data-service/src/index.ts b/packages/base-data-service/src/index.ts index 58090738556..027e2dde2cd 100644 --- a/packages/base-data-service/src/index.ts +++ b/packages/base-data-service/src/index.ts @@ -24,7 +24,10 @@ export type { QueryKey, PersistenceConfiguration, } from './BaseDataService.js'; -export { BaseDataService } from './BaseDataService.js'; +export { + BaseDataService, + DEFAULT_HYDRATION_TIMEOUT, +} from './BaseDataService.js'; export { DEFAULT_CIRCUIT_BREAK_DURATION, From 723029d824a4c85a1314b7f5ccc7d64956940e62 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:09:06 -0500 Subject: [PATCH 26/38] fix: ignore hotlist diffs for unrecognized list types applyDiffs previously indexed listSets with the diff's list type and threw a TypeError for any type it did not know, and the stricter hotlist struct in this branch turned the same case into a rejected hotlist response that left phishingLists empty. Unknown list types are now skipped per diff. Also adds normalizeScanAddress for lowercasing EVM addresses in scan keys. Co-Authored-By: Claude Opus 5 --- .../phishing-controller/src/utils.test.ts | 34 +++++++++++++++++++ packages/phishing-controller/src/utils.ts | 25 ++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/packages/phishing-controller/src/utils.test.ts b/packages/phishing-controller/src/utils.test.ts index 5bc2b890a1d..97d7ae4942d 100644 --- a/packages/phishing-controller/src/utils.test.ts +++ b/packages/phishing-controller/src/utils.test.ts @@ -1,5 +1,6 @@ import { ListKeys, ListNames } from './PhishingController.js'; import type { PhishingListState } from './PhishingController.js'; +import type { Hotlist } from './types.js'; import { applyDiffs, domainToParts, @@ -15,6 +16,7 @@ import { isPhishingDetectionPathBasedHostname, isTokenScanSupportedChain, matchPartsAgainstList, + normalizeScanAddress, processConfigs, processDomainList, resolveChainName, @@ -206,6 +208,24 @@ describe('applyDiffs', () => { name: ListNames.MetaMask, }); }); + + it('ignores diffs that target an unrecognized list type without advancing lastUpdated', () => { + const unknownTypeDiff = { + targetList: 'eth_phishing_detect_config.newlist', + url: 'https://example-new-list-item.com', + timestamp: exampleAddDiff.timestamp + 10, + } as unknown as Hotlist[number]; + const result = applyDiffs( + exampleListState, + [exampleAddDiff, unknownTypeDiff], + ListKeys.EthPhishingDetectConfig, + ); + expect(result).toStrictEqual({ + ...exampleListState, + blocklist: [...exampleListState.blocklist, exampleBlockedUrlTwo], + lastUpdated: exampleAddDiff.timestamp, + }); + }); // New tests for handling C2 domain blocklist it('should add hashes to the current C2 domain blocklist', () => { exampleListState.c2DomainBlocklist = ['hash1', 'hash2']; @@ -1326,3 +1346,17 @@ describe('getHostnameAndPathComponents', () => { expect(result).toStrictEqual(expected); }); }); + +describe('normalizeScanAddress', () => { + it('lowercases EVM addresses', () => { + expect( + normalizeScanAddress('0xAbCdEf0000000000000000000000000000000001'), + ).toBe('0xabcdef0000000000000000000000000000000001'); + }); + + it('leaves non-EVM addresses unchanged', () => { + expect( + normalizeScanAddress('So11111111111111111111111111111111111111112'), + ).toBe('So11111111111111111111111111111111111111112'); + }); +}); diff --git a/packages/phishing-controller/src/utils.ts b/packages/phishing-controller/src/utils.ts index 198a6c16b23..f4cbc550d50 100644 --- a/packages/phishing-controller/src/utils.ts +++ b/packages/phishing-controller/src/utils.ts @@ -121,6 +121,16 @@ export const applyDiffs = ( for (const { isRemoval, targetList, url, timestamp } of diffsToApply) { const targetListType = splitStringByPeriod(targetList)[1]; + // Diffs for list types this client does not know about (for example a + // list introduced server-side after this release) are ignored rather than + // failing the whole update. They do not advance `lastUpdated`, matching + // how diffs for other list keys are treated. + if ( + targetListType !== 'blocklistPaths' && + !Object.hasOwn(listSets, targetListType) + ) { + continue; + } if (timestamp > latestDiffTimestamp) { latestDiffTimestamp = timestamp; } @@ -428,6 +438,21 @@ export const getPhishingDetectionScanUrlParam = ( return [scanUrlParam, true]; }; +const EVM_ADDRESS_REGEX = /^0x[0-9a-fA-F]{40}$/u; + +/** + * Normalizes an address for use in scan requests and cache keys. EVM addresses + * are case-insensitive and are lowercased so that differently-cased inputs + * share one cache entry and match the API's lowercase response keys. Any other + * address (for example a base58 Solana address) is case-sensitive and is + * returned unchanged. + * + * @param address - The address to normalize. + * @returns The normalized address. + */ +export const normalizeScanAddress = (address: string): string => + EVM_ADDRESS_REGEX.test(address) ? address.toLowerCase() : address; + export const getPathnameFromUrl = (url: string): string => { try { const { pathname } = new URL(url); From 9acaa9fcb866f3773fd24cef4f1815a7b195564e Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:33:44 -0500 Subject: [PATCH 27/38] fix: validate phishing responses per entry instead of per response A single unrecognized verdict or malformed nested field previously rejected the whole bulk URL, bulk token, or approvals response, dropping every BLOCK and Malicious verdict in the batch. Entries are now validated individually: malformed URL results are reported per URL, malformed token results and approvals are omitted. Hotlist diffs may target unknown list types and the C2 blocklist no longer requires lastFetchedAt, which the controller never reads. Bulk-seeded cache entries now carry a hostname so scanUrl always returns one. Co-Authored-By: Claude Opus 5 --- .../src/PhishingController.test.ts | 55 +++++ .../src/PhishingDataService.test.ts | 204 ++++++++++++++++-- .../src/PhishingDataService.ts | 89 +++++--- packages/phishing-controller/src/types.ts | 4 +- 4 files changed, 298 insertions(+), 54 deletions(-) diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index da18cf1877d..ba3bc6e396b 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -2087,6 +2087,61 @@ describe('PhishingController', () => { ]); }); + it('ignores hotlist diffs that target an unrecognized list type while applying the rest', async () => { + const testBlockedDomain = 'some-test-blocked-url.com'; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/${0}`) + .reply(200, { + data: [ + { + targetList: 'eth_phishing_detect_config.newlist', + url: 'some-new-list-url.com', + timestamp: 2, + }, + { + targetList: 'eth_phishing_detect_config.blocklist', + url: testBlockedDomain, + timestamp: 1, + }, + ], + }); + + const { controller } = getPhishingController({ + state: { + phishingLists: [ + { + allowlist: [], + blocklist: [], + c2DomainBlocklist: [], + blocklistPaths: {}, + fuzzylist: [], + tolerance: 3, + version: 1, + name: ListNames.MetaMask, + lastUpdated: 0, + }, + ], + }, + }); + await controller.updateHotlist(); + + // The unknown-type diff is skipped and does not advance lastUpdated; + // the recognized diff is still applied. + expect(controller.state.phishingLists).toStrictEqual([ + { + allowlist: [], + blocklist: [testBlockedDomain], + c2DomainBlocklist: [], + blocklistPaths: {}, + fuzzylist: [], + tolerance: 3, + name: ListNames.MetaMask, + version: 1, + lastUpdated: 1, + }, + ]); + }); + it('should not update phishing lists if hotlist fetch returns 404', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(`${METAMASK_HOTLIST_DIFF_FILE}/${0}`) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 0033460aec0..6a1bc3908f5 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -198,17 +198,34 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual(diffs); }); + it('passes through diffs for unrecognized target lists', async () => { + const diffs = { + data: [ + { + url: 'phishing.example.com', + timestamp: 1700000001, + targetList: 'unexpected.blocklist', + }, + ], + }; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, diffs); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getHotlistDiffs', + 1700000000, + ); + + expect(response).toStrictEqual(diffs); + }); + it('throws if the API returns a malformed response', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) .reply(200, { - data: [ - { - url: 'phishing.example.com', - timestamp: 1700000001, - targetList: 'unexpected.blocklist', - }, - ], + data: [{ url: 'phishing.example.com', timestamp: 'soon' }], }); const { rootMessenger } = createService(); @@ -259,10 +276,21 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual(blocklist); }); + it('accepts a response without lastFetchedAt', async () => { + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, { recentlyAdded: ['abc'], recentlyRemoved: [] }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), + ).toStrictEqual({ recentlyAdded: ['abc'], recentlyRemoved: [] }); + }); + it('throws if the API returns a malformed response', async () => { nock(CLIENT_SIDE_DETECION_BASE_URL) .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) - .reply(200, { recentlyAdded: [], recentlyRemoved: [] }); + .reply(200, { recentlyAdded: 'abc', recentlyRemoved: [] }); const { rootMessenger } = createService(); await expect( @@ -522,15 +550,44 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual(apiResponse); }); - it('throws if the API returns a malformed response', async () => { + it('reports a malformed result for that URL and keeps the others', async () => { nock(PHISHING_DETECTION_BASE_URL) .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) .reply(200, { - results: { 'https://example1.com': {} }, + results: { + 'https://example1.com': {}, + 'https://example2.com': { recommendedAction: 'BLOCK' }, + }, errors: {}, }); const { rootMessenger } = createService(); + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com', 'https://example2.com'], + ); + + expect(response).toStrictEqual({ + results: { + 'https://example2.com': { + recommendedAction: 'BLOCK', + hostname: 'example2.com', + }, + }, + errors: { + 'https://example1.com': [ + 'Malformed result returned by bulk URL scan endpoint', + ], + }, + }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { results: 'nope' }); + const { rootMessenger } = createService(); + await expect( rootMessenger.call('PhishingDataService:bulkScanUrls', [ 'https://example1.com', @@ -540,6 +597,49 @@ describe('PhishingDataService', () => { ); }); + it('accepts a response without an errors object', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { 'https://example1.com': { recommendedAction: 'NONE' } }, + }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: { + 'https://example1.com': { + recommendedAction: 'NONE', + hostname: 'example1.com', + }, + }, + errors: {}, + }); + }); + + it('stores a hostname on cache entries shared with scanUrl', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { + 'https://evil.com/some/path': { recommendedAction: 'BLOCK' }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://evil.com/some/path', + ]); + + // No GET interceptor is registered, so this must be a cache hit. + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'evil.com'), + ).toStrictEqual({ recommendedAction: 'BLOCK', hostname: 'evil.com' }); + }); + it('sends each path separately for path-sensitive hosts', async () => { const urls = ['https://ipfs.io/ipfs/AAA', 'https://ipfs.io/ipfs/BBB']; nock(PHISHING_DETECTION_BASE_URL) @@ -852,16 +952,60 @@ describe('PhishingDataService', () => { expect(response).toStrictEqual({ results: {} }); }); - it('throws if the API returns a malformed response', async () => { + it('omits a malformed result for that token and keeps the others', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(TOKEN_BULK_SCANNING_ENDPOINT) .reply(200, { results: { '0x1234567890123456789012345678901234567890': {}, + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd': { + result_type: 'Malicious', + }, }, }); const { rootMessenger } = createService(); + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [ + '0x1234567890123456789012345678901234567890', + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd', + ], + ); + + expect(response).toStrictEqual({ + results: { + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd': { + result_type: 'Malicious', + }, + }, + }); + }); + + it('throws if every result in the batch is malformed', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { + results: { '0x1234567890123456789012345678901234567890': {} }, + }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + ]), + ).rejects.toThrow( + 'Malformed result returned by bulk token scan endpoint', + ); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { results: 'nope' }); + const { rootMessenger } = createService(); + await expect( rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ '0x1234567890123456789012345678901234567890', @@ -1140,22 +1284,44 @@ describe('PhishingDataService', () => { expect(scope.isDone()).toBe(true); }); - it('throws if the API returns a malformed response', async () => { + it('omits malformed approvals and keeps valid ones', async () => { + const validApproval = { + allowance: {}, + asset: { + address: '0xtoken', + symbol: 'TKN', + name: 'Token', + decimals: 18, + }, + exposure: { value: '100', raw_value: '100000000000000000000' }, + spender: { address: '0xspender' }, + verdict: 'Malicious', + }; nock(SECURITY_ALERTS_BASE_URL) .post(APPROVALS_ENDPOINT) .reply(200, { approvals: [ - { - allowance: {}, - asset: {}, - exposure: {}, - spender: {}, - verdict: 'Benign', - }, + validApproval, + { ...validApproval, asset: { ...validApproval.asset, name: null } }, ], }); const { rootMessenger } = createService(); + expect( + await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ approvals: [validApproval] }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(200, { approvals: 'nope' }); + const { rootMessenger } = createService(); + await expect( rootMessenger.call( 'PhishingDataService:getApprovals', diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 3dcfed4f045..3cbc2a3b7a3 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -25,6 +25,7 @@ import { string, type, union, + unknown, } from '@metamask/superstruct'; import { Duration, getErrorMessage, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; @@ -266,14 +267,6 @@ const ApprovalFeatureTypeStruct = union([ literal(ApprovalFeatureType.Info), ]); -const HotlistTargetListStruct = union([ - literal('eth_phishing_detect_config.allowlist'), - literal('eth_phishing_detect_config.blocklist'), - literal('eth_phishing_detect_config.blocklistPaths'), - literal('eth_phishing_detect_config.fuzzylist'), - literal('eth_phishing_detect_config.c2DomainBlocklist'), -]); - const StalelistResponseStruct = type({ data: type({ allowlist: array(string()), @@ -291,7 +284,9 @@ const HotlistDiffsResponseStruct = type({ type({ url: string(), timestamp: number(), - targetList: HotlistTargetListStruct, + // Kept open so that a list added server-side does not reject the whole + // hotlist; `applyDiffs` ignores entries for unknown list types. + targetList: string(), isRemoval: optional(boolean()), }), ), @@ -300,7 +295,9 @@ const HotlistDiffsResponseStruct = type({ const C2DomainBlocklistResponseStruct = type({ recentlyAdded: array(string()), recentlyRemoved: array(string()), - lastFetchedAt: number(), + // The controller never reads this field, so its absence must not reject + // the whole blocklist response. + lastFetchedAt: optional(number()), }); const ScanUrlResponseStruct = type({ @@ -309,9 +306,11 @@ const ScanUrlResponseStruct = type({ fetchError: optional(string()), }); +// Entries are validated individually by `bulkScanUrls`, so that one malformed +// entry is reported for that URL rather than discarding every verdict. const BulkScanUrlsResponseStruct = type({ - results: record(string(), ScanUrlResponseStruct), - errors: record(string(), array(string())), + results: record(string(), unknown()), + errors: optional(record(string(), array(string()))), }); const TokenScanResultStruct = type({ @@ -320,8 +319,11 @@ const TokenScanResultStruct = type({ address: optional(string()), }); +// Entries are validated individually by the token batch loader, so that one +// malformed entry is reported for that token rather than discarding every +// verdict. const BulkScanTokensResponseStruct = type({ - results: optional(record(string(), TokenScanResultStruct)), + results: optional(record(string(), unknown())), }); const ScanAddressResponseStruct = type({ @@ -361,8 +363,10 @@ const ApprovalStruct = type({ verdict: ApprovalResultTypeStruct, }); +// Entries are validated individually by `getApprovals`, so that one malformed +// approval does not empty the whole list. const ApprovalsResponseStruct = type({ - approvals: array(ApprovalStruct), + approvals: array(unknown()), }); // === BATCH LOADING === @@ -754,20 +758,30 @@ export class PhishingDataService extends BaseDataService< jsonResponse, BulkScanUrlsResponseStruct, 'bulk URL scan', - ) as BulkPhishingDetectionScanResponse; + ); // URLs the endpoint reported an error for are rejected rather than // resolved, so that the failure is surfaced to the caller instead of // being cached as a "no result" verdict for the stale time. const itemErrors: Record = {}; - for (const [key, messages] of Object.entries(response.errors)) { + for (const [key, messages] of Object.entries(response.errors ?? {})) { itemErrors[key] = new BatchItemError(messages.join(', ')); } const results: Record = {}; for (const [key, result] of Object.entries(response.results)) { - if (result.fetchError) { + if (!is(result, ScanUrlResponseStruct)) { + itemErrors[key] = new BatchItemError( + 'Malformed result returned by bulk URL scan endpoint', + ); + } else if (result.fetchError) { itemErrors[key] = new BatchItemError(result.fetchError); } else { - results[key] = result as Json; + // Entries are shared with `scanUrl`, whose results always carry a + // hostname, so fill it in before the entry reaches the cache. + const [hostname] = getHostnameFromWebUrl(key); + results[key] = { + ...result, + hostname: result.hostname ?? hostname, + } as Json; } } for (const url of batchUrls) { @@ -784,7 +798,7 @@ export class PhishingDataService extends BaseDataService< }, }); - const requested: { url: string; hostname: string }[] = []; + const requested: string[] = []; const entries: Promise[] = []; for (const url of urls) { const [scanUrlParam, ok] = getPhishingDetectionScanUrlParam(url); @@ -792,8 +806,7 @@ export class PhishingDataService extends BaseDataService< addError(url, 'url is not a valid web URL'); continue; } - const [hostname] = getHostnameFromWebUrl(url); - requested.push({ url, hostname }); + requested.push(url); entries.push( // Keyed by the scan parameter rather than the bare hostname so that // path-sensitive hosts (see @@ -814,7 +827,7 @@ export class PhishingDataService extends BaseDataService< let requestFailure: { reason: unknown } | undefined; for (const [index, outcome] of settled.entries()) { - const { url, hostname } = requested[index]; + const url = requested[index]; if (outcome.status === 'rejected') { addError(url, getErrorMessage(outcome.reason)); @@ -824,13 +837,7 @@ export class PhishingDataService extends BaseDataService< continue; } - const scanResult = outcome.value as PhishingDetectionScanResult; - // Entries seeded by single-URL scans hold the raw scan response, - // which may not include the hostname; fill it in from the URL. - results[url] = { - ...scanResult, - hostname: scanResult.hostname ?? hostname, - }; + results[url] = outcome.value as PhishingDetectionScanResult; } // A request-level failure that produced nothing at all is surfaced to the @@ -930,8 +937,19 @@ export class PhishingDataService extends BaseDataService< jsonResponse, BulkScanTokensResponseStruct, 'bulk token scan', - ) as TokenScanApiResponse; - return { results: (response.results ?? {}) as Record }; + ); + const results: Record = {}; + const errors: Record = {}; + for (const [key, result] of Object.entries(response.results ?? {})) { + if (is(result, TokenScanResultStruct)) { + results[key] = result as Json; + } else { + errors[key] = new BatchItemError( + 'Malformed result returned by bulk token scan endpoint', + ); + } + } + return { results, errors }; }, }); } @@ -1014,11 +1032,16 @@ export class PhishingDataService extends BaseDataService< ), ); - return this.#validate( + const response = this.#validate( jsonResponse, ApprovalsResponseStruct, 'approvals', - ) as ApprovalsResponse; + ); + return { + approvals: response.approvals.filter((approval) => + is(approval, ApprovalStruct), + ), + } as ApprovalsResponse; } /** diff --git a/packages/phishing-controller/src/types.ts b/packages/phishing-controller/src/types.ts index 5e8e1e246f9..e4dff45a0ff 100644 --- a/packages/phishing-controller/src/types.ts +++ b/packages/phishing-controller/src/types.ts @@ -40,12 +40,12 @@ export type EthPhishingResponse = { * * @property recentlyAdded - List of c2 domains recently added to the blocklist * @property recentlyRemoved - List of c2 domains recently removed from the blocklist - * @property lastFetchedAt - Unix timestamp, in seconds, of the last fetch request + * @property lastFetchedAt - Unix timestamp, in seconds, of the last fetch request. Not read by the controller. */ export type C2DomainBlocklistResponse = { recentlyAdded: string[]; recentlyRemoved: string[]; - lastFetchedAt: number; + lastFetchedAt?: number; }; /** From acd27f1179abffa0600721430fc57e2af1b9c13e Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:34:08 -0500 Subject: [PATCH 28/38] fix: start data service queries synchronously when init was not called The bounded rehydration wait added an asynchronous hop before every query even for services that never call init(). That hop delays request timers relative to callers' own timeouts; in the phishing controller's timeout tests the request outlived the test, was aborted during teardown, and TanStack then scheduled a real five-minute gcTime timer on the destroyed query, keeping the Jest process alive. Only await rehydration when an initialization promise exists. Co-Authored-By: Claude Opus 5 --- .../base-data-service/src/BaseDataService.ts | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index daf1730233a..e79a5a03067 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -321,7 +321,9 @@ export class BaseDataService< queryFn: QueryFunction; responseStruct?: TDataStruct; }): Promise { - await this.#waitForInitialization(); + if (this.#initializationPromise) { + await this.#waitForInitialization(this.#initializationPromise); + } return this.#queryClient.fetchQuery({ ...options, @@ -375,7 +377,9 @@ export class BaseDataService< }, pageParam?: TPageParam, ): Promise { - await this.#waitForInitialization(); + if (this.#initializationPromise) { + await this.#waitForInitialization(this.#initializationPromise); + } const cache = this.#queryClient.getQueryCache(); @@ -469,21 +473,25 @@ export class BaseDataService< } /** - * Waits for cache rehydration to finish if `init` has been called, giving up - * after the configured hydration timeout so that a slow or hung storage read - * cannot block queries indefinitely. A late rehydration is still applied by - * TanStack, which only overwrites entries older than the persisted ones. + * Waits for cache rehydration to finish, giving up after the configured + * hydration timeout so that a slow or hung storage read cannot block queries + * indefinitely. A late rehydration is still applied by TanStack, which only + * overwrites entries older than the persisted ones. + * + * Callers must only await this when `init` has been called: an asynchronous + * hop before a query starts changes its timing relative to callers' own + * timers, so services without persistence keep starting queries + * synchronously. + * + * @param initialization - The pending rehydration. */ - async #waitForInitialization(): Promise { - if (!this.#initializationPromise) { - return; - } + async #waitForInitialization(initialization: Promise): Promise { const timeout = this.#persistenceConfig?.hydrationTimeout ?? DEFAULT_HYDRATION_TIMEOUT; let timer: ReturnType | undefined; try { await Promise.race([ - this.#initializationPromise, + initialization, new Promise((resolve) => { timer = setTimeout(resolve, timeout); }), From d4c0676fe037ba2f04d1e0fc60154ee22935597c Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:35:33 -0500 Subject: [PATCH 29/38] fix: normalize EVM addresses in phishing scan requests and cache keys Address lowercasing lived only in PhishingController, while the service's query keys became public API for UI consumers. A mixed-case token or address passed directly to the service missed the API's lowercase response keys and negatively cached a null verdict. The service now lowercases EVM addresses for both the request and the cache key; non-EVM addresses are unchanged. Also corrects the scanToken JSDoc, which claimed concurrent calls coalesce. Co-Authored-By: Claude Opus 5 --- ...PhishingDataService-method-action-types.ts | 15 ++-- .../src/PhishingDataService.test.ts | 87 +++++++++++++++++++ .../src/PhishingDataService.ts | 38 +++++--- 3 files changed, 122 insertions(+), 18 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts index 6942b3b13c9..dba8bb20b5d 100644 --- a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts +++ b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts @@ -74,8 +74,10 @@ export type PhishingDataServiceBulkScanUrlsAction = { /** * Scans a token for malicious activity via the security-alerts API. * - * Requests made while a bulk scan is being assembled are coalesced into a - * single request to the bulk scanning endpoint. + * Each call issues its own request to the bulk scanning endpoint; use + * {@link PhishingDataService.bulkScanTokens} to scan several tokens in one + * request. EVM token addresses are lowercased before being used as the cache + * key and sent to the API; other addresses are used as given. * * @param chain - The chain name (e.g. `ethereum`). * @param token - The token address to scan. @@ -96,8 +98,9 @@ export type PhishingDataServiceScanTokenAction = { * * @param chain - The chain name (e.g. `ethereum`). * @param tokens - The token addresses to scan. - * @returns The token scan results, keyed by token address. Tokens for which - * the API returned no result are omitted. + * @returns The token scan results, keyed by normalized token address (EVM + * addresses are lowercased). Tokens for which the API returned no result + * are omitted. */ export type PhishingDataServiceBulkScanTokensAction = { type: `PhishingDataService:bulkScanTokens`; @@ -105,7 +108,9 @@ export type PhishingDataServiceBulkScanTokensAction = { }; /** - * Scans an address for security alerts via the security-alerts API. + * Scans an address for security alerts via the security-alerts API. EVM + * addresses are lowercased before being used as the cache key and sent to + * the API; other addresses are used as given. * * @param chain - The chain name (e.g. `ethereum`). * @param address - The address to scan. diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index 6a1bc3908f5..c79624314f4 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -851,6 +851,25 @@ describe('PhishingDataService', () => { }); describe('bulkScanTokens', () => { + it('lowercases EVM token addresses and keys results by the normalized address', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [lower], + }) + .reply(200, { results: { [lower]: { result_type: 'Malicious' } } }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + ['0xAbCdEf0000000000000000000000000000000001'], + ), + ).toStrictEqual({ results: { [lower]: { result_type: 'Malicious' } } }); + }); + it('returns the scan results from the API', async () => { const tokens = ['0x1234567890123456789012345678901234567890']; const apiResponse = { @@ -1017,6 +1036,51 @@ describe('PhishingDataService', () => { }); describe('scanToken', () => { + it('lowercases EVM token addresses in the request and cache key', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [lower], + }) + .reply(200, { results: { [lower]: { result_type: 'Malicious' } } }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + '0xAbCdEf0000000000000000000000000000000001', + ); + // No second interceptor is registered, so this must be a cache hit. + const second = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + lower, + ); + + expect(first).toStrictEqual({ result_type: 'Malicious' }); + expect(second).toStrictEqual({ result_type: 'Malicious' }); + }); + + it('preserves the casing of non-EVM token addresses', async () => { + const solanaToken = 'So11111111111111111111111111111111111111112'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'solana', + tokens: [solanaToken], + }) + .reply(200, { results: { [solanaToken]: { result_type: 'Benign' } } }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:scanToken', + 'solana', + solanaToken, + ), + ).toStrictEqual({ result_type: 'Benign' }); + }); + it('returns the scan result for a single token from the bulk API', async () => { const token = '0x1234567890123456789012345678901234567890'; nock(SECURITY_ALERTS_BASE_URL) @@ -1156,6 +1220,29 @@ describe('PhishingDataService', () => { }); describe('scanAddress', () => { + it('lowercases EVM addresses in the request and cache key', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT, { chain: 'ethereum', address: lower }) + .reply(200, { result_type: 'Malicious', label: 'bad' }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0xAbCdEf0000000000000000000000000000000001', + ); + // No second interceptor is registered, so this must be a cache hit. + const second = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + lower, + ); + + expect(first).toStrictEqual({ result_type: 'Malicious', label: 'bad' }); + expect(second).toStrictEqual({ result_type: 'Malicious', label: 'bad' }); + }); + it('returns the scan result from the API', async () => { nock(SECURITY_ALERTS_BASE_URL) .post(ADDRESS_SCAN_ENDPOINT, { diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 3cbc2a3b7a3..54ed7d59f7d 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -53,6 +53,7 @@ import { import { getHostnameFromWebUrl, getPhishingDetectionScanUrlParam, + normalizeScanAddress, } from './utils.js'; /** @@ -854,8 +855,10 @@ export class PhishingDataService extends BaseDataService< /** * Scans a token for malicious activity via the security-alerts API. * - * Requests made while a bulk scan is being assembled are coalesced into a - * single request to the bulk scanning endpoint. + * Each call issues its own request to the bulk scanning endpoint; use + * {@link PhishingDataService.bulkScanTokens} to scan several tokens in one + * request. EVM token addresses are lowercased before being used as the cache + * key and sent to the API; other addresses are used as given. * * @param chain - The chain name (e.g. `ethereum`). * @param token - The token address to scan. @@ -867,7 +870,11 @@ export class PhishingDataService extends BaseDataService< token: string, ): Promise { const loader = this.#createTokenScanLoader(chain); - const result = this.#fetchTokenScanQuery(loader, chain, token); + const result = this.#fetchTokenScanQuery( + loader, + chain, + normalizeScanAddress(token), + ); loader.flush(); return await result; } @@ -881,19 +888,21 @@ export class PhishingDataService extends BaseDataService< * * @param chain - The chain name (e.g. `ethereum`). * @param tokens - The token addresses to scan. - * @returns The token scan results, keyed by token address. Tokens for which - * the API returned no result are omitted. + * @returns The token scan results, keyed by normalized token address (EVM + * addresses are lowercased). Tokens for which the API returned no result + * are omitted. */ async bulkScanTokens( chain: string, tokens: string[], ): Promise { const loader = this.#createTokenScanLoader(chain); - const entries = tokens.map((token) => - this.#fetchTokenScanQuery(loader, chain, token).then( - (result) => [token, result] as const, - ), - ); + const entries = tokens.map((token) => { + const normalizedToken = normalizeScanAddress(token); + return this.#fetchTokenScanQuery(loader, chain, normalizedToken).then( + (result) => [normalizedToken, result] as const, + ); + }); loader.flush(); const results: TokenScanApiResponse['results'] = {}; @@ -977,7 +986,9 @@ export class PhishingDataService extends BaseDataService< } /** - * Scans an address for security alerts via the security-alerts API. + * Scans an address for security alerts via the security-alerts API. EVM + * addresses are lowercased before being used as the cache key and sent to + * the API; other addresses are used as given. * * @param chain - The chain name (e.g. `ethereum`). * @param address - The address to scan. @@ -987,13 +998,14 @@ export class PhishingDataService extends BaseDataService< chain: string, address: string, ): Promise { + const normalizedAddress = normalizeScanAddress(address); const jsonResponse = await this.fetchQuery({ - queryKey: [`${this.name}:scanAddress`, chain, address], + queryKey: [`${this.name}:scanAddress`, chain, normalizedAddress], queryFn: async ({ signal }) => this.#validate( await this.#postJson( `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, - { chain, address }, + { chain, address: normalizedAddress }, { signal, timeout: ADDRESS_SCAN_TIMEOUT }, ), ScanAddressResponseStruct, From 62f02d17e14c7e5d85d9f3da99be1f3af70be17e Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:39:00 -0500 Subject: [PATCH 30/38] fix: retry batched phishing requests as a whole and validate rehydrated scan results With policyOptions.maxRetries enabled, each item query backed by a failed batch retried on its own and de-batched into single-item requests (10 URLs against a failing host produced 31 POSTs). The batch POST now runs under the service policy and item-level batch errors are excluded from retries, so a failed batch is retried intact. Persisted scan results are validated against their endpoint shapes before hydration; entries that fail, and any query other than a scan result, are discarded. Co-Authored-By: Claude Opus 5 --- .../src/PhishingDataService.test.ts | 323 +++++++++++++++++- .../src/PhishingDataService.ts | 132 +++++-- 2 files changed, 430 insertions(+), 25 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index c79624314f4..afddd19b6ae 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -1,4 +1,8 @@ -import { ConstantBackoff } from '@metamask/base-data-service'; +import { + ConstantBackoff, + DEFAULT_HYDRATION_TIMEOUT, + handleWhen, +} from '@metamask/base-data-service'; import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; import type { MessengerActions, @@ -522,6 +526,130 @@ describe('PhishingDataService', () => { }); describe('bulkScanUrls', () => { + it('retries a failed batch as a whole when retries are enabled', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(function (_uri, body) { + batchSizes.push((body as { urls: string[] }).urls.length); + return [500, 'boom']; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ]), + ).rejects.toThrow('500 Internal Server Error'); + expect(batchSizes).toStrictEqual([3, 3, 3]); + }); + + it('still coalesces lookups into one request after init', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(function (_uri, body) { + const { urls } = body as { urls: string[] }; + batchSizes.push(urls.length); + return [ + 200, + { + results: Object.fromEntries( + urls.map((url) => [url, { recommendedAction: 'NONE' }]), + ), + errors: {}, + }, + ]; + }); + const { rootMessenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: jest.fn().mockResolvedValue({ result: null }), + }); + service.init(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ], + ); + + expect(batchSizes).toStrictEqual([3]); + expect(Object.keys(response.results)).toHaveLength(3); + }); + + it('does not retry per-URL errors reported by the endpoint', async () => { + let requests = 0; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(() => { + requests += 1; + return [ + 200, + { results: {}, errors: { 'https://example1.com': ['boom'] } }, + ]; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: {}, + errors: { 'https://example1.com': ['boom'] }, + }); + expect(requests).toBe(1); + }); + + it('never retries item-level batch errors even if a caller-provided retryFilterPolicy would', async () => { + let requests = 0; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(() => { + requests += 1; + return [ + 200, + { results: {}, errors: { 'https://example1.com': ['boom'] } }, + ]; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { + maxRetries: 2, + backoff: new ConstantBackoff(0), + retryFilterPolicy: handleWhen(() => true), + }, + }, + }); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: {}, + errors: { 'https://example1.com': ['boom'] }, + }); + expect(requests).toBe(1); + }); + it('returns the scan results from the API', async () => { const urls = ['https://example1.com', 'https://example2.com']; const apiResponse = { @@ -851,6 +979,30 @@ describe('PhishingDataService', () => { }); describe('bulkScanTokens', () => { + it('retries a failed batch as a whole when retries are enabled', async () => { + const batchSizes: number[] = []; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .times(3) + .reply(function (_uri, body) { + batchSizes.push((body as { tokens: string[] }).tokens.length); + return [500, 'boom']; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd', + ]), + ).rejects.toThrow('500 Internal Server Error'); + expect(batchSizes).toStrictEqual([2, 2, 2]); + }); + it('lowercases EVM token addresses and keys results by the normalized address', async () => { const lower = '0xabcdef0000000000000000000000000000000001'; nock(SECURITY_ALERTS_BASE_URL) @@ -1583,6 +1735,175 @@ describe('PhishingDataService', () => { expect(getItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); expect(removeItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); }); + + it('discards persisted scan results that fail validation or belong to unknown queries', async () => { + const now = Date.now(); + const dehydratedQuery = ( + queryKey: unknown[], + data: unknown, + ): Record => ({ + queryHash: JSON.stringify(queryKey), + queryKey, + state: { + data, + dataUpdateCount: 1, + dataUpdatedAt: now, + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: now, + state: { + mutations: [], + queries: [ + dehydratedQuery(['PhishingDataService:scanUrl', 'good.com'], { + hostname: 'good.com', + recommendedAction: 'BLOCK', + }), + dehydratedQuery(['PhishingDataService:scanUrl', 'evil.com'], { + hostname: 'evil.com', + recommendedAction: 'PWNED', + }), + dehydratedQuery( + ['PhishingDataService:scanToken', 'ethereum', '0xabc'], + null, + ), + dehydratedQuery( + ['PhishingDataService:scanToken', 'ethereum', '0xdef'], + { result_type: 'Malicious' }, + ), + dehydratedQuery( + ['PhishingDataService:scanAddress', 'ethereum', '0xabc'], + { result_type: 'Benign' }, + ), + dehydratedQuery(['PhishingDataService:getStalelist'], { + data: {}, + }), + ], + }, + }, + }); + const evilScope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'evil.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger, messenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: getItem, + }); + const publishSpy = jest.spyOn(messenger, 'publish'); + service.init(); + await flushPromises(); + + const hydratedEvents = publishSpy.mock.calls + .map(([eventType]) => String(eventType)) + .filter((eventType) => + eventType.startsWith('PhishingDataService:cacheUpdated:'), + ); + expect(hydratedEvents).toStrictEqual([ + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanUrl","good.com"]', + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanToken","ethereum","0xabc"]', + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanToken","ethereum","0xdef"]', + ]); + + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'good.com'), + ).toStrictEqual({ hostname: 'good.com', recommendedAction: 'BLOCK' }); + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'evil.com'), + ).toStrictEqual({ hostname: 'evil.com', recommendedAction: 'BLOCK' }); + expect(evilScope.isDone()).toBe(true); + }); + + it('composes a caller-provided shouldHydrateQuery with the built-in validation', async () => { + const shouldHydrateQuery = jest.fn(() => false); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: Date.now(), + state: { + mutations: [], + queries: [ + { + queryHash: '["PhishingDataService:scanUrl","good.com"]', + queryKey: ['PhishingDataService:scanUrl', 'good.com'], + state: { + data: { hostname: 'good.com', recommendedAction: 'BLOCK' }, + dataUpdateCount: 1, + dataUpdatedAt: Date.now(), + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }, + ], + }, + }, + }); + const { messenger, service } = createService({ + options: { + persistenceConfig: { + maxAge: inMilliseconds(5, Duration.Minute), + shouldHydrateQuery, + }, + }, + setItemMock: jest.fn(), + getItemMock: getItem, + }); + const publishSpy = jest.spyOn(messenger, 'publish'); + service.init(); + await flushPromises(); + + expect(shouldHydrateQuery).toHaveBeenCalledTimes(1); + expect(publishSpy).not.toHaveBeenCalled(); + }); + + it('proceeds with a scan when rehydration hangs past the hydration timeout', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: jest.fn(() => new Promise(() => undefined)), + }); + service.init(); + + const resultPromise = rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + await flushPromises(); + expect(scope.isDone()).toBe(false); + + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + + expect(await resultPromise).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(scope.isDone()).toBe(true); + }); }); describe('retry policy', () => { diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 54ed7d59f7d..36d2ac954ef 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -1,4 +1,4 @@ -import { BaseDataService } from '@metamask/base-data-service'; +import { BaseDataService, handleWhen } from '@metamask/base-data-service'; import type { CreateServicePolicyOptions, DataServiceCacheUpdatedEvent, @@ -29,7 +29,7 @@ import { } from '@metamask/superstruct'; import { Duration, getErrorMessage, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; -import type { QueryClientConfig } from '@tanstack/query-core'; +import type { DehydratedState, QueryClientConfig } from '@tanstack/query-core'; import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; import type { @@ -332,6 +332,37 @@ const ScanAddressResponseStruct = type({ label: string(), }); +const PersistedScanUrlResultStruct = type({ + hostname: string(), + recommendedAction: RecommendedActionStruct, +}); + +/** + * Decides whether a persisted query may be restored into the cache. Only scan + * results are retained between sessions, and each is checked against the + * shape its endpoint validation produces, so that a corrupted or tampered + * persisted entry can never be served as a verdict. Anything else, including + * list queries, is discarded. + * + * @param query - The persisted query. + * @returns Whether the query may be hydrated. + */ +function isValidPersistedScanQuery( + query: DehydratedState['queries'][number], +): boolean { + const { data } = query.state; + switch (query.queryKey[0]) { + case `${serviceName}:scanUrl`: + return is(data, PersistedScanUrlResultStruct); + case `${serviceName}:scanToken`: + return data === null || is(data, TokenScanResultStruct); + case `${serviceName}:scanAddress`: + return is(data, ScanAddressResponseStruct); + default: + return false; + } +} + const ApprovalFeatureStruct = type({ feature_id: string(), type: ApprovalFeatureTypeStruct, @@ -380,21 +411,52 @@ const ApprovalsResponseStruct = type({ */ type BatchOutcome = { results: Record; - errors?: Record; + errors: Record; }; +/** + * Base class for errors delivered to items of a batched request. The service + * policy never retries these: item queries are backed by a batched request + * that is itself retried as a whole, so retrying items individually would + * only fan one failed batch out into many single-item requests against a host + * that is already failing. + */ +class BatchError extends Error {} + /** * An error reported by a batch endpoint for one specific item, as opposed to a * failure of the request as a whole. These are surfaced to the caller per item * and never cached, but they do not make the overall call fail. */ -class BatchItemError extends Error { +class BatchItemError extends BatchError { constructor(message: string) { super(message); this.name = 'BatchItemError'; } } +/** + * A failure of a batched request as a whole, delivered to every item in it. + * The original request error is available as `cause` and is what callers + * ultimately receive. + */ +class BatchRequestError extends BatchError { + constructor(cause: unknown) { + super(getErrorMessage(cause), { cause }); + this.name = 'BatchRequestError'; + } +} + +/** + * Returns the request error behind a batch failure, or the error itself. + * + * @param error - An error rejected by a batch item. + * @returns The underlying error. + */ +function unwrapBatchError(error: unknown): unknown { + return error instanceof BatchRequestError ? error.cause : error; +} + type BatchLoader = { /** * Registers an item to be resolved by the next executed batch. @@ -440,7 +502,7 @@ function createBatchLoader({ const executeChunk = async (chunk: PendingItem[]): Promise => { try { - const { results, errors = {} } = await executeBatch( + const { results, errors } = await executeBatch( chunk.map((item) => item.key), ); for (const item of chunk) { @@ -453,7 +515,7 @@ function createBatchLoader({ } } catch (error) { for (const item of chunk) { - item.reject(error); + item.reject(new BatchRequestError(error)); } } }; @@ -547,6 +609,7 @@ export class PhishingDataService extends BaseDataService< policyOptions?: CreateServicePolicyOptions; persistenceConfig?: PersistenceConfiguration | null; }) { + const { retryFilterPolicy, ...restPolicyOptions } = policyOptions; super({ name: serviceName, messenger, @@ -573,17 +636,33 @@ export class PhishingDataService extends BaseDataService< // // Retries are disabled by default for the same reason. The previous // in-controller implementation made a single request per call, and the - // controller's timeouts are sized for one attempt. Retries are also - // unsafe for the batched endpoints: a failed batch rejects every item - // query in it, and each would then retry independently, turning one - // failed request into many single-item requests against a host that is - // already failing. + // controller's timeouts are sized for one attempt. When enabled, retries + // apply to each batched request as a whole; see `retryFilterPolicy`. policyOptions: { maxConsecutiveFailures: Number.MAX_SAFE_INTEGER, maxRetries: 0, - ...policyOptions, + ...restPolicyOptions, + // Item queries backed by a batched request are never retried on their + // own: the batch is retried as a whole inside the loader, and an item + // retry would only fan one failed batch out into many single-item + // requests against a host that is already failing. A caller-provided + // filter is still applied to everything else. + retryFilterPolicy: handleWhen( + (error): boolean => + !(error instanceof BatchError) && + (retryFilterPolicy?.options.errorFilter(error) ?? true), + ), }, - persistenceConfig: persistenceConfig ?? undefined, + persistenceConfig: persistenceConfig + ? { + ...persistenceConfig, + // Persisted scan results are validated before they can be served + // from the cache; a caller-provided filter is applied on top. + shouldHydrateQuery: (query): boolean => + isValidPersistedScanQuery(query) && + (persistenceConfig.shouldHydrateQuery?.(query) ?? true), + } + : undefined, }); this.messenger.registerMethodActionHandlers( @@ -750,10 +829,12 @@ export class PhishingDataService extends BaseDataService< const loader = createBatchLoader({ maxBatchSize: MAX_URLS_PER_SCAN_REQUEST, executeBatch: async (batchUrls) => { - const jsonResponse = await this.#postJson( - `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, - { urls: batchUrls }, - { timeout: BULK_URL_SCAN_TIMEOUT }, + const jsonResponse = await this.executeWithPolicy(() => + this.#postJson( + `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, + { urls: batchUrls }, + { timeout: BULK_URL_SCAN_TIMEOUT }, + ), ); const response = this.#validate( jsonResponse, @@ -831,9 +912,10 @@ export class PhishingDataService extends BaseDataService< const url = requested[index]; if (outcome.status === 'rejected') { - addError(url, getErrorMessage(outcome.reason)); + const reason = unwrapBatchError(outcome.reason); + addError(url, getErrorMessage(reason)); if (!(outcome.reason instanceof BatchItemError)) { - requestFailure ??= { reason: outcome.reason }; + requestFailure ??= { reason }; } continue; } @@ -909,7 +991,7 @@ export class PhishingDataService extends BaseDataService< let firstError: Error | undefined; for (const outcome of await Promise.allSettled(entries)) { if (outcome.status === 'rejected') { - firstError ??= outcome.reason as Error; + firstError ??= unwrapBatchError(outcome.reason) as Error; continue; } @@ -937,10 +1019,12 @@ export class PhishingDataService extends BaseDataService< return createBatchLoader({ maxBatchSize: MAX_TOKENS_PER_SCAN_REQUEST, executeBatch: async (batchTokens) => { - const jsonResponse = await this.#postJson( - `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, - { chain, tokens: batchTokens }, - { timeout: TOKEN_SCAN_TIMEOUT }, + const jsonResponse = await this.executeWithPolicy(() => + this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, + { chain, tokens: batchTokens }, + { timeout: TOKEN_SCAN_TIMEOUT }, + ), ); const response = this.#validate( jsonResponse, From a9d3e6d169e68f77bf95b9363c4333515ea664e4 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Thu, 10 Sep 2026 11:44:50 -0500 Subject: [PATCH 31/38] docs: describe phishing data service validation and retry changes Co-Authored-By: Claude Opus 5 --- packages/phishing-controller/CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index ce2509eee7c..e1dbbf1f70b 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -26,15 +26,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) - Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) - `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in; retries then apply to each batched bulk-scan request as a whole rather than to individual items ([#9914](https://github.com/MetaMask/core/pull/9914)) - Uncached approval requests now use the configured retry and circuit-breaker policy consistently with cached service requests ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Malformed API responses, including incomplete stalelists, unknown verdicts, and malformed nested bulk-scan or approval entries, are now rejected and treated as request failures instead of being passed through, and are not cached ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `PhishingDataService:scanUrl` now always returns a hostname, and C2 blocklist responses now validate `lastFetchedAt` as the numeric Unix timestamp returned by the API ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses are now rejected and treated as request failures instead of being passed through, and are never cached. Bulk URL scan, bulk token scan, and approvals responses are validated per entry: a malformed URL result is reported for that URL in `errors`, and malformed token results and approvals are omitted, so one bad entry no longer discards the other verdicts in the response ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Hotlist diffs that target a list type this client does not recognize are ignored instead of rejecting the whole hotlist response ([#9914](https://github.com/MetaMask/core/pull/9914)) +- **BREAKING:** `C2DomainBlocklistResponse.lastFetchedAt` is now typed as an optional `number` (previously a required `string`), matching the numeric Unix timestamp returned by the API; responses without it are accepted since the controller does not read it ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `PhishingDataService:scanUrl` now always returns a hostname, including for entries seeded by `bulkScanUrls` ([#9914](https://github.com/MetaMask/core/pull/9914)) - URL scan responses containing `fetchError` are now treated as failures and are not cached, allowing subsequent calls to retry the detector ([#9914](https://github.com/MetaMask/core/pull/9914)) - Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) - `bulkScanTokens` now preserves successful cached verdicts when another token lookup fails, instead of discarding every result in the batch ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Token and address scans lowercase EVM addresses before using them as cache keys and sending them to the API, so differently-cased inputs share one cache entry and match the API's response keys; `bulkScanTokens` results are keyed by the normalized address ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Persisted scan results are validated on rehydration; entries that fail validation, and persisted queries other than scan results, are discarded. Queries wait at most `DEFAULT_HYDRATION_TIMEOUT` (1 second, configurable via `persistenceConfig.hydrationTimeout`) for rehydration before proceeding ([#9914](https://github.com/MetaMask/core/pull/9914)) - Timed-out URL, token, and address-security requests are now aborted so later calls can retry instead of remaining attached to the original pending query ([#9914](https://github.com/MetaMask/core/pull/9914)) - Destroying `PhishingDataService` now aborts all pending requests, including batched scans and uncached approval requests ([#9914](https://github.com/MetaMask/core/pull/9914)) From d04bcaa2dac31145aa6b7b45696260d974b67a56 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 12:47:44 -0500 Subject: [PATCH 32/38] refactor: fetch approvals through the query cache and drop redundant response casts getApprovals now goes through fetchQuery with staleTime and gcTime of 0, the same pattern KycService, ShieldApiService, and SubscriptionService use for uncached reads. The precedent in #10007 bypasses fetchQuery for writes, not for POST-shaped reads, so the direct executeWithPolicy path was a special case with no benefit. getStalelist and scanAddress no longer cast through Json: their structs have no optional fields, so the validated type satisfies fetchQuery's Json bound and flows through as the return type. The remaining casts are forced by optional() fields inferring T | undefined, which fetchQuery's TQueryFnData bound rejects; relaxing that lives in base-data-service. Co-Authored-By: Claude Opus 5 --- ...PhishingDataService-method-action-types.ts | 6 +- .../src/PhishingDataService.ts | 73 ++++++++++--------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts index dba8bb20b5d..356a098e4db 100644 --- a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts +++ b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts @@ -123,8 +123,10 @@ export type PhishingDataServiceScanAddressAction = { /** * Gets token approvals for an address with security enrichments via the - * security-alerts API. Approvals reflect live account state and are never - * cached. + * security-alerts API. Approvals reflect live account state, so they are + * always refetched and never retained in the query cache. EVM addresses are + * lowercased before being sent to the API; other addresses are used as + * given. * * @param chain - The chain name (e.g. `ethereum`). * @param address - The address to get approvals for. diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 36d2ac954ef..570ba5a69b8 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -686,21 +686,21 @@ export class PhishingDataService extends BaseDataService< * @returns The stalelist response. */ async getStalelist(): Promise> { - const jsonResponse = await this.fetchQuery({ + // Validated inside the query function so that a malformed response is + // never committed to, or persisted from, the query cache. The struct has + // no optional fields, so its inferred type satisfies `fetchQuery`'s `Json` + // bound and types the result without a cast. + return await this.fetchQuery({ queryKey: [`${this.name}:getStalelist`], - // Validated inside the query function so that a malformed response is - // never committed to, or persisted from, the query cache. queryFn: async ({ signal }) => this.#validate( await this.#getJson(METAMASK_STALELIST_URL, { signal }), StalelistResponseStruct, 'stalelist', - ) as Json, + ), staleTime: 0, gcTime: LIST_GC_TIME, }); - - return jsonResponse as DataResultWrapper; } /** @@ -1083,7 +1083,7 @@ export class PhishingDataService extends BaseDataService< address: string, ): Promise { const normalizedAddress = normalizeScanAddress(address); - const jsonResponse = await this.fetchQuery({ + return await this.fetchQuery({ queryKey: [`${this.name}:scanAddress`, chain, normalizedAddress], queryFn: async ({ signal }) => this.#validate( @@ -1094,18 +1094,18 @@ export class PhishingDataService extends BaseDataService< ), ScanAddressResponseStruct, 'address scan', - ) as Json, + ), staleTime: SCAN_RESULT_STALE_TIME, gcTime: SCAN_RESULT_GC_TIME, }); - - return jsonResponse as AddressScanResult; } /** * Gets token approvals for an address with security enrichments via the - * security-alerts API. Approvals reflect live account state and are never - * cached. + * security-alerts API. Approvals reflect live account state, so they are + * always refetched and never retained in the query cache. EVM addresses are + * lowercased before being sent to the API; other addresses are used as + * given. * * @param chain - The chain name (e.g. `ethereum`). * @param address - The address to get approvals for. @@ -1115,29 +1115,34 @@ export class PhishingDataService extends BaseDataService< chain: string, address: string, ): Promise { - // Deliberately not routed through `fetchQuery`. Approvals reflect live, - // account-specific state that is never cached, so the query cache would - // provide no benefit while publishing the response on the messenger as a - // `cacheUpdated` payload. This matches the handling of non-cached POSTs - // elsewhere in the monorepo. - const jsonResponse = await this.executeWithPolicy(() => - this.#postJson( - `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, - { chain, address }, - { timeout: APPROVALS_TIMEOUT }, - ), - ); + const normalizedAddress = normalizeScanAddress(address); + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getApprovals`, chain, normalizedAddress], + queryFn: async ({ signal }) => { + const response = this.#validate( + await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, + { chain, address: normalizedAddress }, + { signal, timeout: APPROVALS_TIMEOUT }, + ), + ApprovalsResponseStruct, + 'approvals', + ); + // Approvals are validated individually so that one malformed entry + // does not empty the whole list. + return { + approvals: response.approvals.filter((approval) => + is(approval, ApprovalStruct), + ), + } as Json; + }, + // Live account state: always refetch and evict as soon as the call + // settles, as other data services do for uncached reads. + staleTime: 0, + gcTime: 0, + }); - const response = this.#validate( - jsonResponse, - ApprovalsResponseStruct, - 'approvals', - ); - return { - approvals: response.approvals.filter((approval) => - is(approval, ApprovalStruct), - ), - } as ApprovalsResponse; + return jsonResponse as ApprovalsResponse; } /** From 337bbda8793d1f8cf474bfa761a9c6f0b027b087 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 12:47:44 -0500 Subject: [PATCH 33/38] docs: consolidate the phishing-controller changelog Fold new-service behavior into the PhishingDataService entry, move fixes of released behavior to Fixed, list the added dependencies, name the class and method in ambiguous entries, and drop entries that only fixed iterations of this branch that never shipped. Co-Authored-By: Claude Opus 5 --- packages/phishing-controller/CHANGELOG.md | 40 ++++++++++++----------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index e1dbbf1f70b..b954cd2d179 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -11,10 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) ([#9914](https://github.com/MetaMask/core/pull/9914)) - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` - - Requests are wrapped in a shared service policy, configurable via the `policyOptions` constructor option. Retries and circuit breaking are both disabled by default: the service spans four independent API hosts, so a circuit broken by one host would pause phishing-list updates from the others, and the previous in-controller implementation made a single request per call - - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes), including after cache rehydration; bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries; approvals are never cached - - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization; queries started after `init` wait for rehydration to finish. Pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves + - Requests are wrapped in a shared service policy, configurable via the `policyOptions` option of the `PhishingDataService` constructor. Retries and circuit breaking are both disabled by default: the service spans four independent API hosts, so a circuit broken by one host would pause phishing-list updates from the others, and the previous in-controller implementation made a single request per call. Pass `policyOptions.maxRetries` to opt in; retries then apply to each batched bulk-scan request as a whole rather than to individual items + - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes), including after cache rehydration; bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries. Approvals reflect live account state and are always refetched and never retained + - EVM token and address inputs are lowercased before being used as cache keys and sent to the API, so differently-cased inputs share one cache entry and match the API's response keys; `bulkScanTokens` results are keyed by the normalized address + - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization. Queries started after `init` wait at most `DEFAULT_HYDRATION_TIMEOUT` (1 second, configurable via `persistenceConfig.hydrationTimeout`) for rehydration to finish, and persisted scan results are validated on rehydration with anything malformed or unrecognized discarded. Pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves - Fetched lists (stalelist, hotlist diffs, and the C2 domain blocklist) are not retained by the query cache, and so are not persisted; the controller keeps its own copy in state + - Timed-out requests are aborted so later calls can retry, and destroying the service aborts all pending requests - Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys ([#9914](https://github.com/MetaMask/core/pull/9914)) ### Changed @@ -24,29 +26,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **BREAKING:** Remove the `urlScanCache`, `tokenScanCache`, and `addressScanCache` properties from `PhishingControllerState`; scan results are now cached (and persisted) by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) - Client state migrations should remove these properties from persisted `PhishingController` state - **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Requests are no longer retried by default; the previous in-controller implementation made a single request per call, and the controller's timeouts are sized for one attempt. Pass `policyOptions.maxRetries` to opt back in; retries then apply to each batched bulk-scan request as a whole rather than to individual items ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Uncached approval requests now use the configured retry and circuit-breaker policy consistently with cached service requests ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Malformed API responses are now rejected and treated as request failures instead of being passed through, and are never cached. Bulk URL scan, bulk token scan, and approvals responses are validated per entry: a malformed URL result is reported for that URL in `errors`, and malformed token results and approvals are omitted, so one bad entry no longer discards the other verdicts in the response ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Hotlist diffs that target a list type this client does not recognize are ignored instead of rejecting the whole hotlist response ([#9914](https://github.com/MetaMask/core/pull/9914)) - **BREAKING:** `C2DomainBlocklistResponse.lastFetchedAt` is now typed as an optional `number` (previously a required `string`), matching the numeric Unix timestamp returned by the API; responses without it are accepted since the controller does not read it ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `PhishingDataService:scanUrl` now always returns a hostname, including for entries seeded by `bulkScanUrls` ([#9914](https://github.com/MetaMask/core/pull/9914)) -- URL scan responses containing `fetchError` are now treated as failures and are not cached, allowing subsequent calls to retry the detector ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Token, address, and approval scans now accept all verdicts documented by the security-alerts API, including `Verified`, `Trusted`, and API-originated `Error` results ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`; it only rejects when no result could be resolved at all. Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `bulkScanUrls` no longer caches a "no result" verdict for URLs the API reported an error for, so those URLs are retried on the next call instead of being silently skipped for a minute ([#9914](https://github.com/MetaMask/core/pull/9914)) -- `bulkScanTokens` now preserves successful cached verdicts when another token lookup fails, instead of discarding every result in the batch ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Token and address scans lowercase EVM addresses before using them as cache keys and sending them to the API, so differently-cased inputs share one cache entry and match the API's response keys; `bulkScanTokens` results are keyed by the normalized address ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Persisted scan results are validated on rehydration; entries that fail validation, and persisted queries other than scan results, are discarded. Queries wait at most `DEFAULT_HYDRATION_TIMEOUT` (1 second, configurable via `persistenceConfig.hydrationTimeout`) for rehydration before proceeding ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Timed-out URL, token, and address-security requests are now aborted so later calls can retry instead of remaining attached to the original pending query ([#9914](https://github.com/MetaMask/core/pull/9914)) -- Destroying `PhishingDataService` now aborts all pending requests, including batched scans and uncached approval requests ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/base-data-service` `^2.0.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/storage-service` `^2.0.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/superstruct` `^3.4.1` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/utils` `^11.12.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@tanstack/query-core` `^5.62.16` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `PhishingController.scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses are now rejected and treated as request failures instead of being passed through, and are never cached ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Bulk URL scan, bulk token scan, and approvals responses are validated per entry: a malformed URL result is reported for that URL in `errors`, and malformed token results and approvals are omitted, so one bad entry does not discard the other verdicts in the response ### Removed - **BREAKING:** Remove the `CacheEntry` type; the custom cache manager has been replaced by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) - **BREAKING:** Remove the `DEFAULT_URL_SCAN_CACHE_TTL`, `DEFAULT_URL_SCAN_CACHE_MAX_SIZE`, `DEFAULT_TOKEN_SCAN_CACHE_TTL`, `DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE`, `DEFAULT_ADDRESS_SCAN_CACHE_TTL`, and `DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE` constants ([#9914](https://github.com/MetaMask/core/pull/9914)) +### Fixed + +- `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`, and only rejects when nothing could be resolved at all ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs +- Hotlist diffs that target a list type this client does not recognize are now ignored ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Previously such a diff threw a `TypeError` while the hotlist was being applied, leaving the phishing lists un-updated + ## [18.0.0] ### Changed From 2c421e3a0e51b6121134c402e967bad543eb0a20 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 13:19:05 -0500 Subject: [PATCH 34/38] refactor: expose the data service policy instead of wrapping execute Replace the protected executeWithPolicy helper (introduced on this branch) with a protected policy getter. Subclasses can still run uncached requests under the shared retry and circuit-breaker policy, and can now also observe its onBreak, onDegraded, and onRetry events, which hand-rolled services elsewhere in the monorepo forward but BaseDataService subclasses could not reach. Co-Authored-By: Claude Opus 5 --- packages/base-data-service/CHANGELOG.md | 2 +- .../src/BaseDataService.test.ts | 13 +++++++++++++ .../base-data-service/src/BaseDataService.ts | 19 +++++++++---------- .../tests/ExampleDataService.ts | 10 ++++++++++ .../src/PhishingDataService.ts | 4 ++-- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index 9c92baaed15..2ad26d43a8f 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add a protected `executeWithPolicy` helper for applying a data service's retry and circuit-breaker policy to uncached requests ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add a protected `policy` getter exposing the service's retry and circuit-breaker policy, so subclasses can run uncached requests under the same policy and observe its `onBreak`, `onDegraded`, and `onRetry` events ([#9914](https://github.com/MetaMask/core/pull/9914)) - Add `hydrationTimeout` and `shouldHydrateQuery` options to `PersistenceConfiguration`, and export `DEFAULT_HYDRATION_TIMEOUT` ([#9914](https://github.com/MetaMask/core/pull/9914)) - `hydrationTimeout` bounds how long a query waits for cache rehydration after `init` (default 1 second), and `shouldHydrateQuery` filters persisted queries before they are restored into the cache diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index 455211c565d..443c2384721 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -357,6 +357,19 @@ describe('BaseDataService', () => { }); }); + describe('policy', () => { + it('exposes the service policy to subclasses', async () => { + const service = new ExampleDataService(createServiceMessenger()); + const policy = service.getPolicy(); + + expect(await policy.execute(() => 'ok')).toBe('ok'); + expect(typeof policy.onBreak).toBe('function'); + expect(typeof policy.onDegraded).toBe('function'); + + service.destroy(); + }); + }); + describe('persistence', () => { it('persists the cache using the StorageService', async () => { const setItem = jest.fn(); diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index e79a5a03067..9c108096c27 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -328,7 +328,7 @@ export class BaseDataService< return this.#queryClient.fetchQuery({ ...options, queryFn: async (context) => { - const response = await this.executeWithPolicy(() => + const response = await this.#policy.execute(() => options.queryFn(context), ); return processQueryResponse(options.queryKey, response, responseStruct); @@ -396,7 +396,7 @@ export class BaseDataService< ...options, initialPageParam: pageParam ?? options.initialPageParam, queryFn: async (context) => { - const response = await this.executeWithPolicy(async () => + const response = await this.#policy.execute(async () => options.queryFn({ ...context, pageParam: context.meta?.pageParam ?? context.pageParam, @@ -436,16 +436,15 @@ export class BaseDataService< } /** - * Executes an operation using this service's retry and circuit-breaker - * policy without adding the result to the query cache. + * The retry and circuit-breaker policy that wraps every query made through + * this service. Subclasses can run uncached requests under the same policy + * with `this.policy.execute(...)`, or observe its `onBreak`, `onDegraded`, + * and `onRetry` events. * - * @param operation - The asynchronous operation to execute. - * @returns The operation result. + * @returns The service policy. */ - protected async executeWithPolicy( - operation: () => PromiseLike | Result, - ): Promise { - return this.#policy.execute(operation); + protected get policy(): ServicePolicy { + return this.#policy; } /** diff --git a/packages/base-data-service/tests/ExampleDataService.ts b/packages/base-data-service/tests/ExampleDataService.ts index 8f77e10b751..ce2c93dcbbd 100644 --- a/packages/base-data-service/tests/ExampleDataService.ts +++ b/packages/base-data-service/tests/ExampleDataService.ts @@ -21,6 +21,7 @@ import { DataServiceGranularCacheUpdatedEvent, PersistenceConfiguration, } from '../src/BaseDataService.js'; +import type { ServicePolicy } from '../src/createServicePolicy.js'; import { ExampleDataServiceMethodActions } from './ExampleDataService-method-action-types.js'; export const serviceName = 'ExampleDataService'; @@ -109,6 +110,15 @@ export class ExampleDataService extends BaseDataService< ); } + /** + * Exposes the protected service policy for tests. + * + * @returns The service policy. + */ + getPolicy(): ServicePolicy { + return this.policy; + } + async getAssets(assets: string[]): Promise { return this.fetchQuery({ queryKey: [`${this.name}:getAssets`, assets], diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 570ba5a69b8..d4c867d95ef 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -829,7 +829,7 @@ export class PhishingDataService extends BaseDataService< const loader = createBatchLoader({ maxBatchSize: MAX_URLS_PER_SCAN_REQUEST, executeBatch: async (batchUrls) => { - const jsonResponse = await this.executeWithPolicy(() => + const jsonResponse = await this.policy.execute(() => this.#postJson( `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls: batchUrls }, @@ -1019,7 +1019,7 @@ export class PhishingDataService extends BaseDataService< return createBatchLoader({ maxBatchSize: MAX_TOKENS_PER_SCAN_REQUEST, executeBatch: async (batchTokens) => { - const jsonResponse = await this.executeWithPolicy(() => + const jsonResponse = await this.policy.execute(() => this.#postJson( `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, { chain, tokens: batchTokens }, From 8d5a1c9ed6c26388656014d46fd7eb1891547de5 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 13:30:36 -0500 Subject: [PATCH 35/38] docs: explain the per-request abort controller in PhishingDataService Each request combines the query's TanStack signal, the service-wide destroy signal, and its own timeout into the single signal that fetch accepts. Co-Authored-By: Claude Opus 5 --- packages/phishing-controller/src/PhishingDataService.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index d4c867d95ef..4a664a98b4e 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -1203,6 +1203,12 @@ export class PhishingDataService extends BaseDataService< init: RequestInit, timeout?: number, ): Promise { + // `fetch` takes a single signal, so each request gets its own controller + // that funnels three cancellation sources into one: the query's own signal + // from TanStack, the service-wide signal that `destroy` aborts, and this + // request's timeout. The timeout is tracked separately so that it can be + // reported as one. `AbortSignal.any` and `AbortSignal.timeout` would + // express this directly but are not reliably available on React Native. const controller = new AbortController(); const sourceSignal = init.signal; const serviceSignal = this.#abortController.signal; From c6bd755740997d4e7c7891c863b35d23e94ddaad Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 13:31:07 -0500 Subject: [PATCH 36/38] refactor: batch scan lookups with a single deferred flush The batch loader had two ways to send a batch: an explicit flush() called by each bulk method, and a microtask fallback originally added for retried item queries. Item queries are no longer retried, and once a client has called init() every lookup runs after the explicit flush anyway, so the deferred flush was already doing all the work in that configuration. Remove the explicit flush and its API; every lookup made in the same turn is sent in one batch. Co-Authored-By: Claude Opus 5 --- .../src/PhishingDataService.ts | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 4a664a98b4e..836251e1d06 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -466,18 +466,14 @@ type BatchLoader = { * include it. */ load: (key: string) => Promise; - /** - * Executes all pending items, in requests of up to the configured batch - * size. Items registered after a flush (e.g. by a retry) are scheduled for - * a later flush automatically. - */ - flush: () => void; }; /** * Creates a loader that coalesces individual item lookups into batched - * requests. This preserves the per-item caching granularity of the query - * cache while keeping the batched network behavior of the bulk endpoints. + * requests. Every lookup made in the same turn of the event loop is sent in + * one batch (split into requests of up to `maxBatchSize`). This preserves the + * per-item caching granularity of the query cache while keeping the batched + * network behavior of the bulk endpoints. * * @param options - The loader options. * @param options.maxBatchSize - The maximum number of items per request. @@ -538,19 +534,17 @@ function createBatchLoader({ async load(key: string): Promise { return new Promise((resolve, reject) => { pending.push({ key, resolve, reject }); - // Items registered outside an explicit flush (e.g. by the retry - // policy re-running a query) are coalesced via the microtask queue. + // The first lookup in a turn schedules the flush as a microtask, which + // runs once the current synchronous work has finished, so every other + // lookup made in the same turn (for example all the item queries of + // one bulk call, whether they start synchronously or after awaiting + // cache rehydration) joins the same batch. if (!flushScheduled) { flushScheduled = true; - queueMicrotask(() => { - if (flushScheduled) { - flush(); - } - }); + queueMicrotask(flush); } }); }, - flush, }; } @@ -902,7 +896,6 @@ export class PhishingDataService extends BaseDataService< }), ); } - loader.flush(); const settled = await Promise.allSettled(entries); const results: Record = {}; @@ -957,7 +950,6 @@ export class PhishingDataService extends BaseDataService< chain, normalizeScanAddress(token), ); - loader.flush(); return await result; } @@ -985,7 +977,6 @@ export class PhishingDataService extends BaseDataService< (result) => [normalizedToken, result] as const, ); }); - loader.flush(); const results: TokenScanApiResponse['results'] = {}; let firstError: Error | undefined; From 5f89fc129ecb3e2fb35da12c344fa262e3dc31bb Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 13:36:42 -0500 Subject: [PATCH 37/38] fix: share the bounded rehydration wait across queries Each query used to start its own hydration deadline. While storage hung, every query paid the full timeout, and because the per-query timers fire as separate macrotasks the item lookups of one bulk scan resumed in different turns and were sent as single-item requests. One shared deadline per service means all waiters resume together, batches stay intact, and no query waits again once the deadline has passed. Co-Authored-By: Claude Opus 5 --- packages/base-data-service/CHANGELOG.md | 2 +- .../src/BaseDataService.test.ts | 32 ++++++++++++++ .../base-data-service/src/BaseDataService.ts | 33 ++++++++------ .../src/PhishingDataService.test.ts | 43 +++++++++++++++++++ 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index 2ad26d43a8f..babf2dcbbd3 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a protected `policy` getter exposing the service's retry and circuit-breaker policy, so subclasses can run uncached requests under the same policy and observe its `onBreak`, `onDegraded`, and `onRetry` events ([#9914](https://github.com/MetaMask/core/pull/9914)) - Add `hydrationTimeout` and `shouldHydrateQuery` options to `PersistenceConfiguration`, and export `DEFAULT_HYDRATION_TIMEOUT` ([#9914](https://github.com/MetaMask/core/pull/9914)) - - `hydrationTimeout` bounds how long a query waits for cache rehydration after `init` (default 1 second), and `shouldHydrateQuery` filters persisted queries before they are restored into the cache + - `hydrationTimeout` bounds how long queries wait for cache rehydration after `init` (default 1 second, measured once from the first waiting query), and `shouldHydrateQuery` filters persisted queries before they are restored into the cache ### Changed diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index 443c2384721..9c8f60a6b18 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -583,6 +583,38 @@ describe('BaseDataService', () => { service.destroy(); }); + it('does not wait again once the hydration timeout has elapsed', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const first = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + expect(await first).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + // A later query must not start a fresh wait while storage still hangs. + const secondScope = mockAssets(); + await service.invalidateQueries({ + queryKey: ['ExampleDataService:getAssets', MOCK_ASSETS], + }); + expect(await service.getAssets(MOCK_ASSETS)).toHaveLength(3); + expect(secondScope.isDone()).toBe(true); + + service.destroy(); + }); + it('honors a custom hydrationTimeout', async () => { cleanAll(); const networkScope = mockAssets(); diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index 9c108096c27..eba90f8fdf5 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -146,9 +146,10 @@ export type PersistenceConfiguration = { */ maxWriteDelay?: number; /** - * The maximum number of milliseconds a query waits for cache rehydration to - * finish after `init` has been called. Once exceeded, the query proceeds - * without the persisted cache. Defaults to {@link DEFAULT_HYDRATION_TIMEOUT}. + * The maximum number of milliseconds queries wait for cache rehydration to + * finish after `init` has been called, measured from the first query that + * waits. Once exceeded, queries proceed without the persisted cache. + * Defaults to {@link DEFAULT_HYDRATION_TIMEOUT}. */ hydrationTimeout?: number; /** @@ -204,6 +205,8 @@ export class BaseDataService< #initializationPromise?: Promise; + #boundedInitialization?: Promise; + constructor({ name, messenger, @@ -477,6 +480,11 @@ export class BaseDataService< * indefinitely. A late rehydration is still applied by TanStack, which only * overwrites entries older than the persisted ones. * + * The deadline is shared by every waiting query rather than started per + * query: all of them resume in the same turn, so lookups that are batched + * together stay batched, and once the deadline has passed no later query + * waits again while storage remains unresponsive. + * * Callers must only await this when `init` has been called: an asynchronous * hop before a query starts changes its timing relative to callers' own * timers, so services without persistence keep starting queries @@ -485,19 +493,20 @@ export class BaseDataService< * @param initialization - The pending rehydration. */ async #waitForInitialization(initialization: Promise): Promise { - const timeout = - this.#persistenceConfig?.hydrationTimeout ?? DEFAULT_HYDRATION_TIMEOUT; - let timer: ReturnType | undefined; - try { - await Promise.race([ + if (!this.#boundedInitialization) { + let timer: ReturnType | undefined; + this.#boundedInitialization = Promise.race([ initialization, new Promise((resolve) => { - timer = setTimeout(resolve, timeout); + timer = setTimeout( + resolve, + this.#persistenceConfig?.hydrationTimeout ?? + DEFAULT_HYDRATION_TIMEOUT, + ); }), - ]); - } finally { - clearTimeout(timer); + ]).finally(() => clearTimeout(timer)); } + await this.#boundedInitialization; } /** diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts index afddd19b6ae..baac507dd18 100644 --- a/packages/phishing-controller/src/PhishingDataService.test.ts +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -588,6 +588,49 @@ describe('PhishingDataService', () => { expect(Object.keys(response.results)).toHaveLength(3); }); + it('batches a bulk call even when rehydration times out', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(function (_uri, body) { + const { urls } = body as { urls: string[] }; + batchSizes.push(urls.length); + return [ + 200, + { + results: Object.fromEntries( + urls.map((url) => [url, { recommendedAction: 'NONE' }]), + ), + errors: {}, + }, + ]; + }); + const { rootMessenger, service } = createService({ + options: { + persistenceConfig: { + maxAge: inMilliseconds(5, Duration.Minute), + hydrationTimeout: 20, + }, + }, + setItemMock: jest.fn(), + getItemMock: jest.fn(() => new Promise(() => undefined)), + }); + service.init(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ], + ); + + expect(batchSizes).toStrictEqual([3]); + expect(Object.keys(response.results)).toHaveLength(3); + }); + it('does not retry per-URL errors reported by the endpoint', async () => { let requests = 0; nock(PHISHING_DETECTION_BASE_URL) From cff89ad93c1159ae226f96e4b61bf314bb4599e9 Mon Sep 17 00:00:00 2001 From: Alex Donesky Date: Fri, 11 Sep 2026 13:36:43 -0500 Subject: [PATCH 38/38] docs: note that batched requests carry no per-query abort signal Co-Authored-By: Claude Opus 5 --- packages/phishing-controller/src/PhishingDataService.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts index 836251e1d06..05cb61f1a65 100644 --- a/packages/phishing-controller/src/PhishingDataService.ts +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -1195,9 +1195,10 @@ export class PhishingDataService extends BaseDataService< timeout?: number, ): Promise { // `fetch` takes a single signal, so each request gets its own controller - // that funnels three cancellation sources into one: the query's own signal - // from TanStack, the service-wide signal that `destroy` aborts, and this - // request's timeout. The timeout is tracked separately so that it can be + // that funnels up to three cancellation sources into one: the query's own + // signal from TanStack (single-item requests only; batched requests are + // not tied to one query), the service-wide signal that `destroy` aborts, + // and this request's timeout. The timeout is tracked separately so that it can be // reported as one. `AbortSignal.any` and `AbortSignal.timeout` would // express this directly but are not reliably available on React Native. const controller = new AbortController();