diff --git a/README.md b/README.md index 0a4278a3847..13841892673 100644 --- a/README.md +++ b/README.md @@ -597,8 +597,10 @@ linkStyle default opacity:0.5 perps_controller --> transaction_controller; phishing_controller --> address_book_controller; phishing_controller --> base_controller; + phishing_controller --> base_data_service; phishing_controller --> controller_utils; phishing_controller --> messenger; + phishing_controller --> storage_service; phishing_controller --> transaction_controller; platform_api_docs --> utils; polling_controller --> base_controller; diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 49a28c0feeb..b6f688bf643 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1326,16 +1326,6 @@ "count": 1 } }, - "packages/phishing-controller/src/CacheManager.test.ts": { - "@typescript-eslint/explicit-function-return-type": { - "count": 2 - } - }, - "packages/phishing-controller/src/CacheManager.ts": { - "@typescript-eslint/naming-convention": { - "count": 3 - } - }, "packages/phishing-controller/src/PathTrie.ts": { "@typescript-eslint/explicit-function-return-type": { "count": 2 @@ -1343,10 +1333,7 @@ }, "packages/phishing-controller/src/PhishingController.ts": { "@typescript-eslint/explicit-function-return-type": { - "count": 14 - }, - "@typescript-eslint/naming-convention": { - "count": 1 + "count": 11 }, "@typescript-eslint/prefer-nullish-coalescing": { "count": 6 @@ -1377,7 +1364,7 @@ }, "packages/phishing-controller/src/utils.ts": { "@typescript-eslint/explicit-function-return-type": { - "count": 5 + "count": 4 }, "@typescript-eslint/prefer-nullish-coalescing": { "count": 1 diff --git a/packages/base-data-service/CHANGELOG.md b/packages/base-data-service/CHANGELOG.md index 362cf9042f6..babf2dcbbd3 100644 --- a/packages/base-data-service/CHANGELOG.md +++ b/packages/base-data-service/CHANGELOG.md @@ -7,10 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add a protected `policy` getter exposing the service's retry and circuit-breaker policy, so subclasses can run uncached requests under the same policy and observe its `onBreak`, `onDegraded`, and `onRetry` events ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `hydrationTimeout` and `shouldHydrateQuery` options to `PersistenceConfiguration`, and export `DEFAULT_HYDRATION_TIMEOUT` ([#9914](https://github.com/MetaMask/core/pull/9914)) + - `hydrationTimeout` bounds how long queries wait for cache rehydration after `init` (default 1 second, measured once from the first waiting query), and `shouldHydrateQuery` filters persisted queries before they are restored into the cache + ### Changed - Bump `@metamask/utils` from `^11.12.0` to `^12.0.0` ([#10192](https://github.com/MetaMask/core/pull/10192)) +### Fixed + +- Wait for cache rehydration to finish before starting a query when `init` has been called, preventing persisted results from racing the first network request; the wait is bounded by `hydrationTimeout` so that a slow or hung storage read cannot block queries ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Discard persisted caches that fail shape validation instead of attempting to hydrate them ([#9914](https://github.com/MetaMask/core/pull/9914)) + ## [2.0.0] ### Changed diff --git a/packages/base-data-service/src/BaseDataService.test.ts b/packages/base-data-service/src/BaseDataService.test.ts index c647ac6e7f7..9c8f60a6b18 100644 --- a/packages/base-data-service/src/BaseDataService.test.ts +++ b/packages/base-data-service/src/BaseDataService.test.ts @@ -26,7 +26,10 @@ import { TRANSACTIONS_PAGE_2_CURSOR, TRANSACTIONS_PAGE_3_CURSOR, } from '../tests/mocks.js'; -import { STORAGE_SERVICE_KEY } from './BaseDataService.js'; +import { + DEFAULT_HYDRATION_TIMEOUT, + STORAGE_SERVICE_KEY, +} from './BaseDataService.js'; const TEST_ADDRESS = '0x4bbeEB066eD09B7AEd07bF39EEe0460DFa261520'; @@ -354,6 +357,19 @@ describe('BaseDataService', () => { }); }); + describe('policy', () => { + it('exposes the service policy to subclasses', async () => { + const service = new ExampleDataService(createServiceMessenger()); + const policy = service.getPolicy(); + + expect(await policy.execute(() => 'ok')).toBe('ok'); + expect(typeof policy.onBreak).toBe('function'); + expect(typeof policy.onDegraded).toBe('function'); + + service.destroy(); + }); + }); + describe('persistence', () => { it('persists the cache using the StorageService', async () => { const setItem = jest.fn(); @@ -506,6 +522,214 @@ describe('BaseDataService', () => { expect(getItem).toHaveBeenCalledWith(serviceName, STORAGE_SERVICE_KEY); }); + it('waits for cache initialization before fetching a query', async () => { + cleanAll(); + const networkScope = mockAssets(); + const activityScope = mockTransactionsPage1(); + let resolveGetItem: ((value: { result: null }) => void) | undefined; + const getItem = jest.fn( + () => + new Promise<{ result: null }>((resolve) => { + resolveGetItem = resolve; + }), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + + expect(networkScope.isDone()).toBe(false); + resolveGetItem?.({ result: null }); + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + expect(await service.getActivity(TEST_ADDRESS)).toHaveProperty('data'); + expect(activityScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('proceeds with a query when rehydration exceeds the hydration timeout', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('does not wait again once the hydration timeout has elapsed', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger); + + service.init(); + const first = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + expect(await first).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + // A later query must not start a fresh wait while storage still hangs. + const secondScope = mockAssets(); + await service.invalidateQueries({ + queryKey: ['ExampleDataService:getAssets', MOCK_ASSETS], + }); + expect(await service.getAssets(MOCK_ASSETS)).toHaveLength(3); + expect(secondScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('honors a custom hydrationTimeout', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn( + () => new Promise<{ result: null }>(() => undefined), + ); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const service = new ExampleDataService(messenger, { + persistenceConfig: { maxAge: 1000, hydrationTimeout: 50 }, + }); + + service.init(); + const resultPromise = service.getAssets(MOCK_ASSETS); + await new Promise(setImmediate); + expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(50); + + expect(await resultPromise).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + + it('discards a persisted cache that fails shape validation', async () => { + const getItem = jest.fn().mockResolvedValue({ + result: { state: { queries: 'not-an-array' } }, + }); + const removeItem = jest.fn(); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + 'StorageService:removeItem': removeItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const publishSpy = jest.spyOn(messenger, 'publish'); + const service = new ExampleDataService(messenger); + + service.init(); + await new Promise(setImmediate); + + expect(removeItem).toHaveBeenCalledWith(serviceName, STORAGE_SERVICE_KEY); + expect(publishSpy).not.toHaveBeenCalled(); + }); + + it('skips persisted queries rejected by shouldHydrateQuery', async () => { + cleanAll(); + const networkScope = mockAssets(); + const getItem = jest.fn().mockResolvedValue({ + result: { + state: { + queries: [ + { + queryHash: hashKey([ + 'ExampleDataService:getAssets', + MOCK_ASSETS, + ]), + queryKey: ['ExampleDataService:getAssets', MOCK_ASSETS], + state: { + data: [ + { + assetId: 'eip155:1/slip44:60', + decimals: 18, + name: 'Ethereum', + symbol: 'ETH', + }, + ], + dataUpdateCount: 1, + dataUpdatedAt: Date.now(), + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }, + ], + mutations: [], + }, + timestamp: Date.now(), + }, + }); + const rootMessenger = createRootMessenger({ + actionHandlers: { + 'StorageService:getItem': getItem, + }, + }); + const messenger = createServiceMessenger(rootMessenger); + const shouldHydrateQuery = jest.fn(() => false); + const service = new ExampleDataService(messenger, { + persistenceConfig: { maxAge: 1000, shouldHydrateQuery }, + }); + + service.init(); + await new Promise(setImmediate); + + const result = await service.getAssets(MOCK_ASSETS); + + expect(shouldHydrateQuery).toHaveBeenCalledTimes(1); + expect(result).toHaveLength(3); + expect(networkScope.isDone()).toBe(true); + + service.destroy(); + }); + it('discards the cache if it has expired', async () => { const getItem = jest.fn().mockResolvedValue({ result: { diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index 77ffd8981c7..eba90f8fdf5 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -10,7 +10,14 @@ import type { StorageServiceRemoveItemAction, StorageServiceSetItemAction, } from '@metamask/storage-service'; -import { Struct } from '@metamask/superstruct'; +import { + array, + is, + number, + Struct, + type as objectType, + unknown, +} from '@metamask/superstruct'; import { Duration, inMilliseconds } from '@metamask/utils'; import type { Json } from '@metamask/utils'; import { @@ -113,6 +120,13 @@ const QUERY_CLIENT_DEFAULTS: DefaultOptions = { export const STORAGE_SERVICE_KEY = 'cache'; +/** + * How long a query waits for cache rehydration to finish after `init` has been + * called before proceeding without it. Rehydration is an optimization: a slow + * or hung storage read must never block queries indefinitely. + */ +export const DEFAULT_HYDRATION_TIMEOUT = inMilliseconds(1, Duration.Second); + /** * Options for persistence configuration. */ @@ -131,6 +145,20 @@ export type PersistenceConfiguration = { * The maximum number of milliseconds to wait between persistence writes. */ maxWriteDelay?: number; + /** + * The maximum number of milliseconds queries wait for cache rehydration to + * finish after `init` has been called, measured from the first query that + * waits. Once exceeded, queries proceed without the persisted cache. + * Defaults to {@link DEFAULT_HYDRATION_TIMEOUT}. + */ + hydrationTimeout?: number; + /** + * Decides whether a persisted query is restored into the cache during + * rehydration. Queries for which this returns `false` are discarded. Use it + * to validate persisted data before it can be served from the cache. + * Defaults to restoring every persisted query. + */ + shouldHydrateQuery?: (query: DehydratedState['queries'][number]) => boolean; }; type PersistedCache = { @@ -138,6 +166,14 @@ type PersistedCache = { timestamp: number; }; +const PersistedCacheStruct = objectType({ + timestamp: number(), + state: objectType({ + queries: array(unknown()), + mutations: array(unknown()), + }), +}); + export class BaseDataService< ServiceName extends string, ServiceMessenger extends BaseMessenger, @@ -167,6 +203,10 @@ export class BaseDataService< readonly #persistenceConfig?: PersistenceConfiguration; + #initializationPromise?: Promise; + + #boundedInitialization?: Promise; + constructor({ name, messenger, @@ -284,6 +324,10 @@ export class BaseDataService< queryFn: QueryFunction; responseStruct?: TDataStruct; }): Promise { + if (this.#initializationPromise) { + await this.#waitForInitialization(this.#initializationPromise); + } + return this.#queryClient.fetchQuery({ ...options, queryFn: async (context) => { @@ -336,6 +380,10 @@ export class BaseDataService< }, pageParam?: TPageParam, ): Promise { + if (this.#initializationPromise) { + await this.#waitForInitialization(this.#initializationPromise); + } + const cache = this.#queryClient.getQueryCache(); const query = cache.find< @@ -390,6 +438,18 @@ export class BaseDataService< return result.pages[pageIndex]; } + /** + * The retry and circuit-breaker policy that wraps every query made through + * this service. Subclasses can run uncached requests under the same policy + * with `this.policy.execute(...)`, or observe its `onBreak`, `onDegraded`, + * and `onRetry` events. + * + * @returns The service policy. + */ + protected get policy(): ServicePolicy { + return this.#policy; + } + /** * Invalidate queries serviced by this data service. * @@ -408,12 +468,47 @@ export class BaseDataService< * Initialize the service, rehydrating the cache with persisted data if possible. */ init(): void { - this.#loadCache().catch( + this.#initializationPromise ??= this.#loadCache().catch( /* istanbul ignore next */ (error) => this.#messenger.captureException?.(error), ); } + /** + * Waits for cache rehydration to finish, giving up after the configured + * hydration timeout so that a slow or hung storage read cannot block queries + * indefinitely. A late rehydration is still applied by TanStack, which only + * overwrites entries older than the persisted ones. + * + * The deadline is shared by every waiting query rather than started per + * query: all of them resume in the same turn, so lookups that are batched + * together stay batched, and once the deadline has passed no later query + * waits again while storage remains unresponsive. + * + * Callers must only await this when `init` has been called: an asynchronous + * hop before a query starts changes its timing relative to callers' own + * timers, so services without persistence keep starting queries + * synchronously. + * + * @param initialization - The pending rehydration. + */ + async #waitForInitialization(initialization: Promise): Promise { + if (!this.#boundedInitialization) { + let timer: ReturnType | undefined; + this.#boundedInitialization = Promise.race([ + initialization, + new Promise((resolve) => { + timer = setTimeout( + resolve, + this.#persistenceConfig?.hydrationTimeout ?? + DEFAULT_HYDRATION_TIMEOUT, + ); + }), + ]).finally(() => clearTimeout(timer)); + } + await this.#boundedInitialization; + } + /** * Prepares the service for garbage collection. This should be extended * by any subclasses to clean up any additional connections or events. @@ -512,6 +607,14 @@ export class BaseDataService< return; } + if (!is(untypedCache, PersistedCacheStruct)) { + await this.#externalMessenger.call( + 'StorageService:removeItem', + this.name, + STORAGE_SERVICE_KEY, + ); + return; + } const cache = untypedCache as unknown as PersistedCache; if (Date.now() - cache.timestamp >= this.#persistenceConfig.maxAge) { @@ -523,6 +626,15 @@ export class BaseDataService< return; } - hydrate(this.#queryClient, cache.state); + const { shouldHydrateQuery } = this.#persistenceConfig; + hydrate( + this.#queryClient, + shouldHydrateQuery + ? { + ...cache.state, + queries: cache.state.queries.filter(shouldHydrateQuery), + } + : cache.state, + ); } } diff --git a/packages/base-data-service/src/index.ts b/packages/base-data-service/src/index.ts index 58090738556..027e2dde2cd 100644 --- a/packages/base-data-service/src/index.ts +++ b/packages/base-data-service/src/index.ts @@ -24,7 +24,10 @@ export type { QueryKey, PersistenceConfiguration, } from './BaseDataService.js'; -export { BaseDataService } from './BaseDataService.js'; +export { + BaseDataService, + DEFAULT_HYDRATION_TIMEOUT, +} from './BaseDataService.js'; export { DEFAULT_CIRCUIT_BREAK_DURATION, diff --git a/packages/base-data-service/tests/ExampleDataService.ts b/packages/base-data-service/tests/ExampleDataService.ts index 8f77e10b751..ce2c93dcbbd 100644 --- a/packages/base-data-service/tests/ExampleDataService.ts +++ b/packages/base-data-service/tests/ExampleDataService.ts @@ -21,6 +21,7 @@ import { DataServiceGranularCacheUpdatedEvent, PersistenceConfiguration, } from '../src/BaseDataService.js'; +import type { ServicePolicy } from '../src/createServicePolicy.js'; import { ExampleDataServiceMethodActions } from './ExampleDataService-method-action-types.js'; export const serviceName = 'ExampleDataService'; @@ -109,6 +110,15 @@ export class ExampleDataService extends BaseDataService< ); } + /** + * Exposes the protected service policy for tests. + * + * @returns The service policy. + */ + getPolicy(): ServicePolicy { + return this.policy; + } + async getAssets(assets: string[]): Promise { return this.fetchQuery({ queryKey: [`${this.name}:getAssets`, assets], diff --git a/packages/phishing-controller/CHANGELOG.md b/packages/phishing-controller/CHANGELOG.md index c6bc1b0af55..4686b1f1d23 100644 --- a/packages/phishing-controller/CHANGELOG.md +++ b/packages/phishing-controller/CHANGELOG.md @@ -15,6 +15,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Excludes the zero address, a caller-provided `exclude` list (e.g. the signer), and caller-provided top-level `excludeFields`. - Bounds work with a distinct-address cap (default 10, caller-overridable via `maxAddresses`, hard ceiling 50), a traversal depth limit, and a node budget, reporting `overflow` when the message could not be fully walked. Exports `DEFAULT_MAX_SIGNATURE_ADDRESSES` and `MAX_SIGNATURE_ADDRESSES_CEILING`. - Returns the field name each address was found under so callers can attribute alerts. +- Add `PhishingDataService`, a `BaseDataService` subclass that now performs all network requests for `PhishingController` (stalelist, hotlist diffs, C2 domain blocklist, URL/token/address scans, and approvals) ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Exposes the messenger actions `PhishingDataService:getStalelist`, `PhishingDataService:getHotlistDiffs`, `PhishingDataService:getC2DomainBlocklist`, `PhishingDataService:scanUrl`, `PhishingDataService:bulkScanUrls`, `PhishingDataService:scanToken`, `PhishingDataService:bulkScanTokens`, `PhishingDataService:scanAddress`, and `PhishingDataService:getApprovals`, making query results available to the UI via `@metamask/react-data-query` + - Requests are wrapped in a shared service policy, configurable via the `policyOptions` option of the `PhishingDataService` constructor. Retries and circuit breaking are both disabled by default: the service spans four independent API hosts, so a circuit broken by one host would pause phishing-list updates from the others, and the previous in-controller implementation made a single request per call. Pass `policyOptions.maxRetries` to opt in; retries then apply to each batched bulk-scan request as a whole rather than to individual items + - Scan results are cached for `SCAN_RESULT_STALE_TIME` (1 minute, matching the previous cache TTLs), keyed by the scan URL parameter, token, and address, and retained for `SCAN_RESULT_GC_TIME` (5 minutes), including after cache rehydration; bulk scans only request items without a fresh cached result and coalesce them into batched API calls (up to 50 URLs / 100 tokens per request), and single and bulk URL scans share cache entries. Approvals reflect live account state and are always refetched and never retained + - EVM token and address inputs are lowercased before being used as cache keys and sent to the API, so differently-cased inputs share one cache entry and match the API's response keys; `bulkScanTokens` results are keyed by the normalized address + - The query cache is persisted between sessions by default (`persistenceConfig`, max age 5 minutes), which requires the `StorageService:setItem`, `StorageService:getItem`, and `StorageService:removeItem` messenger actions, and an `init` call during client initialization. Queries started after `init` wait at most `DEFAULT_HYDRATION_TIMEOUT` (1 second, configurable via `persistenceConfig.hydrationTimeout`) for rehydration to finish, and persisted scan results are validated on rehydration with anything malformed or unrecognized discarded. Pass `persistenceConfig: null` to disable. `PhishingDataService` is not yet part of `@metamask/wallet`'s default instances, so clients must construct it and call `init` themselves + - Fetched lists (stalelist, hotlist diffs, and the C2 domain blocklist) are not retained by the query cache, and so are not persisted; the controller keeps its own copy in state + - Timed-out requests are aborted so later calls can retry, and destroying the service aborts all pending requests +- Export the `resolveChainName` utility, which maps chain IDs to the chain names used in scan query keys, enabling UI consumers to construct `PhishingDataService` query keys ([#9914](https://github.com/MetaMask/core/pull/9914)) + +### Changed + +- **BREAKING:** `PhishingController` no longer performs network requests directly; a `PhishingDataService` must be registered and its method actions delegated to the controller's messenger ([#9914](https://github.com/MetaMask/core/pull/9914)) + - `PhishingControllerMessenger` now requires the `PhishingDataService` method actions listed above as allowed actions +- **BREAKING:** Remove the `urlScanCache`, `tokenScanCache`, and `addressScanCache` properties from `PhishingControllerState`; scan results are now cached (and persisted) by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Client state migrations should remove these properties from persisted `PhishingController` state +- **BREAKING:** Remove the `urlScanCacheTTL`, `urlScanCacheMaxSize`, `tokenScanCacheTTL`, `tokenScanCacheMaxSize`, `addressScanCacheTTL`, and `addressScanCacheMaxSize` options from `PhishingControllerOptions`; scan result freshness is now controlled by `SCAN_RESULT_STALE_TIME` in `PhishingDataService` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- **BREAKING:** `C2DomainBlocklistResponse.lastFetchedAt` is now typed as an optional `number` (previously a required `string`), matching the numeric Unix timestamp returned by the API; responses without it are accepted since the controller does not read it ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/base-data-service` `^2.0.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/storage-service` `^2.0.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/superstruct` `^3.4.1` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@metamask/utils` `^12.0.0` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Add `@tanstack/query-core` `^5.62.16` as a dependency ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Tokens for which the bulk scanning API returns no result are now negatively cached for `SCAN_RESULT_STALE_TIME` instead of being re-requested on every call ([#9914](https://github.com/MetaMask/core/pull/9914)) +- `PhishingController.scanUrl` now reports the underlying error message in `fetchError` for network errors instead of `'timeout of 8000ms exceeded'` ([#9914](https://github.com/MetaMask/core/pull/9914)) +- Malformed API responses are now rejected and treated as request failures instead of being passed through, and are never cached ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Bulk URL scan, bulk token scan, and approvals responses are validated per entry: a malformed URL result is reported for that URL in `errors`, and malformed token results and approvals are omitted, so one bad entry does not discard the other verdicts in the response + +### Removed + +- **BREAKING:** Remove the `CacheEntry` type; the custom cache manager has been replaced by `PhishingDataService`'s query cache ([#9914](https://github.com/MetaMask/core/pull/9914)) +- **BREAKING:** Remove the `DEFAULT_URL_SCAN_CACHE_TTL`, `DEFAULT_URL_SCAN_CACHE_MAX_SIZE`, `DEFAULT_TOKEN_SCAN_CACHE_TTL`, `DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE`, `DEFAULT_ADDRESS_SCAN_CACHE_TTL`, and `DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE` constants ([#9914](https://github.com/MetaMask/core/pull/9914)) + +### Fixed + +- `bulkScanUrls` now returns the results it was able to resolve even if some lookups fail, reporting the failures per URL in `errors`, and only rejects when nothing could be resolved at all ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Previously a single failed lookup discarded every result in the batch, including cached `BLOCK` verdicts for unrelated URLs +- Hotlist diffs that target a list type this client does not recognize are now ignored ([#9914](https://github.com/MetaMask/core/pull/9914)) + - Previously such a diff threw a `TypeError` while the hotlist was being applied, leaving the phishing lists un-updated ## [18.0.0] diff --git a/packages/phishing-controller/package.json b/packages/phishing-controller/package.json index f988dea2d3a..45bd5847396 100644 --- a/packages/phishing-controller/package.json +++ b/packages/phishing-controller/package.json @@ -52,10 +52,15 @@ "dependencies": { "@metamask/address-book-controller": "^8.0.0", "@metamask/base-controller": "^10.0.0", + "@metamask/base-data-service": "^2.0.0", "@metamask/controller-utils": "^13.0.0", "@metamask/messenger": "^3.0.0", + "@metamask/storage-service": "^2.0.0", + "@metamask/superstruct": "^3.4.1", "@metamask/transaction-controller": "^70.0.0", + "@metamask/utils": "^12.0.0", "@noble/hashes": "^1.8.0", + "@tanstack/query-core": "^5.62.16", "@types/punycode": "^2.1.0", "ethereum-cryptography": "^2.1.2", "fastest-levenshtein": "^1.0.16", diff --git a/packages/phishing-controller/src/BulkTokenScan.test.ts b/packages/phishing-controller/src/BulkTokenScan.test.ts index 7f2ab12fc82..d79a42495cc 100644 --- a/packages/phishing-controller/src/BulkTokenScan.test.ts +++ b/packages/phishing-controller/src/BulkTokenScan.test.ts @@ -1,4 +1,3 @@ -import { safelyExecuteWithTimeout } from '@metamask/controller-utils'; import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; import type { MessengerActions, @@ -16,19 +15,11 @@ import type { PhishingControllerMessenger, PhishingControllerOptions, } from './PhishingController.js'; +import { PhishingDataService } from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; import { TokenScanResultType } from './types.js'; import type { BulkTokenScanRequest, TokenScanApiResponse } from './types.js'; -jest.mock('@metamask/controller-utils', () => ({ - ...jest.requireActual('@metamask/controller-utils'), - safelyExecuteWithTimeout: jest.fn(), -})); - -const mockSafelyExecuteWithTimeout = - safelyExecuteWithTimeout as jest.MockedFunction< - typeof safelyExecuteWithTimeout - >; - const controllerName = 'PhishingController'; type AllPhishingControllerActions = @@ -38,11 +29,13 @@ type AllPhishingControllerEvents = MessengerEvents; type RootMessenger = Messenger< MockAnyNamespace, - AllPhishingControllerActions, - AllPhishingControllerEvents, + AllPhishingControllerActions | MessengerActions, + AllPhishingControllerEvents | MessengerEvents, RootMessenger >; +const createdDataServices: PhishingDataService[] = []; + /** * Creates and returns a root messenger for testing * @@ -55,7 +48,8 @@ function getRootMessenger(): RootMessenger { } /** - * Constructs a messenger with transaction events enabled. + * Constructs a messenger with transaction events enabled, plus a real + * PhishingDataService so that tests exercise the full request path via nock. * * @returns A restricted messenger that can listen to TransactionController events. */ @@ -72,8 +66,34 @@ function getMessengerWithTransactionEvents() { parent: rootMessenger, }); + const dataServiceMessenger = new Messenger< + 'PhishingDataService', + MessengerActions, + MessengerEvents, + RootMessenger + >({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + createdDataServices.push( + new PhishingDataService({ + messenger: dataServiceMessenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + }), + ); + rootMessenger.delegate({ - actions: [], + actions: [ + 'PhishingDataService:getStalelist', + 'PhishingDataService:getHotlistDiffs', + 'PhishingDataService:getC2DomainBlocklist', + 'PhishingDataService:scanUrl', + 'PhishingDataService:bulkScanUrls', + 'PhishingDataService:bulkScanTokens', + 'PhishingDataService:scanAddress', + 'PhishingDataService:getApprovals', + ], events: ['TransactionController:stateChange'], messenger, }); @@ -96,6 +116,26 @@ function getPhishingController(options?: Partial) { }); } +/** + * Mock a fetch that remains pending until its abort signal fires. + * + * @returns The fetch spy. + */ +function mockPendingFetch(): jest.SpiedFunction { + return jest + .spyOn(globalThis, 'fetch') + .mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ), + ), + ); +} + describe('PhishingController - Bulk Token Scanning', () => { let controller: PhishingController; let consoleErrorSpy: jest.SpyInstance; @@ -105,21 +145,14 @@ describe('PhishingController - Bulk Token Scanning', () => { controller = getPhishingController(); consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(); consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(); - - // Reset the mock to its default behavior (pass through to real implementation) - mockSafelyExecuteWithTimeout.mockImplementation( - (fn, throwOnTimeout, timeout) => { - return jest - .requireActual('@metamask/controller-utils') - .safelyExecuteWithTimeout(fn, throwOnTimeout, timeout); - }, - ); }); afterEach(() => { cleanAll(); - consoleErrorSpy.mockRestore(); - consoleWarnSpy.mockRestore(); + jest.restoreAllMocks(); + while (createdDataServices.length > 0) { + createdDataServices.pop()?.destroy(); + } }); describe('bulkScanTokens', () => { @@ -420,22 +453,28 @@ describe('PhishingController - Bulk Token Scanning', () => { }); it('should handle API timeout and return empty results', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); const tokens = ['0x1234567890123456789012345678901234567890']; - - // Mock safelyExecuteWithTimeout to return null (simulating a timeout) - mockSafelyExecuteWithTimeout.mockResolvedValueOnce(null); + const fetchMock = mockPendingFetch(); const request: BulkTokenScanRequest = { chainId: '0x1', tokens, }; - const result = await controller.bulkScanTokens(request); + const promise = controller.bulkScanTokens(request); + jest.advanceTimersByTime(8000); + const result = await promise; expect(result).toStrictEqual({}); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Error scanning tokens: timeout of 8000ms exceeded', ); + expect(fetchMock).toHaveBeenCalledTimes(1); + jest.useRealTimers(); }); }); diff --git a/packages/phishing-controller/src/CacheManager.test.ts b/packages/phishing-controller/src/CacheManager.test.ts deleted file mode 100644 index 5bbf8c92cb5..00000000000 --- a/packages/phishing-controller/src/CacheManager.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { CacheManager } from './CacheManager.js'; -import * as utils from './utils.js'; - -describe('CacheManager', () => { - let updateStateSpy: jest.Mock; - let cache: CacheManager<{ value: string }>; - - beforeEach(() => { - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); - jest - .spyOn(utils, 'fetchTimeNow') - .mockImplementation(() => Math.floor(Date.now() / 1000)); - updateStateSpy = jest.fn(); - cache = new CacheManager<{ value: string }>({ - cacheTTL: 300, // 5 minutes - maxCacheSize: 3, - updateState: updateStateSpy, - }); - }); - - afterEach(() => { - jest.useRealTimers(); - jest.restoreAllMocks(); - }); - - describe('constructor', () => { - it('should initialize with empty cache when no initialCache provided', () => { - const emptyCache = new CacheManager<{ value: string }>({ - // eslint-disable-next-line no-empty-function - updateState: () => {}, - }); - expect(emptyCache.get('test-key')).toBeUndefined(); - }); - - it('should initialize with provided initialCache data', () => { - const now = Math.floor(Date.now() / 1000); - const initialCache = { - 'test-key': { - data: { value: 'test-value' }, - timestamp: now, - }, - }; - - const cacheWithInitialData = new CacheManager<{ value: string }>({ - initialCache, - // eslint-disable-next-line no-empty-function - updateState: () => {}, - }); - - expect(cacheWithInitialData.get('test-key')).toStrictEqual({ - value: 'test-value', - }); - }); - }); - - describe('get', () => { - it('should return undefined for non-existent keys', () => { - expect(cache.get('non-existent')).toBeUndefined(); - }); - - it('should return data for existing keys', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.get('key1')).toStrictEqual({ value: 'value1' }); - }); - - it('should return undefined for expired entries', () => { - cache.set('key1', { value: 'value1' }); - - // Fast forward time past TTL - jest.advanceTimersByTime(301 * 1000); - - expect(cache.get('key1')).toBeUndefined(); - }); - }); - - describe('set', () => { - it('should add new entries', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.get('key1')).toStrictEqual({ value: 'value1' }); - }); - - it('should update existing entries', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key1', { value: 'updated-value' }); - expect(cache.get('key1')).toStrictEqual({ value: 'updated-value' }); - }); - - it('should call updateState when adding entries', () => { - cache.set('key1', { value: 'value1' }); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - - it('should evict oldest entries when cache exceeds max size', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.set('key3', { value: 'value3' }); - cache.set('key4', { value: 'value4' }); // This should evict key1 - - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toStrictEqual({ value: 'value2' }); - expect(cache.get('key3')).toStrictEqual({ value: 'value3' }); - expect(cache.get('key4')).toStrictEqual({ value: 'value4' }); - }); - }); - - describe('delete', () => { - it('should remove entries', () => { - cache.set('key1', { value: 'value1' }); - expect(cache.delete('key1')).toBe(true); - expect(cache.get('key1')).toBeUndefined(); - }); - - it('should return false when deleting non-existent keys', () => { - expect(cache.delete('non-existent')).toBe(false); - }); - - it('should call updateState when deleting entries', () => { - cache.set('key1', { value: 'value1' }); - updateStateSpy.mockClear(); - cache.delete('key1'); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - }); - - describe('clear', () => { - it('should remove all entries', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.clear(); - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toBeUndefined(); - }); - - it('should call updateState', () => { - cache.set('key1', { value: 'value1' }); - updateStateSpy.mockClear(); - cache.clear(); - expect(updateStateSpy).toHaveBeenCalledTimes(1); - }); - }); - - describe('setTTL', () => { - it('should update the TTL', () => { - cache.setTTL(600); - expect(cache.getTTL()).toBe(600); - }); - }); - - describe('setMaxSize', () => { - it('should update the max size', () => { - cache.setMaxSize(5); - expect(cache.getMaxSize()).toBe(5); - }); - - it('should evict entries if new size is smaller than current cache size', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - cache.set('key3', { value: 'value3' }); - cache.setMaxSize(2); // This should evict key1 - - expect(cache.get('key1')).toBeUndefined(); - expect(cache.get('key2')).toStrictEqual({ value: 'value2' }); - expect(cache.get('key3')).toStrictEqual({ value: 'value3' }); - }); - }); - - describe('getSize', () => { - it('should return the current cache size', () => { - expect(cache.getSize()).toBe(0); - cache.set('key1', { value: 'value1' }); - expect(cache.getSize()).toBe(1); - cache.set('key2', { value: 'value2' }); - expect(cache.getSize()).toBe(2); - cache.delete('key1'); - expect(cache.getSize()).toBe(1); - }); - }); - - describe('keys', () => { - it('should return all cache keys', () => { - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - expect(cache.keys()).toStrictEqual(['key1', 'key2']); - }); - }); - - describe('getAllEntries', () => { - it('should return all cache entries', () => { - const now = Math.floor(Date.now() / 1000); - cache.set('key1', { value: 'value1' }); - cache.set('key2', { value: 'value2' }); - const entries = cache.getAllEntries(); - expect(Object.keys(entries)).toStrictEqual(['key1', 'key2']); - expect(entries.key1.data).toStrictEqual({ value: 'value1' }); - expect(entries.key2.data).toStrictEqual({ value: 'value2' }); - expect(entries.key1.timestamp).toBeGreaterThanOrEqual(now); - expect(entries.key2.timestamp).toBeGreaterThanOrEqual(now); - }); - }); -}); diff --git a/packages/phishing-controller/src/CacheManager.ts b/packages/phishing-controller/src/CacheManager.ts deleted file mode 100644 index 9dd4b256353..00000000000 --- a/packages/phishing-controller/src/CacheManager.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { fetchTimeNow } from './utils.js'; - -/** - * Generic cache entry type that wraps the data with a timestamp - */ -export type CacheEntry = { - data: T; - timestamp: number; -}; - -/** - * Configuration options for CacheManager - */ -export type CacheManagerOptions = { - cacheTTL?: number; - maxCacheSize?: number; - initialCache?: Record>; - updateState: (cache: Record>) => void; -}; - -/** - * Generic cache manager with TTL and size limit support - * - * @template T - The type of data to cache - */ -export class CacheManager { - #cacheTTL: number; - - #maxCacheSize: number; - - readonly #cache: Map>; - - readonly #updateState: (cache: Record>) => void; - - /** - * Constructor for CacheManager - * - * @param options - Cache configuration options - * @param options.cacheTTL - Time to live in seconds for cached entries - * @param options.maxCacheSize - Maximum number of entries in the cache - * @param options.initialCache - Initial cache state - * @param options.updateState - Function to update the state when cache changes - */ - constructor({ - cacheTTL = 300, // 5 minutes default - maxCacheSize = 100, - initialCache = {}, - updateState, - }: CacheManagerOptions) { - this.#cacheTTL = cacheTTL; - this.#maxCacheSize = maxCacheSize; - this.#cache = new Map(Object.entries(initialCache)); - this.#updateState = updateState; - this.#evictEntries(); - } - - /** - * Set the time-to-live for cached entries - * - * @param ttl - The TTL in seconds - */ - setTTL(ttl: number): void { - this.#cacheTTL = ttl; - } - - /** - * Get the current TTL setting - * - * @returns The TTL in seconds - */ - getTTL(): number { - return this.#cacheTTL; - } - - /** - * Set the maximum cache size - * - * @param maxSize - The maximum cache size - */ - setMaxSize(maxSize: number): void { - this.#maxCacheSize = maxSize; - this.#evictEntries(); - } - - /** - * Get the current maximum cache size - * - * @returns The maximum cache size - */ - getMaxSize(): number { - return this.#maxCacheSize; - } - - /** - * Get the current cache size - * - * @returns The current number of entries in the cache - */ - getSize(): number { - return this.#cache.size; - } - - /** - * Clear the cache - */ - clear(): void { - this.#cache.clear(); - this.#persistCache(); - } - - /** - * Get a cached result if it exists and is not expired - * - * @param key - The cache key - * @returns The cached data or undefined if not found or expired - */ - get(key: string): T | undefined { - const cacheEntry = this.#cache.get(key); - if (!cacheEntry) { - return undefined; - } - - // Check if the entry is expired - const now = fetchTimeNow(); - if (now - cacheEntry.timestamp > this.#cacheTTL) { - // Entry expired, remove it from cache - this.#cache.delete(key); - this.#persistCache(); - return undefined; - } - - return cacheEntry.data; - } - - /** - * Add an entry to the cache, evicting oldest entries if necessary - * - * @param key - The cache key - * @param data - The data to cache - */ - set(key: string, data: T): void { - this.#cache.set(key, { - data, - timestamp: fetchTimeNow(), - }); - - this.#evictEntries(); - this.#persistCache(); - } - - /** - * Delete a specific entry from the cache - * - * @param key - The cache key - * @returns True if an entry was deleted - */ - delete(key: string): boolean { - const result = this.#cache.delete(key); - if (result) { - this.#persistCache(); - } - return result; - } - - /** - * Get all keys in the cache - * - * @returns Array of cache keys - */ - keys(): string[] { - return Array.from(this.#cache.keys()); - } - - /** - * Get all entries in the cache (including expired ones) - * Useful for debugging or persistence - * - * @returns Record of all cache entries - */ - getAllEntries(): Record> { - return Object.fromEntries(this.#cache); - } - - /** - * Persist the current cache state - */ - #persistCache(): void { - this.#updateState(Object.fromEntries(this.#cache)); - } - - /** - * Evict oldest entries if cache exceeds max size - */ - #evictEntries(): void { - if (this.#cache.size <= this.#maxCacheSize) { - return; - } - - const entriesToRemove = this.#cache.size - this.#maxCacheSize; - let count = 0; - // Delete the oldest entries (Map maintains insertion order) - for (const key of this.#cache.keys()) { - if (count >= entriesToRemove) { - break; - } - this.#cache.delete(key); - count += 1; - } - } -} diff --git a/packages/phishing-controller/src/PhishingController.test.ts b/packages/phishing-controller/src/PhishingController.test.ts index 776f1d6fbd0..f7d91348f43 100644 --- a/packages/phishing-controller/src/PhishingController.test.ts +++ b/packages/phishing-controller/src/PhishingController.test.ts @@ -18,6 +18,10 @@ import { ListNames, METAMASK_HOTLIST_DIFF_FILE, METAMASK_STALELIST_FILE, + METAMASK_HOTLIST_DIFF_URL, + METAMASK_STALELIST_URL, + C2_DOMAIN_BLOCKLIST_URL, + phishingListKeyNameMap, PhishingController, PHISHING_CONFIG_BASE_URL, CLIENT_SIDE_DETECION_BASE_URL, @@ -34,6 +38,11 @@ import type { BulkPhishingDetectionScanResponse, PhishingControllerMessenger, } from './PhishingController.js'; +import { + PhishingDataService, + SCAN_RESULT_STALE_TIME, +} from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; import { createMockStateChangePayload, createMockTransaction, @@ -62,11 +71,61 @@ type AllPhishingControllerEvents = MessengerEvents; type RootMessenger = Messenger< MockAnyNamespace, - AllPhishingControllerActions, - AllPhishingControllerEvents, + AllPhishingControllerActions | MessengerActions, + AllPhishingControllerEvents | MessengerEvents, RootMessenger >; +const PHISHING_DATA_SERVICE_ACTIONS = [ + 'PhishingDataService:getStalelist', + 'PhishingDataService:getHotlistDiffs', + 'PhishingDataService:getC2DomainBlocklist', + 'PhishingDataService:scanUrl', + 'PhishingDataService:bulkScanUrls', + 'PhishingDataService:bulkScanTokens', + 'PhishingDataService:scanAddress', + 'PhishingDataService:getApprovals', +] as const; + +const createdDataServices: PhishingDataService[] = []; + +/** + * Destroys all data services created during the current test, releasing their + * query cache resources. + */ +function destroyDataServices(): void { + while (createdDataServices.length > 0) { + createdDataServices.pop()?.destroy(); + } +} + +/** + * Constructs a real PhishingDataService wired to the given root messenger, so + * that controller tests exercise the full request path via nock. + * + * @param rootMessenger - The root messenger. + * @returns The data service. + */ +function setupDataService(rootMessenger: RootMessenger): PhishingDataService { + const dataServiceMessenger = new Messenger< + 'PhishingDataService', + MessengerActions, + MessengerEvents, + RootMessenger + >({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + + const dataService = new PhishingDataService({ + messenger: dataServiceMessenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + }); + createdDataServices.push(dataService); + return dataService; +} + type SetupMessengerOptions = { transactionControllerState?: TransactionControllerState; addressBookControllerState?: AddressBookControllerState; @@ -134,10 +193,13 @@ function setupMessenger(options: SetupMessengerOptions = {}): { parent: rootMessenger, }); + setupDataService(rootMessenger); + rootMessenger.delegate({ actions: [ 'AddressBookController:getState', 'TransactionController:getState', + ...PHISHING_DATA_SERVICE_ACTIONS, ], events: [ 'AddressBookController:stateChange', @@ -190,10 +252,32 @@ function getPhishingController(options?: Partial): { return { controller, rootMessenger }; } +/** + * Mock a fetch that remains pending until its abort signal fires. + * + * @returns The fetch spy. + */ +function mockPendingFetch(): jest.SpiedFunction { + return jest + .spyOn(globalThis, 'fetch') + .mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ), + ), + ); +} + describe('PhishingController', () => { afterEach(() => { jest.useRealTimers(); + jest.restoreAllMocks(); cleanAll(); + destroyDataServices(); }); it('should have no default phishing lists', () => { @@ -201,6 +285,21 @@ describe('PhishingController', () => { expect(controller.state.phishingLists).toStrictEqual([]); }); + it('re-exports API URLs and list mappings for backwards compatibility', () => { + expect(METAMASK_STALELIST_URL).toBe( + `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`, + ); + expect(METAMASK_HOTLIST_DIFF_URL).toBe( + `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`, + ); + expect(C2_DOMAIN_BLOCKLIST_URL).toBe( + `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`, + ); + expect(phishingListKeyNameMap.eth_phishing_detect_config).toBe( + ListNames.MetaMask, + ); + }); + it('should default to an empty whitelist', () => { const { controller } = getPhishingController(); expect(controller.state.whitelist).toStrictEqual([]); @@ -581,7 +680,6 @@ describe('PhishingController', () => { hotlistLastFetched: 0, stalelistLastFetched: 0, c2DomainBlocklistLastFetched: 0, - urlScanCache: {}, }, }); @@ -1880,6 +1978,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -1916,6 +2015,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -1988,6 +2088,61 @@ describe('PhishingController', () => { ]); }); + it('ignores hotlist diffs that target an unrecognized list type while applying the rest', async () => { + const testBlockedDomain = 'some-test-blocked-url.com'; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/${0}`) + .reply(200, { + data: [ + { + targetList: 'eth_phishing_detect_config.newlist', + url: 'some-new-list-url.com', + timestamp: 2, + }, + { + targetList: 'eth_phishing_detect_config.blocklist', + url: testBlockedDomain, + timestamp: 1, + }, + ], + }); + + const { controller } = getPhishingController({ + state: { + phishingLists: [ + { + allowlist: [], + blocklist: [], + c2DomainBlocklist: [], + blocklistPaths: {}, + fuzzylist: [], + tolerance: 3, + version: 1, + name: ListNames.MetaMask, + lastUpdated: 0, + }, + ], + }, + }); + await controller.updateHotlist(); + + // The unknown-type diff is skipped and does not advance lastUpdated; + // the recognized diff is still applied. + expect(controller.state.phishingLists).toStrictEqual([ + { + allowlist: [], + blocklist: [testBlockedDomain], + c2DomainBlocklist: [], + blocklistPaths: {}, + fuzzylist: [], + tolerance: 3, + name: ListNames.MetaMask, + version: 1, + lastUpdated: 1, + }, + ]); + }); + it('should not update phishing lists if hotlist fetch returns 404', async () => { nock(PHISHING_CONFIG_BASE_URL) .get(`${METAMASK_HOTLIST_DIFF_FILE}/${0}`) @@ -2494,6 +2649,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2532,6 +2688,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2569,6 +2726,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2605,6 +2763,7 @@ describe('PhishingController', () => { data: { allowlist: [], blocklist: [], + blocklistPaths: [], fuzzylist: [], tolerance: 0, version: 0, @@ -2833,7 +2992,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); it('should return the scan result', async () => { @@ -2881,11 +3045,7 @@ describe('PhishingController', () => { ); it('should return a PhishingDetectionScanResult with a fetchError on timeout', async () => { - const scope = nock(PHISHING_DETECTION_BASE_URL) - .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) - .query({ url: 'example.com' }) - .delayConnection(10000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call('PhishingController:scanUrl', testUrl); jest.advanceTimersByTime(8000); @@ -2895,7 +3055,7 @@ describe('PhishingController', () => { recommendedAction: RecommendedAction.None, fetchError: 'timeout of 8000ms exceeded', }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('should only send hostname when URL contains query parameters', async () => { @@ -3088,7 +3248,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { @@ -3182,12 +3347,7 @@ describe('PhishingController', () => { ); it('should handle timeouts correctly', async () => { - const scope = nock(PHISHING_DETECTION_BASE_URL) - .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { - urls: testUrls, - }) - .delayConnection(20000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:bulkScanUrls', @@ -3201,7 +3361,7 @@ describe('PhishingController', () => { network_error: ['timeout of 15000ms exceeded'], }, }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('should process URLs in batches when more than 50 URLs are provided', async () => { @@ -3438,6 +3598,7 @@ describe('PhishingController', () => { // eslint-disable-next-line import-x/no-named-as-default-member expect(nock.pendingMocks()).toHaveLength(0); }); + it('should handle invalid URLs properly when mixed with valid URLs and cache results correctly', async () => { const validUrl = 'https://valid-example.com'; const invalidUrl = 'not-a-url'; @@ -3558,7 +3719,12 @@ describe('PhishingController', () => { rootMessenger = createdMessenger; - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { @@ -3615,13 +3781,7 @@ describe('PhishingController', () => { ); it('will return an AddressScanResult with an ErrorResult on timeout', async () => { - const scope = nock(SECURITY_ALERTS_BASE_URL) - .post(ADDRESS_SCAN_ENDPOINT, { - chain: 'ethereum', - address: testAddress.toLowerCase(), - }) - .delayConnection(10000) - .reply(200, {}); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:scanAddress', @@ -3634,7 +3794,7 @@ describe('PhishingController', () => { result_type: AddressScanResultType.ErrorResult, label: '', }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('will return an AddressScanResult with an ErrorResult when address is missing', async () => { @@ -3922,13 +4082,7 @@ describe('PhishingController', () => { }); it('will return empty approvals on timeout', async () => { - const scope = nock(SECURITY_ALERTS_BASE_URL) - .post(APPROVALS_ENDPOINT, { - chain: 'ethereum', - address: testAddress.toLowerCase(), - }) - .delayConnection(10000) - .reply(200, mockResponse); + const fetchMock = mockPendingFetch(); const promise = rootMessenger.call( 'PhishingController:getApprovals', @@ -3938,7 +4092,7 @@ describe('PhishingController', () => { jest.advanceTimersByTime(5000); const response = await promise; expect(response).toStrictEqual({ approvals: [] }); - expect(scope.isDone()).toBe(false); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it('will normalize address to lowercase before API call', async () => { @@ -3981,11 +4135,17 @@ describe('PhishingController', () => { describe('URL Scan Cache', () => { beforeEach(() => { - jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'], now: 0 }); + // A nonzero epoch is required for the data service's query cache: a + // cached entry with `dataUpdatedAt` of 0 is treated as never fetched. + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); }); afterEach(() => { jest.useRealTimers(); cleanAll(); + destroyDataServices(); }); it('should cache scan results and return them on subsequent calls', async () => { @@ -4030,9 +4190,8 @@ describe('URL Scan Cache', () => { fetchSpy.mockRestore(); }); - it('should expire cache entries after TTL', async () => { + it('should expire cache entries after the scan result stale time', async () => { const testDomain = 'example.com'; - const cacheTTL = 300; // 5 minutes nock(PHISHING_DETECTION_BASE_URL) .get( @@ -4052,25 +4211,23 @@ describe('URL Scan Cache', () => { recommendedAction: RecommendedAction.None, }); - const { rootMessenger } = getPhishingController({ - urlScanCacheTTL: cacheTTL, - }); + const { rootMessenger } = getPhishingController(); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, ); - // Before TTL expires, should use cache - jest.advanceTimersByTime((cacheTTL - 10) * 1000); + // Before the stale time elapses, should use cache + jest.advanceTimersByTime(SCAN_RESULT_STALE_TIME - 10_000); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, ); expect(pendingMocks()).toHaveLength(1); // One mock remaining - // After TTL expires, should fetch again - jest.advanceTimersByTime(11 * 1000); + // After the stale time elapses, should fetch again + jest.advanceTimersByTime(11_000); await rootMessenger.call( 'PhishingController:scanUrl', `https://${testDomain}`, @@ -4078,66 +4235,6 @@ describe('URL Scan Cache', () => { expect(pendingMocks()).toHaveLength(0); // All mocks used }); - it('should evict oldest entries when cache exceeds max size', async () => { - const maxCacheSize = 2; - const domains = ['domain1.com', 'domain2.com', 'domain3.com']; - - // Setup nock to respond to all three domains - domains.forEach((domain) => { - nock(PHISHING_DETECTION_BASE_URL) - .get( - `/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent( - domain, - )}`, - ) - .reply(200, { - recommendedAction: RecommendedAction.None, - }); - }); - - // Setup a second request for the first domain - nock(PHISHING_DETECTION_BASE_URL) - .get( - `/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent( - domains[0], - )}`, - ) - .reply(200, { - recommendedAction: RecommendedAction.Warn, - }); - - const { rootMessenger } = getPhishingController({ - urlScanCacheMaxSize: maxCacheSize, - }); - - // Fill the cache - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[0]}`, - ); - jest.advanceTimersByTime(1000); // Ensure different timestamps - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[1]}`, - ); - - // This should evict the oldest entry (domain1) - jest.advanceTimersByTime(1000); - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[2]}`, - ); - - // Now domain1 should not be in cache and require a new fetch - await rootMessenger.call( - 'PhishingController:scanUrl', - `https://${domains[0]}`, - ); - - // All mocks should be used - expect(isDone()).toBe(true); - }); - it('should handle fetch errors and not cache them', async () => { const testDomain = 'example.com'; @@ -4281,13 +4378,10 @@ describe('URL Scan Cache', () => { ), ).toMatchInlineSnapshot(` { - "addressScanCache": {}, "c2DomainBlocklistLastFetched": 0, "hotlistLastFetched": 0, "phishingLists": [], "stalelistLastFetched": 0, - "tokenScanCache": {}, - "urlScanCache": {}, "whitelist": [], "whitelistPaths": {}, } @@ -4303,13 +4397,7 @@ describe('URL Scan Cache', () => { controller.metadata, 'usedInUi', ), - ).toMatchInlineSnapshot(` - { - "addressScanCache": {}, - "tokenScanCache": {}, - "urlScanCache": {}, - } - `); + ).toMatchInlineSnapshot(`{}`); }); }); }); @@ -4335,6 +4423,7 @@ describe('Transaction Controller State Change Integration', () => { afterEach(() => { bulkScanTokensSpy.mockRestore(); + destroyDataServices(); }); it('triggers bulk token scanning when transaction with token balance changes is added', async () => { @@ -4420,6 +4509,47 @@ describe('Transaction Controller State Change Integration', () => { expect(bulkScanTokensSpy).not.toHaveBeenCalled(); }); + it('groups tokens from multiple transactions on the same chain into one scan', async () => { + const transaction1 = createMockTransaction('test-tx-1', [ + TEST_ADDRESSES.USDC, + ]); + const transaction2 = createMockTransaction('test-tx-2', [ + TEST_ADDRESSES.MOCK_TOKEN_1, + ]); + const stateChangePayload = createMockStateChangePayload([ + transaction1, + transaction2, + ]); + + globalMessenger.publish( + 'TransactionController:stateChange', + stateChangePayload, + [ + { + op: 'add' as const, + path: ['transactions', 0], + value: transaction1, + }, + { + op: 'add' as const, + path: ['transactions', 1], + value: transaction2, + }, + ], + ); + + await new Promise((resolve) => process.nextTick(resolve)); + + expect(bulkScanTokensSpy).toHaveBeenCalledTimes(1); + expect(bulkScanTokensSpy).toHaveBeenCalledWith({ + chainId: transaction1.chainId.toLowerCase(), + tokens: [ + TEST_ADDRESSES.USDC.toLowerCase(), + TEST_ADDRESSES.MOCK_TOKEN_1.toLowerCase(), + ], + }); + }); + it('does not trigger bulk token scanning when transaction has no token balance changes', async () => { const mockTransaction = createMockTransaction('test-tx-1', []); @@ -4571,6 +4701,8 @@ describe('Transaction Controller State Change Integration', () => { }); describe('Address poisoning detection', () => { + afterEach(destroyDataServices); + const ADDRESS_BOOK_RECIPIENT = '0x1234bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb5678' as `0x${string}`; const CONFIRMED_TX_RECIPIENT = diff --git a/packages/phishing-controller/src/PhishingController.ts b/packages/phishing-controller/src/PhishingController.ts index 62f9e132227..06ea99aa48c 100644 --- a/packages/phishing-controller/src/PhishingController.ts +++ b/packages/phishing-controller/src/PhishingController.ts @@ -9,11 +9,7 @@ import type { ControllerGetStateAction, ControllerStateChangeEvent, } from '@metamask/base-controller'; -import { - isValidHexAddress, - safelyExecute, - safelyExecuteWithTimeout, -} from '@metamask/controller-utils'; +import { HttpError, isValidHexAddress } from '@metamask/controller-utils'; import type { Messenger } from '@metamask/messenger'; import type { TransactionControllerGetStateAction, @@ -25,12 +21,11 @@ import { getSendRecipients, TransactionStatus, } from '@metamask/transaction-controller'; +import { getErrorMessage } from '@metamask/utils'; import type { Patch } from 'immer'; import { toASCII } from 'punycode/punycode.js'; import { findSimilarAddresses } from './address-poisoning.js'; -import { CacheManager } from './CacheManager.js'; -import type { CacheEntry } from './CacheManager.js'; import { convertListToTrie, insertToTrie, @@ -42,23 +37,38 @@ import type { PhishingControllerMethodActions, PhishingControllerTestOriginAction, } from './PhishingController-method-action-types.js'; +import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; +import { + ADDRESS_SCAN_TIMEOUT, + APPROVALS_TIMEOUT, + BULK_URL_SCAN_TIMEOUT, + TOKEN_SCAN_TIMEOUT, + URL_SCAN_TIMEOUT, +} from './PhishingDataService.js'; import { PhishingDetector } from './PhishingDetector.js'; import { PhishingDetectorResultType, RecommendedAction, AddressScanResultType, + ListKeys, + phishingListNameKeyMap, + phishingListKeyNameMap, } from './types.js'; import type { PhishingDetectorResult, PhishingDetectionScanResult, - TokenScanCacheData, BulkTokenScanResponse, BulkTokenScanRequest, TokenScanApiResponse, - AddressScanCacheData, AddressScanResult, SimilarAddressMatch, ApprovalsResponse, + BulkPhishingDetectionScanResponse, + C2DomainBlocklistResponse, + DataResultWrapper, + Hotlist, + PhishingListState, + PhishingStalelist, } from './types.js'; import { applyDiffs, @@ -67,8 +77,6 @@ import { roundToNearestMinute, getHostnameFromWebUrl, getPhishingDetectionScanUrlParam, - buildCacheKey, - splitCacheHits, resolveChainName, getPathnameFromUrl, getAddressScanSupportedChain, @@ -76,201 +84,40 @@ import { isTokenScanSupportedChain, } from './utils.js'; -export const PHISHING_CONFIG_BASE_URL = - 'https://phishing-detection.api.cx.metamask.io'; -export const METAMASK_STALELIST_FILE = '/v1/stalelist'; -export const METAMASK_HOTLIST_DIFF_FILE = '/v2/diffsSince'; - -export const CLIENT_SIDE_DETECION_BASE_URL = - 'https://client-side-detection.api.cx.metamask.io'; -export const C2_DOMAIN_BLOCKLIST_ENDPOINT = '/v1/request-blocklist'; - -export const PHISHING_DETECTION_BASE_URL = - 'https://dapp-scanning.api.cx.metamask.io'; -export const PHISHING_DETECTION_SCAN_ENDPOINT = 'v2/scan'; -export const PHISHING_DETECTION_BULK_SCAN_ENDPOINT = 'bulk-scan'; - -export const SECURITY_ALERTS_BASE_URL = - 'https://security-alerts.api.cx.metamask.io'; -export const TOKEN_BULK_SCANNING_ENDPOINT = '/token/scan-bulk'; -export const ADDRESS_SCAN_ENDPOINT = '/address/evm/scan'; -export const APPROVALS_ENDPOINT = '/address/evm/approvals'; - -// Cache configuration defaults -export const DEFAULT_URL_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_URL_SCAN_CACHE_MAX_SIZE = 250; -export const DEFAULT_TOKEN_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE = 1000; -export const DEFAULT_ADDRESS_SCAN_CACHE_TTL = 1 * 60; // 1 minute in seconds -export const DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE = 1000; +export { + PHISHING_CONFIG_BASE_URL, + METAMASK_STALELIST_FILE, + METAMASK_HOTLIST_DIFF_FILE, + CLIENT_SIDE_DETECION_BASE_URL, + C2_DOMAIN_BLOCKLIST_ENDPOINT, + PHISHING_DETECTION_BASE_URL, + PHISHING_DETECTION_SCAN_ENDPOINT, + PHISHING_DETECTION_BULK_SCAN_ENDPOINT, + SECURITY_ALERTS_BASE_URL, + TOKEN_BULK_SCANNING_ENDPOINT, + ADDRESS_SCAN_ENDPOINT, + APPROVALS_ENDPOINT, + METAMASK_STALELIST_URL, + METAMASK_HOTLIST_DIFF_URL, + C2_DOMAIN_BLOCKLIST_URL, +} from './PhishingDataService.js'; +export { ListKeys, ListNames, phishingListKeyNameMap } from './types.js'; +export type { + ListTypes, + EthPhishingResponse, + C2DomainBlocklistResponse, + PhishingStalelist, + PhishingListState, + HotlistDiff, + DataResultWrapper, + Hotlist, + BulkPhishingDetectionScanResponse, +} from './types.js'; export const C2_DOMAIN_BLOCKLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const HOTLIST_REFRESH_INTERVAL = 5 * 60; // 5 mins in seconds export const STALELIST_REFRESH_INTERVAL = 30 * 24 * 60 * 60; // 30 days in seconds -export const METAMASK_STALELIST_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`; -export const METAMASK_HOTLIST_DIFF_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`; -export const C2_DOMAIN_BLOCKLIST_URL = `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`; - -/** - * @type ListTypes - * - * Type outlining the types of lists provided by aggregating different source lists - */ -export type ListTypes = - | 'fuzzylist' - | 'blocklist' - | 'blocklistPaths' - | 'allowlist' - | 'c2DomainBlocklist'; - -/** - * @type EthPhishingResponse - * - * Configuration response from the eth-phishing-detect package - * consisting of approved and unapproved website origins - * - * @property blacklist - List of unapproved origins - * @property fuzzylist - List of fuzzy-matched unapproved origins - * @property tolerance - Fuzzy match tolerance level - * @property version - Version number of this configuration - * @property whitelist - List of approved origins - */ -export type EthPhishingResponse = { - blacklist: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - whitelist: string[]; -}; - -/** - * @type C2DomainBlocklistResponse - * - * Response for blocklist update requests - * - * @property recentlyAdded - List of c2 domains recently added to the blocklist - * @property recentlyRemoved - List of c2 domains recently removed from the blocklist - * @property lastFetchedAt - Timestamp of the last fetch request - */ -export type C2DomainBlocklistResponse = { - recentlyAdded: string[]; - recentlyRemoved: string[]; - lastFetchedAt: string; -}; - -/** - * PhishingStalelist defines the expected type of the stalelist from the API. - * - * allowlist - List of approved origins. - * blocklist - List of unapproved origins (hostname-only entries). - * blocklistPaths - Trie of unapproved origins with paths (hostname + path entries). - * fuzzylist - List of fuzzy-matched unapproved origins. - * tolerance - Fuzzy match tolerance level - * lastUpdated - Timestamp of last update. - * version - Stalelist data structure iteration. - */ -export type PhishingStalelist = { - allowlist: string[]; - blocklist: string[]; - blocklistPaths: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - lastUpdated: number; -}; - -/** - * @type PhishingListState - * - * type defining the persisted list state. This is the persisted state that is updated frequently with `this.maybeUpdateState()`. - * - * @property allowlist - List of approved origins (legacy naming "whitelist") - * @property blocklist - List of unapproved origins (legacy naming "blacklist") - * @property blocklistPaths - Trie of unapproved origins with paths (hostname + path, no query params). - * @property c2DomainBlocklist - List of hashed hostnames that C2 requests are blocked against. - * @property fuzzylist - List of fuzzy-matched unapproved origins - * @property tolerance - Fuzzy match tolerance level - * @property lastUpdated - Timestamp of last update. - * @property version - Version of the phishing list state. - * @property name - Name of the list. Used for attribution. - */ -export type PhishingListState = { - allowlist: string[]; - blocklist: string[]; - blocklistPaths: PathTrie; - c2DomainBlocklist: string[]; - fuzzylist: string[]; - tolerance: number; - version: number; - lastUpdated: number; - name: ListNames; -}; - -/** - * @type HotlistDiff - * - * type defining the expected type of the diffs in hotlist.json file. - * - * @property url - Url of the diff entry. - * @property timestamp - Timestamp at which the diff was identified. - * @property targetList - The list name where the diff was identified. - * @property isRemoval - Was the diff identified a removal type. - */ -export type HotlistDiff = { - url: string; - timestamp: number; - targetList: `${ListKeys}.${ListTypes}`; - isRemoval?: boolean; -}; - -export type DataResultWrapper = { - data: T; -}; - -/** - * @type Hotlist - * - * Type defining expected hotlist.json file. - * - * @property url - Url of the diff entry. - * @property timestamp - Timestamp at which the diff was identified. - * @property targetList - The list name where the diff was identified. - * @property isRemoval - Was the diff identified a removal type. - */ -export type Hotlist = HotlistDiff[]; - -/** - * Enum containing upstream data provider source list keys. - * These are the keys denoting lists consumed by the upstream data provider. - */ -export enum ListKeys { - EthPhishingDetectConfig = 'eth_phishing_detect_config', -} - -/** - * Enum containing downstream client attribution names. - */ -export enum ListNames { - MetaMask = 'MetaMask', -} - -/** - * Maps from downstream client attribution name - * to list key sourced from upstream data provider. - */ -const phishingListNameKeyMap = { - [ListNames.MetaMask]: ListKeys.EthPhishingDetectConfig, -}; - -/** - * Maps from list key sourced from upstream data - * provider to downstream client attribution name. - */ -export const phishingListKeyNameMap = { - [ListKeys.EthPhishingDetectConfig]: ListNames.MetaMask, -}; - const controllerName = 'PhishingController'; const metadata: StateMetadata = { @@ -310,24 +157,6 @@ const metadata: StateMetadata = { includeInDebugSnapshot: false, usedInUi: false, }, - urlScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, - tokenScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, - addressScanCache: { - includeInStateLogs: false, - persist: true, - includeInDebugSnapshot: false, - usedInUi: true, - }, }; /** @@ -343,9 +172,6 @@ const getDefaultState = (): PhishingControllerState => { hotlistLastFetched: 0, stalelistLastFetched: 0, c2DomainBlocklistLastFetched: 0, - urlScanCache: {}, - tokenScanCache: {}, - addressScanCache: {}, }; }; @@ -359,9 +185,6 @@ const getDefaultState = (): PhishingControllerState => { * hotlistLastFetched - timestamp of the last hotlist fetch * stalelistLastFetched - timestamp of the last stalelist fetch * c2DomainBlocklistLastFetched - timestamp of the last c2 domain blocklist fetch - * urlScanCache - cache of URL scan results - * tokenScanCache - cache of token scan results - * addressScanCache - cache of address scan results */ export type PhishingControllerState = { phishingLists: PhishingListState[]; @@ -370,9 +193,6 @@ export type PhishingControllerState = { hotlistLastFetched: number; stalelistLastFetched: number; c2DomainBlocklistLastFetched: number; - urlScanCache: Record>; - tokenScanCache: Record>; - addressScanCache: Record>; }; /** @@ -382,23 +202,11 @@ export type PhishingControllerState = { * stalelistRefreshInterval - Polling interval used to fetch stale list. * hotlistRefreshInterval - Polling interval used to fetch hotlist diff list. * c2DomainBlocklistRefreshInterval - Polling interval used to fetch c2 domain blocklist. - * urlScanCacheTTL - Time to live in seconds for cached scan results. - * urlScanCacheMaxSize - Maximum number of entries in the scan cache. - * tokenScanCacheTTL - Time to live in seconds for cached token scan results. - * tokenScanCacheMaxSize - Maximum number of entries in the token scan cache. - * addressScanCacheTTL - Time to live in seconds for cached address scan results. - * addressScanCacheMaxSize - Maximum number of entries in the address scan cache. */ export type PhishingControllerOptions = { stalelistRefreshInterval?: number; hotlistRefreshInterval?: number; c2DomainBlocklistRefreshInterval?: number; - urlScanCacheTTL?: number; - urlScanCacheMaxSize?: number; - tokenScanCacheTTL?: number; - tokenScanCacheMaxSize?: number; - addressScanCacheTTL?: number; - addressScanCacheMaxSize?: number; messenger: PhishingControllerMessenger; state?: Partial; }; @@ -447,7 +255,8 @@ export type PhishingControllerEvents = PhishingControllerStateChangeEvent; */ type AllowedActions = | AddressBookControllerGetStateAction - | TransactionControllerGetStateAction; + | TransactionControllerGetStateAction + | PhishingDataServiceMethodActions; /** * The external events available to the PhishingController. @@ -462,19 +271,6 @@ export type PhishingControllerMessenger = Messenger< PhishingControllerEvents | AllowedEvents >; -/** - * BulkPhishingDetectionScanResponse - * - * Response for bulk phishing detection scan requests - * results - Record of domain names and their corresponding phishing detection scan results - * - * errors - Record of domain names and their corresponding errors - */ -export type BulkPhishingDetectionScanResponse = { - results: Record; - errors: Record; -}; - /** * Controller that manages community-maintained lists of approved and unapproved website origins. */ @@ -493,12 +289,6 @@ export class PhishingController extends BaseController< readonly #c2DomainBlocklistRefreshInterval: number; - readonly #urlScanCache: CacheManager; - - readonly #tokenScanCache: CacheManager; - - readonly #addressScanCache: CacheManager; - readonly #knownRecipients: Set; readonly #transactionRecipients: Set; @@ -531,12 +321,6 @@ export class PhishingController extends BaseController< * @param config.stalelistRefreshInterval - Polling interval used to fetch stale list. * @param config.hotlistRefreshInterval - Polling interval used to fetch hotlist diff list. * @param config.c2DomainBlocklistRefreshInterval - Polling interval used to fetch c2 domain blocklist. - * @param config.urlScanCacheTTL - Time to live in seconds for cached scan results. - * @param config.urlScanCacheMaxSize - Maximum number of entries in the scan cache. - * @param config.tokenScanCacheTTL - Time to live in seconds for cached token scan results. - * @param config.tokenScanCacheMaxSize - Maximum number of entries in the token scan cache. - * @param config.addressScanCacheTTL - Time to live in seconds for cached address scan results. - * @param config.addressScanCacheMaxSize - Maximum number of entries in the address scan cache. * @param config.messenger - The controller restricted messenger. * @param config.state - Initial state to set on this controller. */ @@ -544,12 +328,6 @@ export class PhishingController extends BaseController< stalelistRefreshInterval = STALELIST_REFRESH_INTERVAL, hotlistRefreshInterval = HOTLIST_REFRESH_INTERVAL, c2DomainBlocklistRefreshInterval = C2_DOMAIN_BLOCKLIST_REFRESH_INTERVAL, - urlScanCacheTTL = DEFAULT_URL_SCAN_CACHE_TTL, - urlScanCacheMaxSize = DEFAULT_URL_SCAN_CACHE_MAX_SIZE, - tokenScanCacheTTL = DEFAULT_TOKEN_SCAN_CACHE_TTL, - tokenScanCacheMaxSize = DEFAULT_TOKEN_SCAN_CACHE_MAX_SIZE, - addressScanCacheTTL = DEFAULT_ADDRESS_SCAN_CACHE_TTL, - addressScanCacheMaxSize = DEFAULT_ADDRESS_SCAN_CACHE_MAX_SIZE, messenger, state = {}, }: PhishingControllerOptions) { @@ -575,36 +353,6 @@ export class PhishingController extends BaseController< this.#onTransactionControllerStateChange.bind(this); this.#addressBookControllerStateChangeHandler = this.#onAddressBookControllerStateChange.bind(this); - this.#urlScanCache = new CacheManager({ - cacheTTL: urlScanCacheTTL, - maxCacheSize: urlScanCacheMaxSize, - initialCache: this.state.urlScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.urlScanCache = cache; - }); - }, - }); - this.#tokenScanCache = new CacheManager({ - cacheTTL: tokenScanCacheTTL, - maxCacheSize: tokenScanCacheMaxSize, - initialCache: this.state.tokenScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.tokenScanCache = cache; - }); - }, - }); - this.#addressScanCache = new CacheManager({ - cacheTTL: addressScanCacheTTL, - maxCacheSize: addressScanCacheMaxSize, - initialCache: this.state.addressScanCache, - updateState: (cache) => { - this.update((draftState) => { - draftState.addressScanCache = cache; - }); - }, - }); this.messenger.registerMethodActionHandlers( this, @@ -1222,58 +970,24 @@ export class PhishingController extends BaseController< const [hostname] = getHostnameFromWebUrl(url); - const cachedResult = this.#urlScanCache.get(scanUrlParam); - if (cachedResult) { - return cachedResult; - } - - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent(scanUrlParam)}`, - { - method: 'GET', - headers: { - Accept: 'application/json', - }, - }, - ); - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - }; - } - const data = await res.json(); - return data; - }, - true, - 8000, - ); - - // Need to do it this way because safelyExecuteWithTimeout returns undefined for both timeouts and errors. - if (!apiResponse) { - return { - hostname: '', - recommendedAction: RecommendedAction.None, - fetchError: 'timeout of 8000ms exceeded', - }; - } else if ((apiResponse as { error?: string }).error) { + let scanResult: PhishingDetectionScanResult; + try { + scanResult = await this.#callWithTimeout( + this.messenger.call('PhishingDataService:scanUrl', scanUrlParam), + URL_SCAN_TIMEOUT, + ); + } catch (error) { return { hostname: '', recommendedAction: RecommendedAction.None, - fetchError: (apiResponse as { error: string }).error, + fetchError: getErrorMessage(error), }; } - const scanResult = apiResponse as PhishingDetectionScanResult; - const result = { + return { hostname, recommendedAction: scanResult.recommendedAction, }; - - this.#urlScanCache.set(scanUrlParam, result); - - return result; } /** @@ -1312,8 +1026,7 @@ export class PhishingController extends BaseController< errors: {}, }; - // Extract hostnames from URLs and check for validity and length constraints - const urlsToHostnames: Record = {}; + // Check URLs for validity and length constraints const urlsToFetch: string[] = []; for (const url of urls) { @@ -1324,22 +1037,13 @@ export class PhishingController extends BaseController< continue; } - const [hostname, ok] = getHostnameFromWebUrl(url); + const [, ok] = getHostnameFromWebUrl(url); if (!ok) { combinedResponse.errors[url] = ['url is not a valid web URL']; continue; } - // Check if result is already in cache - const cachedResult = this.#urlScanCache.get(hostname); - if (cachedResult) { - // Use cached result - combinedResponse.results[url] = cachedResult; - } else { - // Add to list of URLs to fetch - urlsToHostnames[url] = hostname; - urlsToFetch.push(url); - } + urlsToFetch.push(url); } // If there are URLs to fetch, process them in batches @@ -1358,12 +1062,7 @@ export class PhishingController extends BaseController< // Merge results and errors from all batches batchResults.forEach((batchResponse) => { - // Add results to cache and combine with response Object.entries(batchResponse.results).forEach(([url, result]) => { - const hostname = urlsToHostnames[url]; - if (hostname) { - this.#urlScanCache.set(hostname, result); - } combinedResponse.results[url] = result; }); @@ -1391,55 +1090,23 @@ export class PhishingController extends BaseController< chain: string, tokens: string[], ): Promise => { - const timeout = 8000; // 8 seconds - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const response = await fetch( - `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - tokens, - }), - }, - ); - - if (!response.ok) { - return { - error: `${response.status} ${response.statusText}`, - status: response.status, - statusText: response.statusText, - }; - } - - const data = await response.json(); - return data; - }, - true, - timeout, - ); - - if (!apiResponse) { - console.error(`Error scanning tokens: timeout of ${timeout}ms exceeded`); - return null; - } - - if ((apiResponse as { error?: string }).error) { - const { status, statusText } = apiResponse as { - status: number; - statusText: string; - }; - - console.warn(`Token bulk screening API error: ${status} ${statusText}`); + try { + return await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:bulkScanTokens', + chain, + tokens, + ), + TOKEN_SCAN_TIMEOUT, + ); + } catch (error) { + if (error instanceof HttpError) { + console.warn(`Token bulk screening API error: ${error.message}`); + } else { + console.error(`Error scanning tokens: ${getErrorMessage(error)}`); + } return null; } - - return apiResponse as TokenScanApiResponse; }; /** @@ -1471,67 +1138,25 @@ export class PhishingController extends BaseController< }; } - const cacheKey = buildCacheKey(normalizedChainId, normalizedAddress); - const cachedResult = this.#addressScanCache.get(cacheKey); - if (cachedResult) { - return { - result_type: cachedResult.result_type, - label: cachedResult.label, - }; - } - - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - address: normalizedAddress, - }), - }, - ); - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - }; - } - const data: AddressScanResult = await res.json(); - return data; - }, - true, - 5000, - ); - - if (!apiResponse) { + try { + const scanResult = await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:scanAddress', + chain, + normalizedAddress, + ), + ADDRESS_SCAN_TIMEOUT, + ); return { - result_type: AddressScanResultType.ErrorResult, - label: '', + result_type: scanResult.result_type, + label: scanResult.label, }; - } else if ((apiResponse as { error?: string }).error) { + } catch { return { result_type: AddressScanResultType.ErrorResult, label: '', }; } - - const scanResult = apiResponse as AddressScanResult; - const result: AddressScanCacheData = { - result_type: scanResult.result_type, - label: scanResult.label, - }; - - this.#addressScanCache.set(cacheKey, result); - - return { - result_type: scanResult.result_type, - label: scanResult.label, - }; } /** @@ -1557,44 +1182,18 @@ export class PhishingController extends BaseController< return { approvals: [] }; } - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chain, - address: normalizedAddress, - }), - }, - ); - if (!res.ok) { - return { error: `${res.status} ${res.statusText}` }; - } - const data: ApprovalsResponse = await res.json(); - return data; - }, - true, - 5000, - ); - - if (!apiResponse) { - return { approvals: [] }; - } - - if ( - (apiResponse as { error?: string }).error || - !Array.isArray((apiResponse as Partial).approvals) - ) { + try { + return await this.#callWithTimeout( + this.messenger.call( + 'PhishingDataService:getApprovals', + chain, + normalizedAddress, + ), + APPROVALS_TIMEOUT, + ); + } catch { return { approvals: [] }; } - - return apiResponse as ApprovalsResponse; }; /** @@ -1637,50 +1236,26 @@ export class PhishingController extends BaseController< // EVM addresses are case-insensitive; non-EVM addresses (e.g. Solana // base58) are case-sensitive and must not be lowercased. const caseSensitive = !normalizedChainId.startsWith('0x'); + const normalizedTokens = caseSensitive + ? tokens + : tokens.map((tokenAddress) => tokenAddress.toLowerCase()); - // Split tokens into cached results and tokens that need to be fetched - const { cachedResults, tokensToFetch } = splitCacheHits( - this.#tokenScanCache, - normalizedChainId, - tokens, - caseSensitive, - ); + const results: BulkTokenScanResponse = {}; - const results: BulkTokenScanResponse = { ...cachedResults }; - - // If there are tokens to fetch, call the bulk token scan API - if (tokensToFetch.length > 0) { - const apiResponse = await this.#fetchTokenScanBulkResults( - chain, - tokensToFetch, - ); - if (apiResponse?.results) { - // Process API results and update cache - for (const tokenAddress of tokensToFetch) { - const normalizedAddress = caseSensitive - ? tokenAddress - : tokenAddress.toLowerCase(); - const tokenResult = apiResponse.results[normalizedAddress]; - - if (tokenResult?.result_type) { - const result = { - result_type: tokenResult.result_type, - chain: tokenResult.chain || normalizedChainId, - address: tokenResult.address || normalizedAddress, - }; - - // Update cache - const cacheKey = buildCacheKey( - normalizedChainId, - normalizedAddress, - caseSensitive, - ); - this.#tokenScanCache.set(cacheKey, { - result_type: tokenResult.result_type, - }); - - results[normalizedAddress] = result; - } + const apiResponse = await this.#fetchTokenScanBulkResults( + chain, + normalizedTokens, + ); + if (apiResponse?.results) { + for (const normalizedAddress of normalizedTokens) { + const tokenResult = apiResponse.results[normalizedAddress]; + + if (tokenResult?.result_type) { + results[normalizedAddress] = { + result_type: tokenResult.result_type, + chain: tokenResult.chain || normalizedChainId, + address: tokenResult.address || normalizedAddress, + }; } } } @@ -1697,61 +1272,27 @@ export class PhishingController extends BaseController< readonly #processBatch = async ( urls: string[], ): Promise => { - const apiResponse = await safelyExecuteWithTimeout( - async () => { - const res = await fetch( - `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, - { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ urls }), + try { + return await this.#callWithTimeout( + this.messenger.call('PhishingDataService:bulkScanUrls', urls), + BULK_URL_SCAN_TIMEOUT, + ); + } catch (error) { + if (error instanceof HttpError) { + return { + results: {}, + errors: { + api_error: [error.message], }, - ); - - if (!res.ok) { - return { - error: `${res.status} ${res.statusText}`, - status: res.status, - statusText: res.statusText, - }; - } - - const data = await res.json(); - return data; - }, - true, - 15000, - ); - - // Handle timeout or network errors - if (!apiResponse) { - return { - results: {}, - errors: { - network_error: ['timeout of 15000ms exceeded'], - }, - }; - } - - // Handle HTTP error responses - if ((apiResponse as { error?: string }).error) { - const { status, statusText } = apiResponse as { - status: number; - statusText: string; - }; - + }; + } return { results: {}, errors: { - api_error: [`${status} ${statusText}`], + network_error: [getErrorMessage(error)], }, }; } - - return apiResponse as BulkPhishingDetectionScanResponse; }; /** @@ -1765,12 +1306,13 @@ export class PhishingController extends BaseController< let hotlistDiffsResponse: DataResultWrapper | null = null; let c2DomainBlocklistResponse: C2DomainBlocklistResponse | null = null; try { - const stalelistPromise = this.#queryConfig< - DataResultWrapper - >(METAMASK_STALELIST_URL); + const stalelistPromise = this.#safelyCallService(() => + this.messenger.call('PhishingDataService:getStalelist'), + ); - const c2DomainBlocklistPromise = - this.#queryConfig(C2_DOMAIN_BLOCKLIST_URL); + const c2DomainBlocklistPromise = this.#safelyCallService(() => + this.messenger.call('PhishingDataService:getC2DomainBlocklist'), + ); [stalelistResponse, c2DomainBlocklistResponse] = await Promise.all([ stalelistPromise, @@ -1778,10 +1320,14 @@ export class PhishingController extends BaseController< ]); // Fetching hotlist diffs relies on having a lastUpdated timestamp to do `GET /v1/diffsSince/:timestamp`, // so it doesn't make sense to call if there is not a timestamp to begin with. - if (stalelistResponse?.data && stalelistResponse.data.lastUpdated > 0) { - hotlistDiffsResponse = await this.#queryConfig< - DataResultWrapper - >(`${METAMASK_HOTLIST_DIFF_URL}/${stalelistResponse.data.lastUpdated}`); + const stalelistData = stalelistResponse?.data; + if (stalelistData && stalelistData.lastUpdated > 0) { + hotlistDiffsResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getHotlistDiffs', + stalelistData.lastUpdated, + ), + ); } } finally { // Set `stalelistLastFetched` and `hotlistLastFetched` even for failed requests to prevent server @@ -1844,8 +1390,11 @@ export class PhishingController extends BaseController< ...this.state.phishingLists.map(({ lastUpdated }) => lastUpdated), ); - hotlistResponse = await this.#queryConfig>( - `${METAMASK_HOTLIST_DIFF_URL}/${lastDiffTimestamp}`, + hotlistResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getHotlistDiffs', + lastDiffTimestamp, + ), ); } finally { // Set `hotlistLastFetched` even for failed requests to prevent server from being overwhelmed with @@ -1884,12 +1433,12 @@ export class PhishingController extends BaseController< * this function that prevents redundant configuration updates. */ async #updateC2DomainBlocklist() { - const c2DomainBlocklistResponse = - await this.#queryConfig( - `${C2_DOMAIN_BLOCKLIST_URL}?timestamp=${roundToNearestMinute( - this.state.c2DomainBlocklistLastFetched, - )}`, - ); + const c2DomainBlocklistResponse = await this.#safelyCallService(() => + this.messenger.call( + 'PhishingDataService:getC2DomainBlocklist', + roundToNearestMinute(this.state.c2DomainBlocklistLastFetched), + ), + ); if (!c2DomainBlocklistResponse) { return; @@ -1920,22 +1469,51 @@ export class PhishingController extends BaseController< this.updatePhishingDetector(); } - async #queryConfig( - input: RequestInfo, - ): Promise { - const response = await safelyExecute( - () => fetch(input, { cache: 'no-cache' }), - true, - ); - - switch (response?.status) { - case 200: { - return await response.json(); - } + /** + * Calls the data service, returning `null` instead of throwing if the call + * fails for any reason (network error, non-2xx response, or malformed + * response). + * + * @param call - The service call to execute. + * @returns The result of the call, or `null` if it failed. + */ + async #safelyCallService( + call: () => Promise, + ): Promise { + try { + return await call(); + } catch (error) { + console.error(error); + return null; + } + } - default: { - return null; - } + /** + * Awaits a promise, rejecting if it does not settle within the given + * timeout. On timeout, any eventual rejection of the original promise is + * suppressed to avoid unhandled rejections. + * + * @param promise - The promise to await. + * @param timeout - The timeout in milliseconds. + * @returns The result of the promise. + */ + async #callWithTimeout( + promise: Promise, + timeout: number, + ): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + promise.catch(() => undefined); + reject(new Error(`timeout of ${timeout}ms exceeded`)); + }, timeout); + }), + ]); + } finally { + clearTimeout(timer); } } } diff --git a/packages/phishing-controller/src/PhishingDataService-method-action-types.ts b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts new file mode 100644 index 00000000000..356a098e4db --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService-method-action-types.ts @@ -0,0 +1,152 @@ +/** + * This file is auto generated. + * Do not edit manually. + */ + +import type { PhishingDataService } from './PhishingDataService.js'; + +/** + * Fetches the full phishing detection stalelist. + * + * @returns The stalelist response. + */ +export type PhishingDataServiceGetStalelistAction = { + type: `PhishingDataService:getStalelist`; + handler: PhishingDataService['getStalelist']; +}; + +/** + * Fetches the hotlist diffs recorded since the given timestamp. + * + * @param timestamp - The timestamp (in seconds) to fetch diffs since. + * @returns The hotlist diffs response. + */ +export type PhishingDataServiceGetHotlistDiffsAction = { + type: `PhishingDataService:getHotlistDiffs`; + handler: PhishingDataService['getHotlistDiffs']; +}; + +/** + * Fetches the C2 domain blocklist changes recorded since the given + * timestamp, or the current blocklist if no timestamp is given. + * + * @param timestamp - The timestamp (in seconds) to fetch changes since. + * @returns The C2 domain blocklist response. + */ +export type PhishingDataServiceGetC2DomainBlocklistAction = { + type: `PhishingDataService:getC2DomainBlocklist`; + handler: PhishingDataService['getC2DomainBlocklist']; +}; + +/** + * Scans a URL for phishing via the dapp-scanning API. + * + * @param url - The prepared URL parameter to scan (hostname, or hostname + * plus path for shared gateways). + * @returns The phishing detection scan result. + */ +export type PhishingDataServiceScanUrlAction = { + type: `PhishingDataService:scanUrl`; + handler: PhishingDataService['scanUrl']; +}; + +/** + * Scans a batch of URLs for phishing via the dapp-scanning API. + * + * Results are cached under the same query keys as + * {@link PhishingDataService.scanUrl}, so results are shared between single + * and bulk scans, including for the path-sensitive hosts listed in + * `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`. Only URLs without a fresh + * cached result are sent to the API, in requests of up to 50 URLs. + * + * If some lookups fail, the results that did resolve are still returned and + * the failures are reported per URL. The call only rejects when nothing at + * all could be resolved. + * + * @param urls - The URLs to scan. + * @returns The scan results, keyed by URL, and any per-URL errors. + */ +export type PhishingDataServiceBulkScanUrlsAction = { + type: `PhishingDataService:bulkScanUrls`; + handler: PhishingDataService['bulkScanUrls']; +}; + +/** + * Scans a token for malicious activity via the security-alerts API. + * + * Each call issues its own request to the bulk scanning endpoint; use + * {@link PhishingDataService.bulkScanTokens} to scan several tokens in one + * request. EVM token addresses are lowercased before being used as the cache + * key and sent to the API; other addresses are used as given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result + * for the token. + */ +export type PhishingDataServiceScanTokenAction = { + type: `PhishingDataService:scanToken`; + handler: PhishingDataService['scanToken']; +}; + +/** + * Scans a batch of tokens for malicious activity via the security-alerts + * API. + * + * Results are cached per token; only tokens without a fresh cached result + * are sent to the API, in requests of up to 100 tokens. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param tokens - The token addresses to scan. + * @returns The token scan results, keyed by normalized token address (EVM + * addresses are lowercased). Tokens for which the API returned no result + * are omitted. + */ +export type PhishingDataServiceBulkScanTokensAction = { + type: `PhishingDataService:bulkScanTokens`; + handler: PhishingDataService['bulkScanTokens']; +}; + +/** + * Scans an address for security alerts via the security-alerts API. EVM + * addresses are lowercased before being used as the cache key and sent to + * the API; other addresses are used as given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to scan. + * @returns The address scan result. + */ +export type PhishingDataServiceScanAddressAction = { + type: `PhishingDataService:scanAddress`; + handler: PhishingDataService['scanAddress']; +}; + +/** + * Gets token approvals for an address with security enrichments via the + * security-alerts API. Approvals reflect live account state, so they are + * always refetched and never retained in the query cache. EVM addresses are + * lowercased before being sent to the API; other addresses are used as + * given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to get approvals for. + * @returns The approvals response. + */ +export type PhishingDataServiceGetApprovalsAction = { + type: `PhishingDataService:getApprovals`; + handler: PhishingDataService['getApprovals']; +}; + +/** + * Union of all PhishingDataService action types. + */ +export type PhishingDataServiceMethodActions = + | PhishingDataServiceGetStalelistAction + | PhishingDataServiceGetHotlistDiffsAction + | PhishingDataServiceGetC2DomainBlocklistAction + | PhishingDataServiceScanUrlAction + | PhishingDataServiceBulkScanUrlsAction + | PhishingDataServiceScanTokenAction + | PhishingDataServiceBulkScanTokensAction + | PhishingDataServiceScanAddressAction + | PhishingDataServiceGetApprovalsAction; diff --git a/packages/phishing-controller/src/PhishingDataService.test.ts b/packages/phishing-controller/src/PhishingDataService.test.ts new file mode 100644 index 00000000000..baac507dd18 --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService.test.ts @@ -0,0 +1,2078 @@ +import { + ConstantBackoff, + DEFAULT_HYDRATION_TIMEOUT, + handleWhen, +} from '@metamask/base-data-service'; +import { Messenger, MOCK_ANY_NAMESPACE } from '@metamask/messenger'; +import type { + MessengerActions, + MessengerEvents, + MockAnyNamespace, +} from '@metamask/messenger'; +import { Duration, inMilliseconds } from '@metamask/utils'; +import nock, { cleanAll } from 'nock'; + +import { flushPromises } from '../../../tests/helpers.js'; +import { + PhishingDataService, + C2_DOMAIN_BLOCKLIST_ENDPOINT, + CLIENT_SIDE_DETECION_BASE_URL, + METAMASK_HOTLIST_DIFF_FILE, + METAMASK_STALELIST_FILE, + PHISHING_CONFIG_BASE_URL, + PHISHING_DETECTION_BASE_URL, + PHISHING_DETECTION_BULK_SCAN_ENDPOINT, + PHISHING_DETECTION_SCAN_ENDPOINT, + SECURITY_ALERTS_BASE_URL, + TOKEN_BULK_SCANNING_ENDPOINT, + ADDRESS_SCAN_ENDPOINT, + APPROVALS_ENDPOINT, + SCAN_RESULT_GC_TIME, + SCAN_RESULT_STALE_TIME, + URL_SCAN_TIMEOUT, +} from './PhishingDataService.js'; +import type { PhishingDataServiceMessenger } from './PhishingDataService.js'; +import { TokenScanResultType } from './types.js'; +import type { TokenScanApiResponse } from './types.js'; + +const createdServices: PhishingDataService[] = []; + +const STALELIST_RESPONSE = { + data: { + allowlist: [], + blocklist: ['phishing.example.com'], + blocklistPaths: [], + fuzzylist: [], + tolerance: 2, + version: 1, + lastUpdated: 1700000000, + }, +}; + +describe('PhishingDataService', () => { + afterEach(() => { + jest.useRealTimers(); + cleanAll(); + while (createdServices.length > 0) { + createdServices.pop()?.destroy(); + } + }); + + describe('constructor', () => { + it('applies default options when only a messenger is given', () => { + const rootMessenger = createRootMessenger(); + const messenger: PhishingDataServiceMessenger = new Messenger({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + const service = new PhishingDataService({ messenger }); + createdServices.push(service); + + expect(service.name).toBe('PhishingDataService'); + }); + }); + + describe('getStalelist', () => { + it('returns the stalelist from the API', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getStalelist', + ); + + expect(response).toStrictEqual(STALELIST_RESPONSE); + }); + + it('throws if the API returns a non-200 status', async () => { + nock(PHISHING_CONFIG_BASE_URL).get(METAMASK_STALELIST_FILE).reply(500); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getStalelist'), + ).rejects.toThrow('500 Internal Server Error'); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, { data: { lastUpdated: 1700000000 } }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getStalelist'), + ).rejects.toThrow('Malformed response received from stalelist endpoint'); + }); + + it('aborts a pending list request when the service is destroyed', async () => { + let requestSignal: AbortSignal | null | undefined; + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + requestSignal = init?.signal; + requestSignal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + }), + ); + const { service } = createService(); + const pendingRequest = service.getStalelist().catch((error) => error); + + try { + await flushPromises(); + service.destroy(); + + expect(requestSignal?.aborted).toBe(true); + await pendingRequest; + } finally { + fetchMock.mockRestore(); + } + }); + + it('aborts batched and uncached POST requests when destroyed', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask', 'setImmediate'], + }); + const requestSignals: AbortSignal[] = []; + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + if (signal) { + requestSignals.push(signal); + if (signal.aborted) { + reject(new Error('aborted')); + } else { + signal.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + } + } + }), + ); + const { service } = createService(); + const pendingBulkScan = service + .bulkScanUrls(['https://example.com']) + .catch((error) => error); + const pendingApprovals = service + .getApprovals('ethereum', '0x1234567890123456789012345678901234567890') + .catch((error) => error); + + try { + await flushPromises(); + expect(requestSignals).toHaveLength(2); + + service.destroy(); + + expect(requestSignals.every((signal) => signal.aborted)).toBe(true); + await Promise.all([pendingBulkScan, pendingApprovals]); + } finally { + fetchMock.mockRestore(); + } + }); + }); + + describe('getHotlistDiffs', () => { + it('returns the hotlist diffs recorded since the given timestamp', async () => { + const diffs = { + data: [ + { + url: 'phishing.example.com', + timestamp: 1700000001, + targetList: 'eth_phishing_detect_config.blocklist', + }, + ], + }; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, diffs); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getHotlistDiffs', + 1700000000, + ); + + expect(response).toStrictEqual(diffs); + }); + + it('passes through diffs for unrecognized target lists', async () => { + const diffs = { + data: [ + { + url: 'phishing.example.com', + timestamp: 1700000001, + targetList: 'unexpected.blocklist', + }, + ], + }; + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, diffs); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getHotlistDiffs', + 1700000000, + ); + + expect(response).toStrictEqual(diffs); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(`${METAMASK_HOTLIST_DIFF_FILE}/1700000000`) + .reply(200, { + data: [{ url: 'phishing.example.com', timestamp: 'soon' }], + }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getHotlistDiffs', 1700000000), + ).rejects.toThrow( + 'Malformed response received from hotlist diffs endpoint', + ); + }); + }); + + describe('getC2DomainBlocklist', () => { + it('returns the C2 domain blocklist when no timestamp is given', async () => { + const blocklist = { + recentlyAdded: ['0415f1f1'], + recentlyRemoved: [], + lastFetchedAt: 1700000000, + }; + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, blocklist); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getC2DomainBlocklist', + ); + + expect(response).toStrictEqual(blocklist); + }); + + it('passes the given timestamp to the API', async () => { + const blocklist = { + recentlyAdded: [], + recentlyRemoved: ['0415f1f1'], + lastFetchedAt: 1700000000, + }; + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .query({ timestamp: 1700000000 }) + .reply(200, blocklist); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:getC2DomainBlocklist', + 1700000000, + ); + + expect(response).toStrictEqual(blocklist); + }); + + it('accepts a response without lastFetchedAt', async () => { + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, { recentlyAdded: ['abc'], recentlyRemoved: [] }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), + ).toStrictEqual({ recentlyAdded: ['abc'], recentlyRemoved: [] }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(CLIENT_SIDE_DETECION_BASE_URL) + .get(C2_DOMAIN_BLOCKLIST_ENDPOINT) + .reply(200, { recentlyAdded: 'abc', recentlyRemoved: [] }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:getC2DomainBlocklist'), + ).rejects.toThrow( + 'Malformed response received from C2 domain blocklist endpoint', + ); + }); + }); + + describe('scanUrl', () => { + it('returns the scan result from the API', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { hostname: 'example.com', recommendedAction: 'NONE' }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + + expect(response).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'NONE', + }); + }); + + it('serves a repeated scan of the same URL from the cache within the stale time', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + const response1 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + const response2 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + expect(response1).toStrictEqual(response2); + + // Once the result goes stale, the URL is scanned again. + jest.advanceTimersByTime(SCAN_RESULT_STALE_TIME + 1); + const response3 = await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + expect(response3).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + }); + + it('throws if the API returns a non-200 status', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(404); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('404 Not Found'); + }); + + it('aborts a timed-out request so the next scan can retry', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + }); + const fetchMock = jest.spyOn(globalThis, 'fetch'); + fetchMock + .mockImplementationOnce( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new Error('aborted')), + { once: true }, + ); + }), + ) + .mockResolvedValueOnce( + new globalThis.Response( + JSON.stringify({ recommendedAction: 'BLOCK' }), + { + status: 200, + }, + ), + ); + const { rootMessenger } = createService(); + + try { + const timedOutScan = rootMessenger + .call('PhishingDataService:scanUrl', 'example.com') + .catch((error) => error); + await jest.advanceTimersByTimeAsync(URL_SCAN_TIMEOUT); + expect(await timedOutScan).toMatchObject({ + message: `timeout of ${URL_SCAN_TIMEOUT}ms exceeded`, + }); + + expect( + await rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ), + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + fetchMock.mockRestore(); + } + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'INVALID' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('Malformed response received from URL scan endpoint'); + }); + + it('validates optional URL scan response fields when present', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { + hostname: 123, + recommendedAction: 'NONE', + fetchError: false, + }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('Malformed response received from URL scan endpoint'); + }); + + it('does not cache a malformed response', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { unexpected: 'shape' }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('Malformed response received from URL scan endpoint'); + + // The malformed body must not have been committed to the cache, so the + // next call re-requests and sees the real verdict. + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(scope.isDone()).toBe(true); + }); + + it('does not cache a response containing a fetch error', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { + recommendedAction: 'NONE', + fetchError: 'detector unavailable', + }) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('detector unavailable'); + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(scope.isDone()).toBe(true); + }); + + it('refetches once a cached result passes its garbage collection time', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .times(2) + .reply(200, { recommendedAction: 'NONE' }); + const { rootMessenger } = createService(); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1000); + await flushPromises(); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + // The entry is collected rather than being retained forever, which is + // what TanStack Query would otherwise do in a service worker. + expect(scope.isDone()).toBe(true); + }); + }); + + describe('bulkScanUrls', () => { + it('retries a failed batch as a whole when retries are enabled', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(function (_uri, body) { + batchSizes.push((body as { urls: string[] }).urls.length); + return [500, 'boom']; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ]), + ).rejects.toThrow('500 Internal Server Error'); + expect(batchSizes).toStrictEqual([3, 3, 3]); + }); + + it('still coalesces lookups into one request after init', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(function (_uri, body) { + const { urls } = body as { urls: string[] }; + batchSizes.push(urls.length); + return [ + 200, + { + results: Object.fromEntries( + urls.map((url) => [url, { recommendedAction: 'NONE' }]), + ), + errors: {}, + }, + ]; + }); + const { rootMessenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: jest.fn().mockResolvedValue({ result: null }), + }); + service.init(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ], + ); + + expect(batchSizes).toStrictEqual([3]); + expect(Object.keys(response.results)).toHaveLength(3); + }); + + it('batches a bulk call even when rehydration times out', async () => { + const batchSizes: number[] = []; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(function (_uri, body) { + const { urls } = body as { urls: string[] }; + batchSizes.push(urls.length); + return [ + 200, + { + results: Object.fromEntries( + urls.map((url) => [url, { recommendedAction: 'NONE' }]), + ), + errors: {}, + }, + ]; + }); + const { rootMessenger, service } = createService({ + options: { + persistenceConfig: { + maxAge: inMilliseconds(5, Duration.Minute), + hydrationTimeout: 20, + }, + }, + setItemMock: jest.fn(), + getItemMock: jest.fn(() => new Promise(() => undefined)), + }); + service.init(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [ + 'https://example1.com', + 'https://example2.com', + 'https://example3.com', + ], + ); + + expect(batchSizes).toStrictEqual([3]); + expect(Object.keys(response.results)).toHaveLength(3); + }); + + it('does not retry per-URL errors reported by the endpoint', async () => { + let requests = 0; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(() => { + requests += 1; + return [ + 200, + { results: {}, errors: { 'https://example1.com': ['boom'] } }, + ]; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: {}, + errors: { 'https://example1.com': ['boom'] }, + }); + expect(requests).toBe(1); + }); + + it('never retries item-level batch errors even if a caller-provided retryFilterPolicy would', async () => { + let requests = 0; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(3) + .reply(() => { + requests += 1; + return [ + 200, + { results: {}, errors: { 'https://example1.com': ['boom'] } }, + ]; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { + maxRetries: 2, + backoff: new ConstantBackoff(0), + retryFilterPolicy: handleWhen(() => true), + }, + }, + }); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: {}, + errors: { 'https://example1.com': ['boom'] }, + }); + expect(requests).toBe(1); + }); + + it('returns the scan results from the API', async () => { + const urls = ['https://example1.com', 'https://example2.com']; + const apiResponse = { + results: { + 'https://example1.com': { + hostname: 'example1.com', + recommendedAction: 'NONE', + }, + 'https://example2.com': { + hostname: 'example2.com', + recommendedAction: 'BLOCK', + }, + }, + errors: {}, + }; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + urls, + ); + + expect(response).toStrictEqual(apiResponse); + }); + + it('reports a malformed result for that URL and keeps the others', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { + 'https://example1.com': {}, + 'https://example2.com': { recommendedAction: 'BLOCK' }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com', 'https://example2.com'], + ); + + expect(response).toStrictEqual({ + results: { + 'https://example2.com': { + recommendedAction: 'BLOCK', + hostname: 'example2.com', + }, + }, + errors: { + 'https://example1.com': [ + 'Malformed result returned by bulk URL scan endpoint', + ], + }, + }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { results: 'nope' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).rejects.toThrow( + 'Malformed response received from bulk URL scan endpoint', + ); + }); + + it('accepts a response without an errors object', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { 'https://example1.com': { recommendedAction: 'NONE' } }, + }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).toStrictEqual({ + results: { + 'https://example1.com': { + recommendedAction: 'NONE', + hostname: 'example1.com', + }, + }, + errors: {}, + }); + }); + + it('stores a hostname on cache entries shared with scanUrl', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { + 'https://evil.com/some/path': { recommendedAction: 'BLOCK' }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://evil.com/some/path', + ]); + + // No GET interceptor is registered, so this must be a cache hit. + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'evil.com'), + ).toStrictEqual({ recommendedAction: 'BLOCK', hostname: 'evil.com' }); + }); + + it('sends each path separately for path-sensitive hosts', async () => { + const urls = ['https://ipfs.io/ipfs/AAA', 'https://ipfs.io/ipfs/BBB']; + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { urls }) + .reply(200, { + results: { + [urls[0]]: { hostname: urls[0], recommendedAction: 'NONE' }, + [urls[1]]: { hostname: urls[1], recommendedAction: 'BLOCK' }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + urls, + ); + + // Both paths must be scanned; they must not share one verdict. + expect(response.results[urls[0]].recommendedAction).toBe('NONE'); + expect(response.results[urls[1]].recommendedAction).toBe('BLOCK'); + }); + + it('reports invalid URLs without calling the API', async () => { + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['not-a-url'], + ); + + expect(response).toStrictEqual({ + results: {}, + errors: { 'not-a-url': ['url is not a valid web URL'] }, + }); + }); + + it('does not cache URLs the API reported an error for', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(2) + .reply(200, { + results: {}, + errors: { 'https://example1.com': ['upstream failure'] }, + }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + + // The error must be reported both times rather than being cached as a + // silent "no result" for the stale time. + expect(first.errors['https://example1.com']).toStrictEqual([ + 'upstream failure', + ]); + expect(second.errors['https://example1.com']).toStrictEqual([ + 'upstream failure', + ]); + expect(scope.isDone()).toBe(true); + }); + + it('does not cache URL results containing a fetch error', async () => { + const blockedUrl = 'https://blocked.com'; + const failedUrl = 'https://failed.com'; + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { + urls: [blockedUrl, failedUrl], + }) + .reply(200, { + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'NONE', + fetchError: 'detector unavailable', + }, + }, + errors: {}, + }) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, { + urls: [failedUrl], + }) + .reply(200, { + results: { + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'WARN', + }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [blockedUrl, failedUrl], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + [blockedUrl, failedUrl], + ); + + expect(first).toStrictEqual({ + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + }, + errors: { [failedUrl]: ['detector unavailable'] }, + }); + expect(second).toStrictEqual({ + results: { + [blockedUrl]: { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + [failedUrl]: { + hostname: 'failed.com', + recommendedAction: 'WARN', + }, + }, + errors: {}, + }); + expect(scope.isDone()).toBe(true); + }); + + it('does not cache a URL omitted from the API response', async () => { + const scope = nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .times(2) + .reply(200, { results: {}, errors: {} }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + const second = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://example1.com'], + ); + + expect(first.errors['https://example1.com']).toStrictEqual([ + 'No result returned by bulk URL scan endpoint', + ]); + expect(second.errors['https://example1.com']).toStrictEqual([ + 'No result returned by bulk URL scan endpoint', + ]); + expect(scope.isDone()).toBe(true); + }); + + it('keeps fresh cached results when another lookup fails', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(200, { + results: { + 'https://blocked.com': { + hostname: 'blocked.com', + recommendedAction: 'BLOCK', + }, + }, + errors: {}, + }); + const { rootMessenger } = createService(); + + await rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://blocked.com', + ]); + + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(500, 'boom'); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanUrls', + ['https://blocked.com', 'https://uncached.com'], + ); + + expect(response.results['https://blocked.com'].recommendedAction).toBe( + 'BLOCK', + ); + expect(response.errors['https://uncached.com']).toStrictEqual([ + '500 Internal Server Error', + ]); + }); + + it('rejects when no URL could be resolved', async () => { + nock(PHISHING_DETECTION_BASE_URL) + .post(`/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`) + .reply(500, 'boom'); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanUrls', [ + 'https://example1.com', + ]), + ).rejects.toThrow('500 Internal Server Error'); + }); + }); + + describe('bulkScanTokens', () => { + it('retries a failed batch as a whole when retries are enabled', async () => { + const batchSizes: number[] = []; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .times(3) + .reply(function (_uri, body) { + batchSizes.push((body as { tokens: string[] }).tokens.length); + return [500, 'boom']; + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 2, backoff: new ConstantBackoff(0) }, + }, + }); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd', + ]), + ).rejects.toThrow('500 Internal Server Error'); + expect(batchSizes).toStrictEqual([2, 2, 2]); + }); + + it('lowercases EVM token addresses and keys results by the normalized address', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [lower], + }) + .reply(200, { results: { [lower]: { result_type: 'Malicious' } } }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + ['0xAbCdEf0000000000000000000000000000000001'], + ), + ).toStrictEqual({ results: { [lower]: { result_type: 'Malicious' } } }); + }); + + it('returns the scan results from the API', async () => { + const tokens = ['0x1234567890123456789012345678901234567890']; + const apiResponse = { + results: { + [tokens[0]]: { result_type: 'Benign' }, + }, + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { chain: 'ethereum', tokens }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + tokens, + ); + + expect(response).toStrictEqual(apiResponse); + }); + + it('accepts the Verified verdict returned by the token API', async () => { + const token = '0x1234567890123456789012345678901234567890'; + const apiResponse = { + results: { + [token]: { result_type: 'Verified' }, + }, + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, apiResponse); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [token], + ), + ).toStrictEqual(apiResponse); + }); + + it('preserves cached results when an uncached token fails', async () => { + const cachedToken = '0x1234567890123456789012345678901234567890'; + const uncachedToken = '0x0987654321098765432109876543210987654321'; + const cachedResult = { result_type: 'Malicious' }; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [cachedToken], + }) + .reply(200, { + results: { + [cachedToken]: cachedResult, + }, + }); + const { rootMessenger } = createService(); + await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + cachedToken, + ); + + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [uncachedToken], + }) + .reply(500, 'boom'); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [cachedToken, uncachedToken], + ); + + expect(response).toStrictEqual({ + results: { + [cachedToken]: cachedResult, + }, + }); + }); + + it('accepts a response without a results field', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, {}); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + ['0x1234567890123456789012345678901234567890'], + ); + + expect(response).toStrictEqual({ results: {} }); + }); + + it('omits a malformed result for that token and keeps the others', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { + results: { + '0x1234567890123456789012345678901234567890': {}, + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd': { + result_type: 'Malicious', + }, + }, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [ + '0x1234567890123456789012345678901234567890', + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd', + ], + ); + + expect(response).toStrictEqual({ + results: { + '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd': { + result_type: 'Malicious', + }, + }, + }); + }); + + it('throws if every result in the batch is malformed', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { + results: { '0x1234567890123456789012345678901234567890': {} }, + }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + ]), + ).rejects.toThrow( + 'Malformed result returned by bulk token scan endpoint', + ); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { results: 'nope' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call('PhishingDataService:bulkScanTokens', 'ethereum', [ + '0x1234567890123456789012345678901234567890', + ]), + ).rejects.toThrow( + 'Malformed response received from bulk token scan endpoint', + ); + }); + }); + + describe('scanToken', () => { + it('lowercases EVM token addresses in the request and cache key', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [lower], + }) + .reply(200, { results: { [lower]: { result_type: 'Malicious' } } }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + '0xAbCdEf0000000000000000000000000000000001', + ); + // No second interceptor is registered, so this must be a cache hit. + const second = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + lower, + ); + + expect(first).toStrictEqual({ result_type: 'Malicious' }); + expect(second).toStrictEqual({ result_type: 'Malicious' }); + }); + + it('preserves the casing of non-EVM token addresses', async () => { + const solanaToken = 'So11111111111111111111111111111111111111112'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'solana', + tokens: [solanaToken], + }) + .reply(200, { results: { [solanaToken]: { result_type: 'Benign' } } }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:scanToken', + 'solana', + solanaToken, + ), + ).toStrictEqual({ result_type: 'Benign' }); + }); + + it('returns the scan result for a single token from the bulk API', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, { + results: { + [token]: { result_type: 'Benign' }, + }, + }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toStrictEqual({ result_type: 'Benign' }); + }); + + it('returns null if the API returned no result for the token', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { results: {} }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toBeNull(); + }); + + it('shares cached results with bulkScanTokens', async () => { + const token = '0x1234567890123456789012345678901234567890'; + nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: [token], + }) + .reply(200, { + results: { + [token]: { result_type: 'Malicious' }, + }, + }); + const { rootMessenger } = createService(); + + await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + [token], + ); + + // Served from the cache; there is no remaining nock interceptor, so a + // fetch would throw. + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(response).toStrictEqual({ result_type: 'Malicious' }); + }); + }); + + describe('batching', () => { + it('splits large token batches into requests of up to 100 tokens', async () => { + const tokens = Array.from( + { length: 120 }, + (_, index) => `0x${index.toString().padStart(40, '0')}`, + ); + const firstChunk = tokens.slice(0, 100); + const secondChunk = tokens.slice(100); + const buildResults = (chunk: string[]): TokenScanApiResponse['results'] => + Object.fromEntries( + chunk.map((token) => [ + token, + { result_type: TokenScanResultType.Benign }, + ]), + ); + + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: firstChunk, + }) + .reply(200, { results: buildResults(firstChunk) }) + .post(TOKEN_BULK_SCANNING_ENDPOINT, { + chain: 'ethereum', + tokens: secondChunk, + }) + .reply(200, { results: buildResults(secondChunk) }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:bulkScanTokens', + 'ethereum', + tokens, + ); + + expect(scope.isDone()).toBe(true); + expect(Object.keys(response.results ?? {})).toHaveLength(120); + }); + + it('coalesces retried queries into a new batched request', async () => { + const token = '0x1234567890123456789012345678901234567890'; + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(500) + .post(TOKEN_BULK_SCANNING_ENDPOINT) + .reply(200, { + results: { + [token]: { result_type: 'Benign' }, + }, + }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 1, backoff: new ConstantBackoff(0) }, + }, + }); + + const response = await rootMessenger.call( + 'PhishingDataService:scanToken', + 'ethereum', + token, + ); + + expect(scope.isDone()).toBe(true); + expect(response).toStrictEqual({ result_type: 'Benign' }); + }); + }); + + describe('scanAddress', () => { + it('lowercases EVM addresses in the request and cache key', async () => { + const lower = '0xabcdef0000000000000000000000000000000001'; + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT, { chain: 'ethereum', address: lower }) + .reply(200, { result_type: 'Malicious', label: 'bad' }); + const { rootMessenger } = createService(); + + const first = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0xAbCdEf0000000000000000000000000000000001', + ); + // No second interceptor is registered, so this must be a cache hit. + const second = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + lower, + ); + + expect(first).toStrictEqual({ result_type: 'Malicious', label: 'bad' }); + expect(second).toStrictEqual({ result_type: 'Malicious', label: 'bad' }); + }); + + it('returns the scan result from the API', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, { result_type: 'Benign', label: '' }); + const { rootMessenger } = createService(); + + const response = await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + + expect(response).toStrictEqual({ result_type: 'Benign', label: '' }); + }); + + it.each(['Verified', 'Trusted', 'Error'])( + 'accepts the %s verdict returned by the address API', + async (resultType) => { + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT) + .reply(200, { result_type: resultType, label: '' }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ result_type: resultType, label: '' }); + }, + ); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(ADDRESS_SCAN_ENDPOINT) + .reply(200, { result_type: 'Benign' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call( + 'PhishingDataService:scanAddress', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).rejects.toThrow( + 'Malformed response received from address scan endpoint', + ); + }); + }); + + describe('getApprovals', () => { + it('returns the approvals from the API without caching them', async () => { + const firstResponse = { approvals: [] }; + const secondResponse = { + approvals: [ + { + allowance: {}, + asset: { + address: '0xtoken', + symbol: 'TKN', + name: 'Token', + decimals: 18, + }, + exposure: { + value: '100', + raw_value: '100000000000000000000', + }, + spender: { + address: '0xspender', + }, + verdict: 'Verified', + }, + ], + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, firstResponse) + .post(APPROVALS_ENDPOINT, { + chain: 'ethereum', + address: '0x1234567890123456789012345678901234567890', + }) + .reply(200, secondResponse); + const { rootMessenger } = createService(); + + const response1 = await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + const response2 = await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ); + + expect(response1).toStrictEqual(firstResponse); + expect(response2).toStrictEqual(secondResponse); + }); + + it('applies the configured service policy', async () => { + const scope = nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(500) + .post(APPROVALS_ENDPOINT) + .reply(200, { approvals: [] }); + const { rootMessenger } = createService({ + options: { + policyOptions: { maxRetries: 1, backoff: new ConstantBackoff(0) }, + }, + }); + + expect( + await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ approvals: [] }); + expect(scope.isDone()).toBe(true); + }); + + it('omits malformed approvals and keeps valid ones', async () => { + const validApproval = { + allowance: {}, + asset: { + address: '0xtoken', + symbol: 'TKN', + name: 'Token', + decimals: 18, + }, + exposure: { value: '100', raw_value: '100000000000000000000' }, + spender: { address: '0xspender' }, + verdict: 'Malicious', + }; + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(200, { + approvals: [ + validApproval, + { ...validApproval, asset: { ...validApproval.asset, name: null } }, + ], + }); + const { rootMessenger } = createService(); + + expect( + await rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).toStrictEqual({ approvals: [validApproval] }); + }); + + it('throws if the API returns a malformed response', async () => { + nock(SECURITY_ALERTS_BASE_URL) + .post(APPROVALS_ENDPOINT) + .reply(200, { approvals: 'nope' }); + const { rootMessenger } = createService(); + + await expect( + rootMessenger.call( + 'PhishingDataService:getApprovals', + 'ethereum', + '0x1234567890123456789012345678901234567890', + ), + ).rejects.toThrow('Malformed response received from approvals endpoint'); + }); + }); + + describe('persistence', () => { + it('persists the query cache using the StorageService by default', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + const setItem = jest.fn(); + const { rootMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + // The persistence write is debounced; advance past the write delay. + jest.advanceTimersByTime(15_000); + await flushPromises(); + + expect(setItem).toHaveBeenCalledWith( + 'PhishingDataService', + 'cache', + expect.objectContaining({ + timestamp: expect.any(Number), + state: expect.any(Object), + }), + ); + }); + + it('does not persist fetched lists', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + const setItem = jest.fn(); + const { rootMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + + await rootMessenger.call('PhishingDataService:getStalelist'); + await rootMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(15_000); + await flushPromises(); + + const written = JSON.stringify(setItem.mock.calls.at(-1)?.[2]); + // The scan result is persisted, but the (multi-megabyte) list is not. + expect(written).toContain('scanUrl'); + expect(written).not.toContain('phishing.example.com'); + }); + + it('rehydrates the cache from the StorageService on init', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'NONE' }); + + // Populate a cache using one service, then hand it to a second one. + const setItem = jest.fn(); + const { rootMessenger: firstMessenger } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: setItem, + }); + await firstMessenger.call('PhishingDataService:scanUrl', 'example.com'); + + jest.advanceTimersByTime(15_000); + await flushPromises(); + + const persisted = setItem.mock.calls.at(-1)?.[2]; + expect(persisted).toBeDefined(); + + let resolveGetItem: + | ((value: { result: typeof persisted }) => void) + | undefined; + const getItem = jest.fn( + () => + new Promise<{ result: typeof persisted }>((resolve) => { + resolveGetItem = resolve; + }), + ); + const { rootMessenger: secondMessenger, service } = createService({ + options: { + persistenceConfig: undefined, + queryClientConfig: { + defaultOptions: { queries: { gcTime: Infinity } }, + }, + }, + setItemMock: jest.fn(), + getItemMock: getItem, + }); + service.init(); + + const networkScope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const resultPromise = secondMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + await flushPromises(); + expect(networkScope.isDone()).toBe(false); + + resolveGetItem?.({ result: persisted }); + const result = await resultPromise; + expect(result).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'NONE', + }); + expect(networkScope.isDone()).toBe(false); + + jest.advanceTimersByTime(SCAN_RESULT_GC_TIME + 1); + await flushPromises(); + + expect( + await secondMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ), + ).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(networkScope.isDone()).toBe(true); + }); + + it('discards and removes a persisted cache older than maxAge', async () => { + const removeItem = jest.fn(); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: Date.now() - inMilliseconds(10, Duration.Minute), + state: { queries: [], mutations: [] }, + }, + }); + const { service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: getItem, + removeItemMock: removeItem, + }); + + service.init(); + await flushPromises(); + + expect(getItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); + expect(removeItem).toHaveBeenCalledWith('PhishingDataService', 'cache'); + }); + + it('discards persisted scan results that fail validation or belong to unknown queries', async () => { + const now = Date.now(); + const dehydratedQuery = ( + queryKey: unknown[], + data: unknown, + ): Record => ({ + queryHash: JSON.stringify(queryKey), + queryKey, + state: { + data, + dataUpdateCount: 1, + dataUpdatedAt: now, + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: now, + state: { + mutations: [], + queries: [ + dehydratedQuery(['PhishingDataService:scanUrl', 'good.com'], { + hostname: 'good.com', + recommendedAction: 'BLOCK', + }), + dehydratedQuery(['PhishingDataService:scanUrl', 'evil.com'], { + hostname: 'evil.com', + recommendedAction: 'PWNED', + }), + dehydratedQuery( + ['PhishingDataService:scanToken', 'ethereum', '0xabc'], + null, + ), + dehydratedQuery( + ['PhishingDataService:scanToken', 'ethereum', '0xdef'], + { result_type: 'Malicious' }, + ), + dehydratedQuery( + ['PhishingDataService:scanAddress', 'ethereum', '0xabc'], + { result_type: 'Benign' }, + ), + dehydratedQuery(['PhishingDataService:getStalelist'], { + data: {}, + }), + ], + }, + }, + }); + const evilScope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'evil.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger, messenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: getItem, + }); + const publishSpy = jest.spyOn(messenger, 'publish'); + service.init(); + await flushPromises(); + + const hydratedEvents = publishSpy.mock.calls + .map(([eventType]) => String(eventType)) + .filter((eventType) => + eventType.startsWith('PhishingDataService:cacheUpdated:'), + ); + expect(hydratedEvents).toStrictEqual([ + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanUrl","good.com"]', + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanToken","ethereum","0xabc"]', + 'PhishingDataService:cacheUpdated:["PhishingDataService:scanToken","ethereum","0xdef"]', + ]); + + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'good.com'), + ).toStrictEqual({ hostname: 'good.com', recommendedAction: 'BLOCK' }); + expect( + await rootMessenger.call('PhishingDataService:scanUrl', 'evil.com'), + ).toStrictEqual({ hostname: 'evil.com', recommendedAction: 'BLOCK' }); + expect(evilScope.isDone()).toBe(true); + }); + + it('composes a caller-provided shouldHydrateQuery with the built-in validation', async () => { + const shouldHydrateQuery = jest.fn(() => false); + const getItem = jest.fn().mockResolvedValue({ + result: { + timestamp: Date.now(), + state: { + mutations: [], + queries: [ + { + queryHash: '["PhishingDataService:scanUrl","good.com"]', + queryKey: ['PhishingDataService:scanUrl', 'good.com'], + state: { + data: { hostname: 'good.com', recommendedAction: 'BLOCK' }, + dataUpdateCount: 1, + dataUpdatedAt: Date.now(), + error: null, + errorUpdateCount: 0, + errorUpdatedAt: 0, + fetchFailureCount: 0, + fetchFailureReason: null, + fetchMeta: null, + fetchStatus: 'idle', + isInvalidated: false, + status: 'success', + }, + }, + ], + }, + }, + }); + const { messenger, service } = createService({ + options: { + persistenceConfig: { + maxAge: inMilliseconds(5, Duration.Minute), + shouldHydrateQuery, + }, + }, + setItemMock: jest.fn(), + getItemMock: getItem, + }); + const publishSpy = jest.spyOn(messenger, 'publish'); + service.init(); + await flushPromises(); + + expect(shouldHydrateQuery).toHaveBeenCalledTimes(1); + expect(publishSpy).not.toHaveBeenCalled(); + }); + + it('proceeds with a scan when rehydration hangs past the hydration timeout', async () => { + jest.useFakeTimers({ + doNotFake: ['nextTick', 'queueMicrotask'], + now: 1_000_000, + }); + const scope = nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .reply(200, { recommendedAction: 'BLOCK' }); + const { rootMessenger, service } = createService({ + options: { persistenceConfig: undefined }, + setItemMock: jest.fn(), + getItemMock: jest.fn(() => new Promise(() => undefined)), + }); + service.init(); + + const resultPromise = rootMessenger.call( + 'PhishingDataService:scanUrl', + 'example.com', + ); + await flushPromises(); + expect(scope.isDone()).toBe(false); + + jest.advanceTimersByTime(DEFAULT_HYDRATION_TIMEOUT); + + expect(await resultPromise).toStrictEqual({ + hostname: 'example.com', + recommendedAction: 'BLOCK', + }); + expect(scope.isDone()).toBe(true); + }); + }); + + describe('retry policy', () => { + it('does not retry failed requests by default', async () => { + let attempts = 0; + nock(PHISHING_DETECTION_BASE_URL) + .get(`/${PHISHING_DETECTION_SCAN_ENDPOINT}`) + .query({ url: 'example.com' }) + .times(5) + .reply(() => { + attempts += 1; + return [500, 'boom']; + }); + // Build the service without the test helper's `maxRetries: 0` override + // so that the shipped defaults are what is exercised here. + const { rootMessenger } = createService({ + options: { policyOptions: {} }, + }); + + await expect( + rootMessenger.call('PhishingDataService:scanUrl', 'example.com'), + ).rejects.toThrow('500 Internal Server Error'); + expect(attempts).toBe(1); + }); + }); + + describe('direct method calls', () => { + it('does the same thing as the messenger action', async () => { + nock(PHISHING_CONFIG_BASE_URL) + .get(METAMASK_STALELIST_FILE) + .reply(200, STALELIST_RESPONSE); + const { service } = createService(); + + const response = await service.getStalelist(); + + expect(response).toStrictEqual(STALELIST_RESPONSE); + }); + }); +}); + +/** + * The type of the messenger populated with all external actions and events + * required by the service under test. + */ +type RootMessenger = Messenger< + MockAnyNamespace, + MessengerActions, + MessengerEvents +>; + +/** + * Constructs the messenger populated with all external actions and events + * required by the service under test. + * + * @returns The root messenger. + */ +function createRootMessenger(): RootMessenger { + return new Messenger({ namespace: MOCK_ANY_NAMESPACE }); +} + +/** + * Constructs the service under test. + * + * @param args - The arguments to this function. + * @param args.options - The options that the service constructor takes. All + * are optional and will be filled in with defaults as needed (including + * `messenger`). + * @param args.setItemMock - Optional mock `StorageService:setItem` handler to + * register and delegate to the service messenger, enabling persistence. + * @param args.getItemMock - Optional mock `StorageService:getItem` handler to + * register and delegate to the service messenger, enabling rehydration. + * @param args.removeItemMock - Optional mock `StorageService:removeItem` + * handler to register and delegate to the service messenger. + * @returns The new service, root messenger, and service messenger. + */ +function createService({ + options = {}, + setItemMock, + getItemMock, + removeItemMock, +}: { + options?: Partial[0]>; + setItemMock?: jest.Mock; + getItemMock?: jest.Mock; + removeItemMock?: jest.Mock; +} = {}): { + service: PhishingDataService; + rootMessenger: RootMessenger; + messenger: PhishingDataServiceMessenger; +} { + const rootMessenger = createRootMessenger(); + const messenger: PhishingDataServiceMessenger = new Messenger({ + namespace: 'PhishingDataService', + parent: rootMessenger, + }); + if (setItemMock) { + rootMessenger.registerActionHandler('StorageService:setItem', setItemMock); + rootMessenger.delegate({ + actions: ['StorageService:setItem'], + messenger, + }); + } + if (getItemMock) { + rootMessenger.registerActionHandler('StorageService:getItem', getItemMock); + rootMessenger.delegate({ + actions: ['StorageService:getItem'], + messenger, + }); + } + if (removeItemMock) { + rootMessenger.registerActionHandler( + 'StorageService:removeItem', + removeItemMock, + ); + rootMessenger.delegate({ + actions: ['StorageService:removeItem'], + messenger, + }); + } + const service = new PhishingDataService({ + messenger, + policyOptions: { maxRetries: 0 }, + persistenceConfig: null, + ...options, + }); + createdServices.push(service); + + return { service, rootMessenger, messenger }; +} diff --git a/packages/phishing-controller/src/PhishingDataService.ts b/packages/phishing-controller/src/PhishingDataService.ts new file mode 100644 index 00000000000..05cb61f1a65 --- /dev/null +++ b/packages/phishing-controller/src/PhishingDataService.ts @@ -0,0 +1,1281 @@ +import { BaseDataService, handleWhen } from '@metamask/base-data-service'; +import type { + CreateServicePolicyOptions, + DataServiceCacheUpdatedEvent, + DataServiceGranularCacheUpdatedEvent, + DataServiceInvalidateQueriesAction, + PersistenceConfiguration, +} from '@metamask/base-data-service'; +import { HttpError } from '@metamask/controller-utils'; +import type { Messenger } from '@metamask/messenger'; +import type { + StorageServiceGetItemAction, + StorageServiceRemoveItemAction, + StorageServiceSetItemAction, +} from '@metamask/storage-service'; +import type { Infer, Struct } from '@metamask/superstruct'; +import { + array, + boolean, + is, + literal, + number, + optional, + record, + string, + type, + union, + unknown, +} from '@metamask/superstruct'; +import { Duration, getErrorMessage, inMilliseconds } from '@metamask/utils'; +import type { Json } from '@metamask/utils'; +import type { DehydratedState, QueryClientConfig } from '@tanstack/query-core'; + +import type { PhishingDataServiceMethodActions } from './PhishingDataService-method-action-types.js'; +import type { + AddressScanResult, + ApprovalsResponse, + BulkPhishingDetectionScanResponse, + C2DomainBlocklistResponse, + DataResultWrapper, + Hotlist, + PhishingDetectionScanResult, + PhishingStalelist, + TokenScanApiResponse, +} from './types.js'; +import { + AddressScanResultType, + ApprovalFeatureType, + ApprovalResultType, + RecommendedAction, + TokenScanResultType, +} from './types.js'; +import { + getHostnameFromWebUrl, + getPhishingDetectionScanUrlParam, + normalizeScanAddress, +} from './utils.js'; + +/** + * A single token's scan result as returned by the bulk token scanning + * endpoint. + */ +export type TokenScanResultResponse = TokenScanApiResponse['results'][string]; + +// === GENERAL === + +/** + * The name of the {@link PhishingDataService}, used to namespace the service's + * actions and events. + */ +export const serviceName = 'PhishingDataService'; + +export const PHISHING_CONFIG_BASE_URL = + 'https://phishing-detection.api.cx.metamask.io'; +export const METAMASK_STALELIST_FILE = '/v1/stalelist'; +export const METAMASK_HOTLIST_DIFF_FILE = '/v2/diffsSince'; + +export const CLIENT_SIDE_DETECION_BASE_URL = + 'https://client-side-detection.api.cx.metamask.io'; +export const C2_DOMAIN_BLOCKLIST_ENDPOINT = '/v1/request-blocklist'; + +export const PHISHING_DETECTION_BASE_URL = + 'https://dapp-scanning.api.cx.metamask.io'; +export const PHISHING_DETECTION_SCAN_ENDPOINT = 'v2/scan'; +export const PHISHING_DETECTION_BULK_SCAN_ENDPOINT = 'bulk-scan'; + +export const SECURITY_ALERTS_BASE_URL = + 'https://security-alerts.api.cx.metamask.io'; +export const TOKEN_BULK_SCANNING_ENDPOINT = '/token/scan-bulk'; +export const ADDRESS_SCAN_ENDPOINT = '/address/evm/scan'; +export const APPROVALS_ENDPOINT = '/address/evm/approvals'; + +export const METAMASK_STALELIST_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_STALELIST_FILE}`; +export const METAMASK_HOTLIST_DIFF_URL = `${PHISHING_CONFIG_BASE_URL}${METAMASK_HOTLIST_DIFF_FILE}`; +export const C2_DOMAIN_BLOCKLIST_URL = `${CLIENT_SIDE_DETECION_BASE_URL}${C2_DOMAIN_BLOCKLIST_ENDPOINT}`; + +// Request timeouts, in milliseconds. +export const URL_SCAN_TIMEOUT = 8000; +export const BULK_URL_SCAN_TIMEOUT = 15000; +export const TOKEN_SCAN_TIMEOUT = 8000; +export const ADDRESS_SCAN_TIMEOUT = 5000; +export const APPROVALS_TIMEOUT = 5000; + +/** + * The maximum number of URLs sent to the bulk dapp-scanning endpoint in one + * request. + */ +const MAX_URLS_PER_SCAN_REQUEST = 50; + +/** + * The maximum number of tokens sent to the bulk token scanning endpoint in + * one request. + */ +const MAX_TOKENS_PER_SCAN_REQUEST = 100; + +/** + * How long scan results (URL, bulk URL, token, and address scans) are + * considered fresh by the query cache. Mirrors the 1-minute TTL previously + * enforced by the controller's scan caches; scan verdicts can change quickly, + * so this value is a security parameter and should not be raised casually. + */ +export const SCAN_RESULT_STALE_TIME = inMilliseconds(1, Duration.Minute); + +/** + * How long a scan result is retained by the query cache before it is eligible + * for garbage collection. This is set explicitly because TanStack Query + * defaults `gcTime` to `Infinity` when it detects a server environment, which + * includes the extension's MV3 service worker (`window` is undefined there). + * Without it, the cache would grow without bound for the life of the worker, + * whereas the cache this service replaces was explicitly size-bounded. + */ +export const SCAN_RESULT_GC_TIME = inMilliseconds(5, Duration.Minute); + +/** + * How long a fetched list is retained by the query cache. The lists are always + * refetched (`staleTime: 0`) and the controller keeps its own copy in + * `phishingLists`, so retaining them here has no benefit and a large cost: the + * stalelist is several megabytes, and the persisted cache is rewritten + * whenever any query changes, including on every scan. + */ +const LIST_GC_TIME = 0; + +/** + * Default persistence configuration for the service's query cache. The max + * age matches the longest useful lifetime of any cached entry: scan results + * go stale after {@link SCAN_RESULT_STALE_TIME} and list queries are always + * refetched, so a persisted cache older than this holds nothing usable. + */ +export const DEFAULT_PHISHING_PERSISTENCE_CONFIG: PersistenceConfiguration = { + maxAge: inMilliseconds(5, Duration.Minute), +}; + +// === MESSENGER === + +/** + * All of the methods within {@link PhishingDataService} that are exposed via + * the messenger. + */ +const MESSENGER_EXPOSED_METHODS = [ + 'getStalelist', + 'getHotlistDiffs', + 'getC2DomainBlocklist', + 'scanUrl', + 'bulkScanUrls', + 'scanToken', + 'bulkScanTokens', + 'scanAddress', + 'getApprovals', +] as const; + +/** + * Invalidates cached queries for {@link PhishingDataService}. + */ +export type PhishingDataServiceInvalidateQueriesAction = + DataServiceInvalidateQueriesAction; + +/** + * Actions that {@link PhishingDataService} exposes to other consumers. + */ +export type PhishingDataServiceActions = + | PhishingDataServiceMethodActions + | PhishingDataServiceInvalidateQueriesAction; + +/** + * Actions from other messengers that {@link PhishingDataService} calls. + * The StorageService actions are required for query cache persistence. + */ +type AllowedActions = + | StorageServiceGetItemAction + | StorageServiceSetItemAction + | StorageServiceRemoveItemAction; + +/** + * Published when {@link PhishingDataService}'s cache is updated. + */ +export type PhishingDataServiceCacheUpdatedEvent = DataServiceCacheUpdatedEvent< + typeof serviceName +>; + +/** + * Published when a key within {@link PhishingDataService}'s cache is updated. + */ +export type PhishingDataServiceGranularCacheUpdatedEvent = + DataServiceGranularCacheUpdatedEvent; + +/** + * Events that {@link PhishingDataService} exposes to other consumers. + */ +export type PhishingDataServiceEvents = + | PhishingDataServiceCacheUpdatedEvent + | PhishingDataServiceGranularCacheUpdatedEvent; + +/** + * Events from other messengers that {@link PhishingDataService} subscribes to. + */ +type AllowedEvents = never; + +/** + * The messenger which is restricted to actions and events accessed by + * {@link PhishingDataService}. + */ +export type PhishingDataServiceMessenger = Messenger< + typeof serviceName, + PhishingDataServiceActions | AllowedActions, + PhishingDataServiceEvents | AllowedEvents +>; + +// === RESPONSE VALIDATION === + +const RecommendedActionStruct = union([ + literal(RecommendedAction.None), + literal(RecommendedAction.Warn), + literal(RecommendedAction.Block), + literal(RecommendedAction.Verified), +]); + +const TokenScanResultTypeStruct = union([ + literal(TokenScanResultType.Verified), + literal(TokenScanResultType.Benign), + literal(TokenScanResultType.Warning), + literal(TokenScanResultType.Malicious), + literal(TokenScanResultType.Spam), +]); + +const AddressScanResultTypeStruct = union([ + literal(AddressScanResultType.Verified), + literal(AddressScanResultType.Trusted), + literal(AddressScanResultType.Benign), + literal(AddressScanResultType.Warning), + literal(AddressScanResultType.Malicious), + literal(AddressScanResultType.ErrorResult), + literal(AddressScanResultType.ApiError), +]); + +const ApprovalResultTypeStruct = union([ + literal(ApprovalResultType.Verified), + literal(ApprovalResultType.Trusted), + literal(ApprovalResultType.Benign), + literal(ApprovalResultType.Warning), + literal(ApprovalResultType.Malicious), + literal(ApprovalResultType.ErrorResult), +]); + +const ApprovalFeatureTypeStruct = union([ + literal(ApprovalFeatureType.Benign), + literal(ApprovalFeatureType.Warning), + literal(ApprovalFeatureType.Malicious), + literal(ApprovalFeatureType.Info), +]); + +const StalelistResponseStruct = type({ + data: type({ + allowlist: array(string()), + blocklist: array(string()), + blocklistPaths: array(string()), + fuzzylist: array(string()), + tolerance: number(), + version: number(), + lastUpdated: number(), + }), +}); + +const HotlistDiffsResponseStruct = type({ + data: array( + type({ + url: string(), + timestamp: number(), + // Kept open so that a list added server-side does not reject the whole + // hotlist; `applyDiffs` ignores entries for unknown list types. + targetList: string(), + isRemoval: optional(boolean()), + }), + ), +}); + +const C2DomainBlocklistResponseStruct = type({ + recentlyAdded: array(string()), + recentlyRemoved: array(string()), + // The controller never reads this field, so its absence must not reject + // the whole blocklist response. + lastFetchedAt: optional(number()), +}); + +const ScanUrlResponseStruct = type({ + hostname: optional(string()), + recommendedAction: RecommendedActionStruct, + fetchError: optional(string()), +}); + +// Entries are validated individually by `bulkScanUrls`, so that one malformed +// entry is reported for that URL rather than discarding every verdict. +const BulkScanUrlsResponseStruct = type({ + results: record(string(), unknown()), + errors: optional(record(string(), array(string()))), +}); + +const TokenScanResultStruct = type({ + result_type: TokenScanResultTypeStruct, + chain: optional(string()), + address: optional(string()), +}); + +// Entries are validated individually by the token batch loader, so that one +// malformed entry is reported for that token rather than discarding every +// verdict. +const BulkScanTokensResponseStruct = type({ + results: optional(record(string(), unknown())), +}); + +const ScanAddressResponseStruct = type({ + result_type: AddressScanResultTypeStruct, + label: string(), +}); + +const PersistedScanUrlResultStruct = type({ + hostname: string(), + recommendedAction: RecommendedActionStruct, +}); + +/** + * Decides whether a persisted query may be restored into the cache. Only scan + * results are retained between sessions, and each is checked against the + * shape its endpoint validation produces, so that a corrupted or tampered + * persisted entry can never be served as a verdict. Anything else, including + * list queries, is discarded. + * + * @param query - The persisted query. + * @returns Whether the query may be hydrated. + */ +function isValidPersistedScanQuery( + query: DehydratedState['queries'][number], +): boolean { + const { data } = query.state; + switch (query.queryKey[0]) { + case `${serviceName}:scanUrl`: + return is(data, PersistedScanUrlResultStruct); + case `${serviceName}:scanToken`: + return data === null || is(data, TokenScanResultStruct); + case `${serviceName}:scanAddress`: + return is(data, ScanAddressResponseStruct); + default: + return false; + } +} + +const ApprovalFeatureStruct = type({ + feature_id: string(), + type: ApprovalFeatureTypeStruct, + description: string(), +}); + +const ApprovalStruct = type({ + allowance: type({ + value: optional(string()), + usd_price: optional(string()), + }), + asset: type({ + address: string(), + symbol: string(), + name: string(), + decimals: number(), + logo_url: optional(string()), + type: optional(string()), + }), + exposure: type({ + usd_price: optional(string()), + value: string(), + raw_value: string(), + }), + spender: type({ + address: string(), + label: optional(string()), + features: optional(array(ApprovalFeatureStruct)), + }), + verdict: ApprovalResultTypeStruct, +}); + +// Entries are validated individually by `getApprovals`, so that one malformed +// approval does not empty the whole list. +const ApprovalsResponseStruct = type({ + approvals: array(unknown()), +}); + +// === BATCH LOADING === + +/** + * The outcome of one batched request: results keyed by item, plus any + * per-item errors reported by the endpoint. Items listed in `errors` are + * rejected rather than resolved, so that an endpoint-reported failure is not + * cached as a "no result" verdict. + */ +type BatchOutcome = { + results: Record; + errors: Record; +}; + +/** + * Base class for errors delivered to items of a batched request. The service + * policy never retries these: item queries are backed by a batched request + * that is itself retried as a whole, so retrying items individually would + * only fan one failed batch out into many single-item requests against a host + * that is already failing. + */ +class BatchError extends Error {} + +/** + * An error reported by a batch endpoint for one specific item, as opposed to a + * failure of the request as a whole. These are surfaced to the caller per item + * and never cached, but they do not make the overall call fail. + */ +class BatchItemError extends BatchError { + constructor(message: string) { + super(message); + this.name = 'BatchItemError'; + } +} + +/** + * A failure of a batched request as a whole, delivered to every item in it. + * The original request error is available as `cause` and is what callers + * ultimately receive. + */ +class BatchRequestError extends BatchError { + constructor(cause: unknown) { + super(getErrorMessage(cause), { cause }); + this.name = 'BatchRequestError'; + } +} + +/** + * Returns the request error behind a batch failure, or the error itself. + * + * @param error - An error rejected by a batch item. + * @returns The underlying error. + */ +function unwrapBatchError(error: unknown): unknown { + return error instanceof BatchRequestError ? error.cause : error; +} + +type BatchLoader = { + /** + * Registers an item to be resolved by the next executed batch. + * + * @param key - The item key, as understood by the batch endpoint. + * @returns The item's result, or `null` if the batch response did not + * include it. + */ + load: (key: string) => Promise; +}; + +/** + * Creates a loader that coalesces individual item lookups into batched + * requests. Every lookup made in the same turn of the event loop is sent in + * one batch (split into requests of up to `maxBatchSize`). This preserves the + * per-item caching granularity of the query cache while keeping the batched + * network behavior of the bulk endpoints. + * + * @param options - The loader options. + * @param options.maxBatchSize - The maximum number of items per request. + * @param options.executeBatch - Executes one batched request, returning + * results keyed by item. + * @returns The batch loader. + */ +function createBatchLoader({ + maxBatchSize, + executeBatch, +}: { + maxBatchSize: number; + executeBatch: (keys: string[]) => Promise; +}): BatchLoader { + type PendingItem = { + key: string; + resolve: (value: Json | null) => void; + reject: (error: unknown) => void; + }; + let pending: PendingItem[] = []; + let flushScheduled = false; + + const executeChunk = async (chunk: PendingItem[]): Promise => { + try { + const { results, errors } = await executeBatch( + chunk.map((item) => item.key), + ); + for (const item of chunk) { + const itemError = errors[item.key]; + if (itemError) { + item.reject(itemError); + } else { + item.resolve(results[item.key] ?? null); + } + } + } catch (error) { + for (const item of chunk) { + item.reject(new BatchRequestError(error)); + } + } + }; + + const flush = (): void => { + flushScheduled = false; + const batch = pending; + pending = []; + for (let index = 0; index < batch.length; index += maxBatchSize) { + // Errors are routed to the chunk's items, so this promise never + // rejects. + executeChunk(batch.slice(index, index + maxBatchSize)).catch( + /* istanbul ignore next */ + () => undefined, + ); + } + }; + + return { + async load(key: string): Promise { + return new Promise((resolve, reject) => { + pending.push({ key, resolve, reject }); + // The first lookup in a turn schedules the flush as a microtask, which + // runs once the current synchronous work has finished, so every other + // lookup made in the same turn (for example all the item queries of + // one bulk call, whether they start synchronously or after awaiting + // cache rehydration) joins the same batch. + if (!flushScheduled) { + flushScheduled = true; + queueMicrotask(flush); + } + }); + }, + }; +} + +// === SERVICE DEFINITION === + +/** + * This service is responsible for all network requests made on behalf of + * `PhishingController`: fetching the phishing configuration lists (stalelist, + * hotlist diffs, and C2 domain blocklist) and calling the dapp-scanning and + * security-alerts APIs (URL, token, and address scans). + * + * Scan results are cached by the underlying query cache for + * {@link SCAN_RESULT_STALE_TIME} and persisted between sessions when + * `persistenceConfig` is enabled (the default), which requires the + * `StorageService:getItem`, `StorageService:setItem`, and + * `StorageService:removeItem` messenger actions to be delegated to this + * service's messenger, plus a call to `init` during client initialization. + * + * List queries are always refetched when requested; the controller remains + * responsible for deciding when the lists are out of date. + * + * Note that a single retry/circuit-breaker policy is shared across all + * endpoints of this service. The policy only counts consecutive failures, so + * an outage of one API is unlikely to pause requests to the others unless + * failures arrive without any interleaved successes. + */ +export class PhishingDataService extends BaseDataService< + typeof serviceName, + PhishingDataServiceMessenger +> { + readonly #abortController = new AbortController(); + + /** + * Constructs a new PhishingDataService object. + * + * @param args - The constructor arguments. + * @param args.messenger - The messenger suited for this service. + * @param args.queryClientConfig - Configuration for the underlying TanStack + * Query client. + * @param args.policyOptions - Options to pass to `createServicePolicy`, + * which is used to wrap each request. See + * {@link CreateServicePolicyOptions}. + * @param args.persistenceConfig - Configuration for persisting the query + * cache between sessions. Defaults to + * {@link DEFAULT_PHISHING_PERSISTENCE_CONFIG}; pass `null` to disable + * persistence. + */ + constructor({ + messenger, + queryClientConfig = {}, + policyOptions = {}, + persistenceConfig = DEFAULT_PHISHING_PERSISTENCE_CONFIG, + }: { + messenger: PhishingDataServiceMessenger; + queryClientConfig?: QueryClientConfig; + policyOptions?: CreateServicePolicyOptions; + persistenceConfig?: PersistenceConfiguration | null; + }) { + const { retryFilterPolicy, ...restPolicyOptions } = policyOptions; + super({ + name: serviceName, + messenger, + queryClientConfig: { + ...queryClientConfig, + defaultOptions: { + ...queryClientConfig.defaultOptions, + queries: { + ...queryClientConfig.defaultOptions?.queries, + // Hydration reconstructs queries using these defaults. Without an + // explicit value, service workers receive TanStack's server + // default of `Infinity`, which cannot later be reduced by a + // per-query option. + gcTime: SCAN_RESULT_GC_TIME, + }, + }, + }, + // Circuit breaking is disabled by default: this service talks to four + // independent API hosts through a single shared policy, so a broken + // circuit caused by one host's outage would also pause phishing-list + // updates from the others. Protection against hammering a failing host + // comes from the controller's refresh-interval bookkeeping and the scan + // result stale times, matching the previous in-controller behavior. + // + // Retries are disabled by default for the same reason. The previous + // in-controller implementation made a single request per call, and the + // controller's timeouts are sized for one attempt. When enabled, retries + // apply to each batched request as a whole; see `retryFilterPolicy`. + policyOptions: { + maxConsecutiveFailures: Number.MAX_SAFE_INTEGER, + maxRetries: 0, + ...restPolicyOptions, + // Item queries backed by a batched request are never retried on their + // own: the batch is retried as a whole inside the loader, and an item + // retry would only fan one failed batch out into many single-item + // requests against a host that is already failing. A caller-provided + // filter is still applied to everything else. + retryFilterPolicy: handleWhen( + (error): boolean => + !(error instanceof BatchError) && + (retryFilterPolicy?.options.errorFilter(error) ?? true), + ), + }, + persistenceConfig: persistenceConfig + ? { + ...persistenceConfig, + // Persisted scan results are validated before they can be served + // from the cache; a caller-provided filter is applied on top. + shouldHydrateQuery: (query): boolean => + isValidPersistedScanQuery(query) && + (persistenceConfig.shouldHydrateQuery?.(query) ?? true), + } + : undefined, + }); + + this.messenger.registerMethodActionHandlers( + this, + MESSENGER_EXPOSED_METHODS, + ); + } + + /** + * Aborts all requests owned by this service before clearing its query cache + * and messenger registrations. + */ + override destroy(): void { + this.#abortController.abort(); + super.destroy(); + } + + /** + * Fetches the full phishing detection stalelist. + * + * @returns The stalelist response. + */ + async getStalelist(): Promise> { + // Validated inside the query function so that a malformed response is + // never committed to, or persisted from, the query cache. The struct has + // no optional fields, so its inferred type satisfies `fetchQuery`'s `Json` + // bound and types the result without a cast. + return await this.fetchQuery({ + queryKey: [`${this.name}:getStalelist`], + queryFn: async ({ signal }) => + this.#validate( + await this.#getJson(METAMASK_STALELIST_URL, { signal }), + StalelistResponseStruct, + 'stalelist', + ), + staleTime: 0, + gcTime: LIST_GC_TIME, + }); + } + + /** + * Fetches the hotlist diffs recorded since the given timestamp. + * + * @param timestamp - The timestamp (in seconds) to fetch diffs since. + * @returns The hotlist diffs response. + */ + async getHotlistDiffs( + timestamp: number, + ): Promise> { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getHotlistDiffs`, timestamp], + queryFn: async ({ signal }) => + this.#validate( + await this.#getJson(`${METAMASK_HOTLIST_DIFF_URL}/${timestamp}`, { + signal, + }), + HotlistDiffsResponseStruct, + 'hotlist diffs', + ) as Json, + staleTime: 0, + gcTime: LIST_GC_TIME, + }); + + return jsonResponse as DataResultWrapper; + } + + /** + * Fetches the C2 domain blocklist changes recorded since the given + * timestamp, or the current blocklist if no timestamp is given. + * + * @param timestamp - The timestamp (in seconds) to fetch changes since. + * @returns The C2 domain blocklist response. + */ + async getC2DomainBlocklist( + timestamp?: number, + ): Promise { + const url = + timestamp === undefined + ? C2_DOMAIN_BLOCKLIST_URL + : `${C2_DOMAIN_BLOCKLIST_URL}?timestamp=${timestamp}`; + + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getC2DomainBlocklist`, timestamp ?? null], + queryFn: async ({ signal }) => + this.#validate( + await this.#getJson(url, { signal }), + C2DomainBlocklistResponseStruct, + 'C2 domain blocklist', + ) as Json, + staleTime: 0, + gcTime: LIST_GC_TIME, + }); + + return jsonResponse as C2DomainBlocklistResponse; + } + + /** + * Scans a URL for phishing via the dapp-scanning API. + * + * @param url - The prepared URL parameter to scan (hostname, or hostname + * plus path for shared gateways). + * @returns The phishing detection scan result. + */ + async scanUrl(url: string): Promise { + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:scanUrl`, url], + queryFn: async ({ signal }) => { + const response = await this.#fetchJson( + `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_SCAN_ENDPOINT}?url=${encodeURIComponent(url)}`, + { + method: 'GET', + headers: { + Accept: 'application/json', + }, + signal, + }, + URL_SCAN_TIMEOUT, + ); + const scanResult = this.#validate( + response, + ScanUrlResponseStruct, + 'URL scan', + ); + if (scanResult.fetchError) { + throw new Error(scanResult.fetchError); + } + const [hostname] = getHostnameFromWebUrl(`https://${url}`); + return { + ...scanResult, + hostname: scanResult.hostname ?? hostname, + } as Json; + }, + staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, + }); + + return jsonResponse as PhishingDetectionScanResult; + } + + /** + * Scans a batch of URLs for phishing via the dapp-scanning API. + * + * Results are cached under the same query keys as + * {@link PhishingDataService.scanUrl}, so results are shared between single + * and bulk scans, including for the path-sensitive hosts listed in + * `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`. Only URLs without a fresh + * cached result are sent to the API, in requests of up to 50 URLs. + * + * If some lookups fail, the results that did resolve are still returned and + * the failures are reported per URL. The call only rejects when nothing at + * all could be resolved. + * + * @param urls - The URLs to scan. + * @returns The scan results, keyed by URL, and any per-URL errors. + */ + async bulkScanUrls( + urls: string[], + ): Promise { + const errors: Record = {}; + const addError = (key: string, message: string): void => { + errors[key] = [...(errors[key] ?? []), message]; + }; + + const loader = createBatchLoader({ + maxBatchSize: MAX_URLS_PER_SCAN_REQUEST, + executeBatch: async (batchUrls) => { + const jsonResponse = await this.policy.execute(() => + this.#postJson( + `${PHISHING_DETECTION_BASE_URL}/${PHISHING_DETECTION_BULK_SCAN_ENDPOINT}`, + { urls: batchUrls }, + { timeout: BULK_URL_SCAN_TIMEOUT }, + ), + ); + const response = this.#validate( + jsonResponse, + BulkScanUrlsResponseStruct, + 'bulk URL scan', + ); + // URLs the endpoint reported an error for are rejected rather than + // resolved, so that the failure is surfaced to the caller instead of + // being cached as a "no result" verdict for the stale time. + const itemErrors: Record = {}; + for (const [key, messages] of Object.entries(response.errors ?? {})) { + itemErrors[key] = new BatchItemError(messages.join(', ')); + } + const results: Record = {}; + for (const [key, result] of Object.entries(response.results)) { + if (!is(result, ScanUrlResponseStruct)) { + itemErrors[key] = new BatchItemError( + 'Malformed result returned by bulk URL scan endpoint', + ); + } else if (result.fetchError) { + itemErrors[key] = new BatchItemError(result.fetchError); + } else { + // Entries are shared with `scanUrl`, whose results always carry a + // hostname, so fill it in before the entry reaches the cache. + const [hostname] = getHostnameFromWebUrl(key); + results[key] = { + ...result, + hostname: result.hostname ?? hostname, + } as Json; + } + } + for (const url of batchUrls) { + if (!Object.hasOwn(results, url) && !Object.hasOwn(itemErrors, url)) { + itemErrors[url] = new BatchItemError( + 'No result returned by bulk URL scan endpoint', + ); + } + } + return { + results, + errors: itemErrors, + }; + }, + }); + + const requested: string[] = []; + const entries: Promise[] = []; + for (const url of urls) { + const [scanUrlParam, ok] = getPhishingDetectionScanUrlParam(url); + if (!ok) { + addError(url, 'url is not a valid web URL'); + continue; + } + requested.push(url); + entries.push( + // Keyed by the scan parameter rather than the bare hostname so that + // path-sensitive hosts (see + // `PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS`) get one entry per + // path, and so that entries are shared with `scanUrl`. + this.fetchQuery({ + queryKey: [`${this.name}:scanUrl`, scanUrlParam], + queryFn: async () => loader.load(url), + staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, + }), + ); + } + + const settled = await Promise.allSettled(entries); + const results: Record = {}; + let requestFailure: { reason: unknown } | undefined; + + for (const [index, outcome] of settled.entries()) { + const url = requested[index]; + + if (outcome.status === 'rejected') { + const reason = unwrapBatchError(outcome.reason); + addError(url, getErrorMessage(reason)); + if (!(outcome.reason instanceof BatchItemError)) { + requestFailure ??= { reason }; + } + continue; + } + + results[url] = outcome.value as PhishingDetectionScanResult; + } + + // A request-level failure that produced nothing at all is surfaced to the + // caller, matching the previous behavior. If anything did resolve, keep + // it, including fresh cache hits, so that one failed lookup cannot + // discard a cached BLOCK verdict for a different URL. + if (requestFailure && Object.keys(results).length === 0) { + throw requestFailure.reason; + } + + return { results, errors }; + } + + /** + * Scans a token for malicious activity via the security-alerts API. + * + * Each call issues its own request to the bulk scanning endpoint; use + * {@link PhishingDataService.bulkScanTokens} to scan several tokens in one + * request. EVM token addresses are lowercased before being used as the cache + * key and sent to the API; other addresses are used as given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result + * for the token. + */ + async scanToken( + chain: string, + token: string, + ): Promise { + const loader = this.#createTokenScanLoader(chain); + const result = this.#fetchTokenScanQuery( + loader, + chain, + normalizeScanAddress(token), + ); + return await result; + } + + /** + * Scans a batch of tokens for malicious activity via the security-alerts + * API. + * + * Results are cached per token; only tokens without a fresh cached result + * are sent to the API, in requests of up to 100 tokens. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param tokens - The token addresses to scan. + * @returns The token scan results, keyed by normalized token address (EVM + * addresses are lowercased). Tokens for which the API returned no result + * are omitted. + */ + async bulkScanTokens( + chain: string, + tokens: string[], + ): Promise { + const loader = this.#createTokenScanLoader(chain); + const entries = tokens.map((token) => { + const normalizedToken = normalizeScanAddress(token); + return this.#fetchTokenScanQuery(loader, chain, normalizedToken).then( + (result) => [normalizedToken, result] as const, + ); + }); + + const results: TokenScanApiResponse['results'] = {}; + let firstError: Error | undefined; + for (const outcome of await Promise.allSettled(entries)) { + if (outcome.status === 'rejected') { + firstError ??= unwrapBatchError(outcome.reason) as Error; + continue; + } + + const [token, result] = outcome.value; + if (result !== null) { + results[token] = result; + } + } + + if (Object.keys(results).length === 0 && firstError !== undefined) { + throw firstError; + } + + return { results }; + } + + /** + * Creates a batch loader that resolves token scans through the bulk + * scanning endpoint. + * + * @param chain - The chain name (e.g. `ethereum`). + * @returns The batch loader. + */ + #createTokenScanLoader(chain: string): BatchLoader { + return createBatchLoader({ + maxBatchSize: MAX_TOKENS_PER_SCAN_REQUEST, + executeBatch: async (batchTokens) => { + const jsonResponse = await this.policy.execute(() => + this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${TOKEN_BULK_SCANNING_ENDPOINT}`, + { chain, tokens: batchTokens }, + { timeout: TOKEN_SCAN_TIMEOUT }, + ), + ); + const response = this.#validate( + jsonResponse, + BulkScanTokensResponseStruct, + 'bulk token scan', + ); + const results: Record = {}; + const errors: Record = {}; + for (const [key, result] of Object.entries(response.results ?? {})) { + if (is(result, TokenScanResultStruct)) { + results[key] = result as Json; + } else { + errors[key] = new BatchItemError( + 'Malformed result returned by bulk token scan endpoint', + ); + } + } + return { results, errors }; + }, + }); + } + + /** + * Fetches a single token scan query backed by the given batch loader. + * + * @param loader - The batch loader used to resolve cache misses. + * @param chain - The chain name (e.g. `ethereum`). + * @param token - The token address to scan. + * @returns The token scan result, or `null` if the API returned no result. + */ + async #fetchTokenScanQuery( + loader: BatchLoader, + chain: string, + token: string, + ): Promise { + const result = await this.fetchQuery({ + queryKey: [`${this.name}:scanToken`, chain, token], + queryFn: async () => loader.load(token), + staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, + }); + return result as TokenScanResultResponse | null; + } + + /** + * Scans an address for security alerts via the security-alerts API. EVM + * addresses are lowercased before being used as the cache key and sent to + * the API; other addresses are used as given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to scan. + * @returns The address scan result. + */ + async scanAddress( + chain: string, + address: string, + ): Promise { + const normalizedAddress = normalizeScanAddress(address); + return await this.fetchQuery({ + queryKey: [`${this.name}:scanAddress`, chain, normalizedAddress], + queryFn: async ({ signal }) => + this.#validate( + await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${ADDRESS_SCAN_ENDPOINT}`, + { chain, address: normalizedAddress }, + { signal, timeout: ADDRESS_SCAN_TIMEOUT }, + ), + ScanAddressResponseStruct, + 'address scan', + ), + staleTime: SCAN_RESULT_STALE_TIME, + gcTime: SCAN_RESULT_GC_TIME, + }); + } + + /** + * Gets token approvals for an address with security enrichments via the + * security-alerts API. Approvals reflect live account state, so they are + * always refetched and never retained in the query cache. EVM addresses are + * lowercased before being sent to the API; other addresses are used as + * given. + * + * @param chain - The chain name (e.g. `ethereum`). + * @param address - The address to get approvals for. + * @returns The approvals response. + */ + async getApprovals( + chain: string, + address: string, + ): Promise { + const normalizedAddress = normalizeScanAddress(address); + const jsonResponse = await this.fetchQuery({ + queryKey: [`${this.name}:getApprovals`, chain, normalizedAddress], + queryFn: async ({ signal }) => { + const response = this.#validate( + await this.#postJson( + `${SECURITY_ALERTS_BASE_URL}${APPROVALS_ENDPOINT}`, + { chain, address: normalizedAddress }, + { signal, timeout: APPROVALS_TIMEOUT }, + ), + ApprovalsResponseStruct, + 'approvals', + ); + // Approvals are validated individually so that one malformed entry + // does not empty the whole list. + return { + approvals: response.approvals.filter((approval) => + is(approval, ApprovalStruct), + ), + } as Json; + }, + // Live account state: always refetch and evict as soon as the call + // settles, as other data services do for uncached reads. + staleTime: 0, + gcTime: 0, + }); + + return jsonResponse as ApprovalsResponse; + } + + /** + * Performs a GET request against a phishing configuration endpoint. + * + * @param url - The URL to fetch. + * @param options - Request cancellation options. + * @param options.signal - A signal that cancels the request. + * @returns The parsed JSON response. + */ + async #getJson( + url: string, + { signal }: { signal?: AbortSignal }, + ): Promise { + return this.#fetchJson(url, { cache: 'no-cache', signal }); + } + + /** + * Performs a POST request with a JSON body. + * + * @param url - The URL to fetch. + * @param body - The request body, serialized as JSON. + * @param options - Request cancellation options. + * @param options.signal - A signal that cancels the request. + * @param options.timeout - The request timeout, in milliseconds. + * @returns The parsed JSON response. + */ + async #postJson( + url: string, + body: Record, + { signal, timeout }: { signal?: AbortSignal; timeout?: number }, + ): Promise { + return this.#fetchJson( + url, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, + }, + timeout, + ); + } + + /** + * Performs a fetch request and parses its JSON response. + * + * @param url - The URL to fetch. + * @param init - The fetch request options. + * @param timeout - The optional request timeout, in milliseconds. + * @returns The parsed JSON response. + */ + async #fetchJson( + url: string, + init: RequestInit, + timeout?: number, + ): Promise { + // `fetch` takes a single signal, so each request gets its own controller + // that funnels up to three cancellation sources into one: the query's own + // signal from TanStack (single-item requests only; batched requests are + // not tied to one query), the service-wide signal that `destroy` aborts, + // and this request's timeout. The timeout is tracked separately so that it can be + // reported as one. `AbortSignal.any` and `AbortSignal.timeout` would + // express this directly but are not reliably available on React Native. + const controller = new AbortController(); + const sourceSignal = init.signal; + const serviceSignal = this.#abortController.signal; + let didTimeout = false; + const abort = (): void => controller.abort(); + const timer = + timeout === undefined + ? undefined + : setTimeout(() => { + didTimeout = true; + controller.abort(); + }, timeout); + + sourceSignal?.addEventListener('abort', abort, { once: true }); + serviceSignal.addEventListener('abort', abort, { once: true }); + /* istanbul ignore next -- service actions are removed during destruction */ + if (sourceSignal?.aborted || serviceSignal.aborted) { + controller.abort(); + } + + try { + const response = await fetch(url, { + ...init, + signal: controller.signal, + }); + return await this.#toJson(response); + } catch (error) { + if (didTimeout) { + throw new Error(`timeout of ${timeout}ms exceeded`, { cause: error }); + } + throw error; + } finally { + clearTimeout(timer); + sourceSignal?.removeEventListener('abort', abort); + serviceSignal.removeEventListener('abort', abort); + } + } + + /** + * Parses a response as JSON, throwing an {@link HttpError} for non-2xx + * responses. The error message intentionally matches the + * ` ` format historically produced by + * `PhishingController` so that consumers relying on it keep working. + * + * @param response - The response to parse. + * @returns The parsed JSON response. + */ + async #toJson(response: Response): Promise { + if (!response.ok) { + throw new HttpError( + response.status, + `${response.status} ${response.statusText}`, + ); + } + return response.json(); + } + + /** + * Validates a response against a struct, throwing if it is malformed. + * + * @param response - The response to validate. + * @param struct - The struct to validate against. + * @param endpointName - The name of the endpoint, used in error messages. + * @returns The validated response. + */ + #validate( + response: unknown, + struct: Struct, + endpointName: string, + ): Infer> { + if (!is(response, struct)) { + throw new Error( + `Malformed response received from ${endpointName} endpoint`, + ); + } + return response; + } +} diff --git a/packages/phishing-controller/src/index.ts b/packages/phishing-controller/src/index.ts index 5a656acf38c..0826113ee76 100644 --- a/packages/phishing-controller/src/index.ts +++ b/packages/phishing-controller/src/index.ts @@ -31,12 +31,12 @@ export { ApprovalResultType, ApprovalFeatureType, } from './types.js'; -export type { CacheEntry } from './CacheManager.js'; export { PHISHING_DETECTION_PATH_BASED_ROOT_DOMAINS, getPhishingDetectionScanUrlParam, isAddressScanSupportedChainId, isPhishingDetectionPathBasedHostname, + resolveChainName, } from './utils.js'; export { extractSignatureAddresses, @@ -60,3 +60,30 @@ export type { PhishingControllerGetApprovalsAction, PhishingControllerCheckAddressPoisoningAction, } from './PhishingController-method-action-types.js'; + +export { + PhishingDataService, + SCAN_RESULT_GC_TIME, + SCAN_RESULT_STALE_TIME, + DEFAULT_PHISHING_PERSISTENCE_CONFIG, +} from './PhishingDataService.js'; +export type { TokenScanResultResponse } from './PhishingDataService.js'; +export type { + PhishingDataServiceActions, + PhishingDataServiceEvents, + PhishingDataServiceMessenger, + PhishingDataServiceInvalidateQueriesAction, + PhishingDataServiceCacheUpdatedEvent, + PhishingDataServiceGranularCacheUpdatedEvent, +} from './PhishingDataService.js'; +export type { + PhishingDataServiceGetStalelistAction, + PhishingDataServiceGetHotlistDiffsAction, + PhishingDataServiceGetC2DomainBlocklistAction, + PhishingDataServiceScanUrlAction, + PhishingDataServiceBulkScanUrlsAction, + PhishingDataServiceScanTokenAction, + PhishingDataServiceBulkScanTokensAction, + PhishingDataServiceScanAddressAction, + PhishingDataServiceGetApprovalsAction, +} from './PhishingDataService-method-action-types.js'; diff --git a/packages/phishing-controller/src/types.ts b/packages/phishing-controller/src/types.ts index 44a9f677d02..e4dff45a0ff 100644 --- a/packages/phishing-controller/src/types.ts +++ b/packages/phishing-controller/src/types.ts @@ -1,4 +1,178 @@ /* eslint-disable @typescript-eslint/naming-convention */ +import type { PathTrie } from './PathTrie.js'; + +/** + * @type ListTypes + * + * Type outlining the types of lists provided by aggregating different source lists + */ +export type ListTypes = + | 'fuzzylist' + | 'blocklist' + | 'blocklistPaths' + | 'allowlist' + | 'c2DomainBlocklist'; + +/** + * @type EthPhishingResponse + * + * Configuration response from the eth-phishing-detect package + * consisting of approved and unapproved website origins + * + * @property blacklist - List of unapproved origins + * @property fuzzylist - List of fuzzy-matched unapproved origins + * @property tolerance - Fuzzy match tolerance level + * @property version - Version number of this configuration + * @property whitelist - List of approved origins + */ +export type EthPhishingResponse = { + blacklist: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + whitelist: string[]; +}; + +/** + * @type C2DomainBlocklistResponse + * + * Response for blocklist update requests + * + * @property recentlyAdded - List of c2 domains recently added to the blocklist + * @property recentlyRemoved - List of c2 domains recently removed from the blocklist + * @property lastFetchedAt - Unix timestamp, in seconds, of the last fetch request. Not read by the controller. + */ +export type C2DomainBlocklistResponse = { + recentlyAdded: string[]; + recentlyRemoved: string[]; + lastFetchedAt?: number; +}; + +/** + * PhishingStalelist defines the expected type of the stalelist from the API. + * + * allowlist - List of approved origins. + * blocklist - List of unapproved origins (hostname-only entries). + * blocklistPaths - Trie of unapproved origins with paths (hostname + path entries). + * fuzzylist - List of fuzzy-matched unapproved origins. + * tolerance - Fuzzy match tolerance level + * lastUpdated - Timestamp of last update. + * version - Stalelist data structure iteration. + */ +export type PhishingStalelist = { + allowlist: string[]; + blocklist: string[]; + blocklistPaths: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + lastUpdated: number; +}; + +/** + * @type PhishingListState + * + * type defining the persisted list state. This is the persisted state that is updated frequently with `this.maybeUpdateState()`. + * + * @property allowlist - List of approved origins (legacy naming "whitelist") + * @property blocklist - List of unapproved origins (legacy naming "blacklist") + * @property blocklistPaths - Trie of unapproved origins with paths (hostname + path, no query params). + * @property c2DomainBlocklist - List of hashed hostnames that C2 requests are blocked against. + * @property fuzzylist - List of fuzzy-matched unapproved origins + * @property tolerance - Fuzzy match tolerance level + * @property lastUpdated - Timestamp of last update. + * @property version - Version of the phishing list state. + * @property name - Name of the list. Used for attribution. + */ +export type PhishingListState = { + allowlist: string[]; + blocklist: string[]; + blocklistPaths: PathTrie; + c2DomainBlocklist: string[]; + fuzzylist: string[]; + tolerance: number; + version: number; + lastUpdated: number; + name: ListNames; +}; + +/** + * @type HotlistDiff + * + * type defining the expected type of the diffs in hotlist.json file. + * + * @property url - Url of the diff entry. + * @property timestamp - Timestamp at which the diff was identified. + * @property targetList - The list name where the diff was identified. + * @property isRemoval - Was the diff identified a removal type. + */ +export type HotlistDiff = { + url: string; + timestamp: number; + targetList: `${ListKeys}.${ListTypes}`; + isRemoval?: boolean; +}; + +export type DataResultWrapper = { + data: T; +}; + +/** + * @type Hotlist + * + * Type defining expected hotlist.json file. + * + * @property url - Url of the diff entry. + * @property timestamp - Timestamp at which the diff was identified. + * @property targetList - The list name where the diff was identified. + * @property isRemoval - Was the diff identified a removal type. + */ +export type Hotlist = HotlistDiff[]; + +/** + * Enum containing upstream data provider source list keys. + * These are the keys denoting lists consumed by the upstream data provider. + */ +export enum ListKeys { + EthPhishingDetectConfig = 'eth_phishing_detect_config', +} + +/** + * Enum containing downstream client attribution names. + */ +export enum ListNames { + MetaMask = 'MetaMask', +} + +/** + * Maps from downstream client attribution name + * to list key sourced from upstream data provider. + */ +export const phishingListNameKeyMap = { + [ListNames.MetaMask]: ListKeys.EthPhishingDetectConfig, +}; + +/** + * Maps from list key sourced from upstream data + * provider to downstream client attribution name. + */ +export const phishingListKeyNameMap = { + [ListKeys.EthPhishingDetectConfig]: ListNames.MetaMask, +}; + +/** + * BulkPhishingDetectionScanResponse + * + * Response for bulk phishing detection scan requests + * results - Record of domain names and their corresponding phishing detection scan results + * + * errors - Record of domain names and their corresponding errors + */ +export type BulkPhishingDetectionScanResponse = { + results: Record; + errors: Record; +}; + /** * Represents the result of checking a domain. */ @@ -136,6 +310,7 @@ export type BulkTokenScanRequest = { * Result type of a token scan */ export enum TokenScanResultType { + Verified = 'Verified', Benign = 'Benign', Warning = 'Warning', Malicious = 'Malicious', @@ -243,6 +418,14 @@ export type ChainIdToNameMap = typeof DEFAULT_CHAIN_ID_TO_NAME; * Result type of an address scan */ export enum AddressScanResultType { + /** + * Address is verified by internal trust signals + */ + Verified = 'Verified', + /** + * Address is trusted by internal trust signals + */ + Trusted = 'Trusted', /** * Address is benign/safe */ @@ -259,6 +442,10 @@ export enum AddressScanResultType { * Error occurred during scan */ ErrorResult = 'ErrorResult', + /** + * Error returned by the security alerts API + */ + ApiError = 'Error', } /** @@ -437,6 +624,8 @@ export enum ApprovalResultType { Malicious = 'Malicious', Warning = 'Warning', Benign = 'Benign', + Trusted = 'Trusted', + Verified = 'Verified', ErrorResult = 'Error', } diff --git a/packages/phishing-controller/src/utils.test.ts b/packages/phishing-controller/src/utils.test.ts index 873968e6284..97d7ae4942d 100644 --- a/packages/phishing-controller/src/utils.test.ts +++ b/packages/phishing-controller/src/utils.test.ts @@ -1,9 +1,8 @@ import { ListKeys, ListNames } from './PhishingController.js'; import type { PhishingListState } from './PhishingController.js'; -import type { TokenScanResultType } from './types.js'; +import type { Hotlist } from './types.js'; import { applyDiffs, - buildCacheKey, domainToParts, fetchTimeNow, generateParentDomains, @@ -17,12 +16,12 @@ import { isPhishingDetectionPathBasedHostname, isTokenScanSupportedChain, matchPartsAgainstList, + normalizeScanAddress, processConfigs, processDomainList, resolveChainName, roundToNearestMinute, sha256Hash, - splitCacheHits, validateConfig, } from './utils.js'; @@ -209,6 +208,24 @@ describe('applyDiffs', () => { name: ListNames.MetaMask, }); }); + + it('ignores diffs that target an unrecognized list type without advancing lastUpdated', () => { + const unknownTypeDiff = { + targetList: 'eth_phishing_detect_config.newlist', + url: 'https://example-new-list-item.com', + timestamp: exampleAddDiff.timestamp + 10, + } as unknown as Hotlist[number]; + const result = applyDiffs( + exampleListState, + [exampleAddDiff, unknownTypeDiff], + ListKeys.EthPhishingDetectConfig, + ); + expect(result).toStrictEqual({ + ...exampleListState, + blocklist: [...exampleListState.blocklist, exampleBlockedUrlTwo], + lastUpdated: exampleAddDiff.timestamp, + }); + }); // New tests for handling C2 domain blocklist it('should add hashes to the current C2 domain blocklist', () => { exampleListState.c2DomainBlocklist = ['hash1', 'hash2']; @@ -1185,43 +1202,6 @@ describe('generateParentDomains', () => { }); }); -describe('buildCacheKey', () => { - it('should create cache key with lowercase chainId and address', () => { - const chainId = '0x1'; - const address = '0x1234ABCD'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0x1:0x1234abcd'); - }); - - it('should handle already lowercase inputs', () => { - const chainId = '0xa'; - const address = '0xdeadbeef'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0xa:0xdeadbeef'); - }); - - it('should handle mixed case inputs', () => { - const chainId = '0X89'; - const address = '0XaBcDeF123456'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('0x89:0xabcdef123456'); - }); - - it('should preserve address casing when caseSensitive is true', () => { - const chainId = 'solana'; - const address = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - const result = buildCacheKey(chainId, address, true); - expect(result).toBe('solana:Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'); - }); - - it('should lowercase address when caseSensitive is false (default)', () => { - const chainId = 'solana'; - const address = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - const result = buildCacheKey(chainId, address); - expect(result).toBe('solana:gh9zwemdlj8dsckntktqpbnwlnnbjuszag9vp2kgtkjr'); - }); -}); - describe('resolveChainName', () => { it('should resolve known chain IDs to chain names', () => { expect(resolveChainName('0x1')).toBe('ethereum'); @@ -1337,133 +1317,6 @@ describe('isAddressScanSupportedChainId', () => { }); }); -describe('splitCacheHits', () => { - const mockCache = { - get: jest.fn(), - }; - - beforeEach(() => { - mockCache.get.mockClear(); - }); - - it('should split tokens correctly when some are cached', () => { - const chainId = '0x1'; - const tokens = ['0xTOKEN1', '0xTOKEN2', '0xTOKEN3']; - - // Mock cache to return data for token1 only - const mockResponses = new Map([ - ['0x1:0xtoken1', { result_type: 'Benign' as TokenScanResultType }], - ]); - mockCache.get.mockImplementation((key: string) => mockResponses.get(key)); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({ - '0xtoken1': { - result_type: 'Benign', - chain: '0x1', - address: '0xtoken1', - }, - }); - expect(result.tokensToFetch).toStrictEqual(['0xtoken2', '0xtoken3']); - }); - - it('should handle all tokens being cached', () => { - const chainId = '0x89'; - const tokens = ['0xTOKEN1', '0xTOKEN2']; - - mockCache.get.mockReturnValue({ - result_type: 'Warning' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({ - '0xtoken1': { - result_type: 'Warning', - chain: '0x89', - address: '0xtoken1', - }, - '0xtoken2': { - result_type: 'Warning', - chain: '0x89', - address: '0xtoken2', - }, - }); - expect(result.tokensToFetch).toStrictEqual([]); - }); - - it('should handle no tokens being cached', () => { - const chainId = '0xa'; - const tokens = ['0xTOKEN1', '0xTOKEN2']; - - mockCache.get.mockReturnValue(undefined); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({}); - expect(result.tokensToFetch).toStrictEqual(['0xtoken1', '0xtoken2']); - }); - - it('should handle empty token list', () => { - const chainId = '0x1'; - const tokens: string[] = []; - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(result.cachedResults).toStrictEqual({}); - expect(result.tokensToFetch).toStrictEqual([]); - expect(mockCache.get).not.toHaveBeenCalled(); - }); - - it('should normalize addresses to lowercase', () => { - const chainId = '0X1'; - const tokens = ['0XTOKEN1']; - - mockCache.get.mockReturnValue({ - result_type: 'Malicious' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, tokens); - - expect(mockCache.get).toHaveBeenCalledWith('0x1:0xtoken1'); - expect(result.cachedResults).toHaveProperty('0xtoken1'); - expect(result.cachedResults['0xtoken1'].address).toBe('0xtoken1'); - }); - - it('should preserve address casing when caseSensitive is true', () => { - const chainId = 'solana'; - const tokens = ['Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr']; - - mockCache.get.mockReturnValue(undefined); - - const result = splitCacheHits(mockCache, chainId, tokens, true); - - // tokensToFetch should preserve original casing - expect(result.tokensToFetch).toStrictEqual([ - 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr', - ]); - }); - - it('should return cached result with preserved casing when caseSensitive is true', () => { - const chainId = 'solana'; - const token = 'Gh9ZwEmdLJ8DscKNTkTqPbNwLNNBjuSzaG9Vp2KGtKJr'; - - mockCache.get.mockReturnValue({ - result_type: 'Benign' as TokenScanResultType, - }); - - const result = splitCacheHits(mockCache, chainId, [token], true); - - expect(result.cachedResults[token]).toStrictEqual({ - result_type: 'Benign', - chain: 'solana', - address: token, - }); - expect(result.tokensToFetch).toStrictEqual([]); - }); -}); - describe('getHostnameAndPathComponents', () => { it.each([ [ @@ -1493,3 +1346,17 @@ describe('getHostnameAndPathComponents', () => { expect(result).toStrictEqual(expected); }); }); + +describe('normalizeScanAddress', () => { + it('lowercases EVM addresses', () => { + expect( + normalizeScanAddress('0xAbCdEf0000000000000000000000000000000001'), + ).toBe('0xabcdef0000000000000000000000000000000001'); + }); + + it('leaves non-EVM addresses unchanged', () => { + expect( + normalizeScanAddress('So11111111111111111111111111111111111111112'), + ).toBe('So11111111111111111111111111111111111111112'); + }); +}); diff --git a/packages/phishing-controller/src/utils.ts b/packages/phishing-controller/src/utils.ts index 5cd354b9795..f4cbc550d50 100644 --- a/packages/phishing-controller/src/utils.ts +++ b/packages/phishing-controller/src/utils.ts @@ -2,8 +2,6 @@ import { bytesToHex } from '@noble/hashes/utils'; import { sha256 } from 'ethereum-cryptography/sha256'; import { deleteFromTrie, insertToTrie, deepCopyPathTrie } from './PathTrie.js'; -import type { Hotlist, PhishingListState } from './PhishingController.js'; -import { ListKeys, phishingListKeyNameMap } from './PhishingController.js'; import type { PhishingDetectorList, PhishingDetectorConfiguration, @@ -12,13 +10,15 @@ import { ADDRESS_SCAN_SUPPORTED_CHAINS, APPROVAL_SUPPORTED_CHAINS, DEFAULT_CHAIN_ID_TO_NAME, + ListKeys, + phishingListKeyNameMap, TOKEN_SCAN_SUPPORTED_CHAINS, } from './types.js'; import type { AddressScanSupportedChain, ApprovalSupportedChain, - TokenScanCacheData, - TokenScanResult, + Hotlist, + PhishingListState, TokenScanSupportedChain, } from './types.js'; @@ -121,6 +121,16 @@ export const applyDiffs = ( for (const { isRemoval, targetList, url, timestamp } of diffsToApply) { const targetListType = splitStringByPeriod(targetList)[1]; + // Diffs for list types this client does not know about (for example a + // list introduced server-side after this release) are ignored rather than + // failing the whole update. They do not advance `lastUpdated`, matching + // how diffs for other list keys are treated. + if ( + targetListType !== 'blocklistPaths' && + !Object.hasOwn(listSets, targetListType) + ) { + continue; + } if (timestamp > latestDiffTimestamp) { latestDiffTimestamp = timestamp; } @@ -428,6 +438,21 @@ export const getPhishingDetectionScanUrlParam = ( return [scanUrlParam, true]; }; +const EVM_ADDRESS_REGEX = /^0x[0-9a-fA-F]{40}$/u; + +/** + * Normalizes an address for use in scan requests and cache keys. EVM addresses + * are case-insensitive and are lowercased so that differently-cased inputs + * share one cache entry and match the API's lowercase response keys. Any other + * address (for example a base58 Solana address) is case-sensitive and is + * returned unchanged. + * + * @param address - The address to normalize. + * @returns The normalized address. + */ +export const normalizeScanAddress = (address: string): string => + EVM_ADDRESS_REGEX.test(address) ? address.toLowerCase() : address; + export const getPathnameFromUrl = (url: string): string => { try { const { pathname } = new URL(url); @@ -485,25 +510,6 @@ export const generateParentDomains = ( return domains; }; -/** - * Builds a cache key for a token scan result. - * - * @param chainId - The chain ID. - * @param address - The token address. - * @param caseSensitive - When `true`, the address is kept as-is (for chains - * like Solana where addresses are case-sensitive). When `false` (default), - * the address is lowercased (appropriate for EVM). - * @returns The cache key. - */ -export const buildCacheKey = ( - chainId: string, - address: string, - caseSensitive = false, -) => { - const normalizedAddress = caseSensitive ? address : address.toLowerCase(); - return `${chainId.toLowerCase()}:${normalizedAddress}`; -}; - /** * Determines whether a chain name is supported for token approval scanning. * @@ -583,45 +589,3 @@ export const getAddressScanSupportedChain = ( */ export const isAddressScanSupportedChainId = (chainId: string): boolean => getAddressScanSupportedChain(chainId) !== null; - -/** - * Split tokens into cached results and tokens that need to be fetched. - * - * @param cache - Cache-like object with get method. - * @param cache.get - Method to retrieve cached data by key. - * @param chainId - The chain ID. - * @param tokens - Array of token addresses. - * @param caseSensitive - When `true`, token addresses are kept as-is (for - * chains like Solana where addresses are case-sensitive). When `false` - * (default), addresses are lowercased (appropriate for EVM). - * @returns Object containing cached results and tokens to fetch. - */ -export const splitCacheHits = ( - cache: { get: (key: string) => TokenScanCacheData | undefined }, - chainId: string, - tokens: string[], - caseSensitive = false, -): { - cachedResults: Record; - tokensToFetch: string[]; -} => { - const cachedResults: Record = {}; - const tokensToFetch: string[] = []; - - for (const address of tokens) { - const normalizedAddress = caseSensitive ? address : address.toLowerCase(); - const key = buildCacheKey(chainId, normalizedAddress, caseSensitive); - const hit = cache.get(key); - if (hit) { - cachedResults[normalizedAddress] = { - result_type: hit.result_type, - chain: chainId, - address: normalizedAddress, - }; - } else { - tokensToFetch.push(normalizedAddress); - } - } - - return { cachedResults, tokensToFetch }; -}; diff --git a/packages/phishing-controller/tsconfig.build.json b/packages/phishing-controller/tsconfig.build.json index 73d31ef5e42..113f5e27c9f 100644 --- a/packages/phishing-controller/tsconfig.build.json +++ b/packages/phishing-controller/tsconfig.build.json @@ -5,20 +5,26 @@ "rootDir": "./src" }, "references": [ + { + "path": "../address-book-controller/tsconfig.build.json" + }, { "path": "../base-controller/tsconfig.build.json" }, { - "path": "../controller-utils/tsconfig.build.json" + "path": "../base-data-service/tsconfig.build.json" }, { - "path": "../transaction-controller/tsconfig.build.json" + "path": "../controller-utils/tsconfig.build.json" }, { "path": "../messenger/tsconfig.build.json" }, { - "path": "../address-book-controller/tsconfig.build.json" + "path": "../storage-service/tsconfig.build.json" + }, + { + "path": "../transaction-controller/tsconfig.build.json" } ], "include": ["../../types", "./src"] diff --git a/packages/phishing-controller/tsconfig.json b/packages/phishing-controller/tsconfig.json index 9a8ac800349..0397ba64be3 100644 --- a/packages/phishing-controller/tsconfig.json +++ b/packages/phishing-controller/tsconfig.json @@ -1,20 +1,26 @@ { "extends": "../../tsconfig.packages.json", "references": [ + { + "path": "../address-book-controller" + }, { "path": "../base-controller" }, { - "path": "../controller-utils" + "path": "../base-data-service" }, { - "path": "../transaction-controller" + "path": "../controller-utils" }, { "path": "../messenger" }, { - "path": "../address-book-controller" + "path": "../storage-service" + }, + { + "path": "../transaction-controller" } ], "include": ["../../types", "./src", "./tests"] diff --git a/yarn.lock b/yarn.lock index f7c6d47c78d..f1e7797eb0b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8478,11 +8478,16 @@ __metadata: "@metamask/address-book-controller": "npm:^8.0.0" "@metamask/auto-changelog": "npm:^6.1.0" "@metamask/base-controller": "npm:^10.0.0" + "@metamask/base-data-service": "npm:^2.0.0" "@metamask/controller-utils": "npm:^13.0.0" "@metamask/eth-sig-util": "npm:^9.0.0" "@metamask/messenger": "npm:^3.0.0" + "@metamask/storage-service": "npm:^2.0.0" + "@metamask/superstruct": "npm:^3.4.1" "@metamask/transaction-controller": "npm:^70.0.0" + "@metamask/utils": "npm:^12.0.0" "@noble/hashes": "npm:^1.8.0" + "@tanstack/query-core": "npm:^5.62.16" "@types/jest": "npm:^30.0.0" "@types/punycode": "npm:^2.1.0" "@typescript/native": "npm:typescript@^7.0.2"